from flask import Blueprint, request, render_template, jsonify

from mod_memory.models import Memory

mod_app = Blueprint('app',__name__, url_prefix='/app')

@mod_app.route('/home', methods=['GET'])
def home():
    return "Hello World home"

@mod_app.route('/mems', methods=['GET', 'POST'])
def get_memories():
    print("Hello there I am here")
    print(request.json)
    my_mems = Memory.query.all()

    print(my_mems[0])
    if (not my_mems):
        return "{}"
    else:
        print("mooooooooooooooooooooooooooooooo")
        return jsonify(memories = [{'mem_name': my_mems[0].mem_name,
                                    'mem_info': my_mems[0].mem_info},
                                    {'mem_name': my_mems[0].mem_name,
                                    'mem_info': my_mems[0].mem_info}])


"""
A script for computing various agreement metrics for a given task

* How do individuals compare to aggregated annotations from different demographic groups?
  --> output: raw JSON results, PDF of boxplots, ttest results
  --> location: output/agreement/{task}/agreement_with_aggregate

TO ADD NEW TASKS: add to the dictionaries in src/config/agreement.py
"""
import argparse
import json
import os
from multiprocessing import Pool

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from scipy.stats import mode, ttest_ind

from src.config.agreement import PAIRWISE_AGGREGATE_AGREEMENT_FN_MAP, PAIRWISE_AGREEMENT_FN_MAP
from src.config.data import DEMOGRAPHICS_FN_MATRIX_MAP, REALIABILITY_FN_MATRIX_MAP
from src.agreement.demographic_agreement import agreement_with_aggregate

AGGREGATION_STR_TO_FN = {
    "mean": np.mean,
    "median": np.nanmedian,
    "mode": lambda x: mode(x).mode[0]
}

CONFIG_MAPS = [DEMOGRAPHICS_FN_MATRIX_MAP, PAIRWISE_AGREEMENT_FN_MAP, REALIABILITY_FN_MATRIX_MAP]

def _output_distribution_results(raw_data, output_dir, boxplot_title, boxplot_figname, ttest_pairs):
    """
    Output results where there is a range of values - such that we can perform t-tests and output boxplots
    """
    # save raw results
    with open(os.path.join(output_dir, f"raw_results_{boxplot_figname}.json"), "w") as f:
        json.dump(raw_data, f)

    # create boxplot
    plt.boxplot(raw_data.values(), labels=raw_data.keys())
    plt.title(boxplot_title)
    plt.savefig(os.path.join(output_dir, f"{boxplot_figname}.pdf"))
    plt.close()

    # output results of t-tests
    # NOTE: pvals not corrected for multiple comparisons (corrected in final table creation code)
    ttest_results = []
    for stat0, stat1 in ttest_pairs:
        stat, pval = ttest_ind(raw_data[stat0], raw_data[stat1])
        ttest_results.append((stat0, stat1, stat, pval))
    pd.DataFrame(ttest_results, columns=["dist1", "dist2", "tval", "pval"]).to_csv(
        os.path.join(output_dir, f"ttest_{boxplot_figname}.csv"), index=False)

def _create_dir(output_path: str) -> str:
    os.makedirs(output_path, exist_ok=True)
    return output_path

class AgreementComputer:
    OUTPUT_DIR = "output/agreement"

    def __init__(self, task, n_processes):
        self.task = task
        self.reliability_matrix = REALIABILITY_FN_MATRIX_MAP[task]()
        self.reliability_matrix.sort_index(inplace=True)
        self.demographics = DEMOGRAPHICS_FN_MATRIX_MAP[task]()
        self.pairwise_agreement_fn = PAIRWISE_AGREEMENT_FN_MAP[task]
        # pairwise aggregate fn only needs to be defined if it differs from the typical pairwise fn
        self.pairwise_agreement_aggregate_fn = PAIRWISE_AGGREGATE_AGREEMENT_FN_MAP.get(task, self.pairwise_agreement_fn)

        self.pool = Pool(n_processes) if n_processes > 1 else None

    def agreement_with_aggregate(self, aggregation):
        # if not using mean, shouldn't use pairwise_agreement_aggregate_fn (that switches to interval measure)
        agreement_fn = self.pairwise_agreement_aggregate_fn \
            if aggregation == "mean" else self.pairwise_agreement_fn

        # compute agreement scores
        agreement_data = agreement_with_aggregate(
            self.reliability_matrix, self.demographics, aggregation=AGGREGATION_STR_TO_FN[aggregation],
            agreement_fn=agreement_fn, mp_pool=self.pool)

        out_dir = _create_dir(os.path.join(self.OUTPUT_DIR, self.task, "agreement_with_aggregate"))
        _output_distribution_results(agreement_data, out_dir, f"Agreement with {aggregation}",
            f"agreement_{aggregation}", [("M-ALLM", "M-ALLF"), ("F-ALLF", "F-ALLM"), ("F-ALL", "M-ALL")])

def _parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--task",
                        choices=set.intersection(*[set(x.keys()) for x in CONFIG_MAPS]),
                        required=True,
                        help="The task to run on. Must provide demographics function and reliability function in config files.")
    parser.add_argument("--aggregation",
                        choices=AGGREGATION_STR_TO_FN.keys(),
                        required=True,
                        help="The way to aggregate individual annotator's annotations.")
    parser.add_argument("--n_processes",
                        type=int,
                        default=1,
                        help="The number of processes to run.")
    return parser.parse_args()

def main():
    args = _parse_args()
    ac = AgreementComputer(args.task, args.n_processes)
    ac.agreement_with_aggregate(args.aggregation)

if __name__ == "__main__":
    main()


import os
import random
import numpy as np

import tensorflow as tf
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam

from src.dqnCartPoolM.CartPool import CartPole
from src.dqnCartPoolM.controllers.CartController import CartController

class DqnCartController(CartController):
    def __init__(self,
                 train_model=False,
                 gamma=0.85,
                 epsilon=1.0,
                 epsilon_decay=0.995,
                 epsilon_min=0.01,
                 learning_rate=0.005,
                 batch_size=32,
                 tau=0.125,
                 save_freq=10,
                 load_path=None,
                 save_path=None,
                 log_path="data/logs/dqn_v1",
                 episode_record=False,
                 print_model=False,
                 label="",
                 render_freq=10
                 ):
        super().__init__(episode_record=episode_record, label=label)
        # self.log_path = log_path
        self.render_freq = render_freq
        self.print_model = print_model
        self.save_path = save_path
        self.load_path = load_path
        self.save_freq = save_freq

        self.train_model = train_model

        self.gamma = gamma  # discount factor
        self.epsilon = epsilon  # amount of randomness in e-greedy policy
        self.epsilon_min = epsilon_min
        self.epsilon_decay = epsilon_decay  # exponential decay
        self.learning_rate = learning_rate
        self.batch_size = batch_size
        self.tau = tau  # target model update

        self.model = self.create_model()
        self.target_model = self.create_model()
        self.target_model.set_weights(self.model.get_weights())

        if load_path:
            self.load_model(load_path=self.load_path)

        self.max_reward = 0

        # self.summaries = {}
        # self.data_log = {"reward": [], "loss": []}

        if not train_model:
            self.cart_pole.set_render_true()

    def act(self, test=False):
        states = self.cart_pole.stored_states.reshape((1, self.cart_pole.space_size * self.cart_pole.time_steps))
        self.epsilon *= self.epsilon_decay
        self.epsilon = max(self.epsilon_min, self.epsilon)
        epsilon = 0.01 if test else self.epsilon  # use epsilon = 0.01 when testing
        q_values = self.model.predict(states)[0]
        self.summaries['q_val'] = max(q_values)
        if np.random.random() < epsilon:
            return self.cart_pole.env.action_space.sample()  # sample random action
        return np.argmax(q_values)

    def create_model(self):
        model = Sequential()
        model.add(Dense(24, input_dim=self.cart_pole.space_size * self.cart_pole.time_steps, activation="relu"))
        model.add(Dense(32, activation="relu"))
        model.add(Dense(48, activation="relu"))
        model.add(Dense(self.cart_pole.a_space_size))
        model.compile(loss="mean_squared_error", optimizer=Adam(lr=self.learning_rate))
        if self.print_model:
            model.summary()
        return model

    def episode_start(self):
        super().episode_start()
        if self.train_model:
            self.cart_pole.set_render_false()
            if self.cart_pole.episode_num % self.render_freq == 0 and self.render_freq > 0:
                self.cart_pole.set_render_true()
        else:
            self.cart_pole.set_render_true()

    def update(self):
        if self.train_model:
            self.replay()  # iterates default (prediction) model through memory replay
            self.target_update()  # iterates target model

    def episode_end(self):
        super().episode_end()
        total_reward = self.cart_pole.total_reward
        episode_num = self.cart_pole.episode_num

        if self.save_path:
            # if episode_num % self.save_freq == 0:
            #     full_path = os.path.join(self.save_path, "model_episode_{}_reward_{}.h5".format(episode_num,
            #                                                                                total_reward))
            #     self.save_model(full_path)

            if total_reward > self.max_reward or total_reward == self.max_reward:
                self.max_reward = total_reward
                full_path = os.path.join(self.save_path, "model_max_episode_{}_reward_{}.h5".format(episode_num,
                                                                                int(total_reward)))
                self.save_model(full_path)

    def replay(self):
        if len(self.cart_pole.memory) < self.batch_size:
            return

        samples = random.sample(self.cart_pole.memory, self.batch_size)
        states, action, reward, new_states, done = map(np.asarray, zip(*samples))
        batch_states = np.array(states).reshape(self.batch_size, -1)
        batch_new_states = np.array(new_states).reshape(self.batch_size, -1)
        batch_target = self.target_model.predict(batch_states)
        q_future = self.target_model.predict(batch_new_states).max(axis=1)
        batch_target[range(self.batch_size), action] = reward + (1 - done) * q_future * self.gamma
        hist = self.model.fit(batch_states, batch_target, epochs=1, verbose=0)
        self.summaries['loss'] = np.mean(hist.history['loss'])

    def end(self):
        if self.train_model:
            super().end()

    def target_update(self):
        weights = self.model.get_weights()
        target_weights = self.target_model.get_weights()
        for i in range(len(target_weights)):  # set tau% of target model to be new weights
            target_weights[i] = weights[i] * self.tau + target_weights[i] * (1 - self.tau)
        self.target_model.set_weights(target_weights)

    def save_model(self, save_path):
        if not self.train_model:
            return
        # save model to file, give file name with .h5 extension
        print("Model saved to: {}".format(save_path))
        self.model.save(save_path)

    def load_model(self, load_path):
        # load model from .h5 file
        self.model = tf.keras.models.load_model(load_path)
        self.target_model = self.create_model()
        self.target_model.set_weights(self.model.get_weights())


"""
this library contains a number of useful functions that are general purpose
data processing functions.
"""

###############################################################################
### import libraries
#from __init__ import (_np,_copy,_pd,_plt,_plot,_math)

### import libraries

# common libraries
import numpy as _np
import matplotlib.pyplot as _plt
import copy as _copy
import math as _math

# hbtepLib libraries
import _plotTools as _plot

###############################################################################
### misc functions

def convertDataToStairstepData(x,y):
    """
    When dealting with discrete data, often it makes sense to show the data
    look like stair-steps insteady of a the more typical smooth plot.  This
    function makes this happen #TODO(John) there is a better way to explain
    this...

    Example
    -------
    x=np.arange(0,1,0.01)
    y=np.sin(2*np.pi*12.345*x)
    (xOut,yOut)=hbt.process.convertDataToStaircaseDate(x,y)
    plt.plot(xOut,yOut)

    """

    xOut=_np.zeros(len(x)*2)
    yOut=_np.zeros(len(x)*2)
    dx=x[1]-x[0];
    for i in range(0,len(x)):
#        print yOut[i*2]
#        print y[i]
        xOut[2*i]=x[i]
        if i == len(x)-1:
            xOut[2*i+1]=xOut[2*i]+dx
        else:
            xOut[2*i+1]=x[i+1]
        yOut[2*i]=y[i]
        yOut[2*i+1]=y[i]
    return (xOut,yOut)

def findNearest(array,value):
    """
    search through `array` and returns the `index` of the cell closest to the
    `value`.   `array` should be sorted in ascending order

    Parameters
    ----------
    array : numpy.array
        data array to search through
    value : float (or int)
        value to look for in array

    Return
    ------
    index : int
        index of value in array that is closest to value

    References
    ----------
    http://stackoverflow.com/questions/2566412/find-nearest-value-in-numpy-array
    """
    index = (_np.abs(array-value)).argmin()
    # value = array[index]
    return index
    # return index, value # uncomment to return both the index AND the value

def rmse(data, targets=0):
    """
    Root mean square error function.
    If targets = 0, this is also the root mean square function.

    Parameters
    ----------
    data : numpy.array
        Data being considered
    target : numpy.array of floats
        values being compared against

    Return
    ------
    : numpy.array
        root mean square error of data

    References
    ----------
    # http://stackoverflow.com/questions/17197492/root-mean-square-error-in-python
    # http://statweb.stanford.edu/~susan/courses/s60/split/node60.html
    """
    return _np.sqrt(((data - targets) ** 2).mean())

def rms(data):
    """
    Root mean square function.

    Parameters
    ----------
    data : numpy.array
        Data to be processed

    Return
    ------
    : numpy.array
        root mean square of data

    References
    ----------
    # http://stackoverflow.com/questions/17197492/root-mean-square-error-in-python
    # http://statweb.stanford.edu/~susan/courses/s60/split/node60.html
    """
    return _np.sqrt(((data - 0) ** 2).mean())

def rejectOutliers(data, sigma=2):
    """
    remove outliers from set of data

    Parameters
    ----------
    data : numpy.ndarray
        data array being considered
    sigma : int
        the number of std. devs. about which to reject data.  E.g. sigma=2
        rejects outliers outside of +-2*sigma

    Return
    ------
     : numpy.ndarray
        Same as databut missing entires considered as outliers
    indicesToKeep : numpy.ndarray (of bool)
        Boolean indices associated with entries of data that are kept

    References
    ----------
    http://stackoverflow.com/questions/11686720/is-there-a-numpy-builtin-to-reject-outliers-from-a-list
    """
    indicesToKeep=abs(data - _np.mean(data)) < sigma* _np.std(data)
    return data[indicesToKeep],indicesToKeep

def rmPhaseJumps(time,data,cut=_np.pi):
    """
    Removed phase jumps of greater than "cut", defaults to pi
    Assumes phase is wrapped.
    Inserts NaN at jump points in data and time vector
    """
    indx=_np.argwhere(_np.abs(_np.diff(data))>=cut)+1
    if indx[-1]==len(data):indx[-1]-=1# in case last point jumps
    data=list(data)
    time=list(time)
    # Iterate over jump points, insert NaNs
    for i in indx:
        data.insert(i,_np.NaN)
        time.insert(i,_np.NaN)
        indx += 1
    return _np.array(time),_np.array(data)

def wrapPhase(data):
    """
    Wraps phase data so that it repeats every 2pi.
    This is important for phase data when you want it to all fit nicely on a
    plot.

    Parameters
    ----------
    data : numpy.ndarray
        data being wrapped

    Return
    ------
    outData : numpy.ndarray
        wrapped data array

    """
    inData=data*1.0
    outData=_np.zeros(inData.size);
    inData-=_np.pi;
    for i in range(0,_np.size(inData)-1):
        a=_np.floor(inData[i]/(_np.pi*2))
        outData[i]=inData[i]-(a+1)*2*_np.pi+_np.pi
    return outData

def unwrapPhase(inData):
    """
    Takes in phase array (in radians).  I think it needs to be centered about 0.
    Unwraps phase data so that it is continuous.
    This is important for phase data when you want to take it's derivative to
    get frequency.

    Parameters
    ----------
    data : numpy.ndarray
        data being unwrapped

    Return
    ------
    outData : numpy.ndarray
        unwrapped data array

    """
    outData=_np.zeros(_np.size(inData));
    offset=0;
    outData[0]=inData[0];
    for i in range(0,_np.size(inData)-1):
        if inData[i] > _np.pi/4 and inData[i+1] < -_np.pi/4:
            offset=offset+2*_np.pi;
        elif inData[i] < -_np.pi/4 and inData[i+1] > _np.pi/4:
            offset=offset-2*_np.pi;
        outData[i+1]=inData[i+1]+offset;
    return outData

def hasNan(inArray):
    """
    searches array, inArray, for any occurances of NaN.  returns True if
    yes, returns False if no.

    Parameters
    ----------
    inArray : numpy.ndarray
        data array being considered for NaN entries

    Return
    ------
        : bool
        True if NaNs are in array, False otherwise
    """
    count = 0;
    for i in range(0,len(inArray)):
        if _math.isnan(inArray[i]):
            count+=1;

    print("There was/were %d instances of NaN" % count)

    if count == 0:
        return False
    if count != 0:
        return True

def sort2Arrays(array1, array2):
    """
    sorts array1 and array2 in ascending order of array1

    outdated:  replaced by sortArrays() below
    """
    array1, array2 = zip(*sorted(zip(array1, array2)));
    array1=_np.array(array1);
    array2=_np.array(array2);
    return array1, array2

def sortArrays(arrays,sortIndex):
    """
    sorts a list of n arrays.  sortIndex is the index of the array to sort all arrays.

    example use:  [V, I, phi]=sortArrays([V,I,phi],1) to sort all arrays by ascending I

    reference:  https://stackoverflow.com/questions/6618515/sorting-list-based-on-values-from-another-list
    """
    sortedIndices=arrays[sortIndex].argsort()
    for i in range(len(arrays)):
        arrays[i]=arrays[i][sortedIndices]
    return arrays

def downSampleData(downX,upX,data):
    """
    Down samples data by finding the nearest x values on data that match the
    downselecting x (downX)

    Sometimes, you want to compare two different sets of data that do not have
    the same time basis.  This function down-samples the data set with more
    data points so that its x-data matches the other

    Parameters
    ----------
    downX : numpy.ndarray
        the x-data that will provide the down smpaled reference to the
        upsampled x-data
    upX : numpy.ndarray
        the upsampled x-data that will be downsampled
    data : list (of numpy.array)
        the upsampled y-data that will be downsampled

    Returns
    -------
    out : list (of np.ndarray)
        list of trimmed y-data

    Notes
    -----
    upX is not actually trimmed in this instance.  it is assumed that you user
    will use downX as their new time basis
    """
    if type(data) is not list:
        data=[data]

    m=len(downX)
    indices=_np.zeros(m,dtype=_np.int16)
    out = []

    for i in range(0,m):
        indices[i]=int(findNearest(upX,downX[i]))

    for i in range(0,len(data)):
        out.append(data[i][indices])
    return out

def upSampleData(upX,downX,data):
    """
    Up samples data by linear interpolating

    Similar to downSampleData() but up-samples instead

    Parameters
    ----------
    upX : np.ndarray
        the x-data that will be used in up-sampling the under-sampled data
    downX : np.ndarray
        the undersampled x-data
    data : list (of np.ndarray)
        the y-data to be up-samples.  downX is its time-base before
        up-sampling.  upX will be its time-base after up-sampling

    Returns
    -------
    out : list (of np.ndarray)
        list of up-sampled y-data
    """
    out = _np.interp(upX,downX,data)
    return out

def linearizeDataMatrix(data):
    """
    data is assumed to be a list of arrays

    this function converts the data to a single, appended array
    """
    m=len(data);
    temp=_np.array([])
    for i in range(0,m):
        temp=_np.append(temp,data[i])
    return temp

###############################################################################
### filters and smoothing algorithms

def downSample(data,time,new_dT,shftStart=False):
    if new_dT%_np.mean(_np.diff(time))>=1e-7:
        raise SyntaxError("Sampling rate must be integer multiple of original: %e vs %e"%(_np.mean(_np.diff(time)),new_dT))
    dS=(new_dT/_np.mean(_np.diff(time))).astype(int)

    # Verify data shape
    if data.shape[0] != len(time):data=data.T

    # Make smoothing kernel
    boxMat=_np.zeros((len(time)/dS,len(time)))
    for i in range(len(boxMat)):boxMat[i,i*dS:(i+1)*dS]=1./dS

    return _np.matmul(boxMat,data),_np.matmul(boxMat,time)-shftStart*new_dT*(3./2)#(dS/2)#*_np.mean(_np.diff(time))

def nPoleFilter(data,xData=None,numPoles=1,alpha=0.0625,filterType='lowPass',plot=False):
    """
    n-pole filter.

    Parameters
    ----------
    data : numpy.ndarray
        data to be smoothed
    xData : numpy.ndarray or NoneType
        (optional) array of x-data
    numPoles : int
        number of poles for the filter
    alpha : float
        weight of the filter, float between 0 and 1.  Close to zero for a low
        pass and close to 1 for a high pass
    filterType : str
        'lowPass' - Low pass filter
        'highPhass' - High pass filter
    plot : bool
        plots results if true

    Returns
    -------
    filteredData : 2D numpy.ndarray
        filtered data

    References
    ----------
    http://techteach.no/simview/lowpass_filter/doc/filter_algorithm.pdf
    https://en.wikipedia.org/wiki/Low-pass_filter#Discrete-time_realization
    https://en.wikipedia.org/wiki/High-pass_filter#Discrete-time_realization

    Notes
    -----
    this method is pulled from Qian Peng's 2016 GPU code.
    his highpass filter does NOT follow this code

    Example #1
    ----------
    t=np.arange(0,.01,6e-8);
    y2=np.sin(2*np.pi*300*t)+np.sin(2*np.pi*30000*t)
    alpha=0.00625;
    hbt.process.nPoleFilter(y2,t,numPoles=2,filterType='lowPass',plot=True,alpha=alpha)
    hbt.process.nPoleFilter(y2,t,numPoles=2,filterType='highPass',plot=True,alpha=1.0-0.000625)

    Example #2
    ----------
    t=np.arange(0,.01,6e-6);
    from scipy.signal import square
    y=square(t,0.001)
    hbt.process.nPoleFilter(y,t,numPoles=2,filterType='lowPass',plot=True)
    hbt.process.nPoleFilter(y,t,numPoles=2,filterType='highPass',plot=True)

    """

    # initialize data arrays
    procData=_np.zeros((numPoles+1,len(data)));
    procData[0,:]=data;

    # filter.  for loop controls the number of poles
    for i in range(1,numPoles+1):

        if filterType=='lowPass':
            for j in range(0,len(data)-1):
                procData[i,j+1]=procData[i,j]+alpha*(procData[i-1,j]-procData[i,j])
        elif filterType=='highPass':
            for j in range(0,len(data)-1):
                procData[i,j+1]=alpha*(procData[i,j]+procData[i-1,j+1]-procData[i-1,j])

    # plot results
    if plot==True:
        p1=_plot.plot(title=str(numPoles)+" pole "+filterType+" filter, alpha="+str(alpha),
                   xLabel="x-axis",yLabel='y-axis')

        if type(xData)==type(None):
            xData=_np.arange(0,len(data));

        p1.addTrace(xData=xData,yData=data,yLegendLabel='raw')

        for i in range(1,numPoles+1):
            p1.addTrace(xData=xData,yData=procData[i,:],yLegendLabel=str(i))

        p1.plot()

    # return results
    return procData

def savgolFilter(data,numPoints,polynomialOrder,plot=False):
    """
    Ssavitzky-Golay moving average smoothing filter.  Applies a nth order
    polynomial to a moving window of data.

    Parameters
    ----------
    data : numpy.ndarray
        data to be smoothed
    numPoints : int
        number of points for smoothing
    method : str
        method to use
        'box' - box car type of smoothing
        'gaussian' - guassian or normal smoothing
    plot : bool
        plots results if true

    Returns
    -------
    smoothedData : numpy.ndarray
        smoothed data

    References
    ----------
    https://docs.scipy.org/doc/scipy-0.16.1/reference/generated/scipy.signal.savgol_filter.html
    https://stackoverflow.com/questions/20618804/how-to-smooth-a-curve-in-the-right-way

    Notes
    -----
    this is a wrapper for scipy.signal.savgol_filter

    I think I prefer the gaussian convolution algorithm over this one (john)
    """

    def plotResults():
        """
        plots results (before and after smoothing)
        """
        p1=_plot.plot()
        x=_np.arange(0,len(data))
        p1.xData=[x,x]
        p1.yData=[data,smoothedData]
        p1.yLegendLabel=['raw data','smoothed data']
        p1.marker=['.','']
        p1.linestyle=['','-']
        p1.title='%d point, Savitzky-Golay smoothing of order %s'% (numPoints,polynomialOrder)
        p1.plot()

    # import savgol package
    from scipy.signal import savgol_filter

    # perform filter
    smoothedData=savgol_filter(data,numPoints,polynomialOrder)

    # plot results if requested
    if plot==True or plot=='all':
        plotResults()

    return smoothedData

def convolutionSmoothing(data,numPoints,method='gaussian',plot=False):
    """
    Convolution moving average smoothing filter

    Parameters
    ----------
    data : numpy.ndarray
        data to be smoothed
    numPoints : int
        number of points for smoothing.  should be an odd number
    method : str
        method to use
        'box' - box car type of smoothing.  keywords: boxcar
        'gaussian' - guassian or normal smoothing
    plot : bool
        plots results if true

    Returns
    -------
    smoothedData : numpy.ndarray
        smoothed data

    References
    ----------
    https://stackoverflow.com/questions/20618804/how-to-smooth-a-curve-in-the-right-way

    Example 1
    ---------
    # randon noise on sine way
    x = np.arange(0,2*np.pi,.1)
    y1 = np.sin(x) + np.random.random(len(x)) * 0.8
    convolutionSmoothing(y,numPoints,method='box',plot='all')

    Example 2
    ---------
    # step function
    x = np.arange(0,2*np.pi,.1)
    y2=np.zeros(len(x))
    y2[np.where(x>np.pi)[0]]=1
    convolutionSmoothing(y,numPoints,method='box',plot='all')

    Notes
    -----
    I want to justiy some of my weird code below.  When numPoints is an even
    number, the filter centers itself 1.5 points ahead of itself in the window
    which results in a small time/phase shift in the filtered data.  When it's
    an odd number, it centers itself 1.0 points ahead in the window and
    smaller time/phase shift.  By requiring that numPoints is odd and
    temporarily removing the first point in the data (I put it back in the
    end), I center the filter in the window and have no time/phase shift.

    The convolution filters "mess up" the last set of points (on the order of
    numPoints).  If you want to get around this, you should give it more data
    than you actually want and trim the end off.
    """
    def plotResults():
        """
        plots before and after of smoothing
        """
        p1=_plot.plot()
        x=_np.arange(0,len(data))
        p1.xData=[x,x]
        p1.yData=[data,smoothedData]
        p1.yLegendLabel=['raw data','smoothed data']
        p1.marker=['.','']
        p1.linestyle=['','-']
        p1.title='%d point, %s smoothing'% (numPoints,method)
        p1.plot()

    def plotSmoothingFunction():
        """
        plots smoothing function
        """
        p1=_plot.plot()
        x=_np.arange(0,len(smoothingFunction))
        p1.xData=[x]
        p1.yData=[smoothingFunction]
        p1.yLegendLabel=['smoothing function']
        p1.marker=['o']
        p1.linestyle=['-']
        p1.title='%d point, %s smoothing function'% (numPoints,method)
        p1.plot()

    # make sure numPoints is an odd number
    if (numPoints % 2 == 0): # is an even number
        numPoints+=1;
        print("Warning: numPoints was not an odd number.  +1 was added.  "
              "numPoints is now %d" % numPoints)

    # temporarily remove first point (see Notes)
    tempPoint=data[0]
    data=data[1:]

    # box smoothing
    if method=='box':
        smoothingFunction=_np.ones(numPoints)/numPoints

    # gaussian smoothing
    elif method=='gaussian' or method == 'normal':
        temp=_np.arange(numPoints)
        sigma=numPoints/(2*_np.pi);
        smoothingFunction=_np.exp(-((temp-(numPoints-1)/2.)/sigma)**2/2)
        smoothingFunction/=_np.sum(smoothingFunction)

    # perform smoothing
    smoothedData = _np.convolve(data, smoothingFunction, mode='same')

    # plot if requested
    if plot==True:
        plotResults()
    if plot=='all':
        plotSmoothingFunction()
        plotResults()

    # add point back to smoothedData (see Notes
    smoothedData=_np.append(tempPoint,smoothedData)

    return smoothedData

def gaussianFilter(t,y,timeFWHM,filterType='high',plot=False,plotGaussian=False):
    """
    Low and pass filters using scipy's gaussian convolution filter

    Parameters
    ----------
    t : numpy.array
        time
    y : numpy.array
        time dependent data
    timeFWHM : float
        full width at half maximum of the gaussian with units in time.  this
        effectively sets the corner frequency of the filter
    filterType : str
        'high' - high-pass filter
        'low' - low-pass filter
    plot : bool
        plots the results
    plotGaussian : bool
        plots the gaussian distribution used for the filter

    Returns
    -------
    yFiltered : numpy.array
        filtered time dependent data

    References
    ----------
    https://en.wikipedia.org/wiki/Full_width_at_half_maximum
    https://docs.scipy.org/doc/scipy-0.19.0/reference/generated/scipy.signal.gaussian.html
    https://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.gaussian_filter1d.html
    """

    dt=t[1]-t[0]

#    from scipy import signal

    from scipy.ndimage import gaussian_filter1d

    def fwhmToGaussFilterStd(fwhm,dt):

        std=1.0/_np.sqrt(8*_np.log(2))*fwhm/dt
        return std

    std=fwhmToGaussFilterStd(timeFWHM,dt)
#    yFiltered=signal.gaussian(len(t), std=std)

#    if filterType=='low':
    yFiltered=gaussian_filter1d(y*1.0,std,mode='nearest')
#    elif filterType=='high':
#        yFiltered=y-gaussian_filter1d(y*1.0,std)

    if plot==True:

        _plt.figure()
        _plt.plot(t,y,label='Raw')
        _plt.plot(t,yFiltered,label='Low-pass')
        _plt.plot(t,y-yFiltered,label='High-pass')
        _plt.legend()

    if plotGaussian==True:

        from scipy import signal
        _plt.figure()
        _plt.plot(t,signal.gaussian(len(t), std=std),label='gaussian')
        _plt.legend()

    if filterType=='low':
        return yFiltered
    else:
        return y-yFiltered

def gaussianLowPassFilter(y,t,timeWidth=1./20000,plot=False,plotGaussian=False):
    """
    Low pass filter using scipy's gaussian filters

    Parameters
    ----------
    y : numpy.array
        time dependent data
    t : numpy.array
        time
    timeWidth : float
        full width at half maximum of the gaussian with units in time.  this
        effectively sets the corner frequency of the filter
        (f_{corner} \approx 1/timeWidth)
    plot : bool
        plots the results
    plotGaussian : bool
        plots the gaussian distribution used for the filter

    Returns
    -------
    yFiltered : numpy.array
        filtered time dependent data

    References
    ----------
    https://en.wikipedia.org/wiki/Full_width_at_half_maximum
    https://docs.scipy.org/doc/8scipy-0.19.0/reference/generated/scipy.signal.gaussian.html
    https://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.gaussian_filter1d.html

    Example
    -------
    import numpy as np
    t=np.arange(0,10e-3,2e-6)
    y = np.random.randn(len(t)).cumsum()
    y+=np.sin(2*np.pi*1000+np.pi*2*np.random.rand())
    y+=np.sin(2*np.pi*3300+np.pi*2*np.random.rand())
    y+=np.sin(2*np.pi*10000+np.pi*2*np.random.rand())
    y+=np.sin(2*np.pi*33000+np.pi*2*np.random.rand())
    gaussianLowPassFilter(y,t,timeWidth=1e-4,plot=True,plotGaussian=True)
    """

    from scipy.ndimage import gaussian_filter1d

    dt=t[1]-t[0]
    #1/(dt*timeWidth*2*_np.pi)#
    sigma= (1./(2*_np.pi))*timeWidth/dt#2.355*timeWidth/dt#  #TODO(John)  This equation is wrong.  Should be dividing by 2.355, not multiplying.  Fix here and with all dependencies
    yFiltered=gaussian_filter1d(y,sigma)

    if plot==True:

        _plt.figure()
        _plt.plot(t,y,label='Raw')
        _plt.plot(t,yFiltered,label='Filtered')
        _plt.legend()
        _plt.grid()

    if plotGaussian==True:

        from scipy import signal
        _plt.figure()
        _plt.plot(t,signal.gaussian(len(t), std=sigma),label='gaussian')
        _plt.legend()

    return yFiltered

def gaussianHighPassFilter(y,t,timeWidth=1./20000,plot=False,plotGaussian=False):
    """
    High pass filter using scipy's gaussian filters

    Parameters
    ----------
    y : numpy.array
        time dependent data
    t : numpy.array
        time
    timeWidth : float
        full width at half maximum of the gaussian with units in time.  this
        effectively sets the corner frequency of the filter
        (f_{corner} \approx 1/timeWidth)
    plot : bool
        plots the results
    plotGaussian : bool
        plots the gaussian distribution used for the filter

    Returns
    -------
    yFiltered : numpy.array
        filtered time dependent data

    References
    ----------
    https://en.wikipedia.org/wiki/Full_width_at_half_maximum
    https://docs.scipy.org/doc/scipy-0.19.0/reference/generated/scipy.signal.gaussian.html
    https://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.gaussian_filter1d.html

    Example
    -------
    import numpy as np
    t=np.arange(0,10e-3,2e-6)
    y = np.random.randn(len(t)).cumsum()
    y+=np.sin(2*np.pi*1000+np.pi*2*np.random.rand())
    y+=np.sin(2*np.pi*3300+np.pi*2*np.random.rand())
    y+=np.sin(2*np.pi*10000+np.pi*2*np.random.rand())
    y+=np.sin(2*np.pi*33000+np.pi*2*np.random.rand())
    gaussianHighPassFilter(y,t,timeWidth=1./20000,plot=True,plotGaussian=True)
    """
    fit=gaussianLowPassFilter(y,t,timeWidth,plot=False,plotGaussian=plotGaussian)
    yFiltered= y-fit

    if plot==True:

        _plt.figure()
        _plt.plot(t,y,label='Raw')
        _plt.plot(t,fit,label='Fit')
        _plt.plot(t,yFiltered,label='Filtered')
        _plt.legend()
        _plt.grid()

    return yFiltered, fit

def butterworthFilter(y, x,filterOrder=2, samplingRate=1/(2*1e-6),
                      cutoffFreq=20*1e3, filterType='low',plot=False):
    """
    Apply a digital butterworth filter on your data

    Parameters
    ----------
    y : numpy.ndarray
        unfiltered dependent data
    x : numpy.ndarray
        independent data
    filterOrder : int
        Butterworth filter order
    samplingRate : float
        Data sampling rate.
    cutoffFreq : float
        cutoff frequency for the filter
    filterType : str
        filter type.  'low' is lowpass filter
    plot : bool or str
        - True - plots filter results.
        - 'all'- plots filter results and filter response (psuedo-BODE plot)

    Returns
    -------
    filteredData : numpy.ndarray
        Filtered dependent data

    References
    ----------
    https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.freqz.html
    https://stackoverflow.com/questions/25191620/creating-lowpass-filter-in-scipy-understanding-methods-and-units
    """

    from scipy.signal import butter, lfilter, freqz

    def butter_lowpass(cutoff, fs, order=5):
        nyq = 0.5 * fs
        normal_cutoff = cutoff / nyq
        b, a = butter(order, normal_cutoff, btype=filterType, analog=False)
        return b, a

    def butter_lowpass_filter(data, cutoff, fs, order=5):
        b, a = butter_lowpass(cutoff, fs, order=order)
        y = lfilter(b, a, data)
        return y

    def plotOfFreqResponse():

        # Get the filter coefficients so we can check its frequency response.
        b, a = butter_lowpass(cutoffFreq, samplingRate, filterOrder)

        # calc frequency response
        w, h = freqz(b, a, worN=8000)
        gain=_np.abs(h)
        phase=_np.unwrap(_np.angle(h))*180/_np.pi

        # generate gain plot
        p1=_plot.plot()
        p1.xLim=[0, 0.5*samplingRate]
        p1.xLabel='Frequency [Hz]'
        p1.subtitle="Gain Response"
        p1.yLabel='Gain'
        p1.yLim=[0,1.]
        p1.title='Butterworth filter. Order=%d. Cutoff Freq=%.1f. Sampling Rate = %.1f Hz. Type = %s.' % (filterOrder, cutoffFreq, samplingRate, filterType)

        p1.xData.append(0.5*samplingRate*w/_np.pi)
        p1.yData.append(gain)
        p1.yLegendLabel.append('Frequency Response')

        p1.xData.append(_np.array([cutoffFreq,cutoffFreq]))
        p1.yData.append(_np.array([0,1])) #0.5*_np.sqrt(2)
        p1.yLegendLabel.append('Cuttoff Frequency')

        # generate phase plot
        p2=_plot.plot()
        p2.xLim=[0, 0.5*samplingRate]
        p2.xLabel='Frequency [Hz]'
        p2.subtitle="Phase Response"
        p2.yLabel='Phase [Degrees]'
        p2.yLim=[_np.min(phase),0]

        p2.xData.append(0.5*samplingRate*w/_np.pi)
        p2.yData.append(phase)
        p2.yLegendLabel.append('Frequency Response')

        p2.xData.append(_np.array([cutoffFreq,cutoffFreq]))
        p2.yData.append(_np.array([_np.min(phase),0])) #0.5*_np.sqrt(2)
        p2.yLegendLabel.append('Cuttoff Frequency')

        # combine into subplot
        sp1=_plot.subPlot([p1,p2],plot=False)

        return sp1

    def plotOfResults():
        p1=_plot.plot()

        p1.xData.append(x)
        p1.yData.append(y)
        p1.yLegendLabel.append('Unfiltered Data')

        p1.xData.append(x)
        p1.yData.append(filteredData)
        p1.yLegendLabel.append('Filtered Data')

        return p1

    filteredData=butter_lowpass_filter(y, cutoffFreq,
                                       samplingRate, filterOrder)

    if plot==True:
        plotOfResults().plot()

    if plot == 'all':
        plotOfResults().plot()
        plotOfFreqResponse().plot()

    return filteredData

###############################################################################
### fitting functions and related

class polyFitData:
    """
    Polynomial fit function.

    Parameters
    ----------
    yData : 'numpy.array'
        dependent variable
    xData : 'numpy.array'
        independent variable
    order : int
        order of polynomial fit.  1 = linear, 2 = quadratic, etc.
    plot : bool
        Causes a plot of the fit to be generated

    Attributes
    ----------
    coefs : 'numpy.array'
        array of fit coefficients, starts at highest order.
    ffit : 'numpy.lib.polynomial.poly1d'
        function that returns yFit data given ANY numpy.array of x values
    fitData : 'numpy.array'
        yFit data corresponding to xData
    plotOfFit :
        custom plot class of data.

    Notes
    -----
    This is merely a wrapper function for the numpy.polyfit function.  However,
    it also plots the result automatically.

    output:
    fitData is the y fit data.
    """

    def __init__(self, yData, xData,order=2, plot=True):
        title = str(order)+' order Polynomial fit'

        ### do fit
        self.coefs=_np.polyfit(xData, yData, order)
        self.ffit = _np.poly1d(self.coefs)
        self.fitData=self.ffit(xData)

        ### generate plot
        self.plotOfFit=_plot.plot();
        self.plotOfFit.xLabel='x'
        self.plotOfFit.yLabel='y'
        self.plotOfFit.title=title;

        ### raw data
        self.plotOfFit.xData.append(xData)
        self.plotOfFit.yData.append(yData)
        self.plotOfFit.marker.append('.')
        self.plotOfFit.linestyle.append('')
        self.plotOfFit.alpha.append(.15)
        self.plotOfFit.yLegendLabel.append('raw data')

        ### fit
        x=_np.linspace(_np.min(xData),_np.max(xData),1000);
        self.plotOfFit.xData.append(x)
        self.plotOfFit.yData.append(self.ffit(x))
        self.plotOfFit.marker.append('')
        self.plotOfFit.linestyle.append('-')
        self.plotOfFit.alpha.append(1.)
        self.plotOfFit.yLegendLabel.append('poly fit order %d'%order)

        if plot==True:
            self.plotOfFit.plot()

class genericCurveFit:
    """
    generic curve fitting function that uses scipy.optimize.curve_fit solution

    I "think" I like the genericLeastSquaresFit code better than this function.  See below.

    func = fit function
    indepVars = independent variables.  for multivariable, use indepVars = (x,y) etc.
    depVars = dependent variable.  this is the single dependent variable that we are trying to model
    guess = guess parameters.  use: guess = 8., 2., 7. etc.

    note:  this function CAN be upgraded to include bounds

    references:
        https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.curve_fit.html

    Example use
    -----------
    def dumbModel(variables,a,b,c):
        x,y=variables
        return _np.log(a) + b*_np.log(x) + c*_np.log(y)

    # some artificially noisy data to fit
    x = _np.linspace(0.1,1.1,101)
    y = _np.linspace(1.,2., 101)
    a, b, c = 10., 4., 6.
    z = dumbModel((x,y), a, b, c) * 1 + _np.random.random(101) / 100

    # initial guesses for a,b,c:
    p0 = 8., 2., 7.

    # solve
    indepVars=(x,y)
    a=genericCurveFit(dumbModel, indepVars, z, p0)

    """

    def __init__(self,func, indepVars,depVar,guess, bounds=None,plot=True , maxNumIterations = None, fileName=''):
        self.indepVars=indepVars;
        self.depVars=depVar
        self.fileName=fileName

        from scipy.optimize import curve_fit

        if bounds == None:
            self.fitParams,self.covMatrix = curve_fit(func, indepVars, depVar, guess) #, max_nfev = 1e2 , max_nfev=maxNumIterations
        else:
            self.fitParams,self.covMatrix = curve_fit(func, indepVars, depVar, guess, bounds=bounds) #, max_nfev = 1e2 , max_nfev=maxNumIterations
#        print self.fitParams
#        print self.covMatrix
        self.fitSoln=func(indepVars, *self.fitParams)
#        print fitSoln

        self.R2=rSquared(indepVars, self.fitSoln)
        print(r"$R^2$ = %.3E" % self.R2)

        if plot==True:
            self.plot();

    def plot(self):
        _plt.figure()
        _plt.plot(self.fitSoln, self.depVars, '.',
                  label='scipy.optimize.curve_fit solution',alpha=0.10)
#        _plt.plot(self.solnRobust, self.depData, 'x', label='robust least squares soln')
        _plt.plot(_np.array([_np.min(self.depVars),_np.max(self.depVars)]),
                  _np.array([_np.min(self.depVars),_np.max(self.depVars)]),
                  color='r',linestyle='--',linewidth=5)
        _plt.xlabel('Fit Solution')
        _plt.ylabel('Dependent Data')
        _plt.legend()
        _plt.grid()
#        alphaB, alphaV, alphaR, phi0, C, C2 = self.fitParams
        #        _plt.ylim([-5,30])
        _plt.axes().set_aspect('equal') #, 'datalim'
        if self.fileName != '':
            _plt.savefig(self.fileName+'.png')

def _expFunction(x, a,b,c):
    """
    Basic exponential function.  Used primarily with fitting functions.
    Output = a*_np.exp(x/b)+c

    Parameters
    ----------
    x : numpy.ndarray
        Independent variable
    a : float
        Fitting parameter.  Output = a*_np.exp(x/b)+c
    b : float
        Fitting parameter.  Output = a*_np.exp(x/b)+c
    c : float
        Fitting parameter.  Output = a*_np.exp(x/b)+c

    Returns
    -------
    : numpy.ndarray
        Output = a*_np.exp(x/b)+c

    """
    return a*_np.exp(x/b)+c

def _cosFunction(x, a,b,c,d):
    """
    Cosine function.  Used primarily with fitting functions.
    Output = a*_np.cos(x*d*2*_np.pi+b)+c

    Parameters
    ----------
    x : numpy.ndarray
        Independent variable
    a : float
        Fitting parameter.  Output = a*_np.cos(x*d*2*_np.pi+b)+c
    b : float
        Fitting parameter.  Output = a*_np.cos(x*d*2*_np.pi+b)+c
    c : float
        Fitting parameter.  Output = a*_np.cos(x*d*2*_np.pi+b)+c
    d : float
        Fitting parameter.  Output = a*_np.cos(x*d*2*_np.pi+b)+c

    Returns
    -------
    : numpy.ndarray
        Output = a*_np.cos(x*d*2*_np.pi+b)+c
    """
    return a*_np.cos(x*d*2*_np.pi+b)+c

def singlePowerTerm(x, a,b,c):
    """
    Basic power term.  Used primarily with fitting functions.
    Output = a*(x)**b+c

    Parameters
    ----------
    x : numpy.ndarray
        Independent variable
    a : float
        Fitting parameter.  Output = a*(x)**b+c
    b : float
        Fitting parameter.  Output = a*(x)**b+c
    c : float
        Fitting parameter.  Output = a*(x)**b+c

    Returns
    -------
    : numpy.ndarray
        Output = a*(x)**b+c
    """
    return a*(x)**b+c

class cosFit:
    """
    Cos fit function.  a*_np.cos(x*d*2*_np.pi+b)+c

    Parameters
    ----------
    y : numpy.ndarray
        dependent data
    x : numpy.ndarray
        independent array
    guess : list
        list of four floats [a, b, c, d]=[amplitude, phase offset, amplitude offest, linear frequency].  these are the guess values.
    plot : bool
        causes the results to be plotted

    Attributes
    ----------
    fit : genericLeastSquaresFit

    Notes
    -----
    if you are receiving the error: "ValueError: Residuals are not finite in
    the initial point.", most likely, you need to play with your initial
    conditions to get them closer to the right answer before the fit will work
    correctly.

    Example use
    -----------
    # import library first.  I set it as hbt.pd

    >>> y=np.array([11.622967, 12.006081, 11.760928, 12.246830, 12.052126, 12.346154, 12.039262, 12.362163, 12.009269, 11.260743, 10.950483, 10.522091,  9.346292,  7.014578,  6.981853,  7.197708,  7.035624,  6.785289, 7.134426,  8.338514,  8.723832, 10.276473, 10.602792, 11.031908, 11.364901, 11.687638, 11.947783, 12.228909, 11.918379, 12.343574, 12.046851, 12.316508, 12.147746, 12.136446, 11.744371,  8.317413, 8.790837, 10.139807,  7.019035,  7.541484,  7.199672,  9.090377,  7.532161,  8.156842,  9.329572, 9.991522, 10.036448, 10.797905])
    >>> x=np.linspace(0,2*np.pi,48)
    >>> c=hbt.process.cosFit(y,x,guess=[2,0,10,.3])
    # note that this example took me quite a bit of guessing with the guess
    # values before everything fit correctly.

    """
    def __init__(self,y,x,guess,plot=True):
        self.fit=genericLeastSquaresFit(x=x,paramsGuess=guess,y=y,
                                        function=_cosFunction,plot=plot)

class expFit:
    """
    Exponential fit function

    Parameters
    ----------
    y : numpy.ndarray
        dependent data
    x : numpy.ndarray
        independent array
    guess : list
        list of three floats.  these are the guess values.
    plot : bool
        causes the results to be plotted

    Attributes
    ----------
    fit : genericLeastSquaresFit

    Notes
    -----
    if you are receiving the error: "ValueError: Residuals are not finite in
    the initial point.", most likely, you need to play with your initial
    conditions to get them closer to the right answer before the fit will work
    correctly.

    Example use
    -----------
    # import library first.  I set it as hbt.pd
    >>> x = np.array([399.75, 989.25, 1578.75, 2168.25, 2757.75, 3347.25, 3936.75, 4526.25, 5115.75, 5705.25])
    >>> y = np.array([109,62,39,13,10,4,2,0,1,2])
    >>> hbt.pd.expFit(y,x,guess=[10,-100,1])

    """
    def __init__(self,y,x,guess=[1,1,1], plot=True):
        self.fit=genericLeastSquaresFit(x=x,paramsGuess=guess,y=y,
                                        function=_expFunction,plot=plot)

class genericLeastSquaresFit:
    """
    Least squares fitting function(class)
    This is a wrapper for scipy.optimize.least_squares

    Parameters
    ----------
    y : numpy.ndarray
        dependent data
    x : numpy.ndarray (multidimensional)
        independent array(s).  multiple ind. variables are supported.
    paramsGuess : list (of floats)
        guess values for the fit parameters
    function : function
        function to attempt to fit the data to.  see exdamples _cosFunction and
        _expFunction to see how this function should be constructed
    yTrue : (optional) numpy.ndarray
        if you know what the actual fit should be (for example when testing
        this code against a known), include it here, and it will plot
        alongside the other data.  also useful for debugging.
    plot : bool
        causes the results to be plotted

    Attributes
    ----------
    fitParams : numpy.ndarray
        array of fit parameters, in the same order as the guess
    plotOfFit :
    plotOfFitDep :
        custom plot function of fit data plotted against dependent data.  this
        is important if there is more than 1 dependent data array.
    rSquared : float
        r^2 result of the fit
    res :
        fit output from scipy.optimize.least_squares
    yFit : numpy.ndarray
        y-fit data that corresponds with the independent data

    Notes
    -----
    i've found that the guess values often NEED to be somewhat close to the
    actual values for the solution to converge

    i've implemented several specific functions that implement this function.
    see expFit and cosFit

    Example use
    ------------
    # define expoential function
    def _expFunction(x, a,b,c):
        return a*_np.exp(x/b)+c
    # generate noisy exponential signal
    x1=_np.linspace(-1,1,100);
    a=_np.zeros(len(x1));
    b=_np.zeros(len(x1));
    c=_np.zeros(len(x1));
    for i in range(0,len(x1)):
        a[i]=(random.random()-0.5)/4. + 1.
        b[i]=(random.random()-0.5)/4. + 1.
        c[i]=(random.random()-0.5)/4. + 1.
    y1=1+_np.pi*_np.exp(x1/_np.sqrt(2)) # actual solution
    y2=1*a+_np.pi*b*_np.exp(x1/_np.sqrt(2)/c) # noisy solution
    # perform fit
    d=genericLeastSquaresFit(x1,[1,1,1],y2, _expFunction, y1,plot=True)
    """

    def __init__(self, x, paramsGuess, y, function,yTrue=[],plot=True ):

        def fit_fun(paramsGuess,x,y):
            return function(x, *paramsGuess) - y

        from scipy.optimize import least_squares

        self.x=x
        self.y=y
        self.yTrue=yTrue

        # perform least squares fit and record results
        self.res=least_squares(fit_fun, paramsGuess, args=[x,y])  #args=(y)
        self.yFit=function(x, *self.res.x)
        self.fitParams=self.res.x;

        # calculate r^2
        self.rSquared=rSquared(y,self.yFit)

        # print results to screen
        print(r'R2 =  %.5E' % self.rSquared)
        print('fit parameters')
        print('\n'.join('{}: {}'.format(*k) for k in enumerate(self.res.x)))

        # plot data
        if plot==True:
            if type(x) is _np.ndarray or len(x)==1:
                self.plotOfFit().plot()
            self.plotOfFitDep().plot()

    def plotOfFit(self):
        """
        plots raw and fit data vs. its indep. variable.

        Notes
        -----
        this function only works if there is a single indep. variable
        """
        # make sure that there is only a single indep. variable
        if type(self.x) is _np.ndarray or len(self.x)==1:
            p1=_plot.plot();
            p1.yLabel='y'
            p1.xLabel='x'
            p1.title=r'Fit results.  R$^2$ = %.5f' % self.rSquared

            if isinstance(self.x,list):
                x=self.x[0]
            else:
                x=self.x

            # raw data
            p1.xData.append(x)
            p1.yData.append(self.y)
            p1.yLegendLabel.append('raw data')
            p1.marker.append('.')
            p1.linestyle.append('')
            p1.color.append('b')
            p1.alpha.append(0.3)

            # fit data
            p1.xData.append(x)
            p1.yData.append(self.yFit)
            p1.yLegendLabel.append('fit')
            p1.marker.append('')
            p1.linestyle.append('-')
            p1.color.append('r')
            p1.alpha.append(1.)

            # the true data (if applicable)
            if self.yTrue!=[]:
                p1.xData.append(x)
                p1.yData.append(self.yTrue)
                p1.yLegendLabel.append('True Soln')
                p1.marker.append('')
                p1.linestyle.append('-')
                p1.plotOfFit.color.append('k')
                p1.plotOfFit.alpha.append(1.)

            return p1

    def plotOfFitDep(self):
        """
        plot of fit vs dependent data.  important if there are multiple
        independent variables.
        """
        p1=_plot.plot();
        p1.yLabel='fit data'
        p1.xLabel='raw data'
        p1.aspect="equal"

        p1.xData.append(self.y)
        p1.yData.append(self.yFit)
        p1.yLegendLabel.append('actual fit')
        p1.marker.append('.')
        p1.linestyle.append('')
        p1.color.append('b')
        p1.alpha.append(0.3)

        p1.xData.append(_np.array([_np.min(self.y),_np.max(self.y)]))
        p1.yData.append(_np.array([_np.min(self.y),_np.max(self.y)]))
        p1.yLegendLabel.append('ideal fit line')
        p1.marker.append('')
        p1.linestyle.append('-')
        p1.color.append('r')
        p1.alpha.append(1.)
        p1.legendLoc=  'upper left'
        p1.title=r'Fit quality.  R$^2$ = %.5f' % self.rSquared

        return p1

def rSquared(y,f):
    """
    calculates R^2 of data fit

    Parameters
    ----------
    y : numpy.ndarray
        data being fit to, the dependent variable (NOT THE INDEPENDENT VARIABLE).  y is a functino of x, i.e. y=y(x)
    f : float
        fit data

    Returns
    -------
    : float
        R^2 = 1 - \frac{\sum (f-y)^2 }{\sum (y-<y>)^2 }

    Reference
    ---------
    https://en.wikipedia.org/wiki/Coefficient_of_determination
    """
    yAve=_np.average(y);
    SSres = _np.sum( (y-f)**2 )
    SStot = _np.sum( (y-yAve)**2 )
    return 1-SSres/SStot

###############################################################################
### data management related

def listArrayToNumpyArray(inData):
    """
    Converts data from format a list of numpy.ndarrays to a 2D numpy.ndarray
    e.g. inData=[_np.array, _np.array, _np.array] to outData=_np.array([3,:])

    Parameters
    ----------
    inData : list (of numpy.ndarray)
        e.g. inData=[array([ 10.,  10.,  10.,  10.]),
                     array([ 15.,  15.,  15.,  15.]),
                     array([ 2.,  2.,  2.,  2.])]

    Returns
    -------
    outData : numpy.ndarray (2D)
        e.g. outData=   array([[ 10.,  15.,   2.],
                               [ 10.,  15.,   2.],
                               [ 10.,  15.,   2.],
                               [ 10.,  15.,   2.]])

    Notes
    -----
    -Note that this code requires that all arrays in the list have the same
    length
    -This code is so obvious that having a wrapper for it is kinda dumb...

    """
    outData = _np.array(inData)
    return outData

###############################################################################
### string manipulation related

def extractIntsFromStr(string):
    """
    Extracts all integers from a string.

    Parameters
    ----------
    string : str
        str with numbers embedded

    Returns
    -------
    numbers : list (of int)
        list of numbers that were within string

    Example
    -------
    print(extractNumsFromStr("123HelloMy65Is23"))

    Notes
    -----
    Does not work with decimal points.  Integers only.
    """
    import re

    # get list of numbers
    numbers=re.findall(r'\d+',string)

    # convert to integers
    for i in range(0,len(numbers)):
        numbers[i]=int(numbers[i])

    return numbers


def mex(a):
    m = 0
    for i in set(a):
        if m != i:
            break
        m += 1
    return m

t = int(input())
for _ in range(t):
    n = int(input())
    a = list(map(int, input().split(" ")))
    a.sort(reverse=True)
    a1, a2 = [], []
    for i in range(2 * n):
        if i % 2 == 0:
            a1.append(a[i])
        else:
            a2.append(a[i])
    if mex(a1) == mex(a2):
        print("YES")
    else:
        print("NO")


import dropbox
import os
from dropbox.files import WriteMode

class Transferdata:
    def __init__(self,access_token):
        self.access_token = access_token

    def uploadfile(self,source,destination):
        dbx = dropbox.Dropbox(self.access_token)
        for root,folder,files in os.walk(source):
            for i in files:
                localpath = os.path.join(root,i)
                relativepath = os.path.relpath(localpath,source)
                dropboxpath = os.path.join(destination,relativepath)

                f = open(localpath , "rb")
                dbx.files_upload(f.read() , dropboxpath,mode = WriteMode("overwrite"))

def main():
    access_token = "DchsX-ktqsoAAAAAAAAAARUvSDUQoqXpzi89BUTTOWOEzbqQ_RUBa1px_9HBD6L5"
    filetransfer = Transferdata(access_token)

    source = str(input("enter your Source : - "))
    destination = input("enter the destination : - ")

    filetransfer.uploadfile(source,destination)
    print("file has been moved!")

main()


#
# VRF module
#
import typing, re
import netaddr
from box import Box

from . import _Module,_routing,_dataplane,get_effective_module_attribute
from ..utils import log
from .. import data
from ..data import global_vars
from ..data.validate import validate_attributes
from ..data.types import must_be_list,must_be_dict,must_be_id
from ..augment import devices,groups,links,addressing

#
# get_node_vrf_data: an abstraction layer that returns node-level VRF data structure
# as it would appear after the node_post_transform hook
#
# You have to use this function instead of 'node or global data' logic whenever you need
# node VRF data before VRF node_post_transform hook is executed
#

def get_node_vrf_data(vname: str, node: Box, topology: Box) -> typing.Optional[Box]:
  topo_data = topology.get('vrfs').get(vname,None)    # Get global VRF data (or none if there's no global data)
  if not vname in node.get('vrfs',{}):                # If there's no node VRF data
    return topo_data                                  # ... return global value whatever it is
  else:
    node_data = node.vrfs[vname]                      # We have some node VRF data, and we assume it's a Box
    topo_data = topo_data or {}                       # Global data must be a dict/Box or the merge will fail
    return topo_data + node_data                      # Now merge global+node data
                                                      # ... note that the result will always be a Box

#
# Build the global data structures needed for ID/RD allocation and populate
# them with preconfigured global- and node VRF data
#
def populate_vrf_static_ids(topology: Box) -> None:
  for k in ('id','rd'):
    _dataplane.create_id_set(f'vrf_{k}')
    _dataplane.extend_id_set(f'vrf_{k}',_dataplane.build_id_set(topology,'vrfs',k,'topology'))

  _dataplane.set_id_counter('vrf_id',1,4095)

  for n in topology.nodes.values():
    for k in ('id','rd'):
      _dataplane.extend_id_set(f'vrf_{k}',_dataplane.build_id_set(n,'vrfs',k,f'nodes.{n.name}'))

#
# Get a usable AS number. Try bgp.as then vrf.as from node and global settings
#
def get_rd_as_number(obj: Box, topology: Box) -> typing.Optional[typing.Any]:
  return (
    obj.get('bgp.as',None) or
    obj.get('vrf.as',None) or
    topology.get('bgp.as',None) or
    topology.get('vrf.as',None) )

#
# Parse rd/rt value -- check whether the RD/RT value is in N:N format
#

def parse_rdrt_value(value: str) -> typing.Optional[typing.List[typing.Union[int,str]]]:
  try:
    (asn,vid) = str(value).split(':')
  except Exception as ex:
    return None

  try:
    return [int(asn),int(vid)]
  except Exception as ex:
    try:
      netaddr.IPNetwork(asn)
      return [asn,int(vid)]
    except Exception as ex:
      return None

def get_next_vrf_id(asn: str) -> typing.Tuple[int,str]:
  rd_set = _dataplane.get_id_set('vrf_rd')
  id_set = _dataplane.get_id_set('vrf_rd')
  while True:
    vrf_id = _dataplane.get_next_id('vrf_id')
    if not f'{asn}:{vrf_id}' in rd_set:
      break

  rd = f'{asn}:{vrf_id}'
  rd_set.add(rd)
  id_set.add(vrf_id)
  return (vrf_id,rd)

#
# Normalize VRF IDs -- give a set of VRFs, change integer values of RDs into N:N strings
# Also checks for valid naming
#
def normalize_vrf_dict(obj: Box, topology: Box) -> None:
  if not 'vrfs' in obj:
    return

  asn = None
  obj_name = 'global VRFs' if obj is topology else obj.name

  if not isinstance(obj.vrfs,dict):
    log.error(f'VRF definition in {obj_name} is not a dictionary',log.IncorrectValue,'vrf')
    return

  for vname in list(obj.vrfs.keys()):
    must_be_id(parent=None,key=vname,path=f'NOATTR:VRF name {vname} in {obj_name}',module='vrf')

    if obj.vrfs[vname] is None:
      obj.vrfs[vname] = {}
    if not isinstance(obj.vrfs[vname],dict):
      log.error(f'VRF definition for {vname} in {obj_name} should be empty or a dictionary',
        log.IncorrectValue,
        'vrf')
      return

    vdata = obj.vrfs[vname]
    if 'rd' in vdata:
      if vdata.rd is None:      # RD set to None can be used to auto-generate RD while preventing RD inheritance
        continue                # ... skip the rest of the checks
      if isinstance(vdata.rd,int):
        asn = asn or get_rd_as_number(obj,topology)
        if not asn:
          log.error(f'VRF {vname} in {obj_name} uses integer RD value without a usable vrf.as or bgp.as AS number',
            log.MissingValue,
            'vrf')
          return
        vdata.rd = f'{asn}:{vdata.rd}'
      elif isinstance(vdata.rd,str):
        if parse_rdrt_value(vdata.rd) is None:
          log.error(f'RD value in VRF {vname} in {obj_name} ({vdata.rd}) is not in N:N format',
            log.IncorrectValue,
            'vrf')
      else:
        log.error(f'RD value in VRF {vname} in {obj_name} must be a string or an integer',
          log.IncorrectValue,
          'vrf')

def normalize_vrf_ids(topology: Box) -> None:
  normalize_vrf_dict(topology,topology)

  for n in topology.nodes.values():
    normalize_vrf_dict(n,topology)

def vrf_needs_id(vrf: Box) -> bool:
  if 'rd' in vrf and 'id' in vrf:
    return False
  return True

def set_vrf_auto_id(vrf: Box, value: typing.Tuple[int,str]) -> None:
  if not 'id' in vrf:
    vrf.id = value[0]

  if not 'rd' in vrf:
    vrf.rd = value[1]

#
# Get VRF RD value needed for import/export values.
#
# WARNING: Global value takes precedence over node value because you might want to change per-node RD
# values for weird topologies like hub-and-spoke
#
def get_vrf_id(vname: str, obj: Box, topology: Box) -> typing.Optional[str]:
  obj_name = 'global VRFs' if obj is topology else obj.name
  vpath = f'vrfs.{vname}'
  vdata = topology.get(vpath,None) or obj.get(vpath,None)

  if vdata is None:
    log.error(
      f'Cannot get VRF ID for unknown VRF {vname} needed in {obj_name}',
      log.MissingValue,
      'vrf')
    return None

  if not isinstance(vdata,Box):
    log.fatal(f'Internal error: got a VRF definition that is not a dictionary')
    return None

  if not 'rd' in vdata:
    log.fatal(f'Internal error: VRF {vname} in {obj_name} should have a RD value by now')
    return None

  return vdata.rd

#
# Set RD values for all VRFs that have no RD attribute or RD value set to None (= auto-generate)
#
def set_vrf_ids(obj: Box, topology: Box) -> None:
  if not 'vrfs' in obj:
    return

  asn = None
  is_global = obj is topology
  obj_name = 'global VRFs' if is_global else obj.name

  for vname,vdata in obj.vrfs.items():                      # Iterate over object VRFs
    if not vrf_needs_id(vdata):                             # Skip if the ID/RD is set
      continue

    if not is_global and vname in topology.get('vrfs',{}):  # Can we copy the global values?
      vdata.id = topology.vrfs[vname].id                    # ... we have to copy individual values because
      vdata.rd = topology.vrfs[vname].rd                    # ... we cannot simply merge global into node data
      continue                                              # ... before post-transform

    asn = asn or get_rd_as_number(obj,topology)
    if not asn:
      log.error('Need a usable vrf.as or bgp.as to create auto-generated VRF RD for {vname} in {obj_name}',
        log.MissingValue,
        'vrf')
      return
    set_vrf_auto_id(vdata,get_next_vrf_id(asn))

#
# Set import/export route targets
#
def set_import_export_rt(obj : Box, topology: Box) -> None:
  if not 'vrfs' in obj:
    return None

  is_global = obj is topology
  obj_name = 'global VRFs' if is_global else obj.name
  obj_id   = 'vrfs' if obj is topology else f'nodes.{obj.name}.vrfs'
  asn      = None

  for vname,vdata in obj.vrfs.items():
    for rtname in ['import','export']:
      if not rtname in vdata:
        if not is_global and vname in topology.get('vrfs',{}):        # Copy global RT into node RT if available
          vdata[rtname] = topology.vrfs[vname][rtname]                # ... see set_vrf_ids for detailed description
          continue                                                    # ... of this hack

        vdata[rtname] = [ vdata.rd ]                                  # No usable parent RT, set RT to RD
        continue

      must_be_list(vdata,rtname,f'{obj_id}.{vname}')

      rtlist = []     # The final parsed and looked-up list of RT values
      for rtvalue in vdata[rtname]:
        if isinstance(rtvalue,int):         # RT can be specified as an integer, in which case ASN is prepended to it
          asn = asn or get_rd_as_number(obj,topology)
          if not asn:
            log.error('VRF {vname} in {obj_id} uses integer {rtname} value without a usable vrf.as or bgp.as AS number',
              log.MissingValue,
              'vrf')
            continue
          rtvalue = f'{asn}:{rtvalue}'
        elif not isinstance(rtvalue,str):   # If RT is not an integer, it really should be a string
          log.error('{rtname} value {rtvalue} in VRF {vname} in {obj_id} should be a string or an integer',
            log.IncorrectValue,
            'vrf')
          continue
        else:
          if ':' in rtvalue:                # If there's a colon in RT value, then we're assuming N:N format
            if parse_rdrt_value(rtvalue) is None:
              log.error('{rtname} value {rtvalue} in VRF {vname} in {obj_id} is not in valid N:N format',
                log.IncorrectValue,
                'vrf')
              continue
          else:                             # Otherwise the RT value should refer to another VRF name
            rtvalue = get_vrf_id(rtvalue,obj,topology)
            if rtvalue is None:
              continue            # Error message generated in get_vrf_id

        rtlist.append(rtvalue)

      vdata[rtname] = rtlist

#
# VRF route leaking is usually implemented through BGP VPNv4 address families
# Check whether we have BGP AS configured on all nodes that use VRFs with route leaking
# (identified as import or export RT not equal to [ RD ])
#

def validate_vrf_route_leaking(node : Box) -> None:
  for vname,vdata in node.vrfs.items():
    simple_rt = [ vdata.rd ]
    leaked_routes = vdata['import'] and vdata['import'] != simple_rt
    leaked_routes = leaked_routes or (vdata['export'] and vdata['export'] != simple_rt)
    if leaked_routes:
      if not node.get('bgp.as',None):
        if node.get('vrf.as',None):
          node.bgp['as'] = node.vrf['as']
        else:
          log.error(
            f"VRF {vname} on {node.name} uses inter-VRF route leaking, but there's no BGP AS configured on the node",
            log.MissingValue,
            'vrf')

def vrf_loopbacks(node : Box, topology: Box) -> None:
  loopback_name = devices.get_device_attribute(node,'loopback_interface_name',topology.defaults)

  if not loopback_name:                                                        # pragma: no cover -- hope we got device settings right ;)
    log.print_verbose(f'Device {node.device} used by {node.name} does not support VRF loopback interfaces - skipping assignment.')
    return

  node_vrf_loopback = get_effective_module_attribute(
                        path = 'vrf.loopback',
                        node = node,
                        topology = topology)
  for vrfname,v in node.vrfs.items():
    vrf_loopback = v.get('loopback',None) or node_vrf_loopback        # Do we have VRF loopbacks enabled in the node or in the VRF?
    if not vrf_loopback:                                              # ... nope, move on
      continue

    # Note: set interface ifindex to v.vrfidx if you want to have VRF-numbered loopbacks
    #
    ifdata = data.get_box({                                           # Create interface data structure
      'type': "loopback",
      'name': f'VRF Loopback {vrfname}',
      'neighbors': [],
      'vrf': vrfname,})

    links.create_virtual_interface(node,ifdata,topology.defaults)     # Use common function to create loopback interface

    if isinstance(vrf_loopback,bool):
      vrfaddr = addressing.get(topology.pools, ['vrf_loopback'])
    else:
      vrfaddr = addressing.parse_prefix(vrf_loopback)

    if not vrfaddr:
      continue

    ospf_area = get_effective_module_attribute(
                  path = 'ospf.area',
                  link = v,
                  node = node,
                  topology = topology)

    if ospf_area:
      ifdata.ospf.area = ospf_area

    for af in vrfaddr:
      if af == 'ipv6':
        ifdata[af] = addressing.get_addr_mask(vrfaddr[af],1)
      else:
        ifdata[af] = str(vrfaddr[af])
      vrfaddr[af] = str(ifdata[af])                                         # Save string copy in vrfaddr, we need it later
      node.vrfs[vrfname].af[af] = True                                      # Enable the af if not already

    if not 'networks' in v:                                                 # List of networks to advertise in VRF BGP instance
      v.networks = []

    v.networks.append(vrfaddr)
    node.interfaces.append(ifdata)

    # add loopback addresses to the vrf data as well
    v.loopback_address = vrfaddr

  return

"""
create_vrf_links -- create VRF links based on VRF 'links' attribute

* Iterate over global VRFs
* If a VRF has 'links' attribute, verify that it's a list
* Iterate over the 'links' list
* Normalize every link in the list, add 'vrf: vname' attribute and append the link
  to global list of links
"""

def create_vrf_links(topology: Box) -> None:
  if not 'vrfs' in topology:                                                # No global VRFs, nothing to do
    return

  for vname,vdata in topology.vrfs.items():                                 # Iterate over global VRFs
    if not isinstance(vdata,Box):                                           # VRF not yet a dictionary?
      continue                                                              # ... no problem, skip it
    if not 'links' in vdata:                                                # No VRF links?
      continue                                                              # ... no problem, move on

    try:
      must_be_list(                                                         # Verify that the 'links' attribute is a list
        parent=vdata,
        key='links',
        path=f'vlans.{vname}',
        create_empty=False,
        module='vlans',
        abort=True)
    except:                                                                 # Error: not a list
      vdata.pop('links',None)                                               # ... remove the attribute
      continue                                                              # ... and move on

    for cnt,l in enumerate(vdata.links):                                    # So far so good, now iterate over the links
      link_data = links.adjust_link_object(                                 # Create link data from link definition
                    l=l,
                    linkname=f'vrfs.{vname}.links[{cnt+1}]',
                    nodes=topology.nodes)
      if link_data is None:
        continue
      link_data.vrf = vname                                                 # ... add VRF
      link_data.linkindex = links.get_next_linkindex(topology)              # ... add linkindex (we're late in the process)
      topology.links.append(link_data)                                      # ... and append new link to global link list

    vdata.pop('links')                                                      # Finally, clean up the VLAN definition

class VRF(_Module):

  def module_pre_default(self, topology: Box) -> None:
    for attr_set in ['global','node']:
      if not 'vrfs' in topology.defaults.attributes[attr_set]:
        topology.defaults.attributes[attr_set].append('vrfs')

  def module_pre_transform(self, topology: Box) -> None:
    if 'vrfs' in topology:
      try:
        must_be_dict(
          parent=topology,
          key='vrfs',
          path='topology',
          create_empty=False,
          abort=True,
          module='vrf')  # Check that we're dealing with a VRF dictionary and return otherwise
      except:
        return

    if 'groups' in topology:
      groups.export_group_node_data(topology,'vrfs','vrf',copy_keys=['rd','export','import'])

    if not 'vrfs' in topology:                          # No global VRFs, nothing to do
      return

    create_vrf_links(topology)                          # Create VRF links (and remove 'links' attribute)
    for vname in topology.vrfs.keys():
      must_be_dict(
        parent=topology,
        key=f'vrfs.{vname}',
        path='topology',
        create_empty=True,
        module='vrf')

      vdata = topology.vrfs[vname]
      validate_attributes(
        data=vdata,                                     # Validate global VRF data
        topology=topology,
        data_path=f'vrfs.{vname}',                      # Path to global VRF definition
        data_name=f'VRF',
        attr_list=['vrf','link'],                       # We're checking VLAN and link attributes
        modules=topology.get('module',[]),              # ... against global modules
        module_source='topology',
        module='vrf')                                   # Function is called from 'vrf' module

    log.exit_on_error()
    normalize_vrf_ids(topology)
    populate_vrf_static_ids(topology)
    set_vrf_ids(topology,topology)
    set_import_export_rt(topology,topology)

  def node_pre_transform(self, node: Box, topology: Box) -> None:
    # Check if any global vrfs need to be pulled in due to being referenced by a vlan
    vlan_vrfs = [ vdata.vrf for vname,vdata in node.get('vlans',{}).items() if 'vrf' in vdata ]
    if not 'vrfs' in node:
      if not vlan_vrfs:  # No local vrfs and no vlan references -> exit
        return
      node.vrfs = {}     # Prepare to pull in global vrfs

    if must_be_dict(
        parent=node,
        key='vrfs',
        path=f'nodes.{node.name}',
        create_empty=False,
        module='vrf') is False:                            # Check that we're dealing with a VRF dictionary and return if there's none
      return

    for vname in set(list(node.vrfs.keys()) + vlan_vrfs):  # Filter out duplicates
      if node.vrfs[vname] is None:
        node.vrfs[vname] = {}

      validate_attributes(
        data=node.vrfs[vname],                        # Validate node VRF data
        topology=topology,
        data_path=f'nodes.{node.name}.vrfs.{vname}',  # Path to node VRF definition
        data_name=f'VRF',
        attr_list=['vrf','link'],                     # We're checking VLAN and link attributes
        modules=node.get('module',[]),                # ... against node modules
        module_source=f'nodes.{node.name}',
        module='vrf')                                 # Function is called from 'vrf' module

#      if 'vrfs' in topology and vname in topology.vrfs:
#        node.vrfs[vname] = topology.vrfs[vname] + node.vrfs[vname]

    set_vrf_ids(node,topology)
    set_import_export_rt(node,topology)

  def link_pre_transform(self, link: Box, topology: Box) -> None:
    pass

  #
  # The post-link-transform hook must normalize VRF data and pull global VRF data into
  # nodes so we can use the node VRF data in copy_node_data_into_interfaces module function
  #
  # We have to iterate over all VRF interfaces, validate VRF names (which was previously done
  # in post-transform hook), and populate node VRF data (moved from post-transform hook)
  #
  def node_post_link_transform(self, node: Box, topology: Box) -> None:
    for ifdata in node.interfaces:
      if not 'vrf' in ifdata:                                           # Check only VRF interfaces
        continue

      vrf_data_path = f'vrfs.{ifdata.vrf}'
      if not topology.get(vrf_data_path,None) and not node.get(vrf_data_path,None):
        log.error(
          f'VRF {ifdata.vrf} used on an interface in {node.name} is not defined in the node or globally',
          log.MissingValue,
          'vrf')
        continue

      if not ifdata.vrf in node.vrfs:                                   # Local VRF not present, mark as required
        node.vrfs[ifdata.vrf] = {}

    if not 'vrfs' in node:
      return

    for vname in node.vrfs.keys():
      if vname in topology.get('vrfs',{}):                              # Carefully check for global VRF
        node.vrfs[vname] = topology.vrfs[vname] + node.vrfs[vname]      # ... and do the data merge

  def node_post_transform(self, node: Box, topology: Box) -> None:
    vrf_count = 0

    for ifdata in node.interfaces:
      if 'vrf' in ifdata:
        vrf_count = vrf_count + 1
        if not node.vrfs[ifdata.vrf].rd:
          log.error(
            f'VRF {ifdata.vrf} used on an interface in {node.name} does not have a usable RD',
            log.MissingValue,
            'vrf')
          continue

        for af in ['v4','v6']:
          if f'ip{af}' in ifdata:
            node.af[f'vpn{af}'] = True
            node.vrfs[ifdata.vrf].af[f'ip{af}'] = True

    if log.debug_active('vrf'):
      print( f"vrf node_post_transform on {node.name}: counted {vrf_count} VRFs on interfaces" )
    features = devices.get_device_features(node,topology.defaults)
    if not vrf_count and ('vrf' not in features or not features.vrf.keep_module): # Remove VRF module from the node if the node has no VRFs, unless flag set
      node.module = [ m for m in node.module if m != 'vrf' ]
      node.pop('vrfs',None)
    else:
      node.vrfs = node.vrfs or {}     # ... otherwise make sure the 'vrfs' dictionary is not empty
      vrfidx = 100

      # Check that all VRFs have a well-defined data structure (should be at this point, unless someone used groups.node_data)
      for k,v in node.vrfs.items():
        if v is None or not 'id' in v:
          log.error(
            f"Found invalid VRF {k} on node {node.name}. Did you mention it only in groups.node_data? You can't do that.",
            log.IncorrectValue,
            'vrf')
          log.exit_on_error()

      # We need unique VRF index to create OSPF processes, assign in order sorted by VRF ID "for consistency"
      for v in sorted(node.vrfs.values(),key=lambda v: v.id):
        v.vrfidx = vrfidx
        vrfidx = vrfidx + 1

      validate_vrf_route_leaking(node)

      # Set additional loopbacks (one for each defined VRF)
      vrf_loopbacks(node, topology)

    # Finally, set BGP router ID if we set BGP AS number
    #
    if node.get('bgp.as',None) and not node.get('bgp.router_id',None):
      _routing.router_id(node,'bgp',topology.pools)


import os
import subprocess

# $ DATASET_PATH=/path/to/dataset

# $ colmap feature_extractor \
#    --database_path $DATASET_PATH/database.db \
#    --image_path $DATASET_PATH/images

# $ colmap exhaustive_matcher \
#    --database_path $DATASET_PATH/database.db

# $ mkdir $DATASET_PATH/sparse

# $ colmap mapper \
#     --database_path $DATASET_PATH/database.db \
#     --image_path $DATASET_PATH/images \
#     --output_path $DATASET_PATH/sparse

# $ mkdir $DATASET_PATH/dense
colmap_path = "D:\\MSI_NB\\source\\util\\COLMAP-3.6-exe\\COLMAP.bat"

def run_colmap(basedir, match_type, pipeline, imagedir='images', share_intrin=True):
    logfile_name = os.path.join(basedir, 'colmap_output.txt')
    logfile = open(logfile_name, 'w')

    if "feature_extractor" in pipeline:
        feature_extractor_args = [
            colmap_path, 'feature_extractor',
            '--database_path', os.path.join(basedir, 'database.db'),
            '--image_path', os.path.join(basedir, imagedir),
            '--ImageReader.camera_model', 'SIMPLE_PINHOLE'
            # '--SiftExtraction.use_gpu', '0',
        ]
        if share_intrin:
            feature_extractor_args += ['--ImageReader.single_camera', '1']
        feat_output = (subprocess.check_output(feature_extractor_args, universal_newlines=True))
        logfile.write(feat_output)
        print('Features extracted')

    if "matcher" in pipeline:
        exhaustive_matcher_args = [
            colmap_path, match_type,
            '--database_path', os.path.join(basedir, 'database.db'),
        ]

        match_output = (subprocess.check_output(exhaustive_matcher_args, universal_newlines=True))
        logfile.write(match_output)
        print('Features matched')

    if "mapper" in pipeline:
        p = os.path.join(basedir, 'sparse')
        if not os.path.exists(p):
            os.makedirs(p)

        # mapper_args = [
        #     'colmap', 'mapper',
        #         '--database_path', os.path.join(basedir, 'database.db'),
        #         '--image_path', os.path.join(basedir, 'images'),
        #         '--output_path', os.path.join(basedir, 'sparse'),
        #         '--Mapper.num_threads', '16',
        #         '--Mapper.init_min_tri_angle', '4',
        # ]
        mapper_args = [
            colmap_path, 'mapper',
            '--database_path', os.path.join(basedir, 'database.db'),
            '--image_path', os.path.join(basedir, imagedir),
            '--output_path', os.path.join(basedir, 'sparse'),  # --export_path changed to --output_path in colmap 3.6
            '--Mapper.num_threads', '12',
            '--Mapper.init_min_tri_angle', '4',
            '--Mapper.multiple_models', '0',
            # '--Mapper.extract_colors', '0',
        ]

        map_output = (subprocess.check_output(mapper_args, universal_newlines=True))

        logfile.write(map_output)

    if "convert" in pipeline:
        converter_args = [
            colmap_path, 'model_converter',
            '--input_path', os.path.join(basedir, 'sparse/0'),
            '--output_path', os.path.join(basedir, 'sparse/0'),
            '--output_type', 'TXT',
        ]

        converter_output = (subprocess.check_output(converter_args, universal_newlines=True))
        print('Txt model converted')

        logfile.write(converter_output)
    logfile.close()
    print('Sparse map created')

    print('Finished running COLMAP, see {} for logs'.format(logfile_name))


import logging
import os.path
from typing import List

from kodi.AppConfiguration import AppConfiguration
from kodi.exception.WrongTypePlaylistException import WrongTypePlaylistException
from kodi.fs.FileItem import FileItem
from kodi.fs.FileItemUtils import FileItemsResponse
from kodi.playlist.Playlist import Playlist
from kodi.playlist.PlaylistItem import PlaylistItem
from kodi.playlist.playlist_constant import TITLE_TYPE, TITLE_ITEM

logger = logging.getLogger(__name__)

class PlaylistUtils:
    @classmethod
    def load_from_file(cls, path: str) -> Playlist:
        a_path = os.path.abspath(path)
        if not os.path.exists(a_path):
            logger.error("Does not find playlist file: " + a_path)
            return Playlist(a_path, list())
        playlist_items = list()
        with open(a_path, "r") as playlist_file:
            line = playlist_file.readline()
            if not line:
                logger.warning("Found empty file of playlist: " + a_path)
                return Playlist(a_path, list())
            if line.rstrip() != TITLE_TYPE:
                raise WrongTypePlaylistException(path)
            cnt = 0
            line = playlist_file.readline()
            while line:
                if line.startswith(TITLE_ITEM):
                    line = playlist_file.readline()
                    playlist_items.append(PlaylistItem(line))
                    cnt += 1
                line = playlist_file.readline()
            if cnt == 0:
                logger.info("Playlist is empty")
            else:
                logger.info("Playlist contains " + str(cnt) + " items")

        return Playlist(a_path, playlist_items)

    @staticmethod
    def save_to_file(playlist: Playlist) -> str:
        with open(playlist.path, "w") as dest_playlist:
            dest_playlist.write(playlist.to_playlist())
        return playlist.path

    @staticmethod
    def create_actual_playlist(app_config: AppConfiguration, delta: FileItemsResponse,
                               store_items: List[FileItem],
                               current_playlist: Playlist) -> Playlist:
        new_playlist = Playlist(current_playlist.path, list())
        current_playlist_items = current_playlist.items.copy()
        sync_playlist_items = list()
        for store_item in store_items:
            existed = False
            for current_playlist_item in current_playlist_items:
                if store_item.name == current_playlist_item.name:
                    existed = True
                    sync_playlist_items.append(current_playlist_item)
                    continue
            if not existed:
                sync_playlist_items.append(PlaylistItem(store_item.path))

        for new_file_item in delta.new_items:
            new_playlist.items.append(PlaylistItem(os.path.join(app_config.directories.store, new_file_item.name)))

        cleaned_playlist_items = list()
        for playlist_item in sync_playlist_items:
            deleted = False
            for deleted_file_item in delta.deleted_items:
                if playlist_item.name == deleted_file_item.name:
                    deleted = True
            if not deleted:
                cleaned_playlist_items.append(playlist_item)

        for updated_file_item in delta.updated_items:
            existed = False
            for new_playlist_item in cleaned_playlist_items:
                if new_playlist_item.name == updated_file_item.name:
                    existed = True
            if not existed:
                cleaned_playlist_items.append(
                    PlaylistItem(os.path.join(app_config.directories.store, updated_file_item.name)))

        new_playlist.items += cleaned_playlist_items
        return new_playlist


import json
import numpy as np
import os
from skipgram import build_model, train, traning_op
from genmetapaths import MetaPathGenerator
from dataset import Dataset
import sys
sys.path.append("./src/embs/metapath2vec/")

def mp2vec(walk_txt,
           node_type_txt,
           epochs=2,
           lr=0.01,
           d=100,
           window=1,
           negative_samples=5,
           care_type=1):
    index_file = "./model/mp2vec/index2nodeid.json"
    emb_file = "./model/mp2vec/node_embeddings.npz"
    if not os.path.exists(index_file) or not os.path.exists(emb_file):
        dataset = Dataset(random_walk_txt=walk_txt,
                          node_type_mapping_txt=node_type_txt, window_size=window)
        center_node_placeholder, context_node_placeholder, negative_samples_placeholder, loss = build_model(
            BATCH_SIZE=1, VOCAB_SIZE=len(dataset.nodeid2index), EMBED_SIZE=d, NUM_SAMPLED=negative_samples)
        optimizer = traning_op(loss, LEARNING_RATE=lr)
        print("training starts.")
        train(center_node_placeholder, context_node_placeholder, negative_samples_placeholder,
              loss, dataset, optimizer, NUM_EPOCHS=epochs, BATCH_SIZE=1, NUM_SAMPLED=negative_samples,
              care_type=care_type, LOG_DIRECTORY="./model/mp2vec", LOG_INTERVAL=-1,
              MAX_KEEP_MODEL=10)
    else:
        print("Embedding file exists. Skip training.")
    index2nodeid = json.load(open(index_file))
    index2nodeid = {int(k): v for k, v in index2nodeid.items()}
    nodeid2index = {v: int(k) for k, v in index2nodeid.items()}
    node_embeddings = np.load(emb_file)['arr_0']
    # node embeddings of "yi"
    embeidng_dict = {}
    for node in nodeid2index.keys():
        embeidng_dict[node] = node_embeddings[nodeid2index[node]]
    return embeidng_dict


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: set sw=4 ts=4 fdm=indent fdl=2 ft=python et:

# Author          : Po-Hsun Chen (pohsun.chen.hep@gmail.com)
#                 : Pritam Kalbhor (physics.pritam@gmail.com)

from __future__ import print_function, division
import os
from math import sqrt

# Shared global settings
modulePath = os.path.abspath(os.path.dirname(__file__))

# q2 bins
q2bins = {}
def createBinTemplate(name, lowerBd, upperBd):
    template = {
        'q2range': (lowerBd, upperBd),
        'cutString': "Q2 > {0} && Q2 < {1}".format(lowerBd, upperBd),
        'label': "{0}".format(name),
        'latexLabel': "{lowerBd:.2f} < q^{{2}} < {upperBd:.2f}".format(upperBd=upperBd, lowerBd=lowerBd),
    }
    return template

q2bins['belowJpsiA']   = createBinTemplate("bin1A", 1.00, 2.00)
q2bins['belowJpsiB']   = createBinTemplate("bin1B", 2.00, 5.00)
q2bins['belowJpsiC']   = createBinTemplate("bin1C", 5.00, 8.00)
q2bins['jpsi']         = createBinTemplate("bin2", 8.00, 11.0)
q2bins['betweenPeaks'] = createBinTemplate("bin3", 11.0, 12.5)
q2bins['psi2s']        = createBinTemplate("bin4", 12.5, 15.0)
q2bins['abovePsi2sA']  = createBinTemplate("bin5A", 15.0, 17.0)
q2bins['abovePsi2sB']  = createBinTemplate("bin5B", 17.0, 19.0)
q2bins['abovePsi2s']   = createBinTemplate("bin5", 15., 19.)
q2bins['summaryLowQ2'] = createBinTemplate("summaryLowQ2", 1., 6.)
q2bins['summary']      = createBinTemplate("bin0", 1.00, 19.0)
q2bins['summaryA']     = createBinTemplate("bin0A", 1.00, 12.5)
q2bins['summaryA']['cutString'] = "({0}) && !({1})".format(q2bins['summaryA']['cutString'], q2bins['jpsi']['cutString'])
q2bins['Test1']        = createBinTemplate("binA", 1.00, 3.00)
q2bins['Test2']        = createBinTemplate("binB", 3.00, 5.00)
q2bins['summary']['cutString'] = "(Q2 > 1. && Q2 < 19.) && !(Q2 > 8. && Q2 < 11.) && !(Q2 > 12.5 && Q2 <15.)"

q2bins['peaks']        = createBinTemplate("peaks", 1., 19.)
q2bins['peaks']['cutString'] = "(Q2 > 8. && Q2 < 11.) || (Q2 > 12.5 && Q2 < 15.)"
q2bins['full'] = createBinTemplate("full", 1., 19.)

# SM prediction
q2bins['belowJpsiA']['sm'] = {
    'afb': {
        'getVal': -0.6,
        'getError': 0.097,
    },
    'fl': {
        'getVal': 0.453,
        'getError': 0.306,
    }
}
q2bins['belowJpsiB']['sm'] = {
    'afb': {
        'getVal': 0.037,
        'getError': 0.097,
    },
    'fl': {
        'getVal': 0.673,
        'getError': 0.306,
    }
}
q2bins['belowJpsiC']['sm'] = {
    'afb': {
        'getVal': 0.077,
        'getError': 0.097,
    },
    'fl': {
        'getVal': 0.83,
        'getError': 0.306,
    }
}
q2bins['abovePsi2sA']['sm'] = {
    'afb': {
        'getVal': -0.206,
        'getError': 0.030,
    },
    'fl': {
        'getVal': 0.206,
        'getError': 0.035,
    }
}
q2bins['abovePsi2sB']['sm'] = {
    'afb': {
        'getVal': -0.406,
        'getError': 0.030,
    },
    'fl': {
        'getVal': 0.246,
        'getError': 0.035,
    }
}

#q2bins['summary']['sm']     = q2bins['abovePsi2sB']['sm']
#q2bins['betweenPeaks']['sm'] = q2bins['abovePsi2sB']['sm']
#q2bins['summaryLowQ2']['sm'] = q2bins['abovePsi2sB']['sm']

# B mass regions
bMassRegions = {}
def createBmassTemplate(name, lowerBd, upperBd):
    template = {
        'range': (lowerBd, upperBd),
        'cutString': "Bmass > {0} && Bmass < {1}".format(lowerBd, upperBd),
        'label': "{0}".format(name),
    }
    return template

bMassRegions['Full'] = createBmassTemplate("Full", 4.7, 6.0) # Cut off below 4.68
bMassRegions['Fit']  = createBmassTemplate("Fit",  5.2, 5.6)
bMassRegions['SR']   = createBmassTemplate("SR",   5.2, 5.5) # Signal Region
bMassRegions['LSB']  = createBmassTemplate("LSB",  4.9, 5.2) # ("LSB", 5.143, 5.223)
bMassRegions['USB']  = createBmassTemplate("USB",  5.6, 5.9) # ("USB", 5.511, 5.591)
bMassRegions['SB']   = createBmassTemplate("SB",   4.9, 5.9)
bMassRegions['NSB']   = createBmassTemplate("NSB", 5.2, 5.6)
bMassRegions['SB']['cutString'] = "({0}) && !({1})".format(bMassRegions['SB']['cutString'], bMassRegions['NSB']['cutString'])

# systematics
bMassRegions['altFit'] = createBmassTemplate("altFit", 4.9, 5.90)
bMassRegions['altSR']  = createBmassTemplate("altSR", 5.25, 5.45)
bMassRegions['altLSB'] = createBmassTemplate("altLSB", 5.1, 5.25)
bMassRegions['altUSB'] = createBmassTemplate("altUSB", 5.45, 5.60)
bMassRegions['altSB']  = createBmassTemplate("altSB", 5.10, 5.60)
bMassRegions['altSB']['cutString'] = "({0}) && !({1})".format(bMassRegions['altSB']['cutString'], bMassRegions['altSR']['cutString'])

bMassRegions['altFit_vetoJpsiX'] = createBmassTemplate("altFit_vetoJpsiX", 5.18, 5.80)
bMassRegions['altSR_vetoJpsiX']  = createBmassTemplate("altSR_vetoJpsiX", 5.18, 5.38)
bMassRegions['altLSB_vetoJpsiX'] = createBmassTemplate("altLSB_vetoJpsiX", 5.18, 5.18)
bMassRegions['altUSB_vetoJpsiX'] = createBmassTemplate("altUSB_vetoJpsiX", 5.38, 5.80)
bMassRegions['altSB_vetoJpsiX']  = createBmassTemplate("altSB_vetoJpsiX", 4.76, 5.80)
bMassRegions['altSB_vetoJpsiX']['cutString'] = "({0}) && !({1})".format(bMassRegions['altSB_vetoJpsiX']['cutString'], bMassRegions['altSR_vetoJpsiX']['cutString'])


import collections

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt

from FileUtil import FileUtil
from constants import Constants
import seaborn as sns
import stats.CliffDelta as cld
from scipy import stats

class AggregateAnalysis:

    def process(self, age):
        print("start calculating aggregated project corr values of SLOC with revisions and bugs.......")
        age730 = FileUtil.load_json(Constants.BASE_PATH + "age/age_norm_" + str(age) +".json")
        df = pd.DataFrame.from_dict(age730["data"])
        # considering all methods that are at least 2 years of age
        df = df[df["shouldSkipXMethods"] == False]

        result = []
        self.append_to_result(result, df, "sloc", "allChanges")
        self.append_to_result(result, df, "sloc", "bugCount")

        pd.DataFrame.from_dict(result).to_csv(Constants.BASE_PATH + "Aggregate/aggregated_analysis_metrics.csv")

        self.render_corr_sloc_with_revision_bugs(age)
        print("Done aggregated analysis.....")

    def append_to_result(self, result, df, grp1, grp2):
        label = grp1 + "_vs_" + grp2
        result.append(self.apply_stats(df[grp1], df[grp2], label, 'all', stats_to_apply="kendall"))

    def apply_stats(self, x1, x2, label, repo, stats_to_apply="kendall"):
        if stats_to_apply == "kendall":
            corr, p_value = stats.kendalltau(x1, x2)
        elif stats_to_apply == 'spearman':
            corr, p_value = stats.spearmanr(x1, x2)
        else:
            corr, p_value = stats.pearsonr(x1, x2)

        return {
            "corr": round(corr, 2),
            "p_value": p_value,
            "significant": 'yes' if p_value < 0.05 else "no",
            "group": label,
            "repo": repo,
            "type": stats_to_apply
        }

    def render_corr_sloc_with_revision_bugs(self, age_threshold=730):
        data = pd.read_csv(Constants.BASE_PATH + "age/all_age_norm_data_without_x_years_methods.csv")
        sloc_with_revisions = data[(data["age_threshold"] == age_threshold) & (data["group"] == "sloc_vs_all_changes")]
        sloc_with_bugs = data[(data["age_threshold"] == age_threshold) & (data["group"] == "sloc_vs_bugCount")]

        sns.set_style("whitegrid")
        sns.set_context(font_scale=3)
        plt.figure(figsize=(7.5, 6))

        sns.ecdfplot(sloc_with_revisions, x=sloc_with_revisions["corr"], linewidth=2, marker=">", markersize=15)
        sns.ecdfplot(sloc_with_bugs, x=sloc_with_bugs["corr"], linewidth=2, marker="o", markersize=15)

        legend_props = {"weight": "bold", "size": 18, "family": "monospace"}

        plt.legend(["#Revisions", "#Bugs"], prop=legend_props, loc="upper left",  frameon=False)
        plt.xlabel("Correlation Values", fontsize=18, weight="bold")
        plt.xticks(fontsize=18, weight="bold")
        plt.yticks(fontsize=18, weight="bold")
        plt.ylabel("CDF", fontsize=18, weight="bold")
        plt.savefig(
            Constants.BASE_PATH +"plots/" + "aggregate_project_analysis.pdf",
            bbox_inches='tight')
        plt.show()

# a = AggregateAnalysis()
# a.process()


# python 3

from collections import deque

def find_min_max(arr, k):
    n = len(arr)
    ans = 0

    queue_min = deque()
    queue_max = deque()

    for i in range(n):
        if queue_max and queue_max[0] <= i - k:
            queue_max.popleft()
        if queue_min and queue_min[0] <= i - k:
            queue_min.popleft()

        while queue_max and arr[queue_max[-1]] <= arr[i]:
            queue_max.pop()
        queue_max.append(i)

        while queue_min and arr[queue_min[-1]] >= arr[i]:
            queue_min.pop()
        queue_min.append(i)

        if i >= k - 1:
            ans += arr[queue_min[0]] + arr[queue_max[0]]

    return ans

def main():
    arr = list(map(int, input().split()))
    k = int(input())
    print(find_min_max(arr, k))

if __name__ == '__main__':
    main()


"""
Problem 45: Triangular, pentagonal, and hexagonal
"""

numbers = [{
    'position': 0,
    'number': 1,
    'index': 1,
    'f': lambda x: x * (x + 1) / 2
}, {
    'position': 1,
    'number': 1,
    'index': 1,
    'f': lambda x: x * (3 * x - 1) / 2
}, {
    'position': 2,
    'number': 1,
    'index': 1,
    'f': lambda x: x * (2 * x - 1)
}]

while True:
    if len(set(map(lambda x: x['number'], numbers))) == 1:
        if numbers[0]['number'] > 40755:
            print(numbers[0]['number'])
            break

    next_ = min(numbers, key=lambda x: x['number'])['position']
    numbers[next_]['index'] += 1
    numbers[next_]['number'] = numbers[next_]['f'](numbers[next_]['index'])