text:
import six.moves.cPickle as pickle
from collections import OrderedDict
import sys, time, os
sys.path.append('../')
import numpy as np
import gzip
import theano
from theano import config
theano.config.compute_test_value = 'warn'
from theano.printing import pydotprint
import theano.tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
from optvaeutils.optimizer import adam
from theano.compile.debugmode import DebugMode
from models import BaseModel
from utils.divergences import KLGaussian,BhattacharryaGaussian
from utils.misc import readPickle

class VAE(BaseModel, object):
    def __init__(self, params, paramFile=None, reloadFile=None, **kwargs):
        super(VAE,self).__init__(params, paramFile=paramFile, reloadFile=reloadFile, **kwargs)
        self.p_names = [k.name for k in self._getModelParams(restrict='p_')]
    def _createParams(self):
        """ _createParams: create weights learned and used in model """
        npWeights = OrderedDict()
        DIM_HIDDEN = self.params['q_dim_hidden']
        #Inference Network
        if self.params['emission_type'] not in ['mlp','res']:
            raise ValueError('emission_type :'+str(self.params['emission_type'])+' not recognized')
        for q_l in range(self.params['q_layers']):
            dim_input     = DIM_HIDDEN
            dim_output    = DIM_HIDDEN
            if q_l==0:
                dim_input     = self.params['dim_observations']
            npWeights['q_'+str(q_l)+'_W'] = self._getWeight((dim_input, dim_output))
            npWeights['q_'+str(q_l)+'_b'] = self._getWeight((dim_output, ))
        if self.params['q_layers']==0:
            DIM_HIDDEN    = self.params['dim_observations']

        npWeights['q_mu_W']  = self._getWeight((DIM_HIDDEN, self.params['dim_stochastic']))
        npWeights['q_logcov_W'] = self._getWeight((DIM_HIDDEN, self.params['dim_stochastic']))
        npWeights['q_mu_b']  = self._getWeight((self.params['dim_stochastic'],))
        npWeights['q_logcov_b'] = self._getWeight((self.params['dim_stochastic'],))
        #Generative Model
        for p_l in range(self.params['p_layers']):
            dim_input     = self.params['p_dim_hidden']
            dim_output    = dim_input
            if p_l==0:
                dim_input     = self.params['dim_stochastic']
            npWeights['p_'+str(p_l)+'_W'] = self._getWeight((dim_input, dim_output))
            npWeights['p_'+str(p_l)+'_b'] = self._getWeight((dim_output, ))
        if self.params['emission_type']=='res':
            npWeights['p_linz_W'] = self._getWeight((self.params['dim_stochastic'],self.params['dim_observations']))
        DIM_HIDDEN     = self.params['p_dim_hidden']
        if self.params['p_layers']==0:
            DIM_HIDDEN = self.params['dim_stochastic']
        npWeights['p_mean_W']     = self._getWeight((DIM_HIDDEN, self.params['dim_observations']))
        npWeights['p_mean_b']     = self._getWeight((self.params['dim_observations'],))
        return npWeights

    def _fakeData(self):
        """ Fake Data for Debugging """
        X = np.random.rand(2, self.params['dim_observations']).astype(config.floatX)
        m1=X>0.5
        m0=X<=0.5
        if self.params['data_type']=='bow':
            X[m0]=0
            X[m1]=400
        else:
            X[m0]=0
            X[m1]=1
        mu = np.random.randn(2, self.params['dim_stochastic']).astype(config.floatX)
        return X,mu

    def _KL(self, mu, logcov, z, keepmat=False):
        """  KL divergence between N(mu,logcov)||N(0,log I) """
        KL      = 0.5*(-logcov-1+T.exp(logcov)+mu**2)
        if keepmat:
            return KL
        else:
            return KL.sum(1,keepdims=True)

    def logsoftmax(self, mat):
        "Logsoftmax along dimension 1 (unless vector)"
        if mat.ndim==1:
            maxval  = mat.max()
            result  = mat-(maxval+T.log(T.sum(T.exp(mat-maxval))+1e-6))
        else:
            maxval  = mat.max(1,keepdims=True)
            result  = mat-(maxval+T.log(T.sum(T.exp(mat-maxval),axis=1,keepdims=True)+1e-6))
        return result

    def _conditionalXgivenZ(self, z, additional = {}):
        """ Mean Probabilities under p_theta(x|z) """
        assert self.params['data_type'] in ['real','binary','bow'],'Only binary data/bow'
        inp_p   = z
        if self.params['emission_type'] in ['res','mlp']:
            for p_l in range(self.params['p_layers']):
                inp_p= self._LinearNL(self.tWeights['p_'+str(p_l)+'_W'], self.tWeights['p_'+str(p_l)+'_b'], inp_p)
            paramMat = T.dot(inp_p,self.tWeights['p_mean_W'])
            if self.params['emission_type']=='res':
                paramMat += T.dot(z,self.tWeights['p_linz_W'])
            #Different kinds of data to model
            if self.params['data_type']=='binary':
                E               = paramMat + self.tWeights['p_mean_b']
                additional['E'] = E
                mean_p          = T.nnet.sigmoid(E)
                return mean_p
            elif self.params['data_type'] == 'bow':
                E = paramMat + self.tWeights['p_mean_b']
                additional['E'] = E
                if self.params['likelihood']=='mult':
                    emb    = self.logsoftmax(E)
                    return emb
                elif self.params['likelihood']=='poisson':
                    loglambda_p=E
                    return loglambda_p
                else:
                    raise ValueError,'Invalid setting of likelihood: '+self.params['likelihood']
            elif self.params['data_type']=='real':
                E = paramMat+self.tWeights['p_mean_b']
                additional['E'] = E
                return (E,T.zeros_like(E))
            else:
                assert False,'Bad data_type:' + str(self.params['data_type'])
        else:
            assert False,'Bad emission_type: '+str(self.params['emission_type'])

    def _negCLL(self, z, X):#, validation = False):
        """Estimate -log p[x|z]"""
        if self.params['data_type']=='binary':
            p_x_z    = self._conditionalXgivenZ(z)
            negCLL_m = T.nnet.binary_crossentropy(p_x_z,X)
        elif self.params['data_type'] =='bow':
            #Likelihood under a multinomial distribution
            if self.params['likelihood'] == 'mult':
                lsf      = self._conditionalXgivenZ(z)
                p_x_z    = T.exp(lsf)
                negCLL_m = -1*(X*lsf)
            elif self.params['likelihood'] =='poisson':
                loglambda_p = self._conditionalXgivenZ(z)
                p_x_z       = T.exp(loglambda_p)
                negCLL_m    = -X*loglambda_p+T.exp(loglambda_p)+T.gammaln(X+1)
            else:
                raise ValueError,'Invalid choice for likelihood: '+self.params['likelihood']
        elif self.params['data_type']=='real':
            params   = self._conditionalXgivenZ(z)
            mu,logvar= params[0], params[1]
            p_x_z    = mu
            negCLL_m = 0.5 * np.log(2 * np.pi) + 0.5*logvar + 0.5 * ((X - mu_p)**2)/T.exp(logvar)
        else:
            assert False,'Bad data_type: '+str(self.params['data_type'])
        return p_x_z, negCLL_m.sum(1,keepdims=True)

    def _inference(self, X, dropout_prob = 0.):
        """ Subgraph to do inference """
        self._p(('Inference with dropout :%.4f')%(dropout_prob))
        if self.params['data_type']=='bow':
            if self.params['input_type']=='counts':
                inp = X/float(self.params['max_word_count'])
            elif self.params['input_type']=='normalize':
                inp  = (X/X.sum(1,keepdims=True))#*float(self.params['max_word_count'])#T.cast(X.shape[1],'float32')
            elif self.params['input_type']=='tfidf':
                #TF-IDF
                assert self.idf is not None,'Expecting tfidf vectors'
                tfidf= (X*self.idf)
                inp  = tfidf/T.sqrt((tfidf**2).sum(1,keepdims=True))
            else:
                assert False,'Invalid input type'
            if dropout_prob>0.1:
                self._p('Applying dropout to bow for sure')
                inp = self._dropout(inp, dropout_prob)
        else:
            inp = self._dropout(X,dropout_prob)

        if self.params['emission_type'] in ['mlp','res']:
            for q_l in range(self.params['q_layers']):
                inp = self._LinearNL(self.tWeights['q_'+str(q_l)+'_W'], self.tWeights['q_'+str(q_l)+'_b'], inp)
        else:
            assert False,'Bad emission_type:'+str(self.params['emission_type'])
        mu      = T.dot(inp,self.tWeights['q_mu_W'])    +self.tWeights['q_mu_b']
        logcov  = T.dot(inp,self.tWeights['q_logcov_W'])+self.tWeights['q_logcov_b']
        return mu, logcov

    ################################    Building Objective Functions #####################
    def _ELBO(self, X, eps = None, mu_q=None, logcov_q=None, anneal = 1.,
              dropout_prob = 0., savedict = None, batch_vec = False):
        """ Wrapper for ELBO """
        if mu_q is None and logcov_q is None:
            mu_q, logcov_q = self._inference(X, dropout_prob)
        if eps is None:
            eps = self.srng.normal(size=mu_q.shape,dtype=config.floatX)
        z              = mu_q+eps*T.exp(logcov_q*0.5)
        mean_p, negCLL = self._negCLL(z, X)#, validation=dropout_prob==0.)
        KL             = self._KL(mu_q, logcov_q, z, keepmat=True)
        #Collect statistics
        if type(savedict) is dict:
            savedict['z']        = z
            savedict['mean_p']   = mean_p
            savedict['mu_q']     = mu_q
            savedict['logcov_q'] = logcov_q
            savedict['negCLL'] = negCLL
            savedict['KLmat']    = KL
            savedict['KL']    = KL
            savedict['elbo_batch']=negCLL+KL.sum(1,keepdims=True)
        bound = (negCLL+ anneal*KL.sum(1,keepdims=True)).sum()
        return bound

    def adamUpdates(self, it_k, paramlist, gradlist, m_list, v_list, plr):
        """Adam for local variational parameters"""
        m_new_list, v_new_list, param_new_list = [], [], []
        b1,b2  = 0.9, 0.001
        fix1   = 1. - (1. - b1)**it_k
        fix2   = 1. - (1. - b2)**it_k
        lr_t   = plr * (T.sqrt(fix2) / fix1)
        for p,g,m,v in zip(paramlist, gradlist, m_list, v_list):
            m_t = (b1 * g) + ((1. - b1) * m)
            v_t = (b2 * T.sqr(g)) + ((1. - b2) * v)
            g_t = m_t / (T.sqrt(v_t) + 1e-8)
            p_t = p - (lr_t * g_t)
            m_new_list.append(m_t)
            v_new_list.append(v_t)
            param_new_list.append(p_t)
        return param_new_list, m_new_list, v_new_list

    def _optimizeVariationalParams(self, X, mu0, logcov0, n_steps, plr, savedict = {}, force_resample = False):
        """             Wrapper to optimize variational parameters
                        mu0,logcov0 - initial variational parameters
                        n_steps     - number of steps to perform optimization
                        plr         - learning rate for variational parameters
        """
        epslist = self.srng.normal(size = (n_steps,X.shape[0],self.params['dim_stochastic']),dtype=config.floatX)
        assert self.params['opt_method']=='adam','expecting adam'
        def optAdam(it_k, eps_k, mu, logcov, m_mu, v_mu, m_logcov, v_logcov, X, plr):
            #if self.params['anneal_finopt_rate']>0:
            #    anneal   = T.switch(T.gt(it_k/float(self.params['anneal_finopt_rate']),1.),
            #            1., it_k/float(self.params['anneal_finopt_rate']))
            #else:
            anneal   = 1.
            L_prev       = self._ELBO(X, eps = eps_k, mu_q = mu, logcov_q = logcov, anneal = anneal)
            gradlist     = T.grad(L_prev,wrt = [mu,logcov])
            paramlist, mlist, vlist = self.adamUpdates(it_k, [mu,logcov], gradlist, [m_mu, m_logcov], [v_mu, v_logcov], plr)

            L_out        = L_prev
            #if self.params['anneal_finopt_rate']>0:
            #    L_out    = self._ELBO(X, eps = eps_k, mu_q = mu, logcov_q = logcov)
            mu_k, logcov_k     = paramlist[0], paramlist[1]
            m_mu_k, v_mu_k         = mlist[0], vlist[0]
            m_logcov_k, v_logcov_k = mlist[1], vlist[1]
            return mu_k, logcov_k, L_out, T.sqrt((gradlist[0]**2).sum(1)).sum(), T.sqrt((gradlist[1]**2).sum(1)).sum(), m_mu_k, v_mu_k, m_logcov_k, v_logcov_k
        self._p('Optimizing variational parameters w/ ADAM')
        m_mu_0     = T.zeros((X.shape[0], self.params['dim_stochastic']))
        v_mu_0     = T.zeros((X.shape[0], self.params['dim_stochastic']))
        m_logcov_0 = T.zeros((X.shape[0], self.params['dim_stochastic']))
        v_logcov_0 = T.zeros((X.shape[0], self.params['dim_stochastic']))
        result,updates = theano.scan(optAdam,
                                 sequences = [T.arange(1,n_steps+1,dtype=config.floatX), epslist],
                                 outputs_info=[mu0,logcov0,None,None,None,m_mu_0,v_mu_0,m_logcov_0,v_logcov_0],
                                 non_sequences = [X, plr],
                                 n_steps = n_steps)
        mu_its, logcov_its, elbo_its, grad_mu_its, grad_logcov_its = result[0],result[1],result[2],result[3],result[4]
        savedict['mu_its']          = mu_its
        savedict['logcov_its']      = logcov_its
        savedict['elbo_its']        = elbo_its
        savedict['gradnorm_mu_its']     = grad_mu_its
        savedict['gradnorm_logcov_its'] = grad_logcov_its
        savedict['n_steps'] = mu_its.shape[0]
        return mu_its[-1], logcov_its[-1], elbo_its[-1]

    def _buildOptimizationFunction(self, X, n_steps, plr):
        mu_0,logcov_0 = self._inference(X)
        optdict = {}
        _, logcov_f, elbo_final = self._optimizeVariationalParams(X, mu_0, logcov_0, n_steps, plr,
                                                                              savedict = optdict)
        diff_elbo, _ = self._estimateELBOEntropy(optdict['elbo_its'][0],optdict['elbo_its'][-1], logcov_0, logcov_f)
        self.optimize_mu_logcov = theano.function([X, theano.In(n_steps, value=self.params['n_steps'], name='n_steps'),
                                                 theano.In(plr, value=self.params['param_lr'], name='plr')],
                                                   [optdict['elbo_its'], optdict['gradnorm_mu_its'],
                                                    optdict['gradnorm_logcov_its'],optdict['elbo_its'].shape[0], diff_elbo],
                                                 name = 'Optimize ELBO wrt mu/cov')
        diff_elbo, _ = self._estimateELBOEntropy(optdict['elbo_its'][0], optdict['elbo_its'][-1], logcov_0, logcov_f)
        self.final_elbo     = theano.function([X, theano.In(n_steps, value=self.params['n_steps'], name='n_steps'),
                                                 theano.In(plr, value=self.params['param_lr'], name='plr')],
                                               [optdict['elbo_its'][0],optdict['elbo_its'][-1], optdict['elbo_its'].shape[0],
                                               optdict['gradnorm_mu_its'][-1],optdict['gradnorm_logcov_its'][-1],
                                               diff_elbo], name = 'Optimize ELBO wrt mu/cov')
        self.init_final_params = theano.function([X, theano.In(n_steps, value=self.params['n_steps'], name='n_steps'),
                                                 theano.In(plr, value=self.params['param_lr'], name='plr')],
                                               [optdict['mu_its'][0],optdict['logcov_its'][0], optdict['mu_its'][-1],
                                                   optdict['logcov_its'][-1]], name = 'init/final params')
    def _estimateELBOEntropy(self, elbo_0, elbo_f, logcov_0, logcov_f):
        """ H = 0.5*log |Sigma| + K/2 + K/2 log (2\pi)
            if the ratio is high, most of the
            We know that elbo_0-elbo_f>0 so the sign of the ratio is given by whether or not there was a contraction
            while the magnitude gives you how much of the change can be attributed to the change in entropy of
            the variational distribution"""
        diff_elbo      = elbo_0-elbo_f #Since we're using upper bounds multiply by -1
        diff_entropy   = 0.5*(logcov_f-logcov_0).sum()
        return diff_elbo, diff_entropy

    ##################                EVALUATION        ######################
    def _buildEvaluationFunctions(self, X,n_steps,plr):
        """ Build functions for evaluation. X: input,evaluation_bound: bound for evaluation
            evaldict: dictionary containing z/mu/logcov and other arrays that might need inspection
            additional_inputs: used to support finopt where you need to have n_steps etc
        """
        self._p('Evaluation: Setting opt_method: ADAM, 100 steps w/ 8e-3 lr')
        evaldict0, evaldictopt, evaldictf     = {}, {}, {}
        elbo_init         = self._ELBO(X,   savedict = evaldict0)

        elbo_init_batch   = evaldict0['elbo_batch']

        mu_f, logcov_f, _ = self._optimizeVariationalParams(X,evaldict0['mu_q'],evaldict0['logcov_q'],
                                                            n_steps, plr, savedict = evaldictopt)
        elbo_final        = self._ELBO(X, mu_q = mu_f, logcov_q = logcov_f, savedict = evaldictf)
        elbo_final_batch  = evaldictf['elbo_batch']

        fxn_inputs = [X]
        init_val = 100
        if self.params['data_type']=='image':
            init_val = 5
        fxn_inputs.append(theano.In(n_steps, value = init_val, name = 'n_steps'))
        fxn_inputs.append(theano.In(plr, value = 8e-3, name = 'plr'))
        diff_elbo, _ = self._estimateELBOEntropy(elbo_init, elbo_final, evaldict0['logcov_q'], evaldictf['logcov_q'])
        self.evaluate   = theano.function(fxn_inputs, [elbo_init, elbo_final,evaldictopt['n_steps'], diff_elbo], name = 'Evaluate')
        self.reconstruct= theano.function([evaldictf['z']], evaldictf['mean_p'], name='Reconstruct')
        self.inference  = theano.function(fxn_inputs, [evaldictf['z'], evaldictf['mu_q'], evaldictf['logcov_q'] ],
                                          name = 'Posterior Inference')
        self.inference0 = theano.function([X], [evaldict0['z'], evaldict0['mu_q'], evaldict0['logcov_q'] ,evaldict0['KL']],
                                          name = 'Posterior Inference 0 ')
        self.inferencef = theano.function(fxn_inputs, [evaldictf['z'],
                                                       evaldictf['mu_q'], evaldictf['logcov_q'] ,evaldictf['KL']],
                                          name = 'Posterior Inference F ')
        #Create a theano input to estimate the Jacobian with respect to
        z0       = T.vector('z')
        z0.tag.test_value = np.random.randn(self.params['dim_stochastic']).astype(config.floatX)
        """
        Estimating Jacobian Vectors
        """
        additional   = {}
        lsf          = self._conditionalXgivenZ(z0,additional=additional) #This computes Jacobian wrt log-probabilities, For poisson models this is the logmean
        if self.params['data_type']=='real':
            lsf = lsf[0]
        #Grad wrt energy
        jacob_energy   = theano.gradient.jacobian(additional['E'],wrt=z0)
        jacob_logprobs = theano.gradient.jacobian(lsf,wrt=z0)
        jacob_probs    = theano.gradient.jacobian(T.exp(lsf),wrt=z0)
        jacob_logprobs_mnist = theano.gradient.jacobian(T.log(lsf),wrt=z0) #For use w/ binarized mnist only
        self.jacobian_logprobs = theano.function([z0],jacob_logprobs,name='Jacobian wrt Log-Probs')
        self.jacobian_probs    = theano.function([z0],jacob_probs,name='Jacobian')
        self.jacobian_energy   = theano.function([z0],jacob_energy,name='Jacobian wrt energy')
        #Evaluating perplexity
        if self.params['data_type']=='bow':
            X_count     = X.sum(1,keepdims=True)
            self.evaluatePerp = theano.function(fxn_inputs, [(elbo_init_batch/X_count).sum(),
                (elbo_final_batch/X_count).sum(), evaldictopt['n_steps'], diff_elbo])
        self.debugModel  = theano.function([X], [evaldict0['elbo_batch'].sum(), evaldict0['negCLL'].sum(),evaldict0['KLmat'].sum()])

    ################################    Building Model #####################
    def _buildModel(self):
        """ Build VAE Graph """
        self.optimizer = adam
        X   = T.matrix('X',   dtype=config.floatX)
        X.tag.test_value, mu_tag  = self._fakeData()

        #Learning rates and annealing objective function
        self._addWeights('iter_ctr', np.asarray(1.))
        iteration_t    = self.tWeights['iter_ctr']
        self._addWeights('anneal', np.asarray(0.,dtype=config.floatX),borrow=False)
        anneal         = self.tWeights['anneal']
        anneal_rate    = self.params['anneal_rate']+1e-5
        anneal_update  = [(iteration_t, iteration_t+1),
                      (anneal,
                       T.switch(0.01+iteration_t/anneal_rate>1,1.,0.01+iteration_t/anneal_rate))]

        #Used in optimizer.py
        self._addWeights('lr',   np.asarray(self.params['lr'],dtype=config.floatX),  borrow=False)
        lr             = self.tWeights['lr']

        n_steps, plr            = T.iscalar('n_steps'), T.fscalar('plr')
        n_steps.tag.test_value, plr.tag.test_value = self.params['n_steps'],self.params['param_lr']

        self._p('Building Functions for Evaluation')
        self._buildOptimizationFunction(X, n_steps, plr)
        self._buildEvaluationFunctions(X,n_steps,plr)
        if 'validate_only' in self.params or 'EVALUATE' in self.params:
            self.updates_ack = True
            self.tOptWeights = []
            self._p('Not building training functions...')
            return
        self._p('Building Functions for Training')
        if self.params['opt_type']=='none':
            """ none: simple vae optimization """
            traindict = {}
            upperbound_train         = self._ELBO(X, anneal = anneal,
                                        dropout_prob = self.params['input_dropout'], savedict=traindict)
            self.updates_ack = True
            if 'GRADONLY' in self.params:
                self._p('Only computing gradients with respect to cost function')
                p_params                 = self._getModelParams('p_')
                p_grads                  = T.grad(upperbound_train, p_params)
                self.getgrads            = theano.function([X],[pp*1. for pp in p_grads])
                self.tOptWeights = []
                return
            model_params             = self._getModelParams()
            optimizer_up, norm_list  = self._setupOptimizer(upperbound_train,  model_params,
                                                        lr = lr,
                                                        grad_noise = self.params['grad_noise'],
                                                        rng = self.srng)#,
            self._p('# additional updates: '+str(len(self.updates)))
            optimizer_up+=anneal_update +self.updates
            fxn_inputs      = [X]
            self.train      = theano.function(fxn_inputs, [upperbound_train, norm_list[0], norm_list[1], norm_list[2],
                                                           anneal.sum(), lr.sum()],
                                              updates = optimizer_up, name = 'Train')
        elif self.params['opt_type'] in ['finopt']:
            ##################                UPDATE  P         ######################
            dictopt,dictf = {},{}
            mu_q_0, logcov_q_0 = self._inference(X)
            mu_q_f, logcov_q_f, _ = self._optimizeVariationalParams(X, mu_q_0, logcov_q_0,
                                                                          n_steps, plr, savedict=dictopt)
            elbo_init      = self._ELBO(X, mu_q=mu_q_0, logcov_q=logcov_q_0, anneal=anneal)
            mu_f_dgrad     = theano.gradient.disconnected_grad(mu_q_f)
            logcov_f_dgrad = theano.gradient.disconnected_grad(logcov_q_f)
            elbo_final     = self._ELBO(X, mu_q=mu_f_dgrad, logcov_q=logcov_f_dgrad, savedict = dictf, anneal=anneal)

            p_params                 = self._getModelParams(restrict='p_')
            self.updates_ack = True
            if 'GRADONLY' in self.params:
                self._p('Only computing gradients with respect to cost function')
                p_grads                  = T.grad(elbo_final, p_params)
                self.getgrads            = theano.function([X,
                                                            theano.In(n_steps, value=self.params['n_steps'], name='n_steps'),
                                                            theano.In(plr, value=self.params['param_lr'], name='plr')],
                                                            [pp*1. for pp in p_grads])
                self.tOptWeights = []
                return
            optimizer_p, norm_list_p = self._setupOptimizer(elbo_final, p_params, lr = lr,
                                                        grad_noise = self.params['grad_noise'],
                                                        rng = self.srng)#,
            self._p('# additional updates: '+str(len(self.updates)))
            optimizer_p += self.updates+anneal_update
            diff_elbo, _ = self._estimateELBOEntropy(elbo_init, elbo_final, logcov_q_0, logcov_q_f)
            self.update_p   = theano.function([X,theano.In(n_steps, value=self.params['n_steps'], name='n_steps'),
                                                  theano.In(plr, value=self.params['param_lr'], name='plr')],
                                              [elbo_init, elbo_final, anneal.sum(),
                                               norm_list_p[0], norm_list_p[1], norm_list_p[2],
                                               dictopt['n_steps'], dictopt['gradnorm_mu_its'][-1],
                                               dictopt['gradnorm_logcov_its'][-1],diff_elbo],#+gdiffs,
                                              updates = optimizer_p, name = 'Train P')
            ##################                UPDATE Q           ######################
            q_params                 = self._getModelParams(restrict='q_')
            elbo                     = self._ELBO(X,  dropout_prob = 0. , anneal = anneal)
            optimizer_q, norm_list_q = self._setupOptimizer(elbo, q_params,lr = lr,
                                                        grad_noise = self.params['grad_noise'],
                                                        rng = self.srng)
            self.update_q   = theano.function([X], elbo, updates = optimizer_q, name = 'Train Q')
        elif self.params['opt_type'] in ['finopt_none','none_finopt']:
            """
            Functions to update with and without optimizing variational parameters
            """
            self._p('Functions to learning while optimizing var.params')
            ##################                UPDATE  P         ######################
            dictopt,dictf = {},{}
            mu_q_0, logcov_q_0 = self._inference(X)
            mu_q_f, logcov_q_f, _ = self._optimizeVariationalParams(X, mu_q_0, logcov_q_0,
                                                                          n_steps, plr, savedict=dictopt)
            elbo_init      = self._ELBO(X, mu_q=mu_q_0, logcov_q=logcov_q_0, anneal=anneal)
            mu_f_dgrad     = theano.gradient.disconnected_grad(mu_q_f)
            logcov_f_dgrad = theano.gradient.disconnected_grad(logcov_q_f)
            elbo_final     = self._ELBO(X, mu_q=mu_f_dgrad, logcov_q=logcov_f_dgrad, savedict = dictf, anneal=anneal)

            p_params                 = self._getModelParams(restrict='p_')
            self.updates_ack = True
            if 'GRADONLY' in self.params:
                self._p('Only computing gradients with respect to cost function')
                p_grads                  = T.grad(elbo_final, p_params)
                self.getgrads            = theano.function([X,
                                                            theano.In(n_steps, value=self.params['n_steps'], name='n_steps'),
                                                            theano.In(plr, value=self.params['param_lr'], name='plr')],
                                                            [pp*1. for pp in p_grads])
                self.tOptWeights = []
                return

            """
            Update P while optimizing var. params
            """
            optimizer_p, norm_list_p = self._setupOptimizer(elbo_final, p_params, lr = lr,
                                                        grad_noise = self.params['grad_noise'],
                                                        rng = self.srng)#,
            self._p('# additional updates: '+str(len(self.updates)))
            optimizer_p += self.updates+anneal_update
            diff_elbo, _ = self._estimateELBOEntropy(elbo_init, elbo_final, logcov_q_0, logcov_q_f)
            self.update_p   = theano.function([X,theano.In(n_steps, value=self.params['n_steps'], name='n_steps'),
                                                  theano.In(plr, value=self.params['param_lr'], name='plr')],
                                              [elbo_init, elbo_final, anneal.sum(),
                                               norm_list_p[0], norm_list_p[1], norm_list_p[2],
                                               dictopt['n_steps'], dictopt['gradnorm_mu_its'][-1],
                                               dictopt['gradnorm_logcov_its'][-1],diff_elbo],#+gdiffs,
                                              updates = optimizer_p, name = 'Train P')
            ##################                UPDATE Q           ######################
            elbo                     = self._ELBO(X,  dropout_prob = 0. , anneal = anneal)
            """
            Update Q
            """
            q_params                 = self._getModelParams(restrict='q_')
            optimizer_q, norm_list_q = self._setupOptimizer(elbo, q_params,lr = lr,
                                                        grad_noise = self.params['grad_noise'],
                                                        rng = self.srng)
            self.update_q            = theano.function([X], elbo, updates = optimizer_q, name = 'Train Q')
            """
            Update P/Q
            """
            self._p('Functions to learning without optimizing var.params')
            model_params             = self._getModelParams()
            optimizer_up, norm_list  = self._setupOptimizer(elbo,  model_params,
                                                        lr = lr,
                                                        grad_noise = self.params['grad_noise'],
                                                        rng = self.srng)#,
            self._p('# additional updates: '+str(len(self.updates)))
            optimizer_up+=anneal_update +self.updates
            fxn_inputs      = [X]
            self.train      = theano.function(fxn_inputs, [elbo, norm_list[0], norm_list[1], norm_list[2], anneal.sum(), lr.sum()],
                                              updates = optimizer_up, name = 'Train')
        else:
            assert False,'Invalid optimization type: '+self.params['opt_type']
        self._p('Done creating functions for training')

if __name__=='__main__':
    print 'Initializing VAE'
    pfile = './tmp'
    from optvaeutils.parse_args_vae import params
    params['dim_observations'] =2000
    params['data_type']   = 'bow'
    params['opt_type']    = 'none'
    params['opt_method']  = 'adam'
    params['anneal_finopt_rate'] = 100
    params['GRADONLY']=True
    vae   = VAE(params, paramFile=pfile)
    from datasets.load import loadDataset
    dataset = loadDataset('binarized_mnist')
    np.random.seed(1)
    idxlist = np.random.permutation(dataset['train'].shape[0])
    X = dataset['train'][idxlist[:200]].astype('float32')
    os.remove(pfile)
    import ipdb;ipdb.set_trace()


text:
<?php

namespace app\components;

use yii\base\Behavior;
use yii\db\ActiveRecord;
/**
 * This command echoes the first argument that you have entered.
 *
 * This command is provided as an example for you to learn how to create console commands.
 *
 * @author Qiang Xue <qiang.xue@gmail.com>
 * @since 2.0
 */
class StringBehavior extends Behavior
{
    public $attributes = array();

    public function events()
    {
        return [
            ActiveRecord::EVENT_BEFORE_INSERT   => 'stringify',
            ActiveRecord::EVENT_BEFORE_UPDATE   => 'stringify',
            ActiveRecord::EVENT_AFTER_FIND      => 'unstringify',
            ActiveRecord::EVENT_AFTER_INSERT    => 'unstringify',
            ActiveRecord::EVENT_AFTER_UPDATE    => 'unstringify',
        ];
    }

    public function stringify()
    {
        foreach ($this->attributes as $attribute) {
            $this->owner->$attribute = json_encode($this->owner->$attribute);
        }
    }

    public function unstringify()
    {
        foreach ($this->attributes as $attribute) {
            $this->owner->setAttribute($attribute, json_decode($this->owner->$attribute, true));
        }
    }
}

redpajama_set_name: RedPajamaGithub


text:
title: "Community, Impact, and Credit: Where Do I Submit My Papers?"
collection: talks
type: "Panelist"
permalink: /talks/2013-02-26-cscw-community-impact/
venue: "ACM Conference on Computer-Supported Cooperative Work (CSCW)"
date: 2013-02-26
location: "San Antonio, TX"
---


text:
layout: default
title: Home
---

  {% for post in paginator.posts %}

        {{ post.title }}

    {{ post.date | date_to_string }}

    {{ post.content }}

  {% endfor %}

{% if paginator.next_page or paginator.previous_page %}

  {% if paginator.next_page %}
    Older
  {% else %}
    Older
  {% endif %}
  {% if paginator.previous_page %}
    {% if paginator.page == 2 %}
      Newer
    {% else %}
      Newer
    {% endif %}
  {% else %}
    Newer
  {% endif %}

{% endif %}


text:
require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))

describe RTurk::UpdateQualificationScore do
  before(:all) do
    aws = YAML.load(File.open(File.join(SPEC_ROOT, 'mturk.yml')))
    RTurk.setup(aws['AWSAccessKeyId'], aws['AWSAccessKey'], :sandbox => true)
    faker('update_qualification_score', :operation => 'UpdateQualificationScore')
  end

  it "should ensure required params" do
    lambda{RTurk::UpdateQualificationScore()}.should raise_error RTurk::MissingParameters
  end

  it "should successfully request the operation" do
    RTurk::Requester.should_receive(:request).once.with(
      hash_including('Operation' => 'UpdateQualificationScore'))
    RTurk::UpdateQualificationScore(:qualification_type_id => '789RVWYBAZW00EXAMPLE',
                                  :subject_id => "ABCDEF1234",
                                  :integer_value => 80) rescue RTurk::InvalidRequest
  end

  it "should parse and return the result" do
    RTurk::UpdateQualificationScore(:qualification_type_id => '789RVWYBAZW00EXAMPLE',
                                  :subject_id => "ABCDEF1234",
                                  :integer_value => 80).elements.should eql(
      {"UpdateQualificationScoreResult"=>{"Request"=>{"IsValid"=>"True"}}})
  end
end


text:
public class MathTest {
	/**
	 * Multiply n by m
	 *
	 * @param n
	 *            the first number
	 * @param m
	 *            the second number
	 * @return n*m
	 */
	public int multiply(int n, int m) {
		return n * m;
	}
}


text:
TODO: Write a gem description

## Installation

Add this line to your application's Gemfile:

    gem 'ninjas-on-a-boat'

And then execute:

    $ bundle

Or install it yourself as:

    $ gem install ninjas-on-a-boat

## Usage

TODO: Write usage instructions here

## Contributing

1. Fork it ( https://github.com/[my-github-username]/ninjas-on-a-boat/fork )
2. Create your feature branch (`git checkout -b my-new-feature`)
3. Commit your changes (`git commit -am 'Add some feature'`)
4. Push to the branch (`git push origin my-new-feature`)
5. Create a new Pull Request


text:
import { triggerToElements } from "../util/event_trigger.ts";
import check from "../util/check.ts";

/**
 * `@emits(event)` decorator
 *
 * This decorator adds the event emission at the end of the method.
 * If the method returns the promise, then the event is emitted when it is resolved.
 * @param event The event name
 */
const emits = (event: string) =>
  (
    // deno-lint-ignore no-explicit-any
    target: any,
    key: string,
    // deno-lint-ignore no-explicit-any
    descriptor: any,
  ) => {
    const method = descriptor.value;
    const constructor = target.constructor;

    check(
      !!event,
      `Unable to emits an empty event: constructor=${constructor.name} key=${key}`,
    );

    descriptor.value = function () {
      const result = method.apply(this, arguments);
      triggerToElements([this.el], event, true, result);
      return result;
    };
  };

export default emits;


text:
<?php

class Twig_Node_Expression_TempName extends Twig_Node_Expression
{
    public function __construct($name, $lineno)
    {
        parent::__construct([], ['name' => $name], $lineno);
    }

    public function compile(Twig_Compiler $compiler)
    {
        $compiler
            ->raw('$_')
            ->raw($this->getAttribute('name'))
            ->raw('_')
        ;
    }
}

class_alias('Twig_Node_Expression_TempName', 'Twig\Node\Expression\TempNameExpression', false);


text:
package ru.radiomayak.http.ssl;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.Socket;
import java.net.URL;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.Principal;
import java.security.PrivateKey;
import java.security.SecureRandom;
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;

import javax.net.ssl.KeyManager;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509ExtendedKeyManager;
import javax.net.ssl.X509TrustManager;

import ru.radiomayak.http.util.Args;

/**
 * Builder for {@link javax.net.ssl.SSLContext} instances.
 *

 * Please note: the default Oracle JSSE implementation of {@link SSLContext#init(KeyManager[], TrustManager[], SecureRandom)}
 * accepts multiple key and trust managers, however only only first matching type is ever used.
 * See for example:
 *
 * SSLContext.html#init
 *
 *
 * @since 4.4
 */
public class SSLContextBuilder {

    static final String TLS   = "TLS";

    private String protocol;
    private final Set<KeyManager> keymanagers;
    private final Set<TrustManager> trustmanagers;
    private SecureRandom secureRandom;

    public static SSLContextBuilder create() {
        return new SSLContextBuilder();
    }

    public SSLContextBuilder() {
        super();
        this.keymanagers = new LinkedHashSet<KeyManager>();
        this.trustmanagers = new LinkedHashSet<TrustManager>();
    }

    public SSLContextBuilder useProtocol(final String protocol) {
        this.protocol = protocol;
        return this;
    }

    public SSLContextBuilder setSecureRandom(final SecureRandom secureRandom) {
        this.secureRandom = secureRandom;
        return this;
    }

    public SSLContextBuilder loadTrustMaterial(
            final KeyStore truststore,
            final TrustStrategy trustStrategy) throws NoSuchAlgorithmException, KeyStoreException {
        final TrustManagerFactory tmfactory = TrustManagerFactory.getInstance(
                TrustManagerFactory.getDefaultAlgorithm());
        tmfactory.init(truststore);
        final TrustManager[] tms = tmfactory.getTrustManagers();
        if (tms != null) {
            if (trustStrategy != null) {
                for (int i = 0; i < tms.length; i++) {
                    final TrustManager tm = tms[i];
                    if (tm instanceof X509TrustManager) {
                        tms[i] = new TrustManagerDelegate(
                                (X509TrustManager) tm, trustStrategy);
                    }
                }
            }
            for (final TrustManager tm : tms) {
                this.trustmanagers.add(tm);
            }
        }
        return this;
    }

    public SSLContextBuilder loadTrustMaterial(
            final TrustStrategy trustStrategy) throws NoSuchAlgorithmException, KeyStoreException {
        return loadTrustMaterial(null, trustStrategy);
    }

    public SSLContextBuilder loadTrustMaterial(
            final File file,
            final char[] storePassword,
            final TrustStrategy trustStrategy) throws NoSuchAlgorithmException, KeyStoreException, CertificateException, IOException {
        Args.notNull(file, "Truststore file");
        final KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
        final FileInputStream instream = new FileInputStream(file);
        try {
            trustStore.load(instream, storePassword);
        } finally {
            instream.close();
        }
        return loadTrustMaterial(trustStore, trustStrategy);
    }

    public SSLContextBuilder loadTrustMaterial(
            final File file,
            final char[] storePassword) throws NoSuchAlgorithmException, KeyStoreException, CertificateException, IOException {
        return loadTrustMaterial(file, storePassword, null);
    }

    public SSLContextBuilder loadTrustMaterial(
            final File file) throws NoSuchAlgorithmException, KeyStoreException, CertificateException, IOException {
        return loadTrustMaterial(file, null);
    }

    public SSLContextBuilder loadTrustMaterial(
            final URL url,
            final char[] storePassword,
            final TrustStrategy trustStrategy) throws NoSuchAlgorithmException, KeyStoreException, CertificateException, IOException {
        Args.notNull(url, "Truststore URL");
        final KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
        final InputStream instream = url.openStream();
        try {
            trustStore.load(instream, storePassword);
        } finally {
            instream.close();
        }
        return loadTrustMaterial(trustStore, trustStrategy);
    }

    public SSLContextBuilder loadTrustMaterial(
            final URL url,
            final char[] storePassword) throws NoSuchAlgorithmException, KeyStoreException, CertificateException, IOException {
        return loadTrustMaterial(url, storePassword, null);
    }

    public SSLContextBuilder loadKeyMaterial(
            final KeyStore keystore,
            final char[] keyPassword,
            final PrivateKeyStrategy aliasStrategy)
            throws NoSuchAlgorithmException, KeyStoreException, UnrecoverableKeyException {
        final KeyManagerFactory kmfactory = KeyManagerFactory.getInstance(
                KeyManagerFactory.getDefaultAlgorithm());
        kmfactory.init(keystore, keyPassword);
        final KeyManager[] kms =  kmfactory.getKeyManagers();
        if (kms != null) {
            if (aliasStrategy != null) {
                for (int i = 0; i < kms.length; i++) {
                    final KeyManager km = kms[i];
                    if (km instanceof X509ExtendedKeyManager) {
                        kms[i] = new KeyManagerDelegate((X509ExtendedKeyManager) km, aliasStrategy);
                    }
                }
            }
            for (final KeyManager km : kms) {
                keymanagers.add(km);
            }
        }
        return this;
    }

    public SSLContextBuilder loadKeyMaterial(
            final KeyStore keystore,
            final char[] keyPassword) throws NoSuchAlgorithmException, KeyStoreException, UnrecoverableKeyException {
        return loadKeyMaterial(keystore, keyPassword, null);
    }

    public SSLContextBuilder loadKeyMaterial(
            final File file,
            final char[] storePassword,
            final char[] keyPassword,
            final PrivateKeyStrategy aliasStrategy) throws NoSuchAlgorithmException, KeyStoreException, UnrecoverableKeyException, CertificateException, IOException {
        Args.notNull(file, "Keystore file");
        final KeyStore identityStore = KeyStore.getInstance(KeyStore.getDefaultType());
        final FileInputStream instream = new FileInputStream(file);
        try {
            identityStore.load(instream, storePassword);
        } finally {
            instream.close();
        }
        return loadKeyMaterial(identityStore, keyPassword, aliasStrategy);
    }

    public SSLContextBuilder loadKeyMaterial(
            final File file,
            final char[] storePassword,
            final char[] keyPassword) throws NoSuchAlgorithmException, KeyStoreException, UnrecoverableKeyException, CertificateException, IOException {
        return loadKeyMaterial(file, storePassword, keyPassword, null);
    }

    public SSLContextBuilder loadKeyMaterial(
            final URL url,
            final char[] storePassword,
            final char[] keyPassword,
            final PrivateKeyStrategy aliasStrategy) throws NoSuchAlgorithmException, KeyStoreException, UnrecoverableKeyException, CertificateException, IOException {
        Args.notNull(url, "Keystore URL");
        final KeyStore identityStore = KeyStore.getInstance(KeyStore.getDefaultType());
        final InputStream instream = url.openStream();
        try {
            identityStore.load(instream, storePassword);
        } finally {
            instream.close();
        }
        return loadKeyMaterial(identityStore, keyPassword, aliasStrategy);
    }

    public SSLContextBuilder loadKeyMaterial(
            final URL url,
            final char[] storePassword,
            final char[] keyPassword) throws NoSuchAlgorithmException, KeyStoreException, UnrecoverableKeyException, CertificateException, IOException {
        return loadKeyMaterial(url, storePassword, keyPassword, null);
    }

    protected void initSSLContext(
            final SSLContext sslcontext,
            final Collection<KeyManager> keyManagers,
            final Collection<TrustManager> trustManagers,
            final SecureRandom secureRandom) throws KeyManagementException {
        sslcontext.init(
                !keyManagers.isEmpty() ? keyManagers.toArray(new KeyManager[keyManagers.size()]) : null,
                !trustManagers.isEmpty() ? trustManagers.toArray(new TrustManager[trustManagers.size()]) : null,
                secureRandom);
    }

    public SSLContext build() throws NoSuchAlgorithmException, KeyManagementException {
        final SSLContext sslcontext = SSLContext.getInstance(
                this.protocol != null ? this.protocol : TLS);
        initSSLContext(sslcontext, keymanagers, trustmanagers, secureRandom);
        return sslcontext;
    }

    static class TrustManagerDelegate implements X509TrustManager {

        private final X509TrustManager trustManager;
        private final TrustStrategy trustStrategy;

        TrustManagerDelegate(final X509TrustManager trustManager, final TrustStrategy trustStrategy) {
            super();
            this.trustManager = trustManager;
            this.trustStrategy = trustStrategy;
        }

        @Override
        public void checkClientTrusted(
                final X509Certificate[] chain, final String authType) throws CertificateException {
            this.trustManager.checkClientTrusted(chain, authType);
        }

        @Override
        public void checkServerTrusted(
                final X509Certificate[] chain, final String authType) throws CertificateException {
            if (!this.trustStrategy.isTrusted(chain, authType)) {
                this.trustManager.checkServerTrusted(chain, authType);
            }
        }

        @Override
        public X509Certificate[] getAcceptedIssuers() {
            return this.trustManager.getAcceptedIssuers();
        }

    }

    static class KeyManagerDelegate extends X509ExtendedKeyManager {

        private final X509ExtendedKeyManager keyManager;
        private final PrivateKeyStrategy aliasStrategy;

        KeyManagerDelegate(final X509ExtendedKeyManager keyManager, final PrivateKeyStrategy aliasStrategy) {
            super();
            this.keyManager = keyManager;
            this.aliasStrategy = aliasStrategy;
        }

        @Override
        public String[] getClientAliases(
                final String keyType, final Principal[] issuers) {
            return this.keyManager.getClientAliases(keyType, issuers);
        }

        public Map<String, ru.radiomayak.http.ssl.PrivateKeyDetails> getClientAliasMap(
                final String[] keyTypes, final Principal[] issuers) {
            final Map<String, ru.radiomayak.http.ssl.PrivateKeyDetails> validAliases = new HashMap<String, ru.radiomayak.http.ssl.PrivateKeyDetails>();
            for (final String keyType: keyTypes) {
                final String[] aliases = this.keyManager.getClientAliases(keyType, issuers);
                if (aliases != null) {
                    for (final String alias: aliases) {
                        validAliases.put(alias,
                                new ru.radiomayak.http.ssl.PrivateKeyDetails(keyType, this.keyManager.getCertificateChain(alias)));
                    }
                }
            }
            return validAliases;
        }

        public Map<String, ru.radiomayak.http.ssl.PrivateKeyDetails> getServerAliasMap(
                final String keyType, final Principal[] issuers) {
            final Map<String, ru.radiomayak.http.ssl.PrivateKeyDetails> validAliases = new HashMap<String, ru.radiomayak.http.ssl.PrivateKeyDetails>();
            final String[] aliases = this.keyManager.getServerAliases(keyType, issuers);
            if (aliases != null) {
                for (final String alias: aliases) {
                    validAliases.put(alias,
                            new ru.radiomayak.http.ssl.PrivateKeyDetails(keyType, this.keyManager.getCertificateChain(alias)));
                }
            }
            return validAliases;
        }

        @Override
        public String chooseClientAlias(
                final String[] keyTypes, final Principal[] issuers, final Socket socket) {
            final Map<String, ru.radiomayak.http.ssl.PrivateKeyDetails> validAliases = getClientAliasMap(keyTypes, issuers);
            return this.aliasStrategy.chooseAlias(validAliases, socket);
        }

        @Override
        public String[] getServerAliases(
                final String keyType, final Principal[] issuers) {
            return this.keyManager.getServerAliases(keyType, issuers);
        }

        @Override
        public String chooseServerAlias(
                final String keyType, final Principal[] issuers, final Socket socket) {
            final Map<String, ru.radiomayak.http.ssl.PrivateKeyDetails> validAliases = getServerAliasMap(keyType, issuers);
            return this.aliasStrategy.chooseAlias(validAliases, socket);
        }

        @Override
        public X509Certificate[] getCertificateChain(final String alias) {
            return this.keyManager.getCertificateChain(alias);
        }

        @Override
        public PrivateKey getPrivateKey(final String alias) {
            return this.keyManager.getPrivateKey(alias);
        }

        @Override
        public String chooseEngineClientAlias(
                final String[] keyTypes, final Principal[] issuers, final SSLEngine sslEngine) {
            final Map<String, ru.radiomayak.http.ssl.PrivateKeyDetails> validAliases = getClientAliasMap(keyTypes, issuers);
            return this.aliasStrategy.chooseAlias(validAliases, null);
        }

        @Override
        public String chooseEngineServerAlias(
                final String keyType, final Principal[] issuers, final SSLEngine sslEngine) {
            final Map<String, ru.radiomayak.http.ssl.PrivateKeyDetails> validAliases = getServerAliasMap(keyType, issuers);
            return this.aliasStrategy.chooseAlias(validAliases, null);
        }

    }

}


text:
require_relative "my_solution"

describe 'welcome' do
  it 'is defined as a method' do
    expect(defined?(welcome)).to eq 'method'
  end
  it 'accepts an address as a single parameter' do
    expect(method(:welcome).arity).to eq 1
  end
  it 'returns "Welcome to California" if the address contains "CA"' do
    expect(welcome("a string with CA in it")).to eq "Welcome to California"
  end
  it 'returns "You should move to California" if the address does not contain "CA"' do
    expect(welcome("nothing like cali in here")).to eq "You should move to California"
    expect(welcome("china")).to eq "You should move to California"
    expect(welcome("paris")).to eq "You should move to California"
  end
end


text:
package org.apache.jena.atlas.lib;

import java.util.HashSet ;
import java.util.Set ;
import java.util.concurrent.ConcurrentHashMap ;
import java.util.stream.Collectors;
import java.util.stream.Stream;

/** Set specific operations */
public class SetUtils
{
    private SetUtils() {}

    /** "ConcurrentHashSet" */
    public static final <X> Set<X> concurrentHashSet() {
        return ConcurrentHashMap.newKeySet() ;
    }

    public static <T> Set<T> intersection(Set<? extends T> setLeft, Set<? extends T> setRight) {
        Set<T> results = new HashSet<>(setLeft) ;
        results.retainAll(setRight) ;
        return results ;
    }

    public static <T> boolean intersectionP(Set<? extends T> s1, Set<? extends T> s2) {
        return s1.stream().anyMatch(s2::contains) ;
    }

    /**
     * @param s1 a {@link Set}
     * @param s2 a {@link Set}
     * @return the set-theoretic union of {@code s1} and {@code s2}
     *
     * @see org.apache.jena.ext.com.google.common.collect.Sets#union(Set, Set) for a view-based alternative
     */
    public static <T> Set<T> union(Set<? extends T> s1, Set<? extends T> s2) {
        Set<T> s3 = new HashSet<>(s1) ;
        s3.addAll(s2) ;
        return s3 ;
    }

    /**
     * @param s1 a {@link Set}
     * @param s2 a {@link Set}
     * @return {@code s1 ∪ s2  \ s1 ∩ s2} : those elements in {@code s1} or {@code s2} but not both
     *
     * @see org.apache.jena.ext.com.google.common.collect.Sets#symmetricDifference(Set, Set) for a view-based alternative
     */
    public static <T> Set<T> symmetricDifference(Set<? extends T> s1, Set<? extends T> s2) {
        Set<T> s3 = new HashSet<>() ;
        s1.forEach(x -> {
            if ( !s2.contains(x) )
                s3.add(x) ;
        }) ;
        s2.forEach(x -> {
            if ( !s1.contains(x) )
                s3.add(x) ;
        }) ;
        return s3 ;
    }

    /**
     * @param s1 a {@link Set}
     * @param s2 a {@link Set}
     * @return {@code s1} \ {@code s2}
     *
     * @see org.apache.jena.ext.com.google.common.collect.Sets#difference(Set, Set) for a view-based alternative
     */
    public static <T> Set<T> difference(Set<? extends T> s1, Set<? extends T> s2) {
        Set<T> s3 = new HashSet<>(s1) ;
        s3.removeAll(s2) ;
        return s3 ;
    }

    /** Stream to set */
    public static <T> Set<T> toSet(Stream<T> stream) {
        return stream.collect(Collectors.toSet());
    }
}


text:
package com.thoughtworks.go.config;

import java.util.List;

import com.thoughtworks.go.domain.ConfigErrors;
import com.thoughtworks.go.domain.config.Admin;

@ConfigTag("user")
public class AdminUser implements Admin {
    @ConfigValue private CaseInsensitiveString name;

    private final ConfigErrors configErrors = new ConfigErrors();

    public AdminUser() {
    }

    public AdminUser(final CaseInsensitiveString name) {
        this.name = name;
    }

    public void validate(ValidationContext validationContext) {
        if(name == null || name.isBlank())
            addError("User cannot be blank.");
    }

    public ConfigErrors errors() {
        return configErrors;
    }

    public void addError(String fieldName, String message) {
        configErrors.add(fieldName, message);
    }

    public CaseInsensitiveString getName() {
        return name;
    }

    public String describe() {
        return "User";
    }

    public boolean belongsTo(Role role) {
        return role.hasMember(name);
    }

    public void addError(String message) {
        errors().add("name", message);
    }

    public boolean isSameAs(Admin admin, List<Role> memberRoles) {
        return admin instanceof AdminUser && name.equals(((AdminUser) admin).name);
    }

    public boolean equals(Object o) {
        if (this == o) {
            return true;
        }
        if (o == null || getClass() != o.getClass()) {
            return false;
        }

        AdminUser adminUser = (AdminUser) o;

        return name.equals(adminUser.name);
    }

    public int hashCode() {
        return name.hashCode();
    }

    public String toString() {
        return CaseInsensitiveString.str(name);
    }
}


text:
Run Prometheus with config from `etc` (assuming that you're in `anycable-go` source directory):

```
docker run -p 9090:9090 -v $(pwd)/etc:/prometheus-data prom/prometheus --config.file=/prometheus-data/prometheus.yml
```

Run Grafana:

```
docker run --name=grafana -p 3100:3000 grafana/grafana
```

And, finally, run `anycable-go` (locally):

```
anycable-go --metrics_http="/metrics"
```


text:
<?xml version="1.0" encoding="UTF-8"?>
<jcr:root xmlns:sling="http://sling.apache.org/jcr/sling/1.0" xmlns:cq="http://www.day.com/jcr/cq/1.0" xmlns:jcr="http://www.jcp.org/jcr/1.0"
    cq:lastReplicated="{Date}2015-01-11T16:12:14.361-08:00"
    cq:lastReplicatedBy="admin"
    cq:lastReplicationAction="Activate"
    jcr:description="login for website"
    jcr:mixinTypes="[cq:ReplicationStatus]"
    jcr:primaryType="cq:Component"
    jcr:title="nblogin"
    sling:resourceSuperType="foundation/components/parbase"
    allowedParents="[*/parsys]"
    componentGroup="nbgardens"/>


text:
package com.amazonaws.geo.util;

import com.amazonaws.geo.GeoDataManagerConfiguration;
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
import com.amazonaws.services.dynamodbv2.model.KeyType;
import com.amazonaws.services.dynamodbv2.model.LocalSecondaryIndex;
import com.amazonaws.services.dynamodbv2.model.Projection;
import com.amazonaws.services.dynamodbv2.model.ProjectionType;
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;

/**
 * Utility class.
 * */
public class GeoTableUtil {

	/**
	 *

	 * Construct a create table request object based on GeoDataManagerConfiguration. The users can update any aspect of
	 * the request and call it.
	 *

	 * Example:
	 *
	 *

	 * AmazonDynamoDBClient ddb = new AmazonDynamoDBClient(new ClasspathPropertiesFileCredentialsProvider());
	 * Region usWest2 = Region.getRegion(Regions.US_WEST_2);
	 * ddb.setRegion(usWest2);
	 *
	 * CreateTableRequest createTableRequest = GeoTableUtil.getCreateTableRequest(config);
	 * CreateTableResult createTableResult = ddb.createTable(createTableRequest);
	 *

	 *
	 * @return Generated create table request.
	 */
	public static CreateTableRequest getCreateTableRequest(GeoDataManagerConfiguration config) {
		CreateTableRequest createTableRequest = new CreateTableRequest()
				.withTableName(config.getTableName())
				.withProvisionedThroughput(
						new ProvisionedThroughput().withReadCapacityUnits(10L).withWriteCapacityUnits(5L))
				.withKeySchema(
						new KeySchemaElement().withKeyType(KeyType.HASH).withAttributeName(
								config.getHashKeyAttributeName()),
						new KeySchemaElement().withKeyType(KeyType.RANGE).withAttributeName(
								config.getRangeKeyAttributeName()))
				.withAttributeDefinitions(
						new AttributeDefinition().withAttributeType(ScalarAttributeType.N).withAttributeName(
								config.getHashKeyAttributeName()),
						new AttributeDefinition().withAttributeType(ScalarAttributeType.S).withAttributeName(
								config.getRangeKeyAttributeName()),
						new AttributeDefinition().withAttributeType(ScalarAttributeType.N).withAttributeName(
								config.getGeohashAttributeName()))
				.withLocalSecondaryIndexes(
						new LocalSecondaryIndex()
								.withIndexName(config.getGeohashIndexName())
								.withKeySchema(
										new KeySchemaElement().withKeyType(KeyType.HASH).withAttributeName(
												config.getHashKeyAttributeName()),
										new KeySchemaElement().withKeyType(KeyType.RANGE).withAttributeName(
												config.getGeohashAttributeName()))
								.withProjection(new Projection().withProjectionType(ProjectionType.ALL)));

		return createTableRequest;
	}
}


text:
#ifndef INFO_COORDINATE_PACKET_H
#define INFO_COORDINATE_PACKET_H

#include "config.h"
#include "MC2Coordinate.h"
#include "Packet.h"

/**
 * Represents a single coordinate-angle pair included in a
 * InfoCoordinateRequestPacket.
 */
class InfoCoordinate
{
public:
   /** @name Constructors */
   //@{
   /**
    * Default constructor. Sets the member variable coord to the
    * default value for a MC2Coordinate, and the angle member variable
    * to MAX_INT32.
    */
   InfoCoordinate() :coord(), angle(MAX_INT32)
   {}
   /**
    * Constructor.
    * @param pCoord The coordinate value.
    * @param pAngle The angle value.
    */
   InfoCoordinate(const MC2Coordinate& pCoord, uint32 pAngle) :
      coord(pCoord), angle(pAngle)
   {}
   //@}

   /**
    * @name Reading and writing to a packet.
    *  4 bytes lat
    *  4 bytes lon
    *  4 bytes angle
    */
   //@{
   /**
    * Write itself into a packet.
    * @param packet The packet to write into.
    * @param pos The position of the packet to start writing at. Will
    *            be updated to point to after the written data.
    */
   void save(Packet* packet, int& pos) const;
   /**
    * Update itself with data read from a packet.
    * @param packet The packet to read from
    * @param pos The position of the packet to start reading from. Will
    *            be updated to point to after the read data.
    */
   void load( const Packet* packet, int& pos) ;
   //@}

   /**@name Member data */
   //@{
   /** The coordinate data */
   MC2Coordinate coord;
   /** The angle */
   uint32 angle;
   //@}
};

/**
 * Requests information regarding coordinate-angle pairs.
 */
class InfoCoordinateRequestPacket: public RequestPacket
{
public:
   /** The container for coordinate-angle pairs. */
   typedef std::vector<InfoCoordinate> InfoCoordCont;
   /** @name Constructors. */
   //@{
   /**
    * Constructor for a packet requesting information about a single
    * coordinate-angle pair.
    * @param The map this packet pertains.
    * @param coord The coordinate.
    * @param angle The angle. Defaults to MAX_INT16.
    */
   InfoCoordinateRequestPacket(uint32 mapID,
                               const MC2Coordinate& coord,
                               uint32 angle = MAX_INT16);

   /**
    * Constructor for a packet requesting information about several
    * coordinate-angle pair.
    * @param The map this packet pertains.
    * @param coords The coordinate-angle pairs.
    */
   InfoCoordinateRequestPacket(uint32 mapID,
                               const InfoCoordCont& coords);

   //@}
   /**
    * Reads the data encoded in the packet into a InfoCoordsCont.
    * @param coords The container that the read InfoCoordinates will be stored.
    * @return The number of InfoCoordinate objects that was read.
    */
   InfoCoordCont::size_type
   readData(InfoCoordCont& coords) const;

private:
   /**
    * Writes the InfoCoordinates from a InfoCoordCont into this
    * packet.
    * @param coords The InfoCoordCont.
    */
   void encodeRequest(const InfoCoordCont& coords);
};

/**
 * Objects of this class are stored in the InfoCoordinateReplyPacket.
 */
class InfoCoordinateReplyData
{
public:
   /** @name Constuctors. */
   //@{
   /**
    * Default constructor. Sets all member variables to MAX_UINT32,
    * which represents an invalid object.
    */
   InfoCoordinateReplyData() :
      mapID(MAX_UINT32), nodeID0(MAX_UINT32), nodeID1(MAX_UINT32),
      distance(MAX_UINT32), offset(MAX_UINT32)
   {}

   /**
    * Constructor.
    */
   InfoCoordinateReplyData(uint32 mapID, uint32 nodeID0, uint32 nodeID1,
                           uint32 distance, uint32 offset) :
      mapID(mapID), nodeID0(nodeID0), nodeID1(nodeID1), distance(distance),
      offset(offset)
   {}

   /**
    * Stores this object in a packet.
    * @param packet The Packet to write into.
    * @param pos The position to start writing at. Will be updated to
    *            point past the written data.
    */
   void save(Packet* packet, int& pos) const;
   /**
    * Updates this object from a packet.
    * @param packet The Packet to read from.
    * @param pos The position to start reading from. Will be updated to
    *            point past the read data.
    */
   void load(Packet* packet, int& pos);
   /**
    * Compares this object with the argument, and updates this object
    * with the best of the two.
    * If the distance member variable is lower in the parameter object
    * the data from that object will be copied into this object.
    * @param data The object to compare to and maybe copy from.
    */
   void merge( const InfoCoordinateReplyData& data );
   /** The id of the map this info is from */
   uint32 mapID;
   /** The first node ID */
   uint32 nodeID0;
   /** The second node ID */
   uint32 nodeID1;
   /** The distance to the segment?*/
   uint32 distance;
   /** The offset into the segment?*/
   uint32 offset;
};

ostream& operator<<(ostream& stream, const InfoCoordinateReplyData& icrd);

/**
 * The reply packet for InfoCoordinateRequestPacket.
 */
class InfoCoordinateReplyPacket: public ReplyPacket
{
public:
   /**
    * The container type used to collect the InfoCoordinateReplyData
    * objects contained in the packet.
    */
   typedef std::vector<InfoCoordinateReplyData> InfoReplyCont;
   /**
    * Constructor.
    * @param request The packet the new object is a reply to.
    * @param status The status of the packet.
    * @param data The objects to store in the packet.
    */
   InfoCoordinateReplyPacket( const InfoCoordinateRequestPacket* request,
                              uint32 status,
                              const InfoReplyCont& data );
   /**
    * Merge the InfoCoordinateReplyData objects in this packet with
    * the ones stored in the InfoReplyCont parameter.  The merge is
    * done by replacing any invalid objects in the parameter with
    * valid from this packet, or if this packet contains better
    * matches they will replace the ones in the parameter see
    * InfoCoordinateReplyData::merge.
    * @param data The container to merge into.
    */
   void mergeData(InfoReplyCont& data);
private:
   /**
    * Write a set of InfoCoordinateReplyData objects into this packet.
    * @param data The objects to write.
    */
   void encodeReply(const InfoReplyCont& data);
};

#endif


text:
{% load i18n static cms_tags sekizai_tags %}

{% addtoblock "js" %}{% endaddtoblock %}
{% addtoblock "ng-requires" %}django.shop.cart{% endaddtoblock %}

{% block shop-cart-table %}

{% endblock shop-cart-table %}


text:
import datetime
import logging
import os
import pathlib

import pandas as pd
import requests
from google.cloud import storage

def main(
    source_url: str,
    source_file: pathlib.Path,
    target_file: pathlib.Path,
    chunksize: str,
    target_gcs_bucket: str,
    target_gcs_path: str,
) -> None:

    logging.info("San Francisco - 311 Service Requests process started")

    pathlib.Path("./files").mkdir(parents=True, exist_ok=True)
    download_file(source_url, source_file)

    chunksz = int(chunksize)

    logging.info(f"Opening source file {source_file}")
    with pd.read_csv(
        source_file,
        engine="python",
        encoding="utf-8",
        quotechar='"',
        sep=",",
        chunksize=chunksz,
    ) as reader:
        for chunk_number, chunk in enumerate(reader):
            target_file_batch = str(target_file).replace(
                ".csv", "-" + str(chunk_number) + ".csv"
            )
            df = pd.DataFrame()
            df = pd.concat([df, chunk])
            process_chunk(df, target_file_batch, target_file, (not chunk_number == 0))

    upload_file_to_gcs(target_file, target_gcs_bucket, target_gcs_path)

    logging.info("San Francisco - 311 Service Requests process completed")

def download_file(source_url: str, source_file: pathlib.Path) -> None:
    logging.info(f"downloading file {source_file} from {source_url}")
    r = requests.get(source_url, stream=True)
    with open(source_file, "wb") as f:
        for chunk in r:
            f.write(chunk)

def process_chunk(
    df: pd.DataFrame, target_file_batch: str, target_file: str, skip_header: bool
) -> None:
    logging.info(f"Processing batch file {target_file_batch}")
    df = rename_headers(df)
    df = remove_empty_key_rows(df, "unique_key")
    df = resolve_datatypes(df)
    df = remove_parenthesis_long_lat(df)
    df = strip_whitespace(df)
    df = resolve_date_format(df)
    df = reorder_headers(df)
    save_to_new_file(df, file_path=str(target_file_batch))
    append_batch_file(target_file_batch, target_file, skip_header)
    logging.info(f"Processing batch file {target_file_batch} completed")

def rename_headers(df: pd.DataFrame) -> pd.DataFrame:
    logging.info("Renaming headers")
    header_names = {
        "CaseID": "unique_key",
        "Opened": "created_date",
        "Closed": "closed_date",
        "Updated": "resolution_action_updated_date",
        "Status": "status",
        "Status Notes": "status_notes",
        "Responsible Agency": "agency_name",
        "Category": "category",
        "Request Type": "complaint_type",
        "Request Details": "descriptor",
        "Address": "incident_address",
        "Supervisor District": "supervisor_district",
        "Neighborhood": "neighborhood",
        "Point": "location",
        "Source": "source",
        "Media URL": "media_url",
        "Latitude": "latitude",
        "Longitude": "longitude",
        "Police District": "police_district",
    }
    df = df.rename(columns=header_names)

    return df

def remove_empty_key_rows(df: pd.DataFrame, key_field: str) -> pd.DataFrame:
    logging.info("Removing rows with empty keys")
    df = df[df[key_field] != ""]

    return df

def resolve_datatypes(df: pd.DataFrame) -> pd.DataFrame:
    logging.info("Resolving datatypes")
    df["supervisor_district"] = df["supervisor_district"].astype("Int64")

    return df

def remove_parenthesis_long_lat(df: pd.DataFrame) -> pd.DataFrame:
    logging.info("Removing parenthesis from latitude and longitude")
    df["latitude"].replace("(", "", regex=False, inplace=True)
    df["latitude"].replace(")", "", regex=False, inplace=True)
    df["longitude"].replace("(", "", regex=False, inplace=True)
    df["longitude"].replace(")", "", regex=False, inplace=True)

    return df

def strip_whitespace(df: pd.DataFrame) -> pd.DataFrame:
    logging.info("Stripping whitespace")
    ws_fields = ["incident_address"]

    for ws_fld in ws_fields:
        df[ws_fld] = df[ws_fld].apply(lambda x: str(x).strip())

    return df

def resolve_date_format(df: pd.DataFrame) -> pd.DataFrame:
    logging.info("Resolving date formats")
    date_fields = [
        "created_date",
        "closed_date",
        "resolution_action_updated_date",
    ]

    for dt_fld in date_fields:
        df[dt_fld] = df[dt_fld].apply(convert_dt_format)

    return df

def convert_dt_format(dt_str: str) -> str:
    if not dt_str or str(dt_str).lower() == "nan" or str(dt_str).lower() == "nat":
        return ""
    elif (
        dt_str.strip()[2] == "/"
    ):  # if there is a '/' in 3rd position, then we have a date format mm/dd/yyyy
        return datetime.datetime.strptime(dt_str, "%m/%d/%Y %H:%M:%S %p").strftime(
            "%Y-%m-%d %H:%M:%S"
        )
    else:
        return str(dt_str)

def reorder_headers(df: pd.DataFrame) -> pd.DataFrame:
    logging.info("Reordering headers")
    df = df[
        [
            "unique_key",
            "created_date",
            "closed_date",
            "resolution_action_updated_date",
            "status",
            "status_notes",
            "agency_name",
            "category",
            "complaint_type",
            "descriptor",
            "incident_address",
            "supervisor_district",
            "neighborhood",
            "location",
            "source",
            "media_url",
            "latitude",
            "longitude",
            "police_district",
        ]
    ]

    return df

def save_to_new_file(df: pd.DataFrame, file_path) -> None:
    df.to_csv(file_path, index=False)

def append_batch_file(
    batch_file_path: str, target_file_path: str, skip_header: bool
) -> None:
    data_file = open(batch_file_path, "r")
    if os.path.exists(target_file_path):
        target_file = open(target_file_path, "a+")
    else:
        target_file = open(target_file_path, "w")
    if skip_header:
        logging.info(
            f"Appending batch file {batch_file_path} to {target_file_path} with skip header"
        )
        next(data_file)
    else:
        logging.info(f"Appending batch file {batch_file_path} to {target_file_path}")
    target_file.write(data_file.read())
    data_file.close()
    target_file.close()
    if os.path.exists(batch_file_path):
        os.remove(batch_file_path)

def upload_file_to_gcs(file_path: pathlib.Path, gcs_bucket: str, gcs_path: str) -> None:
    storage_client = storage.Client()
    bucket = storage_client.bucket(gcs_bucket)
    blob = bucket.blob(gcs_path)
    blob.upload_from_filename(file_path)

if __name__ == "__main__":
    logging.getLogger().setLevel(logging.INFO)

    main(
        source_url=os.environ["SOURCE_URL"],
        source_file=pathlib.Path(os.environ["SOURCE_FILE"]).expanduser(),
        target_file=pathlib.Path(os.environ["TARGET_FILE"]).expanduser(),
        chunksize=os.environ["CHUNKSIZE"],
        target_gcs_bucket=os.environ["TARGET_GCS_BUCKET"],
        target_gcs_path=os.environ["TARGET_GCS_PATH"],
    )


text:
module ASDeprecationTracker
  VERSION = '1.6.0'
end


text:
require_relative 'spec_helper'

describe 'items list' do
  let(:item_list) { ItemsList.new }
  let(:item) { {'danny7' => {'user_name' => 'danny9', 'rep_id' => '100014624', 'profile_id' => '8192'}} }

  before(:each) do
    item_list.items = {
      'admin' => {
        'danny7' => {
          'user_name' => 'danny9', 'rep_id' => '100014620', 'profile_id' => '8190', ItemsList::RESERVE_KEY => false
        },
        'johnny5' => {
          'user_name' => 'danny7', 'rep_id' => '100014624', 'profile_id' => '8192', ItemsList::RESERVE_KEY => false
        },
        '5015806226' => {
          'user_name' => '5015806226', 'rep_id' => '100014698', 'profile_id' => '8193', ItemsList::RESERVE_KEY => false
        }
      }
    }
  end

  describe '#unique_keys_across_items' do
    it 'should raise Item Type Not Defined' do
      expect { item_list.get_item(:blah, 1, 10) }.to raise_error(ItemError::ItemTypeNotDefined)
    end

    it 'should return true if unique keys' do
      expect(item_list.unique_keys_across_items?(item_list.items)).to be true
    end

    it 'should return false if duplicate keys' do
      item_list.items.merge!({'basic' => {'danny7' => {'user_name' => 'danny7'}}})
      expect(item_list.unique_keys_across_items?(item_list.items)).to be false
    end
  end

  describe '#get_item' do
    it 'should raise an error if type not defined' do
      expect { item_list.get_item('foo', 0.01, 3) }.to raise_error ItemError::ItemTypeNotDefined
    end

    it 'should return an item' do
      allow(item_list).to receive(:loop_for_item).with('admin', 0.01, 3).and_return(item)
      expect(item_list.get_item('admin', 0.01, 3)).to eql item
    end
  end

  describe '#release_item' do
    it 'should release an item' do
      item_list.items['admin']['danny7'][ItemsList::RESERVE_KEY] = true
      expect(item_list.release_item('danny7')).to be nil
      expect(item_list.items['admin']['danny7'][ItemsList::RESERVE_KEY]).to be false
      expect(item_list.items['admin']['danny7']).to_not have_key :reserved_time
    end

    it('should raise Invalid Item error when trying to release non reserved item') do
      expect { item_list.release_item('foo') }.to raise_error(ItemError::InvalidItem)
    end

    it('should release an item that is all numeric characters') do
      item_list.items['admin']['5015806226'][ItemsList::RESERVE_KEY] = true
      expect(item_list.release_item(5015806226)).to be nil
      expect(item_list.items['admin']['5015806226'][ItemsList::RESERVE_KEY]).to be false
    end
  end

  describe '#get_reserved_items' do
    it 'should return a string of reserved items' do
      allow(item_list).to receive(:reserved_items).and_return(%w(danny7 johnny5))
      expect(item_list.get_reserved_items).to eql 'danny7, johnny5'
    end
  end

  describe '#get_available_items' do
    it 'should return a string of reserved items' do
      allow(item_list).to receive(:available_items).and_return(%w(danny7))
      expect(item_list.get_available_items).to eql 'danny7'
    end
  end

  describe '#get_all_items' do
    it 'should return a string of reserved items' do
      allow(item_list).to receive(:all_items).and_return(item_list.items)
      response_hash = {"admin" => {"danny7" => {"danny7" => {"user_name" => "danny9", "rep_id" => "100014620", "profile_id" => "8190"}}, "johnny5" => {"johnny5" => {"user_name" => "danny7", "rep_id" => "100014624", "profile_id" => "8192"}}, "5015806226" => {"5015806226" => {'user_name' => '5015806226', 'rep_id' => '100014698', 'profile_id' => '8193'}}}}
      expect(item_list.get_all_items).to eql response_hash
    end
  end

  describe '#release_all_items' do
    it 'should release all items' do
      allow(item_list).to receive(:reserved_items).and_return(['danny7'])
      item_list.items['admin']['danny7'][ItemsList::RESERVE_KEY] = true
      item_list.items['admin']['johnny5'][ItemsList::RESERVE_KEY] = true
      item_list.release_all_items
      expect(item_list.items['admin']['danny7'][ItemsList::RESERVE_KEY]).to be false
      expect(item_list.items['admin']['johnny5'][ItemsList::RESERVE_KEY]).to be false
      expect(item_list.items['admin']['danny7']).to_not have_key :reserved_time
      expect(item_list.items['admin']['johnny5']).to_not have_key :reserved_time
    end

    it('should return true') do
      allow(item_list).to receive(:reserved_items).and_return(['danny7'])
      item_list.items['admin']['danny7'][ItemsList::RESERVE_KEY] = true
      item_list.items['admin']['johnny5'][ItemsList::RESERVE_KEY] = true
      expect(item_list.release_all_items).to be true
    end

    it('should raise No Reserved Items if no items are reserved') do
      expect { item_list.release_all_items }.to raise_error(ItemError::NoReservedItems)
    end
  end

  describe '#reserved_items' do
    it 'should return an array of reserved items' do
      item_list.items['admin']['danny7'][ItemsList::RESERVE_KEY] = true
      expect(item_list.reserved_items).to eql ['danny7']
    end
  end

  describe '#available_items' do
    it 'should return an array of available items' do
      item_list.items['admin']['danny7'][ItemsList::RESERVE_KEY] = true
      expect(item_list.available_items).to eql ['johnny5', '5015806226']
    end
  end

  describe '#loop_for_item' do
    it 'should reserve an item' do
      allow(item_list).to receive(:get_available_item).with('admin').and_return(item)
      allow(item_list).to receive(:reserve_item).with('admin', 'danny7', 3).and_return(item_list.items['admin']['danny7'][ItemsList::RESERVE_KEY] = true)
      allow(item_list).to receive(:sanitize_response).with(item)
      item_list.loop_for_item('admin', 0.01, 3)
      expect(item_list.items['admin']['danny7'][ItemsList::RESERVE_KEY]).to be true
    end

    it 'should return a sanitized response' do
      allow(item_list).to receive(:get_available_item).with('admin').and_return(item)
      allow(item_list).to receive(:reserve_item).with('admin', 'danny7', 3)
      allow(item_list).to receive(:sanitize_response).with(item).and_return(item)
      expect(item_list.loop_for_item('admin', 0.01, 3)['danny7'].keys).to_not include ItemsList::RESERVE_KEY
    end

    it 'should return a hash' do
      allow(item_list).to receive(:get_available_item).with('admin').and_return(item)
      allow(item_list).to receive(:reserve_item).with('admin', 'danny7', 3)
      allow(item_list).to receive(:sanitize_response).with(item).and_return(item)
      expect(item_list.loop_for_item('admin', 0.01, 3)).to be_a_kind_of Hash
    end

    it 'should raise a timeout error if not items available' do
      allow(item_list).to receive(:get_available_item).with('admin').and_return([])
      expect { item_list.loop_for_item('admin', 0.01, 3) }.to raise_error ItemError::NoAvailableItems
    end
  end

  describe '#sanitize_response' do
    it 'should not include reserve key' do
      item = {'danny7' => {'user_name' => 'danny9', 'rep_id' => '100014624', 'profile_id' => '8192', ItemsList::RESERVE_KEY.to_sym => false}}
      expect(item_list.sanitize_response(item)['danny7'].keys).to_not include ItemsList::RESERVE_KEY
    end
  end

  describe '#reserve_item' do
    it 'should reserve an item' do
      expect(item_list.reserve_item('admin', 'danny7', 3)).to be true
      expect(item_list.items['admin']['danny7'][ItemsList::RESERVE_KEY]).to be true
    end
  end

  describe '#get_available_item' do
    it 'should get next available item' do
      item_list.items['admin']['danny7'][ItemsList::RESERVE_KEY] = true
      expect(item_list.get_available_item('admin')).to eql({"johnny5" => {"user_name" => "danny7", "rep_id" => "100014624", "profile_id" => "8192", ItemsList::RESERVE_KEY => false}})
    end
  end

  describe '#load_items!' do
    let(:hash) { {type: {test: {hello: 'world', foo: 'bar'}, basic: {basic1: {}, basic2: {}}}} }
    let(:numeric_hash) { {numeric: {12345 => {hello: 'world', foo: 'bar'}, 967979 => {basic1: {}, basic2: {}}}} }
    let(:duplicate_hash) { {type: {test: {hello: 'world', foo: 'bar'}}, type2: {test: {hello: 'world'}}} }

    it 'should add a unique key 2nd level to the given hash' do
      allow(YAML).to receive(:load_file).with('').and_return(hash)
      expect(ItemsList.new.load_items!('').items["type"]["test"].keys).to include ItemsList::RESERVE_KEY.to_s
    end

    it 'should set @items to provided hash' do
      allow(YAML).to receive(:load_file).with('').and_return(hash)
      items = ItemsList.new
      items.load_items!('')
      expect(items.instance_variable_get('@items')).to eql hash
    end

    it 'should raise an error if duplicate keys' do
      allow(YAML).to receive(:load_file).with('').and_return(duplicate_hash)
      items = ItemsList.new
      expect { items.load_items!('') }.to raise_error ItemError::ItemsNotUnique
    end

    it 'should raise an error on invalid config file' do
      expect { ItemsList.new.load_items!('blah123') }.to raise_error(Errno::ENOENT)
    end

    it 'should convert numeric keys to strings' do
      allow(YAML).to receive(:load_file).with('').and_return(numeric_hash)
      expect(ItemsList.new.load_items!('').items["numeric"].keys).to include(*["12345", "967979"])
    end
  end

  describe '#clear_expire_reservations' do
    it 'should clear expired reservations' do
      item_list.items['admin']['danny7'][:reserved_time] = Time.now - 10
      item_list.items['admin']['danny7'][ItemsList::RESERVE_KEY] = true
      item_list.clear_expired_reservations
      expect(item_list.items['admin']['danny7'][ItemsList::RESERVE_KEY]).to be false
      expect(item_list.items['admin']['danny7']).to_not have_key :reserved_time
    end

    it 'should not clear active reservations' do
      item_list.items['admin']['danny7'][:reserved_time] = Time.now + 180
      item_list.items['admin']['danny7'][ItemsList::RESERVE_KEY] = true
      item_list.clear_expired_reservations
      expect(item_list.items['admin']['danny7'][ItemsList::RESERVE_KEY]).to be true
      expect(item_list.items['admin']['danny7']).to have_key :reserved_time
    end

  end

end