text:
/*************************************************************************************************/

#ifndef GIL_PIXEL_ITERATOR_ADAPTOR_H
#define GIL_PIXEL_ITERATOR_ADAPTOR_H

////////////////////////////////////////////////////////////////////////////////////////
/// \file
/// \brief pixel step iterator, pixel image iterator and pixel dereference iterator
/// \author Lubomir Bourdev and Hailin Jin \n
///         Adobe Systems Incorporated
/// \date   2005-2007 \n Last updated on February 16, 2007
///
////////////////////////////////////////////////////////////////////////////////////////

#include <iterator>
#include <boost/iterator/iterator_facade.hpp>
#include "gil_config.hpp"
#include "gil_concept.hpp"
#include "pixel_iterator.hpp"

namespace boost { namespace gil {

/// \defgroup PixelIteratorModelDerefPtr dereference_iterator_adaptor
/// \ingroup PixelIteratorModel
/// \brief An iterator that invokes a provided function object upon dereference. Models: IteratorAdaptorConcept, PixelIteratorConcept

/// \ingroup PixelIteratorModelDerefPtr PixelBasedModel
/// \brief An adaptor over an existing iterator that provides for custom filter on dereferencing the object. Models: IteratorAdaptorConcept, PixelIteratorConcept

template <typename Iterator,    // Models Iterator
          typename DFn>  // Models Returns the result of dereferencing a given iterator of type Iterator
class dereference_iterator_adaptor : public iterator_adaptor<dereference_iterator_adaptor<Iterator,DFn>,
                                                             Iterator,
                                                             typename DFn::value_type,
                                                             typename std::iterator_traits<Iterator>::iterator_category,
                                                             typename DFn::reference,
                                                             use_default> {
    DFn _deref_fn;
public:
    typedef iterator_adaptor<dereference_iterator_adaptor<Iterator,DFn>,
                                    Iterator,
                                    typename DFn::value_type,
                                    typename std::iterator_traits<Iterator>::iterator_category,
                                    typename DFn::reference,
                                    use_default> parent_t;
    typedef typename DFn::result_type                         reference;
    typedef typename std::iterator_traits<Iterator>::difference_type difference_type;
    typedef DFn                                               dereference_fn;

    dereference_iterator_adaptor() {}
    template <typename Iterator1>
    dereference_iterator_adaptor(const dereference_iterator_adaptor<Iterator1,DFn>& dit) : parent_t(dit.base()), _deref_fn(dit._deref_fn) {}
    dereference_iterator_adaptor(Iterator it, DFn deref_fn=DFn()) : parent_t(it), _deref_fn(deref_fn) {}
    template <typename Iterator1, typename DFn1>
    dereference_iterator_adaptor(const dereference_iterator_adaptor<Iterator1,DFn1>& it) : parent_t(it.base()), _deref_fn(it._deref_fn) {}
    /// For some reason operator[] provided by iterator_facade returns a custom class that is convertible to reference
    /// We require our own reference because it is registered in iterator_traits
    reference operator[](difference_type d) const { return *(*this+d);}

    // although iterator_adaptor defines these, the default implementation computes distance and compares for zero.
    // it is often faster to just apply the relation operator to the base
    bool    operator> (const dereference_iterator_adaptor& p) const { return this->base_reference()> p.base_reference(); }
    bool    operator< (const dereference_iterator_adaptor& p) const { return this->base_reference()< p.base_reference(); }
    bool    operator>=(const dereference_iterator_adaptor& p) const { return this->base_reference()>=p.base_reference(); }
    bool    operator<=(const dereference_iterator_adaptor& p) const { return this->base_reference()<=p.base_reference(); }
    bool    operator==(const dereference_iterator_adaptor& p) const { return this->base_reference()==p.base_reference(); }
    bool    operator!=(const dereference_iterator_adaptor& p) const { return this->base_reference()!=p.base_reference(); }

    Iterator& base()              { return this->base_reference(); }
    const Iterator& base() const  { return this->base_reference(); }
    const DFn& deref_fn() const { return _deref_fn; }
private:
    template <typename Iterator1, typename DFn1>
    friend class dereference_iterator_adaptor;
    friend class boost::iterator_core_access;

    reference dereference() const { return _deref_fn(*(this->base_reference())); }
};

template <typename I, typename DFn>
struct const_iterator_type<dereference_iterator_adaptor<I,DFn> > {
    typedef dereference_iterator_adaptor<typename const_iterator_type<I>::type,typename DFn::const_t> type;
};

template <typename I, typename DFn>
struct iterator_is_mutable<dereference_iterator_adaptor<I,DFn> > : public mpl::bool_<DFn::is_mutable> {};

template <typename I, typename DFn>
struct is_iterator_adaptor<dereference_iterator_adaptor<I,DFn> > : public mpl::true_{};

template <typename I, typename DFn>
struct iterator_adaptor_get_base<dereference_iterator_adaptor<I,DFn> > {
    typedef I type;
};

template <typename I, typename DFn, typename NewBaseIterator>
struct iterator_adaptor_rebind<dereference_iterator_adaptor<I,DFn>,NewBaseIterator> {
    typedef dereference_iterator_adaptor<NewBaseIterator,DFn> type;
};

/////////////////////////////
//  PixelBasedConcept
/////////////////////////////

template <typename I, typename DFn>
struct color_space_type<dereference_iterator_adaptor<I,DFn> > : public color_space_type<typename DFn::value_type> {};

template <typename I, typename DFn>
struct channel_mapping_type<dereference_iterator_adaptor<I,DFn> > : public channel_mapping_type<typename DFn::value_type> {};

template <typename I, typename DFn>
struct is_planar<dereference_iterator_adaptor<I,DFn> > : public is_planar<typename DFn::value_type> {};

template <typename I, typename DFn>
struct channel_type<dereference_iterator_adaptor<I,DFn> > : public channel_type<typename DFn::value_type> {};

/////////////////////////////
//  MemoryBasedIteratorConcept
/////////////////////////////

template <typename Iterator, typename DFn>
struct byte_to_memunit<dereference_iterator_adaptor<Iterator,DFn> > : public byte_to_memunit<Iterator> {};

template <typename Iterator, typename DFn>
inline typename std::iterator_traits<Iterator>::difference_type
memunit_step(const dereference_iterator_adaptor<Iterator,DFn>& p) {
    return memunit_step(p.base());
}

template <typename Iterator, typename DFn>
inline typename std::iterator_traits<Iterator>::difference_type
memunit_distance(const dereference_iterator_adaptor<Iterator,DFn>& p1,
              const dereference_iterator_adaptor<Iterator,DFn>& p2) {
    return memunit_distance(p1.base(),p2.base());
}

template <typename Iterator, typename DFn>
inline void memunit_advance(dereference_iterator_adaptor<Iterator,DFn>& p,
                         typename std::iterator_traits<Iterator>::difference_type diff) {
    memunit_advance(p.base(), diff);
}

template <typename Iterator, typename DFn>
inline dereference_iterator_adaptor<Iterator,DFn>
memunit_advanced(const dereference_iterator_adaptor<Iterator,DFn>& p,
              typename std::iterator_traits<Iterator>::difference_type diff) {
    return dereference_iterator_adaptor<Iterator,DFn>(memunit_advanced(p.base(), diff), p.deref_fn());
}

template <typename Iterator, typename DFn>
inline
typename std::iterator_traits<dereference_iterator_adaptor<Iterator,DFn> >::reference
memunit_advanced_ref(const dereference_iterator_adaptor<Iterator,DFn>& p,
                  typename std::iterator_traits<Iterator>::difference_type diff) {
    return *memunit_advanced(p, diff);
}

/////////////////////////////
//  HasDynamicXStepTypeConcept
/////////////////////////////

template <typename Iterator, typename DFn>
struct dynamic_x_step_type<dereference_iterator_adaptor<Iterator,DFn> > {
    typedef dereference_iterator_adaptor<typename dynamic_x_step_type<Iterator>::type,DFn> type;
};

/// \brief Returns the type (and creates an instance) of an iterator that invokes the given dereference adaptor upon dereferencing
/// \ingroup PixelIteratorModelDerefPtr
template <typename Iterator, typename Deref>
struct iterator_add_deref {
    GIL_CLASS_REQUIRE(Deref, boost::gil, PixelDereferenceAdaptorConcept)

    typedef dereference_iterator_adaptor<Iterator, Deref> type;

    static type make(const Iterator& it, const Deref& d) { return type(it,d); }
};

/// \ingroup PixelIteratorModelDerefPtr
/// \brief For dereference iterator adaptors, compose the new function object after the old one
template <typename Iterator, typename PREV_DEREF, typename Deref>
struct iterator_add_deref<dereference_iterator_adaptor<Iterator, PREV_DEREF>,Deref> {
//    GIL_CLASS_REQUIRE(Deref, boost::gil, PixelDereferenceAdaptorConcept)

    typedef dereference_iterator_adaptor<Iterator, deref_compose<Deref,PREV_DEREF> > type;

    static type make(const dereference_iterator_adaptor<Iterator, PREV_DEREF>& it, const Deref& d) {
        return type(it.base(),deref_compose<Deref,PREV_DEREF>(d,it.deref_fn()));
    }
};

} }  // namespace boost::gil

#endif


text:
// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

namespace Microsoft.CodeAnalysis.Operations
{
    /// <summary>
    /// Represents a local function defined within a method.
    /// <para>
    /// Current usage:
    ///  (1) C# local function statement.
    /// </para>
    /// </summary>
    /// <remarks>
    /// This interface is reserved for implementation by its associated APIs. We reserve the right to
    /// change it in the future.
    /// </remarks>
    public interface ILocalFunctionOperation : IOperation
    {
        /// <summary>
        /// Local function symbol.
        /// </summary>
        IMethodSymbol Symbol { get; }
        /// <summary>
        /// Body of the local function.
        /// </summary>
        /// <remarks>
        /// Expression-bodied local functions are normalized here to be a single <see cref="IOperation"/>
        /// inside of a block statement.
        /// </remarks>
        IBlockOperation Body { get; }
    }
}

redpajama_set_name: RedPajamaGithub


text:
using OpenQA.Selenium;
using OpenQA.Selenium.Firefox;

namespace TestCase
{
    class Program
    {
        static void Main(string[] args)
        {
            IWebDriver driver = new FirefoxDriver();
            driver.Url = "http://localhost:9000/";
        }
    }
}


text:
#ifdef _MSC_VER
#include "stdafx.h"
#else
#include "config.h"
#endif

#include "SocketMonitor.h"
#include "Utility.h"
#include <exception>
#include <set>
#include <algorithm>
#include <iostream>

namespace FIX
{
SocketMonitor::SocketMonitor( int timeout )
: m_timeout( timeout )
{
  socket_init();

  std::pair<int, int> sockets = socket_createpair();
  m_signal = sockets.first;
  m_interrupt = sockets.second;
  socket_setnonblock( m_signal );
  socket_setnonblock( m_interrupt );
  m_readSockets.insert( m_interrupt );

  m_timeval.tv_sec = 0;
  m_timeval.tv_usec = 0;
#ifndef SELECT_DECREMENTS_TIME
  m_ticks = clock();
#endif
}

SocketMonitor::~SocketMonitor()
{
  Sockets::iterator i;
  for ( i = m_readSockets.begin(); i != m_readSockets.end(); ++i ) {
    socket_close( *i );
  }

  socket_close( m_signal );
  socket_term();
}

bool SocketMonitor::addConnect( int s )
{
  socket_setnonblock( s );
  Sockets::iterator i = m_connectSockets.find( s );
  if( i != m_connectSockets.end() ) return false;

  m_connectSockets.insert( s );
  return true;
}

bool SocketMonitor::addRead( int s )
{
  socket_setnonblock( s );
  Sockets::iterator i = m_readSockets.find( s );
  if( i != m_readSockets.end() ) return false;

  m_readSockets.insert( s );
  return true;
}

bool SocketMonitor::addWrite( int s )
{
  if( m_readSockets.find(s) == m_readSockets.end() )
    return false;

  socket_setnonblock( s );
  Sockets::iterator i = m_writeSockets.find( s );
  if( i != m_writeSockets.end() ) return false;

  m_writeSockets.insert( s );
  return true;
}

bool SocketMonitor::drop( int s )
{
  Sockets::iterator i = m_readSockets.find( s );
  Sockets::iterator j = m_writeSockets.find( s );
  Sockets::iterator k = m_connectSockets.find( s );

  if ( i != m_readSockets.end() ||
       j != m_writeSockets.end() ||
       k != m_connectSockets.end() )
  {
    socket_close( s );
    m_readSockets.erase( s );
    m_writeSockets.erase( s );
    m_connectSockets.erase( s );
    m_dropped.push( s );
    return true;
  }
  return false;
}

inline timeval* SocketMonitor::getTimeval( bool poll, double timeout )
{
  if ( poll )
  {
    m_timeval.tv_sec = 0;
    m_timeval.tv_usec = 0;
    return &m_timeval;
  }

  timeout = m_timeout;

  if ( !timeout )
    return 0;
#ifdef SELECT_MODIFIES_TIMEVAL
  if ( !m_timeval.tv_sec && !m_timeval.tv_usec && timeout )
    m_timeval.tv_sec = timeout;
  return &m_timeval;
#else
double elapsed = ( double ) ( clock() - m_ticks ) / ( double ) CLOCKS_PER_SEC;
  if ( elapsed >= timeout || elapsed == 0.0 )
  {
    m_ticks = clock();
    m_timeval.tv_sec = 0;
    m_timeval.tv_usec = (timeout * 1000000);
  }
  else
  {
    m_timeval.tv_sec = 0;
    m_timeval.tv_usec = ( ( timeout - elapsed ) * 1000000 );
  }
  return &m_timeval;
#endif
}

bool SocketMonitor::sleepIfEmpty( bool poll )
{
  if( poll )
    return false;

  if ( m_readSockets.empty() &&
       m_writeSockets.empty() &&
       m_connectSockets.empty() )
  {
    process_sleep( m_timeout );
    return true;
  }
  else
    return false;
}

void SocketMonitor::signal( int socket )
{
  socket_send( m_signal, (char*)&socket, sizeof(socket) );
}

void SocketMonitor::unsignal( int s )
{
  Sockets::iterator i = m_writeSockets.find( s );
  if( i == m_writeSockets.end() ) return;

  m_writeSockets.erase( s );
}

void SocketMonitor::block( Strategy& strategy, bool poll, double timeout )
{
  while ( m_dropped.size() )
  {
    strategy.onError( *this, m_dropped.front() );
    m_dropped.pop();
    if ( m_dropped.size() == 0 )
      return ;
  }

  fd_set readSet;
  FD_ZERO( &readSet );
  buildSet( m_readSockets, readSet );
  fd_set writeSet;
  FD_ZERO( &writeSet );
  buildSet( m_connectSockets, writeSet );
  buildSet( m_writeSockets, writeSet );
  fd_set exceptSet;
  FD_ZERO( &exceptSet );
  buildSet( m_connectSockets, exceptSet );

  if ( sleepIfEmpty(poll) )
  {
    strategy.onTimeout( *this );
    return;
  }

  int result = select( FD_SETSIZE, &readSet, &writeSet, &exceptSet, getTimeval(poll, timeout) );

  if ( result == 0 )
  {
    strategy.onTimeout( *this );
    return;
  }
  else if ( result > 0 )
  {
    processExceptSet( strategy, exceptSet );
    processWriteSet( strategy, writeSet );
    processReadSet( strategy, readSet );
  }
  else
  {
    strategy.onError( *this );
  }
}

void SocketMonitor::processReadSet( Strategy& strategy, fd_set& readSet )
{
#ifdef _MSC_VER
  for ( unsigned i = 0; i < readSet.fd_count; ++i )
  {
    int s = readSet.fd_array[ i ];
    if( s == m_interrupt )
    {
      int socket = 0;
      recv( s, (char*)&socket, sizeof(socket), 0 );
      addWrite( socket );
    }
    else
    {
      strategy.onEvent( *this, s );
    }
  }
#else
    Sockets::iterator i;
    Sockets sockets = m_readSockets;
    for ( i = sockets.begin(); i != sockets.end(); ++i )
    {
      int s = *i;
      if ( !FD_ISSET( *i, &readSet ) )
        continue;
      if( s == m_interrupt )
      {
        int socket = 0;
        recv( s, (char*)&socket, sizeof(socket), 0 );
        addWrite( socket );
      }
      else
      {
        strategy.onEvent( *this, s );
      }
    }
#endif
}

void SocketMonitor::processWriteSet( Strategy& strategy, fd_set& writeSet )
{
#ifdef _MSC_VER
  for ( unsigned i = 0; i < writeSet.fd_count; ++i )
  {
    int s = writeSet.fd_array[ i ];
    if( m_connectSockets.find(s) != m_connectSockets.end() )
    {
      m_connectSockets.erase( s );
      m_readSockets.insert( s );
      strategy.onConnect( *this, s );
    }
    else
    {
      strategy.onWrite( *this, s );
    }
  }
#else
  Sockets::iterator i;
  Sockets sockets = m_connectSockets;
  for( i = sockets.begin(); i != sockets.end(); ++i )
  {
    int s = *i;
    if ( !FD_ISSET( *i, &writeSet ) )
      continue;
    m_connectSockets.erase( s );
    m_readSockets.insert( s );
    strategy.onConnect( *this, s );
  }

  sockets = m_writeSockets;
  for( i = sockets.begin(); i != sockets.end(); ++i )
  {
    int s = *i;
    if ( !FD_ISSET( *i, &writeSet ) )
      continue;
    strategy.onWrite( *this, s );
  }
#endif
}

void SocketMonitor::processExceptSet( Strategy& strategy, fd_set& exceptSet )
{
#ifdef _MSC_VER
  for ( unsigned i = 0; i < exceptSet.fd_count; ++i )
  {
    int s = exceptSet.fd_array[ i ];
    strategy.onError( *this, s );
  }
#else
    Sockets::iterator i;
    Sockets sockets = m_connectSockets;
    for ( i = sockets.begin(); i != sockets.end(); ++i )
    {
      int s = *i;
      if ( !FD_ISSET( *i, &exceptSet ) )
        continue;
      strategy.onError( *this, s );
    }
#endif
}

void SocketMonitor::buildSet( const Sockets& sockets, fd_set& watchSet )
{
  Sockets::const_iterator iter;
  for ( iter = sockets.begin(); iter != sockets.end(); ++iter ) {
    FD_SET( *iter, &watchSet );
  }
}
}


text:
import os
import socket

from sanic import Sanic, response

app = Sanic("Example")

@app.route("/test")
async def test(request):
    return response.text("OK")

if __name__ == "__main__":
    server_address = "./uds_socket"
    # Make sure the socket does not already exist
    try:
        os.unlink(server_address)
    except OSError:
        if os.path.exists(server_address):
            raise
    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    sock.bind(server_address)
    app.run(sock=sock)


text:
<?xml version="1.0"?>
<layers name="" meta_type="ComponentRegistry">
 <layer name="bika.lims"
    interface="bika.lims.interfaces.IBikaLIMS"/>
</layers>


text:
Date: Tue, 26 Nov 1996 02:49:03 GMT
Server: Apache/0.6.5
Content-type: text/html

<title>John Gallagher's Never Ending Story</title>

John's Never Ending Story Page

<!WA0><!WA1>

Last Modified : November 12, 1996

Who I am

John Gallagher, mild mannered computer science graduate student at
Case Western Reserve University who is seemingly unable to decide on
any one story, plan, or home page format....

Where I am

<!WA2>Case Western Reserve University

Crawford Hall, Room 505

Cleveland, OH 44106

(216) 368-5506

<!WA3>johng@alpha.ces.cwru.edu

 What I do

I'm a Ph.D candidate in the
<!WA4>
Department of Computer Engineering and Science
 at
<!WA5>Case Western Reserve University and
a member of the <!WA6>
Autonomous Agents Research Group.  I am also the lab instructor for CWRU's
<!WA7>Introduction to Engineering Class
(only CWRU access for now, sorry)..

I have completed research on the use of
<!WA8>genetic algorithms to
generate dynamical neural network controllers for both simulated and
real hexapod <!WA9>robots..

My current research intrests fall into two catagories:

The use of analog dynamical neural networks to solve
    classically intractable decision problems.

Exploring the neural basis of cognitive behaviors.

Research Related Web Sites

I know it's mostly lame to put lists of "favorite" sites on a home page.
It's sort of egocentric to think that pages I like would be interesting
to anyone else -- especially if I don't bother to do anything clever
with the list, but who has time for clever?  I'm supposed to be burning
that on research -- so that I can put up cool pages like
<!WA10>these.

My Story

In an effort to counteract my talent to avoid "real work",
my
<!WA11>thesis advisor
 often requires I commit to "meeting agendas",
"research proposals", or other administrative nonsense.  Plans are
often nothing more than after the fact rationalizations of behavior.
One would look much more together if he could look into the future so
tomorrow's mistakes could be rationalized today.  For this purpose, I like
<!WA12>
tarot readings  -- as the results are as vague as any good research
<!WA13>proposal
.  In a pinch, I have been known to use a
<!WA14>Magic 8-Ball.
I never use <!WA15>
a ouija board, as I've found that
<!WA16>spirits
rationalize better than I can, and that's really scarry.

Some of my collegues have gone far beyond the automatic
generation of plans, and are into the automatic generation of
results.  What better way to figure out how to build a control
for a walking robot than to crack open the head of a
<!WA17>cockroach
to see how they do it.  Cheating off Ma Nature's test paper.
Who says researchers can't have
<!WA18>slack?

In my spare time, I perform <!WA19>
reconstructive surgery on indigent tubers.  Though I'm not the
<!WA20>worst, I don't think
my skills are quite up to working on
<!WA21>
dead frogs -- I'm trying, though.

to be continued....

<address>johng@alpha.ces.cwru.edu</address>


text:
/**
 * Created by piyushthapa on 11/5/14.
 */
Template.signup.events({
   'submit #signupForm':function(e,t){
       e.preventDefault();
       var user={
           username:t.find('#userName').value,
        email:t.find('#userEmail').value,
        password: t.find('#userPassword').value,
           profile:{name:t.find('#userName').value,avatar:null}
       };

       Accounts.createUser(user,function(err,res){
          if(err){
              console.log(err);
          } else{
             Router.go('/');
          }
       });
   }
});


text:
@addBrand("Bitovi")
export default class Cellphone {
}

function addBrand(brand) {
	return function decorator(klass) {
		klass.prototype.brand = brand;
	};
}


text:
'use strict';

describe('test.http.js', function () {

  var dbs = {};

  beforeEach(function () {
    dbs.name = testUtils.adapterUrl('http', 'test_http');
  });

  afterEach(function (done) {
    testUtils.cleanup([dbs.name], done);
  });

  // TODO: Remove `skipSetup` in favor of `skip_setup` in a future release
  it('Create a pouch without DB setup (skipSetup)', function (done) {
    var instantDB;
    testUtils.isCouchDB(function (isCouchDB) {
      if (!isCouchDB) {
        return done();
      }
      var db = new PouchDB(dbs.name);
      db.destroy(function () {
        instantDB = new PouchDB(dbs.name, { skipSetup: true });
        instantDB.post({ test: 'abc' }, function (err) {
          should.exist(err);
          err.name.should.equal('not_found', 'Skipped setup of database');
          done();
        });
      });
    });
  });

  it('Create a pouch without DB setup (skip_setup)', function (done) {
    var instantDB;
    testUtils.isCouchDB(function (isCouchDB) {
      if (!isCouchDB) {
        return done();
      }
      var db = new PouchDB(dbs.name);
      db.destroy(function () {
        instantDB = new PouchDB(dbs.name, { skip_setup: true });
        instantDB.post({ test: 'abc' }, function (err) {
          should.exist(err);
          err.name.should.equal('not_found', 'Skipped setup of database');
          done();
        });
      });
    });
  });

  it('Issue 1269 redundant _changes requests', function (done) {
    var docs = [];
    var num = 100;
    for (var i = 0; i < num; i++) {
      docs.push({
        _id: 'doc_' + i,
        foo: 'bar_' + i
      });
    }
    var db = new PouchDB(dbs.name);
    db.bulkDocs({ docs: docs }, function () {
      db.info(function (err, info) {
        var update_seq = info.update_seq;

        var callCount = 0;
        var ajax = db._ajax;
        db._ajax = function (opts) {
          if (/_changes/.test(opts.url)) {
            callCount++;
          }
          ajax.apply(this, arguments);
        };
        db.changes({
          since: update_seq
        }).on('change', function () {
        }).on('complete', function () {
          callCount.should.equal(1, 'One _changes call to complete changes');
          db._ajax = ajax;
          done();
        }).on('error', done);
      });
    });
  });

  it('handle ddocs with slashes', function (done) {
    var ddoc = {
      _id: '_design/foo/bar'
    };
    var db = new PouchDB(dbs.name);
    db.bulkDocs({ docs: [ddoc] }, function () {
      db.get(ddoc._id, function (err, doc) {
        should.not.exist(err);
        doc._id.should.equal(ddoc._id, 'Correct doc returned');
        done();
      });
    });
  });

  it('Properly escape url params #4008', function () {
    var db = new PouchDB(dbs.name);
    var ajax = db._ajax;
    db._ajax = function (opts) {
      opts.url.should.not.contain('[');
      ajax.apply(this, arguments);
    };
    return db.changes({doc_ids: ['1']}).then(function () {
      db._ajax = ajax;
    });
  });

  it('Allows the "ajax timeout" to extend "changes timeout"', function (done) {
    var timeout = 120000;
    var db = new PouchDB(dbs.name, {
      skipSetup: true,
      ajax: {
        timeout: timeout
      }
    });

    var ajax = db._ajax;
    var ajaxOpts;
    db._ajax = function (opts) {
      if (/changes/.test(opts.url)) {
        ajaxOpts = opts;
        changes.cancel();
      }
      ajax.apply(this, arguments);
    };

    var changes = db.changes();

    changes.on('complete', function () {
      should.exist(ajaxOpts);
      ajaxOpts.timeout.should.equal(timeout);
      db._ajax = ajax;
      done();
    });

  });

  it('Test custom header', function () {
    var db = new PouchDB(dbs.name, {
      headers: {
        'X-Custom': 'some-custom-header'
      }
    });
    return db.info();
  });

  it('test url too long error for allDocs()', function () {
    var docs = [];
    var numDocs = 75;
    for (var i = 0; i < numDocs; i++) {
      docs.push({
        _id: 'fairly_long_doc_name_' + i
      });
    }
    var db = new PouchDB(dbs.name);
    return db.bulkDocs(docs).then(function () {
      return db.allDocs({
        keys: docs.map(function (x) { return x._id; })
      });
    }).then(function (res) {
      res.rows.should.have.length(numDocs);
    });
  });

  it('4358 db.info rejects when server is down', function () {
    var db = new PouchDB('http://example.com/foo');
    return db.info().then(function () {
      throw new Error('expected an error');
    }).catch(function (err) {
      should.exist(err);
    });
  });

  it('4358 db.destroy rejects when server is down', function () {
    var db = new PouchDB('http://example.com/foo');
    return db.destroy().then(function () {
      throw new Error('expected an error');
    }).catch(function (err) {
      should.exist(err);
    });
  });

  it('5574 Create a pouch with / in name and prefix url', function () {
    // CouchDB Master disallows these characters
    if (testUtils.isCouchMaster()) {
      return true;
    }
    var db = new PouchDB('test/suffix', {
      prefix: testUtils.adapterUrl('http', '')
    });
    return db.info().then(function () {
      return db.destroy();
    });
  });

  it('Issue 6132 - default headers not merged', function () {
    var db = new PouchDB(dbs.name, {
      ajax: {
        // need to use a header that CouchDB allows through CORS
        headers: { "x-csrf-token": "bar" }
      }
    });

    var ajax = db._ajax;
    var tested = false;
    db._ajax = function (opts) {
      if (opts.headers && opts.headers['Content-Type']) {
        if (opts.headers["x-csrf-token"] !== 'bar') {
          throw new Error('default header x-csrf-token expected');
        }
        tested = true;
      }

      ajax.apply(this, arguments);
    };

    return db.putAttachment('mydoc', 'att.txt', testUtils.btoa('abc'), 'text/plain')
    .then(function () {
      if (!tested) {
        throw new Error('header assertion skipped in test');
      }
    });
  });

  it('heartbeart cannot be > request timeout', function (done) {
    var timeout = 500;
    var heartbeat = 1000;
    var CHANGES_TIMEOUT_BUFFER = 5000;
    var db = new PouchDB(dbs.name, {
      skipSetup: true,
      ajax: {
        timeout: timeout
      }
    });

    var ajax = db._ajax;
    var ajaxOpts;
    db._ajax = function (opts) {
      if (/changes/.test(opts.url)) {
        ajaxOpts = opts;
        changes.cancel();
      }
      ajax.apply(this, arguments);
    };

    var changes = db.changes({
        heartbeat: heartbeat
    });

    changes.on('complete', function () {
      should.exist(ajaxOpts);
      ajaxOpts.timeout.should.equal(heartbeat + CHANGES_TIMEOUT_BUFFER);
      ajaxOpts.url.indexOf("heartbeat=" + heartbeat).should.not.equal(-1);
      db._ajax = ajax;
      done();
    });

  });

  it('changes respects seq_interval', function (done) {
    var docs = [
      {_id: '0', integer: 0, string: '0'},
      {_id: '1', integer: 1, string: '1'},
      {_id: '2', integer: 2, string: '2'}
    ];

    var db = new PouchDB(dbs.name);
    var seqCount = 0;
    var changesCount = 0;
    db.bulkDocs(docs).then(function () {
      db.changes({ seq_interval: 4 })
      .on('change', function (change) {
        if (change.seq !== null) {
          seqCount++;
        }
        changesCount++;
      }).on('error', function (err) {
        done(err);
      }).on('complete', function (info) {
        try {
          changesCount.should.equal(3);

          // we can't know in advance which
          // order the changes arrive in so sort them
          // so that nulls appear last
          info.results.sort(function (a, b) {
            if (a.seq !== null && b.seq === null) {
              return -1;
            }

            if (a.seq === null && b.seq !== null) {
              return 1;
            }

            return 0;
          });

          // first change always contains a seq
          should.not.equal(info.results[0].seq, null);
          should.not.equal(info.last_seq, null);

          // CouchDB 1.x should just ignore seq_interval
          // (added in CouchDB 2.0), but not fail with an error
          if (testUtils.isCouchMaster()) {
            // one change (the "first") always contains a seq
            seqCount.should.equal(1);
            should.equal(info.results[1].seq, null);
            should.equal(info.results[2].seq, null);
          }
          else {
            seqCount.should.equal(3);
          }

          done();
        }
        catch (e) {
          done(e);
        }
      });
    });
  });
});


text:
export interface RuntimeInfo {
    architecture: string;
    manifestUrl: string;
    port: number;
    securityRealm?: string;
    version: string;
    args: object;
    chromeVersion: string;
    fdc3AppUuid?: string;
    fdc3ChannelName?: string;
}


text:
import * as Immutable from 'immutable';
import {MultiMap} from '../collections/multimap';
import {Edge, flipEdge} from './edge';
import {UndirectedGraph, normalizeUndirectedEdge} from './undirected_graph';

export class DirectedGraph<V, E> {
    protected constructor(
        public readonly vertices = Immutable.Set<V>(),
        private outgoingEdges = new MultiMap<V, Edge<E, V>>(),
        private destinations = new MultiMap<V, V>(),
        private incomingEdges = new MultiMap<V, Edge<E, V>>(),
        private origins = new MultiMap<V, V>()) {}

    static empty<V, E>(): DirectedGraph<V, E> {
        return new DirectedGraph<V, E>();
    }

    toUndirected(isLessThan: IsLessThan<V>): UndirectedGraph<V, E> {
        return new UndirectedGraph<V, E>(
            isLessThan,
            this.vertices,
            this.outgoingEdges.merge(this.incomingEdges).mapValues(e => normalizeUndirectedEdge(e, isLessThan)),
            this.destinations.merge(this.origins));

    }
    toString() {
        return "DIRECTED GRAPH:\n" +
            this.vertices.map(v => `VERTEX: ${v!.toString()}`).join('\n') + '\n' +
            // this.destinations + '\n' +
            // this.origins + '\n' +
            this.outgoingEdges.map.valueSeq().flatMap(set => set!.map((e: Edge<E, V>) =>
                `EDGE: ${e!.from.toString()} -> ${e!.to.toString()} (${e!.value ? e!.value!.toString() : ''})`)).join('\n') + '\n';
    }

    hasEdge(from: V, to: V): boolean {
        return this.getDestinations(from).contains(to);
    }
    getOutgoingEdges(vertex: V): Immutable.Set<Edge<E, V>> {
        return this.outgoingEdges.get(vertex) || Immutable.Set<V>();
    }
    getIncomingEdges(vertex: V): Immutable.Set<Edge<E, V>> {
        return this.incomingEdges.get(vertex) || Immutable.Set<V>();
    }

    getOrigins(vertex: V): Immutable.Set<V> {
        return this.origins.get(vertex) || Immutable.Set<V>();
    }
    getDestinations(vertex: V): Immutable.Set<V> {
        return this.destinations.get(vertex) || Immutable.Set<V>();
    }
    add({vertices = [], edges = []}: {vertices?: V[], edges?: Edge<E, V>[]}): DirectedGraph<V, E> {
        // if (!edges.every(e => e instanceof Edge)) throw '';

        let newVertices = this.vertices.union(vertices);
        let newOutgoingEdges = this.outgoingEdges;
        let newIncomingEdges = this.incomingEdges;
        let newDestinations = this.destinations;
        let newOrigins = this.origins;

        for (const edge of edges) {
            newOutgoingEdges = newOutgoingEdges.add(edge.from, edge);
            newDestinations = newDestinations.add(edge.from, edge.to);
            newIncomingEdges = newIncomingEdges.add(edge.to, edge);
            newOrigins = newOrigins.add(edge.to, edge.from);
        }
        return new DirectedGraph<V, E>(newVertices, newOutgoingEdges, newDestinations, newIncomingEdges, newOrigins);
    }
    remove({vertices = [], edges = []}: {vertices?: V[], edges?: Edge<E, V>[]}): DirectedGraph<V, E> {
        // if (!edges.every(e => e instanceof Edge)) throw '';

        let newVertices = this.vertices.subtract(vertices);
        let newOutgoingEdges = this.outgoingEdges;
        let newIncomingEdges = this.incomingEdges;
        let newDestinations = this.destinations;
        let newOrigins = this.origins;

        for (const edge of edges) {
            newOutgoingEdges = newOutgoingEdges.remove(edge.from, edge);
            newDestinations = newDestinations.remove(edge.from, edge.to);
            newOrigins = newDestinations.remove(edge.to, edge.from);
            newIncomingEdges = newIncomingEdges.remove(edge.to, edge);
        }
        return new DirectedGraph<V, E>(newVertices, newOutgoingEdges, newDestinations, newIncomingEdges, newOrigins);
    }
}


text:
@interface TJSMyOrderListDataSourceImpl : NSObject<TJSMyOrderListDataSource>

@end


text:
{-  see [NOTE:type-equality-hack] TODO-REBARE-EQ-REPR: GHC shifts the representation of the ~ to something new with 8.4.3 maybe?

 /Users/rjhala/research/liquidhaskell/tests/pos/T1295b.hs:10:5-29: Error: Illegal type specification for `constructor Main.PersonNums`

 10 |   = typ ~ [Int] => PersonNums
          ^^^^^^^^^^^^^^^^^^^^^^^^^

     constructor Main.PersonNums :: forall a##xo .
                                    (~<[]> (TYPE LiftedRep) a##xo [Int]) =>
                                    (EntityFieldPerson {VV : a##xo | len VV > 0})
     Sort Error in Refinement: {VV : typ##arY | len VV > 0}
     Cannot unify typ##arY with (@(42) @(43)) in expression: len VV << ceq2 >>

  -}

{-# LANGUAGE GADTs            #-}
{-# LANGUAGE TypeFamilies     #-}

module TypeEquality00 where

{-@ LIQUID "--exact-data-con" @-}

{-@ data EntityFieldPerson typ where
      PersonNums :: EntityFieldPerson {v:_ | len v > 0}
  @-}
data EntityFieldPerson typ
  = typ ~ [Int] => PersonNums


text:
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See LICENSE in the project root for license information.

using System;
using System.Collections.Generic;
using Microsoft.VisualStudio.Text.Editor;

namespace Microsoft.VisualStudio.R.Package.Interop {
    public sealed class CommandTargetToOleShimVariantStacks {
        private Stack<IntPtr> _variantInStack;
        private Stack<IntPtr> _variantOutStack;
        private Stack<bool> _allocateVariants;

        private CommandTargetToOleShimVariantStacks() {
            _variantInStack = new Stack<IntPtr>();
            _variantOutStack = new Stack<IntPtr>();
            _allocateVariants = new Stack<bool>();
        }

        public static CommandTargetToOleShimVariantStacks EnsureConnected(ITextView textView) {
            CommandTargetToOleShimVariantStacks stacks = null;
            if (!textView.Properties.TryGetProperty(typeof(CommandTargetToOleShimVariantStacks), out stacks)) {
                stacks = new CommandTargetToOleShimVariantStacks();
                textView.Properties.AddProperty(typeof(CommandTargetToOleShimVariantStacks), stacks);
            }

            return stacks;
        }

        public void Push(IntPtr variantIn, IntPtr variantOut, bool allocateVariants) {
            _variantInStack.Push(variantIn);
            _variantOutStack.Push(variantOut);
            _allocateVariants.Push(allocateVariants);
        }

        public void Pop() {
            _variantInStack.Pop();
            _variantOutStack.Pop();
            _allocateVariants.Pop();
        }

        public void Peek(out IntPtr variantIn, out IntPtr variantOut, out bool allocateVariants) {
            variantIn = _variantInStack.Peek();
            variantOut = _variantOutStack.Peek();
            allocateVariants = _allocateVariants.Peek();
        }

        public bool IsEmpty {
            get {
                return _variantInStack.Count == 0;
            }
        }
    }
}


text:
package org.apache.nifi.toolkit.cli.impl.result.nifi;

import org.apache.commons.lang3.Validate;
import org.apache.nifi.toolkit.cli.api.ResultType;
import org.apache.nifi.toolkit.cli.impl.result.AbstractWritableResult;
import org.apache.nifi.toolkit.cli.impl.result.writer.DynamicTableWriter;
import org.apache.nifi.toolkit.cli.impl.result.writer.Table;
import org.apache.nifi.toolkit.cli.impl.result.writer.TableWriter;
import org.apache.nifi.web.api.dto.ReportingTaskDTO;
import org.apache.nifi.web.api.entity.ReportingTaskEntity;
import org.apache.nifi.web.api.entity.ReportingTasksEntity;

import java.io.IOException;
import java.io.PrintStream;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

/**
 * Result for ReportingTasksEntity.
 */
public class ReportingTasksResult extends AbstractWritableResult<ReportingTasksEntity> {

    private final ReportingTasksEntity reportingTasksEntity;

    public ReportingTasksResult(final ResultType resultType, final ReportingTasksEntity reportingTasksEntity) {
        super(resultType);
        this.reportingTasksEntity = reportingTasksEntity;
        Validate.notNull(this.reportingTasksEntity);
    }

    @Override
    protected void writeSimpleResult(final PrintStream output) throws IOException {
        final Set<ReportingTaskEntity> tasksEntities = reportingTasksEntity.getReportingTasks();
        if (tasksEntities == null) {
            return;
        }

        final List<ReportingTaskDTO> taskDTOS = tasksEntities.stream()
                .map(ReportingTaskEntity::getComponent)
                .sorted(Comparator.comparing(ReportingTaskDTO::getName))
                .collect(Collectors.toList());

        final Table table = new Table.Builder()
                .column("#", 3, 3, false)
                .column("Name", 5, 40, true)
                .column("ID", 36, 36, false)
                .column("Type", 5, 40, true)
                .column("Run Status", 10, 20, false)
                .build();

        for (int i = 0; i < taskDTOS.size(); i++) {
            final ReportingTaskDTO taskDTO = taskDTOS.get(i);
            final String[] typeSplit = taskDTO.getType().split("\\.", -1);
            table.addRow(
                    String.valueOf(i + 1),
                    taskDTO.getName(),
                    taskDTO.getId(),
                    typeSplit[typeSplit.length - 1],
                    taskDTO.getState()
            );
        }

        final TableWriter tableWriter = new DynamicTableWriter();
        tableWriter.write(table, output);
    }

    @Override
    public ReportingTasksEntity getResult() {
        return reportingTasksEntity;
    }
}


text:
__author__ = 'thauser'
from argh import arg
from six import iteritems

import logging
from pnc_cli import swagger_client
from pnc_cli import utils

from pnc_cli.swagger_client import BuildsApi

builds_api = BuildsApi()

@arg("-p", "--page-size", help="Limit the amount of builds returned")
@arg("--page-index", help="Select the index of page", type=int)
@arg("-s", "--sort", help="Sorting RSQL")
@arg("-q", help="RSQL query")
def list_builds(page_size=200, page_index=0, sort="", q=""):
    """
    List all builds
    :param page_size: number of builds returned per query
    :param sort: RSQL sorting query
    :param q: RSQL query
    :return:
    """
    response = utils.checked_api_call(builds_api, 'get_all', page_size=page_size, page_index=page_index, sort=sort, q=q)
    if response:
        return response.content


text: ./node_modules/nodeunit/bin/nodeunit test


text: export {InspectorActions} from './InspectorActions';


text:
<?php

/*
	Plugin Name: Limited Registration
	Plugin URI:
	Plugin Description: Provides support for limiting registration based on user information in Question2Answer
	Plugin Version: 1.0
	Plugin Date: 2014-08-16
	Plugin Author: Hai Duong
	Plugin Author URI: http://www.haiduong.me/
	Plugin License: GPLv2
	Plugin Minimum Question2Answer Version: 1.5
	Plugin Update Check URI:
*/

	if (!defined('QA_VERSION')) { // don't allow this page to be requested directly from browser
		header('Location: ../../');
		exit;
	}

	qa_register_plugin_module('filter', 'qa-limited-registration.php', 'qa_limited_registration', 'Limited Registration');

/*
	Omit PHP closing tag to help avoid accidental output
*/


text:
export CLASSPATH=.:lib/repsi-tool.jar:lib/commmons-cli-1.0.jar:lib/commmons-math-1.1.jar:lib/jxl.jar:lib/ojdbc14.jar

export REPSI_JAVA_HOME=$JAVA_HOME

# create the relevant part of the database instance ****************************
$REPSI_JAVA_HOME/bin/java -jar lib/repsi-tool.jar -mode master -fn0 in/halevy_master_db_instance.xls -fn0xls


text:
using namespace std;

extern CCriticalSection cs_orphancache; // from main.h

// Request management
extern CRequestManager requester;

// Any ping < 25 ms is good
unsigned int ACCEPTABLE_PING_USEC = 25*1000;

// When should I request an object from someone else (in microseconds)
unsigned int MIN_TX_REQUEST_RETRY_INTERVAL = 5 * 1000 * 1000;
// When should I request a block from someone else (in microseconds)
unsigned int MIN_BLK_REQUEST_RETRY_INTERVAL = 5 * 1000 * 1000;

// defined in main.cpp.  should be moved into a utilities file but want to make rebasing easier
extern bool CanDirectFetch(const Consensus::Params &consensusParams);
extern void MarkBlockAsInFlight(NodeId nodeid,
    const uint256 &hash,
    const Consensus::Params &consensusParams,
    CBlockIndex *pindex = NULL);
// note mark block as in flight is redundant with the request manager now...

CRequestManager::CRequestManager()
    : inFlightTxns("reqMgr/inFlight", STAT_OP_MAX), receivedTxns("reqMgr/received"), rejectedTxns("reqMgr/rejected"),
      droppedTxns("reqMgr/dropped", STAT_KEEP), pendingTxns("reqMgr/pending", STAT_KEEP),
      requestPacer(512, 256) // Max and average # of requests that can be made per second
      ,
      blockPacer(64, 32) // Max and average # of block requests that can be made per second
{
    inFlight = 0;
    // maxInFlight = 256;

    sendIter = mapTxnInfo.end();
    sendBlkIter = mapBlkInfo.end();
}

void CRequestManager::cleanup(OdMap::iterator &itemIt)
{
    CUnknownObj &item = itemIt->second;
    // Because we'll ignore anything deleted from the map, reduce the # of requests in flight by every request we made
    // for this object
    inFlight -= item.outstandingReqs;
    droppedTxns -= (item.outstandingReqs - 1);
    pendingTxns -= 1;

    LOCK(cs_vNodes);

    // remove all the source nodes
    for (CUnknownObj::ObjectSourceList::iterator i = item.availableFrom.begin(); i != item.availableFrom.end(); ++i)
    {
        CNode *node = i->node;
        if (node)
        {
            i->clear();
            LogPrint("req", "ReqMgr: %s cleanup - removed ref to %d count %d.\n", item.obj.ToString(), node->GetId(),
                node->GetRefCount());
            node->Release();
        }
    }
    item.availableFrom.clear();

    if (item.obj.type == MSG_TX)
    {
        if (sendIter == itemIt)
            ++sendIter;
        mapTxnInfo.erase(itemIt);
    }
    else
    {
        if (sendBlkIter == itemIt)
            ++sendBlkIter;
        mapBlkInfo.erase(itemIt);
    }
}

// Get this object from somewhere, asynchronously.
void CRequestManager::AskFor(const CInv &obj, CNode *from, unsigned int priority)
{
    // LogPrint("req", "ReqMgr: Ask for %s.\n", obj.ToString().c_str());

    LOCK(cs_objDownloader);
    if (obj.type == MSG_TX)
    {
        uint256 temp = obj.hash;
        OdMap::value_type v(temp, CUnknownObj());
        std::pair<OdMap::iterator, bool> result = mapTxnInfo.insert(v);
        OdMap::iterator &item = result.first;
        CUnknownObj &data = item->second;
        data.obj = obj;
        if (result.second) // inserted
        {
            pendingTxns += 1;
            // all other fields are zeroed on creation
        }
        // else the txn already existed so nothing to do

        data.priority = max(priority, data.priority);
        // Got the data, now add the node as a source
        data.AddSource(from);
    }
    else if ((obj.type == MSG_BLOCK) || (obj.type == MSG_THINBLOCK) || (obj.type == MSG_XTHINBLOCK))
    {
        uint256 temp = obj.hash;
        OdMap::value_type v(temp, CUnknownObj());
        std::pair<OdMap::iterator, bool> result = mapBlkInfo.insert(v);
        OdMap::iterator &item = result.first;
        CUnknownObj &data = item->second;
        data.obj = obj;
        // if (result.second)  // means this was inserted rather than already existed
        // { } nothing to do
        data.priority = max(priority, data.priority);
        if (data.AddSource(from))
        {
            LogPrint("blk", "%s available at %s\n", obj.ToString().c_str(), from->addrName.c_str());
        }
    }
    else
    {
        DbgAssert(!"Request manager does not handle objects of this type", return );
    }
}

// Get these objects from somewhere, asynchronously.
void CRequestManager::AskFor(const std::vector<CInv> &objArray, CNode *from, unsigned int priority)
{
    unsigned int sz = objArray.size();
    for (unsigned int nInv = 0; nInv < sz; nInv++)
    {
        AskFor(objArray[nInv], from, priority);
    }
}

// Indicate that we got this object, from and bytes are optional (for node performance tracking)
void CRequestManager::Received(const CInv &obj, CNode *from, int bytes)
{
    int64_t now = GetTimeMicros();
    LOCK(cs_objDownloader);
    if (obj.type == MSG_TX)
    {
        OdMap::iterator item = mapTxnInfo.find(obj.hash);
        if (item == mapTxnInfo.end())
            return; // item has already been removed
        LogPrint("req", "ReqMgr: TX received for %s.\n", item->second.obj.ToString().c_str());
        from->txReqLatency << (now - item->second.lastRequestTime); // keep track of response latency of this node
        // will be decremented in the item cleanup: if (inFlight) inFlight--;
        cleanup(item); // remove the item
        receivedTxns += 1;
    }
    else if ((obj.type == MSG_BLOCK) || (obj.type == MSG_THINBLOCK) || (obj.type == MSG_XTHINBLOCK))
    {
        OdMap::iterator item = mapBlkInfo.find(obj.hash);
        if (item == mapBlkInfo.end())
            return; // item has already been removed
        LogPrint("blk", "%s removed from request queue (received from %s (%d)).\n", item->second.obj.ToString().c_str(),
            from->addrName.c_str(), from->id);
        // from->blkReqLatency << (now - item->second.lastRequestTime);  // keep track of response latency of this node
        cleanup(item); // remove the item
        // receivedTxns += 1;
    }
}

// Indicate that we got this object, from and bytes are optional (for node performance tracking)
void CRequestManager::AlreadyReceived(const CInv &obj)
{
    LOCK(cs_objDownloader);
    OdMap::iterator item = mapTxnInfo.find(obj.hash);
    if (item == mapTxnInfo.end())
    {
        item = mapBlkInfo.find(obj.hash);
        if (item == mapBlkInfo.end())
            return; // Not in any map
    }
    LogPrint("req", "ReqMgr: Already received %s.  Removing request.\n", item->second.obj.ToString().c_str());
    // will be decremented in the item cleanup: if (inFlight) inFlight--;
    cleanup(item); // remove the item
}

// Indicate that we got this object, from and bytes are optional (for node performance tracking)
void CRequestManager::Rejected(const CInv &obj, CNode *from, unsigned char reason)
{
    LOCK(cs_objDownloader);
    OdMap::iterator item;
    if (obj.type == MSG_TX)
    {
        item = mapTxnInfo.find(obj.hash);
        if (item == mapTxnInfo.end())
        {
            LogPrint("req", "ReqMgr: Item already removed. Unknown txn rejected %s\n", obj.ToString().c_str());
            return;
        }
        if (inFlight)
            inFlight--;
        if (item->second.outstandingReqs)
            item->second.outstandingReqs--;

        rejectedTxns += 1;
    }
    else if ((obj.type == MSG_BLOCK) || (obj.type == MSG_THINBLOCK) || (obj.type == MSG_XTHINBLOCK))
    {
        item = mapBlkInfo.find(obj.hash);
        if (item == mapBlkInfo.end())
        {
            LogPrint("req", "ReqMgr: Item already removed. Unknown block rejected %s\n", obj.ToString().c_str());
            return;
        }
    }

    if (reason == REJECT_MALFORMED)
    {
    }
    else if (reason == REJECT_INVALID)
    {
    }
    else if (reason == REJECT_OBSOLETE)
    {
    }
    else if (reason == REJECT_CHECKPOINT)
    {
    }
    else if (reason == REJECT_INSUFFICIENTFEE)
    {
        item->second.rateLimited = true;
    }
    else if (reason == REJECT_DUPLICATE)
    {
        // TODO figure out why this might happen.
    }
    else if (reason == REJECT_NONSTANDARD)
    {
        // Not going to be in any memory pools... does the TX request also look in blocks?
        // TODO remove from request manager (and mark never receivable?)
        // TODO verify that the TX request command also looks in blocks?
    }
    else if (reason == REJECT_DUST)
    {
    }
    else if (reason == REJECT_NONSTANDARD)
    {
    }
    else
    {
        LogPrint("req", "ReqMgr: Unknown TX rejection code [0x%x].\n", reason);
        // assert(0); // TODO
    }
}

CNodeRequestData::CNodeRequestData(CNode *n)
{
    assert(n);
    node = n;
    requestCount = 0;
    desirability = 0;

    const int MaxLatency = 10 * 1000 * 1000; // After 10 seconds latency I don't care

    // Calculate how much we like this node:

    // Prefer thin block nodes over low latency ones when the chain is syncd
    if (node->ThinBlockCapable() && IsChainNearlySyncd())
    {
        desirability += MaxLatency;
    }

    // The bigger the latency (in microseconds), the less we want to request from this node
    int latency = node->txReqLatency.GetTotal().get_int();
    // data has never been requested from this node.  Should we encourage investigation into whether this node is fast,
    // or stick with nodes that we do have data on?
    if (latency == 0)
    {
        latency = 80 * 1000; // assign it a reasonably average latency (80ms) for sorting purposes
    }
    if (latency > MaxLatency)
        latency = MaxLatency;
    desirability -= latency;
}

bool CUnknownObj::AddSource(CNode *from)
{
    // node is not in the request list
    if (std::find_if(availableFrom.begin(), availableFrom.end(), MatchCNodeRequestData(from)) == availableFrom.end())
    {
        LogPrint("req", "%s added ref to node %d.  Current count %d.\n",
                 obj.ToString(), from->GetId(), from->GetRefCount());
        {
            LOCK(cs_vNodes); // This lock is needed to ensure that AddRef happens atomically
            from->AddRef();
        }
        CNodeRequestData req(from);
        for (ObjectSourceList::iterator i = availableFrom.begin(); i != availableFrom.end(); ++i)
        {
            if (i->desirability < req.desirability)
            {
                availableFrom.insert(i, req);
                return true;
            }
        }
        availableFrom.push_back(req);
    }
  return false;
}

bool RequestBlock(CNode *pfrom, CInv obj)
{
    const CChainParams &chainParams = Params();

    // First request the headers preceding the announced block. In the normal fully-synced
    // case where a new block is announced that succeeds the current tip (no reorganization),
    // there are no such headers.
    // Secondly, and only when we are close to being synced, we request the announced block directly,
    // to avoid an extra round-trip. Note that we must *first* ask for the headers, so by the
    // time the block arrives, the header chain leading up to it is already validated. Not
    // doing this will result in the received block being rejected as an orphan in case it is
    // not a direct successor.
    //  NOTE: only download headers if we're not doing IBD.  The IBD process will take care of it's own headers.
    //        Also, we need to always download headers for "regtest". TODO: we need to redesign how IBD is initiated
    //        here.
    if (IsChainNearlySyncd() || chainParams.NetworkIDString() == "regtest")
    {
        LogPrint("net", "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, obj.hash.ToString(), pfrom->id);
        pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), obj.hash);
    }

    {
        // BUIP010 Xtreme Thinblocks: begin section
        CInv inv2(obj);
        CDataStream ss(SER_NETWORK, PROTOCOL_VERSION);
        CBloomFilter filterMemPool;
        if (IsThinBlocksEnabled() && IsChainNearlySyncd())
        {
            if (HaveConnectThinblockNodes() || (HaveThinblockNodes() && thindata.CheckThinblockTimer(obj.hash)))
            {
                // Must download a block from a ThinBlock peer
                if (pfrom->mapThinBlocksInFlight.size() < 1 && CanThinBlockBeDownloaded(pfrom))
                { // We can only send one thinblock per peer at a time
                    {
                        LOCK(pfrom->cs_mapthinblocksinflight);
                        pfrom->mapThinBlocksInFlight.insert(
                            std::pair<uint256, CNode::CThinBlockInFlight>(inv2.hash, CNode::CThinBlockInFlight()));
                    }
                    inv2.type = MSG_XTHINBLOCK;
                    std::vector<uint256> vOrphanHashes;
                    {
                        LOCK(cs_orphancache);
                        for (map<uint256, COrphanTx>::iterator mi = mapOrphanTransactions.begin();
                             mi != mapOrphanTransactions.end(); ++mi)
                            vOrphanHashes.push_back((*mi).first);
                    }
                    BuildSeededBloomFilter(filterMemPool, vOrphanHashes, inv2.hash);
                    ss << inv2;
                    ss << filterMemPool;
                    MarkBlockAsInFlight(pfrom->GetId(), obj.hash, chainParams.GetConsensus());
                    pfrom->PushMessage(NetMsgType::GET_XTHIN, ss);
                    LogPrint("thin", "Requesting Thinblock %s from peer %s (%d)\n", inv2.hash.ToString(),
                        pfrom->addrName.c_str(), pfrom->id);
                    return true;
                }
            }
            else
            {
                // Try to download a thinblock if possible otherwise just download a regular block
                // We can only send one thinblock per peer at a time
                MarkBlockAsInFlight(pfrom->GetId(), obj.hash, chainParams.GetConsensus());
                if (pfrom->mapThinBlocksInFlight.size() < 1 && CanThinBlockBeDownloaded(pfrom))
                {
                    {
                        LOCK(pfrom->cs_mapthinblocksinflight);
                        pfrom->mapThinBlocksInFlight.insert(
                            std::pair<uint256, CNode::CThinBlockInFlight>(inv2.hash, CNode::CThinBlockInFlight()));
                    }
                    inv2.type = MSG_XTHINBLOCK;
                    std::vector<uint256> vOrphanHashes;
                    {
                        LOCK(cs_orphancache);
                        for (map<uint256, COrphanTx>::iterator mi = mapOrphanTransactions.begin();
                             mi != mapOrphanTransactions.end(); ++mi)
                            vOrphanHashes.push_back((*mi).first);
                    }
                    BuildSeededBloomFilter(filterMemPool, vOrphanHashes, inv2.hash);
                    ss << inv2;
                    ss << filterMemPool;
                    pfrom->PushMessage(NetMsgType::GET_XTHIN, ss);
                    LogPrint("thin", "Requesting Thinblock %s from peer %s (%d)\n", inv2.hash.ToString(),
                        pfrom->addrName.c_str(), pfrom->id);
                }
                else
                {
                    LogPrint("thin", "Requesting Regular Block %s from peer %s (%d)\n", inv2.hash.ToString(),
                        pfrom->addrName.c_str(), pfrom->id);
                    std::vector<CInv> vToFetch;
                    inv2.type = MSG_BLOCK;
                    vToFetch.push_back(inv2);
                    pfrom->PushMessage(NetMsgType::GETDATA, vToFetch);
                }
                return true;
            }
        }
        else
        {
            std::vector<CInv> vToFetch;
            inv2.type = MSG_BLOCK;
            vToFetch.push_back(inv2);
            MarkBlockAsInFlight(pfrom->GetId(), obj.hash, chainParams.GetConsensus());
            pfrom->PushMessage(NetMsgType::GETDATA, vToFetch);
            LogPrint("thin", "Requesting Regular Block %s from peer %s (%d)\n", inv2.hash.ToString(),
                pfrom->addrName.c_str(), pfrom->id);
            return true;
        }
        return false; // no block was requested
        // BUIP010 Xtreme Thinblocks: end section
    }
}

void CRequestManager::SendRequests()
{
    int64_t now = 0;

    // TODO: if a node goes offline, rerequest txns from someone else and cleanup references right away
    LOCK(cs_objDownloader);
    if (sendBlkIter == mapBlkInfo.end())
        sendBlkIter = mapBlkInfo.begin();

    // Modify retry interval. If we're doing IBD or if Traffic Shaping is ON we want to have a longer interval because
    // those blocks and txns can take much longer to download.
    unsigned int blkReqRetryInterval = MIN_BLK_REQUEST_RETRY_INTERVAL;
    unsigned int txReqRetryInterval = MIN_TX_REQUEST_RETRY_INTERVAL;
    if ((!IsChainNearlySyncd() && Params().NetworkIDString() != "regtest") || IsTrafficShapingEnabled())
    {
        blkReqRetryInterval *= 6;
        // we want to optimise block DL during IBD (and give lots of time for shaped nodes) so push the TX retry up to 2
        // minutes (default val of MIN_TX is 5 sec)
        txReqRetryInterval *= (12 * 2);
    }

    // Get Blocks if we are not in the middle of a re-org
    while (sendBlkIter != mapBlkInfo.end() && !PV.IsReorgInProgress())
    {
        now = GetTimeMicros();
        OdMap::iterator itemIter = sendBlkIter;
        CUnknownObj &item = itemIter->second;

        ++sendBlkIter; // move it forward up here in case we need to erase the item we are working with.
        if (itemIter == mapBlkInfo.end())
            break;

        // if never requested then lastRequestTime==0 so this will always be true
        if (now - item.lastRequestTime > blkReqRetryInterval)
        {
            if (!item.availableFrom.empty())
            {
                CNodeRequestData next;
                // Go thru the availableFrom list, looking for the first node that isn't disconnected
                while (!item.availableFrom.empty() && (next.node == NULL))
                {
                    next = item.availableFrom.front(); // Grab the next location where we can find this object.
                    item.availableFrom.pop_front();
                    if (next.node != NULL)
                    {
                        // Do not request from this node if it was disconnected or the node pingtime is far beyond
                        // acceptable during initial block download.
                        // We only check pingtime during IBD because we don't want to lock vNodes too often and when the
                        // chain is syncd, waiting
                        // just 5 seconds for a timeout is not an issue, however waiting for a slow node during IBD can
                        // really slow down the process.
                        //   TODO: Eventually when we move away from vNodes or have a different mechanism for tracking
                        //   ping times we can include
                        //   this filtering in all our requests for blocks and transactions.
                        bool release = false;
                        std::string reason;
                        if (next.node->fDisconnect)
                        {
                            reason = "on disconnect";
                            release = true;
                        }
                        else if (!IsChainNearlySyncd() && !IsNodePingAcceptable(next.node))
                        {
                            reason = "bad ping time";
                            release = true;
                        }
                        if (release)
                        {
                            LOCK(cs_vNodes);
                            LogPrint("req", "ReqMgr: %s removed block ref to %s count %d (%s).\n", item.obj.ToString(),
                                next.node->GetLogName(), next.node->GetRefCount(), reason);
                            next.node->Release();
                            next.node = NULL; // force the loop to get another node
                        }
                    }
                }

                if (next.node != NULL)
                {
                    // If item.lastRequestTime is true then we've requested at least once and we'll try a re-request
                    if (item.lastRequestTime)
                    {
                        LogPrint("req", "Block request timeout for %s.  Retrying\n", item.obj.ToString().c_str());
                    }

                    CInv obj = item.obj;
                    item.outstandingReqs++;
                    int64_t then = item.lastRequestTime;
                    item.lastRequestTime = now;
                    LEAVE_CRITICAL_SECTION(cs_objDownloader);  // item and itemIter are now invalid
                    bool reqblkResult = RequestBlock(next.node, obj);
                    ENTER_CRITICAL_SECTION(cs_objDownloader);
                    if (!reqblkResult)
                    {
                        // having released cs_objDownloader, item and itemiter may be invalid.
                        // So in the rare case that we could not request the block we need to
                        // find the item again (if it exists) and set the tracking back to what it was
                        itemIter =  mapBlkInfo.find(obj.hash);
                        if (itemIter != mapBlkInfo.end())
                        {
                            item = itemIter->second;
                            item.outstandingReqs--;
                            item.lastRequestTime = then;
                        }
                    }

                    // If you wanted to remember that this node has this data, you could push it back onto the end of
                    // the availableFrom list like this:
                    // next.requestCount += 1;
                    // next.desirability /= 2;  // Make this node less desirable to re-request.
                    // item.availableFrom.push_back(next);  // Add the node back onto the end of the list

                    // Instead we'll forget about it -- the node is already popped of of the available list so now we'll
                    // release our reference.
                    LOCK(cs_vNodes);
                    LogPrint("req", "ReqMgr: %s removed block ref to %d count %d\n", obj.ToString(),
                        next.node->GetId(), next.node->GetRefCount());
                    next.node->Release();
                    next.node = NULL;
                }
                else
                {
                    // node should never be null... but if it is then there's nothing to do.
                    LogPrint("req", "Block %s has no sources\n", item.obj.ToString());
                }
            }
            else
            {
                // There can be no block sources because a node dropped out.  In this case, nothing can be done so
                // remove the item.
                LogPrint("req", "Block %s has no available sources. Removing\n", item.obj.ToString());
                cleanup(itemIter);
            }
        }
    }

    // Get Transactions
    if (sendIter == mapTxnInfo.end())
        sendIter = mapTxnInfo.begin();
    while ((sendIter != mapTxnInfo.end()) && requestPacer.try_leak(1))
    {
        now = GetTimeMicros();
        OdMap::iterator itemIter = sendIter;
        CUnknownObj &item = itemIter->second;

        ++sendIter; // move it forward up here in case we need to erase the item we are working with.
        if (itemIter == mapTxnInfo.end())
            break;

        // if never requested then lastRequestTime==0 so this will always be true
        if (now - item.lastRequestTime > txReqRetryInterval)
        {
            if (!item.rateLimited)
            {
                // If item.lastRequestTime is true then we've requested at least once, so this is a rerequest -> a txn
                // request was dropped.
                if (item.lastRequestTime)
                {
                    LogPrint("req", "Request timeout for %s.  Retrying\n", item.obj.ToString().c_str());
                    // Not reducing inFlight; it's still outstanding and will be cleaned up when item is removed from
                    // map
                    // note we can never be sure its really dropped verses just delayed for a long time so this is not
                    // authoritative.
                    droppedTxns += 1;
                }

                if (item.availableFrom.empty())
                {
                    // TODO: tell someone about this issue, look in a random node, or something.
                    cleanup(itemIter); // right now we give up requesting it if we have no other sources...
                }
                else // Ok, we have at least on source so request this item.
                {
                    CNodeRequestData next;
                    // Go thru the availableFrom list, looking for the first node that isn't disconnected
                    while (!item.availableFrom.empty() && (next.node == NULL))
                    {
                        next = item.availableFrom.front(); // Grab the next location where we can find this object.
                        item.availableFrom.pop_front();
                        if (next.node != NULL)
                        {
                            if (next.node->fDisconnect) // Node was disconnected so we can't request from it
                            {
                                LOCK(cs_vNodes);
                                LogPrint("req", "ReqMgr: %s removed tx ref to %d count %d (on disconnect).\n",
                                    item.obj.ToString(), next.node->GetId(), next.node->GetRefCount());
                                next.node->Release();
                                next.node = NULL; // force the loop to get another node
                            }
                        }
                    }

                    if (next.node != NULL)
                    {
                        CInv obj = item.obj;
                        if (1)
                        {
                            // from->AskFor(item.obj); basically just shoves the req into mapAskFor
                            // This commented code does skips requesting TX if the node is not synced.  But the req mgr
                            // should not make this decision, the caller should not give the TX to me...
                            // if (!item.lastRequestTime || (item.lastRequestTime && IsChainNearlySyncd()))

                            item.outstandingReqs++;
                            item.lastRequestTime = now;
                            LEAVE_CRITICAL_SECTION(cs_objDownloader);  // do not use "item" after releasing this
                            next.node->mapAskFor.insert(std::make_pair(now, obj));
                            ENTER_CRITICAL_SECTION(cs_objDownloader);
                        }
                        {
                            LOCK(cs_vNodes);
                            LogPrint("req", "ReqMgr: %s removed tx ref to %d count %d\n",
                                obj.ToString(), next.node->GetId(), next.node->GetRefCount());
                            next.node->Release();
                            next.node = NULL;
                        }
                        inFlight++;
                        inFlightTxns << inFlight;
                    }
                }
            }
        }
    }
}

bool CRequestManager::IsNodePingAcceptable(CNode *pfrom)
{
    if (pfrom->nPingUsecTime < ACCEPTABLE_PING_USEC)
        return true;

    // Calculate average ping time of all nodes
    uint16_t nValidNodes = 0;
    std::vector<uint64_t> vPingTimes;
    LOCK(cs_vNodes);
    BOOST_FOREACH (CNode *pnode, vNodes)
    {
        if (!pnode->fDisconnect && pnode->nPingUsecTime > 0)
        {
            nValidNodes++;
            vPingTimes.push_back(pnode->nPingUsecTime);
        }
    }
    if (nValidNodes < 10)  // Take anything if we are poorly connected
        return true;

    // Calculate Standard Deviation and Mean of Ping Time
    using namespace boost::accumulators;
    accumulator_set<double, stats<tag::variance> > acc;
    acc = for_each(vPingTimes.begin(), vPingTimes.end(), acc);
    double nMean = mean(acc);
    double sDeviation = sqrt(variance(acc));

    // If node ping time is greater than the average plus 2 times the standard deviation, or
    // the pong has not been received, then do not request from this node.
    if ((pfrom->nPingUsecTime > (int64_t)(nMean + (2 * sDeviation))) || (pfrom->nPingUsecTime == 0))
    {
        return false;
    }
    return true;
}


text: export { WatsonHealthBrushFreehand24 as default } from "../../../";


text:
""" Creating local ensemble predictions

"""
from .world import world, setup_module, teardown_module, show_doc, \
    show_method
from . import create_source_steps as source_create
from . import create_dataset_steps as dataset_create
from . import create_model_steps as model_create
from . import create_ensemble_steps as ensemble_create
from . import create_prediction_steps as prediction_create
from . import compare_predictions_steps as compare_pred

class TestEnsemblePrediction:
    """Testing local ensemble prediction"""

    def setup_method(self, method):
        """
            Debug information
        """
        self.bigml = {}
        self.bigml["method"] = method.__name__
        print("\n-------------------\nTests in: %s\n" % __name__)

    def teardown_method(self):
        """
            Debug information
        """
        print("\nEnd of tests in: %s\n-------------------\n" % __name__)
        self.bigml = {}

    def test_scenario1(self):
        """
        Scenario: Successfully creating a local prediction from an Ensemble:
            Given I create a data source uploading a "<data>" file
            And I wait until the source is ready less than <source_wait> secs
            And I create a dataset
            And I wait until the dataset is ready less than <dataset_wait> secs
            And I create an ensemble of <number_of_models> models
            And I wait until the ensemble is ready less than <model_wait> secs
            And I create a local Ensemble
            When I create a local ensemble prediction with probabilities for "<input_data>"
            Then the local prediction is "<prediction>"
            And the local prediction's confidence is "<confidence>"
            And the local probabilities are "<probabilities>"
        """
        show_doc(self.test_scenario1)
        headers = ["data", "source_wait", "dataset_wait", "model_wait",
                   "number_of_models", "input_data",
                   "prediction", "confidence", "probabilities"]
        examples = [
            ['data/iris.csv', '10', '10', '50', '5',
             '{"petal width": 0.5}', 'Iris-versicolor', '0.415',
             '["0.3403", "0.4150", "0.2447"]' ]]
        for example in examples:
            example = dict(zip(headers, example))
            show_method(self, self.bigml["method"], example)
            source_create.i_upload_a_file(
                self, example["data"], shared=example["data"])
            source_create.the_source_is_finished(
                self, example["source_wait"], shared=example["data"])
            dataset_create.i_create_a_dataset(self, shared=example["data"])
            dataset_create.the_dataset_is_finished_in_less_than(
                self, example["dataset_wait"], shared=example["data"])
            ensemble_shared = "%s_%s" % (example["data"],
                example["number_of_models"])
            ensemble_create.i_create_an_ensemble(
                self, example["number_of_models"], shared=ensemble_shared)
            ensemble_create.the_ensemble_is_finished_in_less_than(
                self, example["model_wait"], shared=ensemble_shared)
            ensemble_create.create_local_ensemble(self)
            prediction_create.create_local_ensemble_prediction_probabilities(
                self, example["input_data"])
            compare_pred.the_local_prediction_is(self, example["prediction"])
            compare_pred.the_local_prediction_confidence_is(
                self, example["confidence"])
            compare_pred.the_local_probabilities_are(
                self, example["probabilities"])

    def test_scenario2(self):
        """
        Scenario: Successfully obtaining field importance from an Ensemble:
            Given I create a data source uploading a "<data>" file
            And I wait until the source is ready less than <source_wait> secs
            And I create a dataset
            And I wait until the dataset is ready less than <dataset_wait> secs
            And I create a model with "<model_conf1>"
            And I wait until the model is ready less than <model_wait> secs
            And I create a model with "<model_conf2>"
            And I wait until the model is ready less than <model_wait> secs
            And I create a model with "<model_conf3>"
            And I wait until the model is ready less than <model_wait> secs
            When I create a local Ensemble with the last <number_of_models> models
            Then the field importance text is <field_importance>
        """
        show_doc(self.test_scenario2)
        headers = ["data", "source_wait", "dataset_wait", "model_wait",
                   "model_conf1", "model_conf2", "model_conf3",
                   "number_of_models", "field_importance"]
        examples = [
            ['data/iris.csv', '10', '10', '20',
             '{"input_fields": ["000000", "000001","000003", "000004"]}',
             '{"input_fields": ["000000", "000001","000002", "000004"]}',
             '{"input_fields": ["000000", "000001","000002", "000003",'
             ' "000004"]}', '3',
             '[["000002", 0.5269933333333333], ["000003", 0.38936],'
             ' ["000000", 0.04662333333333333],'
             '["000001", 0.037026666666666666]]']]
        for example in examples:
            example = dict(zip(headers, example))
            show_method(self, self.bigml["method"], example)
            source_create.i_upload_a_file(
                self, example["data"], shared=example["data"])
            source_create.the_source_is_finished(
                self, example["source_wait"], shared=example["data"])
            dataset_create.i_create_a_dataset(self, shared=example["data"])
            dataset_create.the_dataset_is_finished_in_less_than(
                self, example["dataset_wait"], shared=example["data"])
            model_create.i_create_a_model_with(self, example["model_conf1"])
            model_create.the_model_is_finished_in_less_than(
                self, example["model_wait"])
            model_create.i_create_a_model_with(self, example["model_conf2"])
            model_create.the_model_is_finished_in_less_than(
                self, example["model_wait"])
            model_create.i_create_a_model_with(self, example["model_conf3"])
            model_create.the_model_is_finished_in_less_than(
                self, example["model_wait"])
            ensemble_create.create_local_ensemble_with_list(
                self, example["number_of_models"])
            ensemble_create.field_importance_print(
                self, example["field_importance"])

    def test_scenario3(self):
        """
        Scenario: Successfully creating a local prediction from an Ensemble adding confidence:
            Given I create a data source uploading a "<data>" file
            And I wait until the source is ready less than <source_wait> secs
            And I create a dataset
            And I wait until the dataset is ready less than <dataset_wait> secs
            And I create an ensemble of <number_of_models> models
            And I wait until the ensemble is ready less than <model_wait> secs
            And I create a local Ensemble
            When I create a local ensemble prediction for "<input_data>" in JSON adding confidence
            Then the local prediction is "<prediction>"
            And the local prediction's confidence is "<confidence>"
        """
        show_doc(self.test_scenario3)
        headers = ["data", "source_wait", "dataset_wait", "model_wait",
                   "number_of_models", "input_data", "prediction",
                   "confidence"]
        examples = [
            ['data/iris.csv', '10', '10', '50', '5',
             '{"petal width": 0.5}', 'Iris-versicolor', '0.415']]
        for example in examples:
            example = dict(zip(headers, example))
            show_method(self, self.bigml["method"], example)
            source_create.i_upload_a_file(
                self, example["data"], shared=example["data"])
            source_create.the_source_is_finished(
                self, example["source_wait"], shared=example["data"])
            dataset_create.i_create_a_dataset(self, shared=example["data"])
            dataset_create.the_dataset_is_finished_in_less_than(
                self, example["dataset_wait"], shared=example["data"])
            ensemble_shared = "%s_%s" % (example["data"],
                example["number_of_models"])
            ensemble_create.i_create_an_ensemble(
                self, example["number_of_models"], shared=ensemble_shared)
            ensemble_create.the_ensemble_is_finished_in_less_than(
                self, example["model_wait"], shared=ensemble_shared)
            ensemble_create.create_local_ensemble(self)
            prediction_create.create_local_ensemble_prediction_add_confidence(
                self, example["input_data"])
            compare_pred.the_local_prediction_is(self, example["prediction"])
            compare_pred.the_local_prediction_confidence_is(
                self, example["confidence"])

    def test_scenario4(self):
        """
        Scenario: Successfully obtaining field importance from an Ensemble created from local models:
            Given I create a data source uploading a "<data>" file
            And I wait until the source is ready less than <source_wait> secs
            And I create a dataset
            And I wait until the dataset is ready less than <dataset_wait> secs
            And I create a model with "<model_conf1>"
            And I wait until the model is ready less than <model_wait> secs
            And I create a model with "<model_conf2>"
            And I wait until the model is ready less than <model_wait> secs
            And I create a model with "<model_conf3>"
            And I wait until the model is ready less than <model_wait> secs
            When I create a local Ensemble with the last <number_of_models> local models
            Then the field importance text is <field_importance>
        """
        show_doc(self.test_scenario4)
        headers = ["data", "source_wait", "dataset_wait", "model_wait",
                   "model_conf1", "model_conf2", "model_conf3",
                   "number_of_models", "field_importance"]
        examples = [
            ['data/iris.csv', '10', '10', '30',
             '{"input_fields": ["000000", "000001","000003", "000004"]}',
             '{"input_fields": ["000000", "000001","000002", "000004"]}',
             '{"input_fields": ["000000", "000001","000002", "000003",'
             ' "000004"]}', '3',
             '[["000002", 0.5269933333333333], ["000003", 0.38936],'
             ' ["000000", 0.04662333333333333], '
             '["000001", 0.037026666666666666]]']]
        for example in examples:
            example = dict(zip(headers, example))
            show_method(self, self.bigml["method"], example)
            source_create.i_upload_a_file(
                self, example["data"], shared=example["data"])
            source_create.the_source_is_finished(
                self, example["source_wait"], shared=example["data"])
            dataset_create.i_create_a_dataset(self, shared=example["data"])
            dataset_create.the_dataset_is_finished_in_less_than(
                self, example["dataset_wait"], shared=example["data"])
            model_create.i_create_a_model_with(self, example["model_conf1"])
            model_create.the_model_is_finished_in_less_than(
                self, example["model_wait"])
            model_create.i_create_a_model_with(self, example["model_conf2"])
            model_create.the_model_is_finished_in_less_than(
                self, example["model_wait"])
            model_create.i_create_a_model_with(self, example["model_conf3"])
            model_create.the_model_is_finished_in_less_than(
                self, example["model_wait"])
            ensemble_create.create_local_ensemble_with_list_of_local_models(
                self, example["number_of_models"])
            ensemble_create.field_importance_print(
                self, example["field_importance"])

    def test_scenario5(self):
        """
        Scenario: Successfully creating a local prediction from an Ensemble:
            Given I create a data source uploading a "<data>" file
            And I wait until the source is ready less than <source_wait> secs
            And I create a dataset
            And I wait until the dataset is ready less than <dataset_wait> secs
            And I create an ensemble of <number_of_models> models
            And I wait until the ensemble is ready less than <model_wait> secs
            And I create a local Ensemble
            When I create a local ensemble prediction using median with confidence for "<input_data>"
            Then the local prediction is "<prediction>"
        """
        show_doc(self.test_scenario5)
        headers = ["data", "source_wait", "dataset_wait", "model_wait",
                   "number_of_models", "input_data", "prediction"]
        examples = [
            ['data/grades.csv', '30', '30', '50', '2', '{}', 69.0934]]
        for example in examples:
            example = dict(zip(headers, example))
            show_method(self, self.bigml["method"], example)
            source_create.i_upload_a_file(
                self, example["data"], shared=example["data"])
            source_create.the_source_is_finished(
                self, example["source_wait"], shared=example["data"])
            dataset_create.i_create_a_dataset(self, shared=example["data"])
            dataset_create.the_dataset_is_finished_in_less_than(
                self, example["dataset_wait"], shared=example["data"])
            ensemble_shared = "%s_%s" % (example["data"],
                example["number_of_models"])
            ensemble_create.i_create_an_ensemble(
                self, example["number_of_models"], shared=ensemble_shared)
            ensemble_create.the_ensemble_is_finished_in_less_than(
                self, example["model_wait"], shared=ensemble_shared)
            ensemble_create.create_local_ensemble(self)
            prediction_create.create_local_ensemble_prediction_using_median_with_confidence(
                self, example["input_data"])
            compare_pred.the_local_prediction_is(self, example["prediction"])


text:
"""Single slice vgg with normalised scale.
"""
import functools

import lasagne as nn
import numpy as np
import theano
import theano.tensor as T

import data_loader
import deep_learning_layers
import image_transform
import layers
import preprocess
import postprocess
import objectives
import theano_printer
import updates
import utils

# Random params
rng = np.random
take_a_dump = False  # dump a lot of data in a pkl-dump file. (for debugging)
dump_network_loaded_data = False  # dump the outputs from the dataloader (for debugging)

# Memory usage scheme
caching = None

# Save and validation frequency
validate_every = 10
validate_train_set = True
save_every = 10
restart_from_save = False

# Training (schedule) parameters
# - batch sizes
batch_size = 2
sunny_batch_size = 4
batches_per_chunk = 16
AV_SLICE_PER_PAT = 11
num_epochs_train = 100

# - learning rate and method
base_lr = .0001
learning_rate_schedule = {
    0: base_lr,
    num_epochs_train*8/10: base_lr/10,
}
momentum = 0.9
build_updates = updates.build_adam_updates

# Preprocessing stuff
cleaning_processes = []
cleaning_processes_post = [functools.partial(preprocess.normalize_contrast_zmuv, z=2)]

preprocess_train = functools.partial(preprocess.preprocess_normscale)
preprocess_validation = functools.partial(preprocess_train, augment=False)
preprocess_test = preprocess_train

augmentation_params = {
    "rotate": (0, 0),
    "shear": (0, 0),
    "zoom_x": (1.0, 1.0),
    "zoom_y": (1.0, 1.0),
    "skew_x": (0, 0),
    "skew_y": (0, 0),
    "translate": (0, 0),
    "flip_vert": (0, 0),
    "roll_time": (0, 0),
    "flip_time": (0, 0)
}

def filter_samples(folders):
    # don't use patients who don't have more than 6 slices
    return [
        folder for folder in folders
        if data_loader.compute_nr_slices(folder) > 6]

sunny_preprocess_train = preprocess.sunny_preprocess_with_augmentation
sunny_preprocess_validation = preprocess.sunny_preprocess_validation
sunny_preprocess_test = preprocess.sunny_preprocess_validation

# Data generators
create_train_gen = data_loader.generate_train_batch
create_eval_valid_gen = functools.partial(data_loader.generate_validation_batch, set="validation")
create_eval_train_gen = functools.partial(data_loader.generate_validation_batch, set="train")
create_test_gen = functools.partial(data_loader.generate_test_batch, set=["validation", "test"])

# Input sizes
image_size = 64
nr_frames = 30
data_sizes = {
    "sliced:data:singleslice:difference:middle": (batch_size, 29, image_size, image_size), # 30 time steps, 30 mri_slices, 100 px wide, 100 px high,
    "sliced:data:singleslice:difference": (batch_size, 29, image_size, image_size), # 30 time steps, 30 mri_slices, 100 px wide, 100 px high,
    "sliced:data:singleslice": (batch_size, nr_frames, image_size, image_size), # 30 time steps, 30 mri_slices, 100 px wide, 100 px high,
    "sliced:data:chanzoom:4ch": (batch_size, nr_frames, image_size, image_size), # 30 time steps, 100 px wide, 100 px high,
    "sliced:data:chanzoom:2ch": (batch_size, nr_frames, image_size, image_size), # 30 time steps, 100 px wide, 100 px high,
    "sliced:data:ax": (batch_size, nr_frames, 15, image_size, image_size), # 30 time steps, 30 mri_slices, 100 px wide, 100 px high,
    "sliced:data:shape": (batch_size, 2,),
    "sunny": (sunny_batch_size, 1, image_size, image_size)
    # TBC with the metadata
}

# Objective
l2_weight = 0.000
l2_weight_out = 0.000
def build_objective(interface_layers):
    # l2 regu on certain layers
    l2_penalty = nn.regularization.regularize_layer_params_weighted(
        interface_layers["regularizable"], nn.regularization.l2)
    # build objective
    return objectives.KaggleObjective(interface_layers["outputs"], penalty=l2_penalty)

# Testing
postprocess = postprocess.postprocess
test_time_augmentations = 20 * AV_SLICE_PER_PAT  # More augmentations since a we only use single slices

def lb_softplus(lb):
    return lambda x: nn.nonlinearities.softplus(x) + lb

init = nn.init.Orthogonal()

rnn_layer = functools.partial(nn.layers.RecurrentLayer,
    W_in_to_hid=init,
    W_hid_to_hid=init,
    b=nn.init.Constant(0.1),
    nonlinearity=nn.nonlinearities.rectify,
    hid_init=nn.init.Constant(0.),
    backwards=False,
    learn_init=True,
    gradient_steps=-1,
    grad_clipping=False,
    unroll_scan=False,
    precompute_input=False,
    mask_input=None,
    only_return_final=True)

# Architecture
def build_model(input_layer = None):

    #################
    # Regular model #
    #################

    l_4ch = nn.layers.InputLayer(data_sizes["sliced:data:chanzoom:4ch"])
    l_2ch = nn.layers.InputLayer(data_sizes["sliced:data:chanzoom:2ch"])

    # Add an axis to concatenate over later
    l_4chr = nn.layers.ReshapeLayer(l_4ch, (batch_size, 1, ) + l_4ch.output_shape[1:])
    l_2chr = nn.layers.ReshapeLayer(l_2ch, (batch_size, 1, ) + l_2ch.output_shape[1:])

    # Cut the images in half, flip the left ones
    l_4ch_left = nn.layers.SliceLayer(l_4chr, indices=slice(image_size//2-1, None, -1), axis=-1)
    l_4ch_right = nn.layers.SliceLayer(l_4chr, indices=slice(image_size//2, None, 1), axis=-1)
    l_2ch_left = nn.layers.SliceLayer(l_2chr, indices=slice(image_size//2-1, None, -1), axis=-1)
    l_2ch_right = nn.layers.SliceLayer(l_2chr, indices=slice(image_size//2, None, 1), axis=-1)

    # Concatenate over second axis
    l_24lr = nn.layers.ConcatLayer([l_4ch_left, l_4ch_right, l_2ch_left, l_2ch_right], axis=1)
    # b, 4, t, h, w

    # Subsample frames
    SUBSAMPLING_FACTOR = 2
    nr_subsampled_frames = nr_frames // SUBSAMPLING_FACTOR
    l_24lr_ss = nn.layers.SliceLayer(l_24lr, indices=slice(None, None, SUBSAMPLING_FACTOR), axis=2)

    # Move frames and halves to batch, process them all in the same way, add channel axis
    l_halves = nn.layers.ReshapeLayer(l_24lr_ss, (batch_size * 4 * nr_subsampled_frames, 1, image_size, image_size//2))

    # First, do some convolutions in all directions
    num_channels = 64
    l1a = nn.layers.dnn.Conv2DDNNLayer(l_halves,  W=nn.init.Orthogonal("relu"), filter_size=(3,3), num_filters=num_channels, stride=(1,1), pad="same", nonlinearity=nn.nonlinearities.rectify)
    l1b = nn.layers.dnn.Conv2DDNNLayer(l1a, W=nn.init.Orthogonal("relu"), filter_size=(3,3), num_filters=num_channels, stride=(1,1), pad="same", nonlinearity=nn.nonlinearities.rectify)

    # Then, put an rnn over the last axis
    l1_shuffle = nn.layers.DimshuffleLayer(l1b, pattern=(0, 2, 3, 1))
    l1_r = nn.layers.ReshapeLayer(l1_shuffle, (batch_size * 4 * nr_subsampled_frames * image_size, image_size//2, num_channels))

    l_rnn = rnn_layer(l1_r, 1024)

    ld3mu = nn.layers.DenseLayer(l_rnn, num_units=1, W=nn.init.Orthogonal("relu"), b=nn.init.Constant(16.0), nonlinearity=None)
    ld3sigma = nn.layers.DenseLayer(l_rnn, num_units=1, W=nn.init.Orthogonal("relu"), b=nn.init.Constant(4.0), nonlinearity=lb_softplus(.01))
    ld3musigma = nn.layers.ConcatLayer([ld3mu, ld3sigma], axis=1)

    # Get the four halves back
    l_24lr_musigma = nn.layers.ReshapeLayer(ld3musigma, (batch_size, 4, nr_subsampled_frames, image_size, 2))
    l_24lr_musigma_shuffle = nn.layers.DimshuffleLayer(l_24lr_musigma, pattern=(0, 2, 1, 3, 4))
    l_24lr_musigma_re = nn.layers.ReshapeLayer(l_24lr_musigma_shuffle, (batch_size * nr_subsampled_frames, 4, image_size, 2))

    l_4ch_left_musigma = nn.layers.SliceLayer(l_24lr_musigma_re, indices=0, axis=1)
    l_4ch_right_musigma = nn.layers.SliceLayer(l_24lr_musigma_re, indices=1, axis=1)
    l_2ch_left_musigma = nn.layers.SliceLayer(l_24lr_musigma_re, indices=2, axis=1)
    l_2ch_right_musigma = nn.layers.SliceLayer(l_24lr_musigma_re, indices=3, axis=1)

    l_4ch_musigma = layers.SumGaussLayer([l_4ch_left_musigma, l_4ch_right_musigma])
    l_2ch_musigma = layers.SumGaussLayer([l_2ch_left_musigma, l_2ch_right_musigma])

    l_musigma_frames = layers.IraLayerNoTime(l_4ch_musigma, l_2ch_musigma)

    # Minmax over time
    print l_musigma_frames.output_shape
    l_musigmas = nn.layers.ReshapeLayer(l_musigma_frames, (-1, nr_subsampled_frames, 2))
    l_musigma_sys = layers.ArgmaxAndMaxLayer(l_musigmas, mode='min')
    l_musigma_dia = layers.ArgmaxAndMaxLayer(l_musigmas, mode='max')

    l_systole = layers.MuSigmaErfLayer(l_musigma_sys)
    l_diastole = layers.MuSigmaErfLayer(l_musigma_dia)

    return {
        "inputs":{
            "sliced:data:chanzoom:4ch": l_4ch,
            "sliced:data:chanzoom:2ch": l_2ch,
        },
        "outputs": {
            "systole": l_systole,
            "diastole": l_diastole,
        },
        "regularizable": {
        },
        "meta_outputs": {
        }
    }