text:
// Copyright (c) 2012, Yahoo! Inc.  All rights reserved.
// Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms.

#include <gearbox/core/logger.h>

#include "SQLStatusImpl.h"
#include <gearbox/store/dbconn.h>
#include <gearbox/core/Errors.h>
#include <gearbox/core/Uri.h>

#include <gearbox/job/JobManager.h>
#include <boost/lexical_cast.hpp>
#include <boost/algorithm/string.hpp>
#include <gearbox/core/logger.h>

using namespace Gearbox;
using namespace Gearbox::Database;
using namespace soci;
using std::string;
using std::vector;

// Plugin Interface
void * sql_status_new(const Gearbox::ConfigFile & cfg, bool collection) {
    if( collection ) {
        return new Gearbox::SQLStatusCollectionImpl(cfg);
    }
    return new Gearbox::SQLStatusImpl(cfg);
}

const char * name() {
    return "sql";
}

#include <soci/exchange-traits.h>
namespace soci
{
    template<> struct type_conversion<Gearbox::SQLStatusImpl>
    {
        typedef values base_type;
        static void from_base(values const & v, indicator /* ind */, Gearbox::SQLStatusImpl & stat) {
            stat.super::name(v.get<std::string>("name"));
            stat.super::operation(v.get<std::string>("operation"));
            stat.super::component(v.get<std::string>("component"));
            stat.super::progress(v.get<int>("progress"));
            stat.super::code(v.get<int>("code"));
            stat.super::resource_uri(v.get<std::string>("resource_uri"));
            stat.super::parent_uri(v.get<std::string>("parent_uri"));
            stat.super::ctime(v.get<int>("ctime"));
            stat.super::ytime(v.get<int>("ytime"));
            stat.super::state( Gearbox::Status::str2state(v.get<std::string>("state")) );
            stat.super::failures( v.get<int>("failures") );

            // this must be set last because any previous assignment
            // will also reset the mtime to "now", so we need this
            // override to happen last
            stat.super::mtime(v.get<int>("mtime"));
        }

        static void to_base( const Gearbox::SQLStatusImpl & stat, values & v, indicator & ind) {
            v.set("name", stat.super::name());
            v.set("operation", stat.super::operation());
            v.set("component", stat.super::component());
            v.set("progress", (int)stat.super::progress());
            v.set("code", stat.super::code());
            v.set("resource_uri", stat.super::resource_uri());
            v.set("parent_uri", stat.super::parent_uri());
            v.set("ctime", (int)stat.super::ctime());
            v.set("mtime", (int)stat.super::mtime());
            v.set("ytime", (int)stat.super::ytime());
            v.set("state", Gearbox::Status::state2str( stat.super::state() ));
            v.set("failures", (int)stat.super::failures());
        }
    };
}

namespace Gearbox {

    SQLStatusImpl::SQLStatusImpl(const ConfigFile & c)
        : super(c),
          messages_loaded(false),
          children_loaded(false),
          meta_loaded(false),
          loaded(false),
          should_load(false) {
        if ( c.as_json().hasKey("status") ) {
            db_init( c.get_json("status"), "status");
        }
    }

    SQLStatusImpl::SQLStatusImpl( const SQLStatusImpl & other )
        : super(other),
          messages_loaded(other.messages_loaded),
          children_loaded(other.children_loaded),
          meta_loaded(other.meta_loaded),
          loaded(other.loaded),
          should_load(other.should_load) {}

    SQLStatusImpl::~SQLStatusImpl() {}

    StatusImpl * SQLStatusImpl::clone() const {
        return new SQLStatusImpl(*this);
    }

    // the load impl is just a stub. we will delay
    // the actual data population for lazy-loading
    // via really_load()
    void SQLStatusImpl::load() {
        should_load = true;
    }

    void SQLStatusImpl::really_load() {
        while( true ) {
            try {
                std::string name,op,comp,uri,parent,state;
                int progress,code,ctime,mtime,ytime,failures;
                LoggedStatement lst("status");
                lst <<
                    "SELECT name, operation, component, progress, code, resource_uri, parent_uri, ctime, mtime, ytime, state, failures " <<
                    "  FROM status " <<
                    " WHERE name = :name ";
                lst.use(this->super::name());
                lst.into(name).into(op).into(comp).into(progress).into(code).into(uri).into(parent).into(ctime).into(mtime).into(ytime).into(state).into(failures);
                lst.execute(true);

                if( !lst.got_data() ) {
                    gbTHROW( ERR_NOT_FOUND("status " + this->name() + " not found") );
                }
                _DEBUG("got:" << " name=" << name<< " op=" << op<< " comp=" << comp<< " progress=" << progress<< " code=" << code<< " uri=" << uri<< " parent=" << parent<< " ctime=" << ctime<< " mtime=" << mtime<< " ytime=" << ytime<< " state=" << state<< " failures=" << failures);
                this->super::name(name);
                this->super::operation(op);
                this->super::component(comp);
                this->super::progress(progress);
                this->super::code(code);
                this->super::resource_uri(uri);
                this->super::parent_uri(parent);
                this->super::ctime(ctime);
                this->super::ytime(ytime);
                this->super::state(Gearbox::Status::str2state(state));
                this->super::failures(failures);

                // this must be set last because any previous assignment
                // will also reset the mtime to "now", so we need this
                // override to happen last
                this->super::mtime(mtime);

                this->loaded = true;
                this->should_load = true;
                break;
            }
            catch( const ERR_NOT_FOUND & err ) {
                throw;
            }
            catch( const std::exception & err ) {
                _WARN("Failed to load status: " << err.what());
                sleep(1);
            }
        }
    }

    void
    SQLStatusImpl::end(int code, Status::State s) {
        if( !this->loaded ) {
            this->really_load();
        }
        else {
            this->sync();
        }

        if( ! this->super::state( s ) ) {
            gbTHROW(
                ERR_INTERNAL_SERVER_ERROR(
                    "Invalid status state change from " + Status::state2str(this->state())
                    + " to " + Status::state2str(s)
                )
            );
        }

        if( this->progress() < 100 ) {
            this->super::code(code);
            this->super::progress(100);
        }

        // state updates are critical, so
        // we keep trying until there are not exceptions
        // which should be pretty rare anyway.
        while( true ) {
            try {
                session & sess = getconn("status");
                dblock("status");

                std::string query_suffix;
                if( sess.get_backend_name() == "mysql" ) {
                    query_suffix = " FOR UPDATE";
                }

                LoggedStatement lst(sess);
                // for mysql we SELECT .. FOR UPDATE to lock the row, with sqlite
                // the dblock() above will do an exclusive database lock
                std::string current_state;
                lst << "SELECT state FROM status WHERE name = :name" << query_suffix;
                lst.use(this->super::name()).into(current_state).execute(true);

                if( ! Status::validStateTransition(Status::str2state(current_state), s) ) {
                    _ERROR("Invalid status state change from " << current_state
                           + " to " + Status::state2str(s));
                    dbrollback("status");
                    return;
                }

                lst << "UPDATE status "
                    << "   SET progress=:progress, "
                    << "       code=:code, "
                    << "       state=:state "
                    << " WHERE name=:name ";
                lst.use(this->super::progress()).use(this->super::code()).use(Status::state2str(this->super::state())).use(this->super::name()).execute(true);

                dbcommit("status");

                return;
            }
            catch( const std::exception & err ) {
                _WARN(err.what());
                dbrollback("status");
                sleep(1);
            }
        }
    }

    void
    SQLStatusImpl::fail( int code ) {
        this->end(code,Status::STATE_COMPLETED);
    }

    void
    SQLStatusImpl::cancel() {
        if ( this->state() != Status::STATE_CANCELLED ) {
            int code = ( this->code() == -1 ) ? ERR_CONFLICT().code() : this->code();
            this->end( code, Status::STATE_CANCELLED );
        }
    }

    void
    SQLStatusImpl::success() {
        this->end(0,Status::STATE_COMPLETED);
    }

    void SQLStatusImpl::on(Status::Event e, const Job & job) {
        while( true ) {
            try {
                LoggedStatement lst("status");
                lst << "REPLACE INTO status_event (status_name, event_type, name, envelope) "
                    << " VALUES (:status_name, :event_type, :name, :envelope)";
                lst.use(this->super::name()).use(Status::event2str(e)).use(job.name()).use(job.serialize()).execute(true);
                break;
            }
            catch( const std::exception & err ) {
                _WARN("Failed to save status event job: " << err.what());
                sleep(1);
            }
        }
    }

    JobPtr SQLStatusImpl::on(Status::Event e, const JobManager & jm) const {
        string name, envelope;
        while( true ) {
            try {
                LoggedStatement lst("status");
                lst << "SELECT name, envelope "
                    << "  FROM status_event "
                    << " WHERE status_name=:status_name "
                    << "   AND event_type=:type ";
                lst.into(name).into(envelope).use(this->super::name()).use(Status::event2str(e)).execute(true);

                if( lst.got_data() ) {
                    return jm.job(name, envelope);
                }
                return JobPtr();
            }
            catch( const std::exception & err ) {
                _WARN("Failed to load status event: " << err.what());
                sleep(1);
            }
        }
    }

    void SQLStatusImpl::insert() {
        while( true ) {
            try {
                LoggedStatement lst( "status" );
                lst << "INSERT INTO status ( "
                    << "    name, operation, component, progress, code, resource_uri, parent_uri, ctime, mtime, ytime, state, failures "
                    << ") VALUES ( "
                    << "    :name, :operation, :component, :progress, :code, :resource_uri, :parent_uri, :ctime, :mtime, :ytime, :state, :failures "
                    << ") ";
                lst.use(this->super::name());
                lst.use(this->super::operation());
                lst.use(this->super::component());
                lst.use(this->super::progress());
                lst.use(this->super::code());
                lst.use(this->super::resource_uri());
                lst.use(this->super::parent_uri());
                lst.use(this->super::ctime());
                lst.use(this->super::mtime());
                lst.use(this->super::ytime());
                lst.use(Status::state2str(this->super::state()));
                lst.use(this->super::failures());
                lst.execute(true);
                return;
            }
            catch( const std::exception & err ) {
                _WARN("failed to insert status: " << err.what());
                sleep(1);
            }
        }
        this->loaded = true;
    }

    const char * SQLStatusImpl::impltype() const {
        return "sql";
    }

    const std::vector<std::string> & SQLStatusImpl::messages() const {
        // if messages are not loaded, we need to load them first
        if( ! this->messages_loaded ) {
            while( true ) {
                try {
                    // make sure messages are empty
                    const_cast<SQLStatusImpl*>(this)->super::messages( std::vector<std::string>() );
                    string tmp;
                    LoggedStatement lst( "status" );
                    lst << "SELECT message FROM messages WHERE status_name = :name ORDER BY id";
                    lst.use(this->super::name()).into(tmp).execute();

                    // cast away constness since we are lazy loading here.
                    SQLStatusImpl * self = const_cast<SQLStatusImpl*>(this);

                    // super::add_message updates mtime.
                    // cache mtime and update the status object
                    time_t mtime = self->super::mtime();

                    while( lst.fetch() ) {
                        self->super::add_message(tmp);
                    }

                    // update with cached mtime
                    self->super::mtime(mtime);

                    self->messages_loaded = true;
                    break;
                }
                catch( const std::exception & err ) {
                    _WARN("Failed to load status messages: " << err.what());
                    sleep(1);
                }
            }
        }
        return this->super::messages();
    }

    void SQLStatusImpl::add_message( const string & message ) {
        if( this->messages_loaded ) {
            // load messages if not loaded yet
            this->messages();
        }
        this->super::add_message(message);
        while( true ) {
            try {
                LoggedStatement lst( "status" );
                lst <<
                    "INSERT INTO messages (status_name, message) " <<
                    "VALUES (:name, :message) ";
                lst.use(this->super::name()).use(message).execute(true);

                lst << "UPDATE status SET mtime = :mtime WHERE name = :name";
                lst.use(this->super::mtime()).use(this->super::name()).execute(true);

                return;
            }
            catch( const std::exception & err ) {
                _WARN("Failed to insert status message: " << err.what());
                sleep(1);
            }
        }
    }

    const Json & SQLStatusImpl::meta() const {
        if( ! this->meta_loaded ) {
            while( true ) {
                try {
                    string name, value_str;
                    LoggedStatement lst( "status" );
                    lst << "SELECT name, value FROM status_meta WHERE status_name = :name";
                    lst.use(this->super::name()).into(name).into(value_str).execute();

                    // cast away constness for lazy loading
                    SQLStatusImpl * self = const_cast<SQLStatusImpl*>(this);

                    // super::meta( name, value) updates mtime.
                    // cache mtime and update the status object
                    time_t mtime = self->super::mtime();

                    while( lst.fetch() ) {
                        Json value;
                        value.parse( value_str );
                        self->super::meta( name, value );
                    }

                    // update with cached mtime
                    self->super::mtime(mtime);

                    self->meta_loaded = true;
                    break;
                }
                catch( const std::exception & err ) {
                    _WARN("Failed to load status meta: " << err.what());
                    sleep(1);
                }
            }
        }
        return this->super::meta();
    }

    const std::vector<std::string> & SQLStatusImpl::children() const {
        // if children are not loaded, then we need to load them first
        if( ! this->children_loaded ) {
            while( true ) {
                try {
                    // make sure children are empty
                    const_cast<SQLStatusImpl*>(this)->super::children( std::vector<std::string>() );
                    string tmp;
                    LoggedStatement lst( "status" );
                    lst << "SELECT uri FROM child_uri WHERE status_name = :name ORDER BY id";
                    lst.use(this->super::name()).into(tmp).execute();

                    // cast away constness for lazy loading
                    SQLStatusImpl * self = const_cast<SQLStatusImpl*>(this);

                    // super::add_child updates mtime.
                    // cache mtime and update the status object
                    time_t mtime = self->super::mtime();

                    while( lst.fetch() ) {
                        self->super::add_child( tmp );
                    }

                    // update with cached mtime
                    self->super::mtime(mtime);

                    self->children_loaded = true;
                    break;
                }
                catch( const std::exception & err ) {
                    _WARN("Failed to load status children: " << err.what());
                    sleep(1);
                }
            }
        }
        return this->super::children();
    }

    void SQLStatusImpl::add_child(const string & child_uri) {
        if( ! this->children_loaded ) {
            // load children if not loaded
            this->children();
        }
        this->super::add_child(child_uri);
        while( true ) {
            try {
                Uri tmp(child_uri);
                string child_path = tmp.path();
                string child_id = tmp.leaf();
                LoggedStatement lst( "status" );
                lst <<
                    "INSERT INTO child_uri (status_name, uri, child_id) " <<
                    "VALUES (:status_name, :uri, :child_id) ";
                lst.use(this->super::name()).use(child_uri).use(child_id).execute(true);

                lst << "UPDATE status SET mtime = :mtime WHERE name = :name";
                lst.use(this->super::mtime()).use(this->super::name()).execute(true);

                return;
            }
            catch( const std::exception & err ) {
                _WARN("Failed to save status child: " << err.what());
                sleep(1);
            }
        }
    }

    void SQLStatusImpl::parent_uri(const string & parent) {
        this->super::parent_uri(parent);
        while( true ) {
            try {
                LoggedStatement lst( "status" );
                lst <<
                    "UPDATE status SET parent_uri=:parent_uri, mtime=:mtime " <<
                    "WHERE name = :name ";
                lst.use(this->super::parent_uri()).use(this->super::mtime()).use(this->super::name()).execute(true);
                return;
            }
            catch( const std::exception & err ) {
                _WARN("Failed to update status parent_uri: " << err.what());
                sleep(1);
            }
        }
    }

    void SQLStatusImpl::meta( const std::string & name, const Json & value ) {
        this->super::meta( name, value );
        while( true ) {
            try {
                LoggedStatement lst( "status" );
                lst <<
                    "REPLACE INTO status_meta (status_name, name, value) " <<
                    "VALUES(:status_name, :name, :value) ";
                lst.use(this->super::name()).use(name).use(value.serialize()).execute(true);

                lst << "UPDATE status SET mtime = :mtime WHERE name = :name";
                lst.use(this->super::mtime()).use(this->super::name()).execute(true);
                return;
            }
            catch( const std::exception & err ) {
                _WARN("Failed to update status meta: " << err.what());
                sleep(1);
            }
        }
    }
    void SQLStatusImpl::meta( const Json & meta ) {
        while( true ) {
            try {
                LoggedStatement lst( "status" );
                lst <<
                    "DELETE FROM status_meta WHERE status_name = :status_name";
                lst.use(this->super::name()).execute(true);

                lst << "UPDATE status SET mtime = :mtime WHERE name = :name";
                lst.use(this->super::mtime()).use(this->super::name()).execute(true);
                break;
            }
            catch( const std::exception & err ) {
                _WARN("Failed to update status meta: " << err.what());
                sleep(1);
            }
        }

        Json::Object & o = meta.as<Json::Object>();
        Json::Object::iterator it = o.begin();
        for( ; it != o.end(); ++it ) {
            this->meta( it->first, *(it->second) );
        }
    }

    void SQLStatusImpl::progress(unsigned int progress) {
        this->super::progress(progress);
        while( true ) {
            try {
                LoggedStatement lst( "status" );
                lst <<
                    "UPDATE status SET progress=:progress, mtime=:mtime " <<
                    "WHERE name = :name AND progress < :progress";
                lst.use(this->super::progress(),"progress").use(this->super::mtime(),"mtime").use(this->super::name(),"name").execute(true);
                return;
            }
            catch( const std::exception & err ) {
                _WARN("Failed to update status progress: " << err.what());
                sleep(1);
            }
        }
    }

    void SQLStatusImpl::code(int code) {
        this->super::code(code);
        while( true ) {
            try {
                LoggedStatement lst( "status" );
                lst <<
                    "UPDATE status SET code=:code, mtime=:mtime " <<
                    "WHERE name = :name ";
                lst.use(this->super::code()).use(this->super::mtime()).use(this->super::name()).execute(true);
                return;
            }
            catch( const std::exception & err ) {
                _WARN("Failed to update status code: " << err.what());
                sleep(1);
            }
        }
    }

    void SQLStatusImpl::ytime( int64_t ytime ) {
        this->super::ytime( ytime );
        while( true ) {
            try {
                LoggedStatement lst( "status" );
                // update ytime unless already in a final ytime
                lst <<
                    "UPDATE status SET ytime=:ytime, mtime=:mtime " <<
                    "WHERE name = :name ";
                lst.use(this->super::ytime()).use(this->super::mtime()).use(this->super::name()).execute(true);
                return;
            }
            catch( const std::exception & err ) {
                _WARN("Failed to update status ytime: " << err.what());
                sleep(1);
            }
        }
    }

    bool SQLStatusImpl::state( Status::State state ) {
        if ( this->should_load && !this->loaded ) this->really_load();

        if( !this->super::state( state ) ) return false;

        // state updates are critical, so
        // we keep trying until there are not exceptions
        // which should be pretty rare anyway.
        while( true ) {
            try {
                session & sess = getconn( "status" );
                LoggedStatement lst(sess);
                dblock("status");

                std::string query_suffix;
                if( sess.get_backend_name() == "mysql" ) {
                    query_suffix = " FOR UPDATE";
                }

                // for mysql we SELECT .. FOR UPDATE to lock the row, with sqlite
                // the dblock() above will do an exclusive database lock
                std::string current_state;
                lst << "SELECT state FROM status WHERE name = :name" << query_suffix;
                lst.use(this->super::name()).into(current_state).execute(true);

                if( ! Status::validStateTransition(Status::str2state(current_state), state) ) {
                    dbrollback("status");
                    return false;
                }

                // update state unless already in a final state
                lst <<
                    "UPDATE status SET state=:state, mtime=:mtime " <<
                    "WHERE name = :name";
                lst.use(Status::state2str(this->super::state())).use(this->super::mtime()).use(this->super::name()).execute(true);

                dbcommit("status");
                return true;
            }
            catch( const std::exception & err ) {
                _WARN(err.what());
                dbrollback("status");
                sleep(1);
            }
        }
    }

    void SQLStatusImpl::failures( uint32_t count )
    {
        this->super::failures( count );
        while( true ) {
            try {
                LoggedStatement lst( "status" );
                // update failures unless already in a final failures
                lst <<
                    "UPDATE status SET failures=:failures, mtime=:mtime " <<
                    "WHERE name = :name";
                lst.use(this->super::failures()).use(this->super::mtime()).use(this->super::name()).execute(true);
                return;
            }
            catch( const std::exception & err ) {
                _WARN("Failed to update status failures: " << err.what());
                sleep(1);
            }
        }
    }

    void SQLStatusImpl::sync() {
        this->super::state(Status::STATE_UNKNOWN);
        this->really_load();
        if( ! this->messages().empty() ) this->super::messages( std::vector<std::string>() );
        this->messages_loaded = false;
        if( ! this->children().empty() ) this->super::children( std::vector<std::string>() );
        this->children_loaded = false;
        if( ! this->meta().empty() ) this->super::meta( Json() );
        this->meta_loaded = false;
    }

    const std::string & SQLStatusImpl::operation() const {
        SQLStatusImpl * self = const_cast<SQLStatusImpl*>(this);
        if( !self->loaded ) self->really_load();
        return this->super::operation();
    }
    const std::string & SQLStatusImpl::component() const {
        SQLStatusImpl * self = const_cast<SQLStatusImpl*>(this);
        if( !self->loaded ) self->really_load();
        return this->super::component();
    }
    const std::string & SQLStatusImpl::resource_uri() const {
        SQLStatusImpl * self = const_cast<SQLStatusImpl*>(this);
        if( !self->loaded ) self->really_load();
        return this->super::resource_uri();
    }
    unsigned int SQLStatusImpl::progress() const {
        SQLStatusImpl * self = const_cast<SQLStatusImpl*>(this);
        if( !self->loaded ) self->really_load();
        return this->super::progress();
    }
    int SQLStatusImpl::code() const {
        SQLStatusImpl * self = const_cast<SQLStatusImpl*>(this);
        if( !self->loaded ) self->really_load();
        return this->super::code();
    }
    const std::string & SQLStatusImpl::parent_uri() const {
        SQLStatusImpl * self = const_cast<SQLStatusImpl*>(this);
        if( !self->loaded ) self->really_load();
        return this->super::parent_uri();
    }
    time_t SQLStatusImpl::ctime() const {
        SQLStatusImpl * self = const_cast<SQLStatusImpl*>(this);
        if( !self->loaded ) self->really_load();
        return this->super::ctime();
    }
    time_t SQLStatusImpl::mtime() const {
        SQLStatusImpl * self = const_cast<SQLStatusImpl*>(this);
        if( !self->loaded ) self->really_load();
        return this->super::mtime();
    }
    int64_t SQLStatusImpl::ytime() const {
        SQLStatusImpl * self = const_cast<SQLStatusImpl*>(this);
        if( !self->loaded ) self->really_load();
        return this->super::ytime();
    }
    Status::State SQLStatusImpl::state() const {
        SQLStatusImpl * self = const_cast<SQLStatusImpl*>(this);
        if( !self->loaded ) self->really_load();
        return this->super::state();
    }
    uint32_t SQLStatusImpl::failures() const {
        SQLStatusImpl * self = const_cast<SQLStatusImpl*>(this);
        if( !self->loaded ) self->really_load();
        return this->super::failures();
    }

    std::string SQLStatusImpl::serialize() const {
        SQLStatusImpl * self = const_cast<SQLStatusImpl*>(this);
        if( !self->loaded ) self->really_load();
        // force lazy loading of messages, children and meta data
        this->messages();
        this->children();
        this->meta();
        return this->super::serialize();
    }

    uint32_t SQLStatusImpl::concurrency() const {
        uint32_t count;
        // update concurrency unless already in a final concurrency
        LoggedStatement lst( "status" );
        lst <<
            "SELECT concurrency FROM status WHERE name = :name";
        lst.into(count).use(this->super::name()).execute(true);
        return count;
    }

    void SQLStatusImpl::starting() {
        while( true ) {
            try {
                LoggedStatement lst( "status" );
                // update concurrency unless already in a final concurrency
                Gearbox::Database::dblock("status");
                lst << "UPDATE status SET concurrency = concurrency + 1 WHERE name = :name";
                lst.use(this->super::name()).execute(true);
                Gearbox::Database::dbcommit("status");
                return;
            }
            catch( const std::exception & err ) {
                _WARN("Failed to update status concurrency: " << err.what());
                sleep(1);
            }
        }
    }

    void SQLStatusImpl::stopping() {
        while( true ) {
            try {
                LoggedStatement lst( "status" );
                // update concurrency unless already in a final concurrency
                Gearbox::Database::dblock("status");
                lst << "UPDATE status SET concurrency = concurrency - 1 WHERE name = :name";
                lst.use(this->super::name()).execute(true);
                Gearbox::Database::dbcommit("status");
                return;
            }
            catch( const std::exception & err ) {
                _WARN("Failed to update status concurrency: " << err.what());
                sleep(1);
            }
        }
    }

    SQLStatusCollectionImpl::SQLStatusCollectionImpl(const ConfigFile & c) :super(c), lst(getconn("status",true)), cursor(c), fetched(false), executed(false), count(10) {
        lst << "SELECT * FROM status WHERE 1=1 ";
    }

    // SQLStatusCollectionImpl::SQLStatusCollectionImpl(
    //     const SQLStatusCollectionImpl & other
    // ) : super(other),
    //     lst(other.lst),
    //     cursor(other.cursor),
    //     fetched(other.fetched),
    //     executed(other.executed),
    //     count(other.count) {}

    SQLStatusCollectionImpl::~SQLStatusCollectionImpl() {
    }

    void
    SQLStatusCollectionImpl::run_query() {
        if( ! this->executed ) {
            this->lst << " ORDER BY ctime DESC ";

            if (this->count) {
                this->lst << " LIMIT :count ";
                this->lst.use(this->count, "count");
            }

            this->lst.into(this->cursor);
            this->lst.execute();
            this->fetched = this->lst.fetch();
            this->executed = true;
        }
    }

    StatusImpl * SQLStatusCollectionImpl::pop() {
        if( ! this->executed ) this->run_query();
        if( ! this->fetched )
            gbTHROW( std::overflow_error("called pop() on empty status collection") );

        StatusImpl * status = new SQLStatusImpl(cursor);
        this->fetched = this->lst.fetch();
        return status;
    }

    bool SQLStatusCollectionImpl::empty() {
        if( ! this->executed ) this->run_query();
        return !this->fetched;
    }

    void SQLStatusCollectionImpl::filter_progress(unsigned int min, unsigned int max) {
        if( min == max ) {
            lst << "AND progress = :progress_min ";
            lst.use(min, "progress_min");
        } else {
            lst << "AND progress >= :progress_min AND progress <= :progress_max ";
            lst.use(min, "progress_min").use(max, "progress_max");
        }
    }

    void SQLStatusCollectionImpl::filter_code(int min, int max) {
        if( min == max ) {
            lst << "AND code = :code_min ";
            lst.use(min, "code_min");
        } else {
            lst << "AND code >= :code_min and code <= :code_max ";
            lst.use(min, "code_min").use(max, "code_max");
        }
    }

    void SQLStatusCollectionImpl::filter_operation(const std::string & op) {
        lst << "AND operation = :op ";
        lst.use(op, "op");
    }

    void SQLStatusCollectionImpl::filter_component(const std::string & c) {
        lst << "AND component = :component ";
        lst.use(c, "component");
    }

    void SQLStatusCollectionImpl::filter_mtime(time_t min, time_t max) {
        if( min == max ) {
            lst << "AND mtime = :mtime_min ";
            lst.use(min, "mtime_min");
        } else {
            lst << "AND mtime >= :mtime_min AND mtime <= :mtime_max ";
            lst.use(min, "mtime_min").use(max, "mtime_max");
        }
    }

    void SQLStatusCollectionImpl::filter_ctime(time_t min, time_t max) {
        if( min == max ) {
            lst << "AND ctime = :ctime_min ";
            lst.use(min, "ctime_min");
        } else {
            lst << "AND ctime >= :ctime_min AND ctime <= :ctime_max ";
            lst.use(min, "ctime_min").use(max, "ctime_max");
        }
    }

    void SQLStatusCollectionImpl::filter_state(const std::string & state) {
        lst << "AND state = :state ";
        lst.use(state, "state");
    }

    void SQLStatusCollectionImpl::filter_uri(const std::string & uri) {
        lst << "AND resource_uri LIKE :uri ";
        lst.use( "%" + uri + "%", "uri");
    }

    void SQLStatusCollectionImpl::limit(unsigned int count) {
        this->count = count;
    }
}


text:
package org.gluu.oxtrust.service;

import java.util.List;

import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;
import javax.inject.Named;

import org.gluu.config.oxtrust.AppConfiguration;
import org.gluu.oxtrust.model.GluuSAMLTrustRelationship;
import org.slf4j.Logger;

/**
 * Perform Shibboleth startup time initialization.
 *
 * @author Yuriy Movchan
 * @version 0.1, 05/04/2017
 */
@ApplicationScoped
@Named
public class ShibbolethInitializer {

	@Inject
	private Logger log;

	@Inject
	private AppConfiguration appConfiguration;

	@Inject
	private TrustService trustService;

	@Inject
	private SamlAcrService samlAcrService;

	@Inject
	private Shibboleth3ConfService shibboleth3ConfService;

	@Inject
	private ShibbolethReloadService shibbolethReloadService;

	public boolean createShibbolethConfiguration() {
		boolean createConfig = appConfiguration.isConfigGeneration();
		log.info("IDP config generation is set to " + createConfig);
		if (createConfig) {
			List<GluuSAMLTrustRelationship> trustRelationships = trustService.getAllActiveTrustRelationships();
			String shibbolethVersion = appConfiguration.getShibbolethVersion();
			log.info("########## shibbolethVersion = " + shibbolethVersion);
			shibboleth3ConfService.generateMetadataFiles();
			shibboleth3ConfService.generateConfigurationFiles(trustRelationships);
			shibboleth3ConfService.generateConfigurationFiles(samlAcrService.getAll());
			if(shibboleth3ConfService.generateGluuAttributeRulesFile()) {
				if(!shibbolethReloadService.reloadAttributeRegistryService()) {
					log.error("Shibboleth attribute registry reload failed. (kindly restart service manually)");
				}
			}
		}

		return true;
	}

}

redpajama_set_name: RedPajamaGithub


text:
let webpack = require('webpack');

let FaviconsWebpackPlugin = require('favicons-webpack-plugin');
let HtmlWebpackPlugin = require('html-webpack-plugin');

let path = require('path');

module.exports = {
  entry: [
    'webpack-dev-server/client?http://localhost:3000',
    'webpack/hot/only-dev-server',
    'react-hot-loader/patch',
    './src/entry.js'
  ],
  output: {
    path: __dirname + '/dist',
    filename: 'bundle.js',
    publicPath: '/'
  },
  devtool: 'source-map',
  module: {
    loaders: [
      { test: /\.json$/, loader: "json-loader" },
      { test: /\.css$/, loader: "style!css" },
      { test: /\.scss$/, loaders: ["style", "css", "sass"] },
      { test: /\.jpe?g$|\.gif$|\.png$|\.svg$|\.woff$|\.ttf$|\.wav$|\.mp3$/, loader: "file" },

      { test: /\.js?$/, exclude: /node_modules/, loader: 'babel' },
      // {enforce: 'pre', test: /\.js$/,  loader: 'eslint-loader', exclude: /node_modules/},
    ]
  },
  plugins: [
    new webpack.HotModuleReplacementPlugin(),
    new FaviconsWebpackPlugin('./src/icon.png'),
    new HtmlWebpackPlugin({
      title: 'Gridlock',
      template: './src/index.html'
    })
  ],
  resolve: {
    root: path.resolve(__dirname, 'src'),
    modules: [
      'node_modules',
      path.resolve(__dirname, 'src')
    ],
  }
};


text:
#ifndef TCPSERVER_H_
#define TCPSERVER_H_

#include "find_object/FindObjectExp.h" // DLL export/import defines

#include "find_object/DetectionInfo.h"
#include <opencv2/opencv.hpp>

#include <QtNetwork/QTcpServer>

namespace find_object {

class QNetworkSession;

class FINDOBJECT_EXP TcpServer : public QTcpServer
{
	Q_OBJECT

public:
	enum Service {
		kAddObject,   // id fileName imageSize image
		kRemoveObject // id
	};

public:
	TcpServer(quint16 port = 0, QObject * parent = 0);

	QHostAddress getHostAddress() const;
	quint16 getPort() const;

public Q_SLOTS:
	void publishDetectionInfo(const find_object::DetectionInfo & info);

private Q_SLOTS:
	void addClient();
	void readReceivedData();
	void displayError(QAbstractSocket::SocketError socketError);
	void connectionLost();

Q_SIGNALS:
	void addObject(const cv::Mat &, int, const QString &);
	void removeObject(int);

private:
	QMap<int, quint64> blockSizes_;
};

} // namespace find_object

#endif /* TCPSERVER_H_ */


text:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using dbqf.Configuration;
using dbqf.Criterion;
using NUnit.Framework;
using Rhino.Mocks;
using Standalone.Core.Serialization.Assemblers;
using Standalone.Core.Serialization.Assemblers.Criterion;
using Standalone.Core.Serialization.DTO;
using Standalone.Core.Serialization.DTO.Criterion;

namespace Standalone.Core.tests.Serialization
{
    [TestFixture]
    public class ParameterAssembler_Fixture
    {
        [Test]
        public void SimpleParameter_Restore()
        {
            // Arrange
            var mockPath = MockRepository.GenerateMock<FieldPath>();
            var mockPathDTO = MockRepository.GenerateMock<FieldPathDTO>();
            var mockPathAssembler = MockRepository.GenerateMock<FieldPathAssembler>((IConfiguration)null);
            mockPathAssembler.Stub(x => x.Restore(mockPathDTO)).Return(mockPath);
            var assemblerUnderTest = new SimpleParameterAssembler(mockPathAssembler);
            var dto = new SimpleParameterDTO()
            {
                Path = mockPathDTO,
                Operator = "=",
                Value = 1
            };

            // Act
            var restored = assemblerUnderTest.Restore(dto);

            // Assert
            Assert.IsNotNull(restored);
            Assert.IsInstanceOf<SimpleParameter>(restored);
            var p = (SimpleParameter)restored;
            Assert.AreSame(mockPath, p.Path);
            Assert.AreEqual(dto.Operator, p.Operator);
            Assert.AreEqual(dto.Value, p.Value);
            mockPathAssembler.VerifyAllExpectations();
        }

        [Test]
        public void SimpleParameter_Create()
        {
            // Arrange
            var mockPath = MockRepository.GenerateMock<FieldPath>();
            var mockPathDTO = MockRepository.GenerateMock<FieldPathDTO>();
            var mockPathAssembler = MockRepository.GenerateMock<FieldPathAssembler>((IConfiguration)null);
            mockPathAssembler.Stub(x => x.Create(mockPath)).Return(mockPathDTO);
            var assemblerUnderTest = new SimpleParameterAssembler(mockPathAssembler);
            var p = new SimpleParameter(mockPath, "=", 1);

            // Act
            var restored = assemblerUnderTest.Create(p);

            // Assert
            Assert.IsNotNull(restored);
            Assert.IsInstanceOf<SimpleParameterDTO>(restored);
            var dto = (SimpleParameterDTO)restored;
            Assert.AreSame(mockPathDTO, dto.Path);
            Assert.AreEqual(p.Operator, dto.Operator);
            Assert.AreEqual(p.Value, dto.Value);
            mockPathAssembler.VerifyAllExpectations();
        }
    }
}


text:
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
    package="com.example.renderthread"
    android:versionCode="1"
    android:versionName="1.0" >

    <application
        android:allowBackup="true"
        android:icon="@drawable/ic_launcher"
        android:label="@string/app_name"
        android:theme="@style/AppTheme" >
        <activity
            android:name=".MainActivity"
            android:label="@string/app_name" >
            <intent-filter>
                <action android:name="android.intent.action.MAIN" />

                <category android:name="android.intent.category.LAUNCHER" />
            </intent-filter>
        </activity>
        <activity android:name=".SubActivity"
            android:theme="@style/AppTheme.Transparent" />
    </application>

</manifest>


text:
/* eslint-disable */
/////////////////////////////////////////////////////////////////////////////////////////
//                                                                                //
// !!! DO NOT EDIT THIS FILE !!!                                                       //
//                                                                                //
// This file is auto-generated by scripts/generate-web-api-types.sh in the repository. //
// Please refer to the script code to learn how to update the source data.             //
//                                                                                //
/////////////////////////////////////////////////////////////////////////////////////////

import { WebAPICallResult } from '../WebClient';
export type UsergroupsDisableResponse = WebAPICallResult & {
  error?:     string;
  needed?:    string;
  ok?:        boolean;
  provided?:  string;
  usergroup?: Usergroup;
};

export interface Usergroup {
  auto_provision?:        boolean;
  channel_count?:         number;
  created_by?:            string;
  date_create?:           number;
  date_delete?:           number;
  date_update?:           number;
  deleted_by?:            string;
  description?:           string;
  enterprise_subteam_id?: string;
  handle?:                string;
  id?:                    string;
  is_external?:           boolean;
  is_subteam?:            boolean;
  is_usergroup?:          boolean;
  name?:                  string;
  prefs?:                 Prefs;
  team_id?:               string;
  updated_by?:            string;
  users?:                 string[];
}

export interface Prefs {
  channels?: string[];
  groups?:   string[];
}


text:
ACCEPTED

#### According to
The Catalogue of Life, 3rd January 2011

#### Published in
Matsush. Mycol. Mem. 10: 94 (2003)

#### Original name
Tholomyces flagellifer Matsush., 2003

### Remarks
null


text:
import {Injectable, Optional} from '@angular/core';

import {CookieOptions} from './base-cookie-options';
import {CookieService} from './cookies.service';

@Injectable()
export class CookieBackendService extends CookieService {
  protected get cookieString(): string {
    return (<any>global).Zone.current.get('req').headers.cookie || '';
  }
  protected set cookieString(val: string) {
    (<any>global).Zone.current.get('req').headers.cookie = val;
    (<any>global).Zone.current.get('res').headers.cookie = val;
  }

  constructor(@Optional() _defaultOptions?: CookieOptions) {
    super(_defaultOptions);
  }
}


text:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Web.Http;
using TodoA.Model.Context;
using TodoA.Model.Entities;
using TodoA.Web.Infrastructure.Mappings;

using TodoA.Web.Models.Todo.Items;
using TodoA.Web.Models.Todo.Lists;

namespace TodoA.Web.Controllers.api
{
    public class TodoListDataController : ApiController
    {
        #region Properties

        /*
         * Seeing as this example is more focused on  front end and Angular we will be using a direct reference
         * to the context object for data management
         */

        private readonly TodoAContext _context;

        #endregion

        #region Constructor

        public TodoListDataController()
        {
            _context = new TodoAContext();
        }

        #endregion

        #region Actions

        /// <summary>
        /// Get all to do lists
        /// </summary>
        /// <returns></returns>
        public IEnumerable<TodoListDisplayDto> Get()
        {
            // Returnt the to do list data from the context
            var todoEntities = _context.TodoLists.ToList();

            // map and return the entities
            var dtos = todoEntities.Select(TodoListMappings.GetDisplayDtoFromEntity);
            return dtos;
        }

        /// <summary>
        /// GET Details for a single to do List
        /// </summary>
        /// <param name="id"></param>
        /// <returns></returns>
        public TodoListDetailsDto Get(int id)
        {
            var list = _context.TodoLists.FirstOrDefault(x => x.Id == id);

            if (list != null)
            {
                var listDto =  TodoListMappings.GetDetailsDtoFromEntity(list);

                return listDto;
            }
            else
            {
                return null;
            }
        }

        /// <summary>
        /// POST Details for creating a new to do list
        /// </summary>
        /// <param name="listCreateModel"></param>
        /// <returns></returns>

        public TodoListDisplayDto Post([FromBody]ListCreateModel listCreateModel)
        {
            var newList = new TodoList()
                          {
                              CreatedOn = DateTime.Now,
                              Title = listCreateModel.Title
                          };

            _context.TodoLists.Add(newList);
            _context.SaveChanges();

            return TodoListMappings.GetDisplayDtoFromEntity(newList);
        }

        /// <summary>
        ///
        /// </summary>
        /// <param name="id"></param>
        /// <param name="data"></param>
        /// <returns></returns>
        public HttpResponseMessage Put(int id, [FromBody] TodoListDisplayDto data)
        {
            var list = _context.TodoLists.FirstOrDefault(x => x.Id == id);

            if (list != null)
            {
                // update the title and save the changes
                list.Title = data.Title;
                _context.SaveChanges();

                return new HttpResponseMessage(HttpStatusCode.OK);
            }
            else
            {
                // The request was accepted, but no changes were made ?
                return new HttpResponseMessage(HttpStatusCode.Accepted);
            }
        }

        public HttpResponseMessage Delete(int id)
        {
            var listToDelete = _context.TodoLists.FirstOrDefault(x => x.Id == id);

            if (listToDelete != null)
            {
                _context.TodoLists.Remove(listToDelete);
                _context.SaveChanges();

                return new HttpResponseMessage(HttpStatusCode.OK);
            }
            else
            {
                return new HttpResponseMessage(HttpStatusCode.Accepted);
            }
        }

        #endregion
    }
}


text:
//
//  ComparatorAppDelegate.h
//  Comparator
//
//  Created by Max on 3/21/11.
//  Copyright 2011 Lis@cintosh. All rights reserved.
//

#import <Cocoa/Cocoa.h>
#import <CoreData/CoreData.h>

#import <IOKit/pwr_mgt/IOPMLib.h> /* For "IOPMAssertionCreateWithName(...)" */

#import "MainViewController.h"
#import "ResultsViewController.h"

#import "MyWindow.h"
#import "MyBigProgressIndicator.h"

@class FileItem;

@class OptionsViewController;
@class ResultsViewController;
@class ApplicationPreferencesWindow;

@class TransitionView;

typedef NS_ENUM(NSUInteger, ContentViewType) {
	ContentViewTypeStart,
	ContentViewTypeAnalysing,
	ContentViewTypeResults
};

@interface ComparatorAppDelegate : NSObject <NSApplicationDelegate, NSTableViewDelegate, NSTableViewDataSource, NSSplitViewDelegate, NSWindowDelegate>
{
    IBOutlet NSWindow * __strong window;

	IBOutlet NSTextField * progressLabel;
	IBOutlet TransitionView * analysingView;

	IBOutlet ApplicationPreferencesWindow * preferencesWindow;

	/*** Main Menu Items ***/
	/* "File" Items  */
	IBOutlet NSMenuItem * __strong chooseSourceMenuItem;
	IBOutlet NSMenuItem * __strong exportAsTextMenuItem, * __strong exportAsXMLMenuItem;

	/* "Edit" Items */
	IBOutlet NSMenuItem * __strong selectAllMenuItem, * __strong selectAllDuplicatesMenuItem, * __strong deselectAllMenuItem, * __strong deleteSelectedMenuItem;

	/* "View" Items */
	IBOutlet NSMenuItem * __strong quickLookMenuItem;

	/* "Window" Items */
	IBOutlet NSMenuItem * __strong mainWindowMenuItem;

	NSOperationQueue * queue;

	NSMutableArray * items;

	NSMutableArray * fileURLs;
	NSArray * sourceURLs;

	//NSMutableArray * emptyFoldersURLs;
	NSMutableArray * emptyFilesURLs;

	NSMutableArray * duplicatesArrays;
	NSMutableArray * emptyItems;
	NSMutableArray * brokenAliasItems;

	@private

	//NSInteger _notificationCount;

	ContentViewType contentViewType;

	BOOL _cancelled, _working;

	IOPMAssertionID preventSleepID;

	/*** Transition View Outlets ***/
	NSTextField * __strong _progressLabel, * __strong _currentFileLabel;
	CenteredView * __strong _transitionCenteredView;
	MyBigProgressIndicator * __strong progressIndicator;

	/*** View's Controllers ***/
	MainViewController * __strong _mainViewController;
	ResultsViewController * __strong _resultsViewController;
}

@property (nonatomic, strong, readonly) NSPersistentStoreCoordinator * persistentStoreCoordinator;
@property (nonatomic, strong, readonly) NSManagedObjectModel * managedObjectModel;
@property (nonatomic, strong, readonly) NSManagedObjectContext * managedObjectContext;

@property (strong) IBOutlet NSWindow * window;

/* Main Menu Items */
@property (strong) IBOutlet NSMenuItem * chooseSourceMenuItem;
@property (strong) IBOutlet NSMenuItem * exportAsTextMenuItem, * exportAsXMLMenuItem;
@property (strong) IBOutlet NSMenuItem * selectAllMenuItem, * selectAllDuplicatesMenuItem, * deselectAllMenuItem, * deleteSelectedMenuItem;
@property (strong) IBOutlet NSMenuItem * quickLookMenuItem;
@property (strong) IBOutlet NSMenuItem * mainWindowMenuItem;

/*** Transition View Outlets ***/
@property (strong) IBOutlet NSTextField * progressLabel, * currentFileLabel;
@property (strong) IBOutlet CenteredView * transitionCenteredView;
@property (strong) IBOutlet MyBigProgressIndicator * progressIndicator;

/*** View's Controllers ***/
@property (strong) IBOutlet MainViewController * mainViewController;
@property (strong) IBOutlet ResultsViewController * resultsViewController;

@property (nonatomic, strong) NSOperationQueue * queue;

//@property (nonatomic, strong) NSMutableArray * fileURLs;

@property (nonatomic, assign) BOOL working;

#pragma mark IBAction

- (IBAction)startAction:(id)sender;
- (IBAction)stopAction:(id)sender;

- (IBAction)backToMainView:(id)sender;
- (IBAction)reopenMainWindow:(id)sender;
- (IBAction)openWebsiteAction:(id)sender;
- (IBAction)openSupportAction:(id)sender;

- (IBAction)showPreferences:(id)sender;

- (IBAction)toggleQuickLook:(id)sender;

/*
- (IBAction)runAnalysing:(id)sender;
- (IBAction)cancelOperation:(id)sender;
*/

//- (IBAction)showOptions:(id)sender;

#pragma mark Export to File
- (IBAction)exportAsText:(id)sender;
- (IBAction)exportAsXML:(id)sender;

#pragma mark Box content
- (void)setContentViewType:(ContentViewType)type;

#pragma mark Comparing Items
- (void)compareType:(NSString *)type forGroup:(NSString *)groupType withOptions:(NSArray *)options;
- (void)compareAllItems;

#pragma mark Automatically Comparing Items
- (void)automaticallyCompare:(NSString *)type;
- (void)automaticallyCompareType:(NSString *)type forGroup:(NSString *)groupType;

@end


text:
if ! [ -z "$1" ]; then
    cd "$1"
fi

echo "Checking for Windows line endings in `pwd`"

if git ls-tree --name-only -r HEAD | xargs file -N | grep CRLF; then
    echo "Some files have CRLF (Windows-style) line endings. Please convert to LF (Unix-style). git can be configured to do this automatically."
    exit 1
fi

exit 0


text:
using System.ComponentModel;

namespace Models
{
    public class SearchOptions : INotifyPropertyChanged
    {
        #region Internal Property Fields

        private bool _exactMatch = false;
        private bool _replaceDash = true;
        private bool _firstOneThousand = true;
        private int _extensionChoice = 0;
        private string _fileName { get; set; } = string.Empty;
        private string _startingDirectory { get; set; } = string.Empty;

        #endregion

        #region Public Properties

        public bool ExactMatchesOnly
        {
            get { return _exactMatch; }
            set
            {
                if (value != _exactMatch)
                {
                    _exactMatch = value;
                    PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(ExactMatchesOnly)));
                }
            }
        }

        public bool ReplaceDashWithUnderscore
        {
            get { return _replaceDash; }
            set
            {
                if (value != _replaceDash)
                {
                    _replaceDash = value;
                    PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(ReplaceDashWithUnderscore)));
                }
            }
        }

        public bool OnlyShowFirstOneThousand
        {
            get { return _firstOneThousand; }
            set
            {
                if (value != _firstOneThousand)
                {
                    _firstOneThousand = value;
                    PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(OnlyShowFirstOneThousand)));
                }
            }
        }

        public int ExtensionChoice
        {
            get { return _extensionChoice; }
            set
            {
                if (value != _extensionChoice)
                {
                    _extensionChoice = value;
                    PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(ExtensionChoice)));
                }
            }
        }

        public string FileName
        {
            get { return _fileName; }
            set
            {
                if (value != _fileName)
                {
                    _fileName = value;
                    PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(FileName)));
                }
            }
        }

        public string StartingDirectory
        {
            get { return _startingDirectory; }
            set
            {
                if (value != _startingDirectory)
                {
                    _startingDirectory = value;
                    PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(StartingDirectory)));
                }
            }
        }

        #endregion

        public event PropertyChangedEventHandler PropertyChanged;
    }
}


text:
CREATE TABLE loo(a int PRIMARY KEY AUTO_INCREMENT, b char(10));

INSERT INTO loo(b) VALUES('AAA');

SELECT ROW_COUNT();

INSERT INTO loo(b) VALUES ('FFF'), ('GGG'), ('HHH');

SELECT ROW_COUNT();

DROP TABLE loo;


text:
<?php

namespace Nelmio\ApiDocBundle\Describer;

use Nelmio\ApiDocBundle\Model\ModelRegistry;

interface ModelRegistryAwareInterface
{
    public function setModelRegistry(ModelRegistry $modelRegistry);
}


text:
package main

import (
	"net/url"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestClientRequest(t *testing.T) {
	Convey("Given ClientRequest", t, func() {
		config := NewConfig()
		values, _ := url.ParseQuery("info_hash=12345678901234567890&peer_id=12345678901234567890&port=1&left=2")
		Convey("Returns new ClientRequest for valid request", func() {
			cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
			So(cr, ShouldNotBeNil)
			So(err, ShouldBeNil)
			So(string(cr.getPeer().InfoHash), ShouldEqual, "12345678901234567890")
			So(string(cr.getPeer().ID), ShouldEqual, "12345678901234567890")
			So(cr.getPeer().Port, ShouldEqual, 1)
			So(cr.getPeer().State, ShouldEqual, stateDownloading)
		})
		Convey("Returns error on invalid info_hash", func() {
			values.Set("info_hash", "123")
			cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
			So(cr, ShouldBeNil)
			So(err, ShouldNotBeNil)
			So(err.Error(), ShouldEqual, "Bad info_hash: 123")
		})
		Convey("Returns error on invalid peer_id", func() {
			values.Set("peer_id", "123")
			cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
			So(cr, ShouldBeNil)
			So(err, ShouldNotBeNil)
			So(err.Error(), ShouldEqual, "Bad peer_id: 123")
		})
		Convey("Returns error on invalid port", func() {
			values.Set("port", "sz")
			cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
			So(cr, ShouldBeNil)
			So(err, ShouldNotBeNil)
			So(err.Error(), ShouldEqual, "d14:failure reason32:client listening port is invalide")
		})
		Convey("Test left", func() {
			Convey("Returns correct state if something is left", func() {
				values.Set("left", "1")
				cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
				So(cr, ShouldNotBeNil)
				So(err, ShouldBeNil)
				So(cr.getPeer().State, ShouldEqual, stateDownloading)
			})
			Convey("Returns correct state if nothing is left", func() {
				values.Set("left", "0")
				cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
				So(cr, ShouldNotBeNil)
				So(err, ShouldBeNil)
				So(cr.getPeer().State, ShouldEqual, stateSeeding)
			})
			Convey("Returns error if left is not a number", func() {
				values.Set("left", "z")
				cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
				So(cr, ShouldBeNil)
				So(err, ShouldNotBeNil)
				So(err.Error(), ShouldEqual, "d14:failure reason33:client data left field is invalide")
			})
		})
		Convey("Test Compact", func() {
			Convey("No compact if 0", func() {
				values.Set("compact", "0")
				config.ForceCompact = false
				cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
				So(cr, ShouldNotBeNil)
				So(err, ShouldBeNil)
				So(cr.isCompact, ShouldBeFalse)
			})
			Convey("No compact if empty", func() {
				values.Set("compact", "")
				config.ForceCompact = false
				cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
				So(cr, ShouldNotBeNil)
				So(err, ShouldBeNil)
				So(cr.isCompact, ShouldBeFalse)
			})
			Convey("Compact if any", func() {
				values.Set("compact", "1")
				config.ForceCompact = false
				cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
				So(cr, ShouldNotBeNil)
				So(err, ShouldBeNil)
				So(cr.isCompact, ShouldBeTrue)
			})
			Convey("Compact if forced", func() {
				values.Set("compact", "0")
				config.ForceCompact = true
				cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
				So(cr, ShouldNotBeNil)
				So(err, ShouldBeNil)
				So(cr.isCompact, ShouldBeTrue)
			})
		})
		Convey("Sets noPeerId", func() {
			values.Set("no_peer_id", "1")
			cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
			So(cr, ShouldNotBeNil)
			So(err, ShouldBeNil)
			So(cr.noPeerId, ShouldBeTrue)
		})
		Convey("Test IP", func() {
			Convey("If correct IPv4 is passed", func() {
				values.Set("ip", "127.0.0.2")
				config.ExternalIp = true
				cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
				So(cr, ShouldNotBeNil)
				So(err, ShouldBeNil)
				So(cr.getPeer().IP, ShouldEqual, "127.0.0.2")
			})
			Convey("If correct IPv6 is passed", func() {
				values.Set("ip", "2001:0db8:0000:0000:0000:ff00:0042:8329")
				config.ExternalIp = true
				cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
				So(cr, ShouldNotBeNil)
				So(err, ShouldBeNil)
				So(cr.getPeer().IP, ShouldEqual, "2001:db8::ff00:42:8329")
			})
			Convey("Error if if Bad IP passed", func() {
				values.Set("ip", "bozo")
				config.ExternalIp = true
				cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
				So(cr, ShouldBeNil)
				So(err, ShouldNotBeNil)
				So(err.Error(), ShouldEqual, "d14:failure reason31:invalid ip, dotted decimal onlye")
			})
			Convey("Use request ip if correct IPv4 is passed and external is disabled", func() {
				values.Set("ip", "127.0.0.2")
				config.ExternalIp = false
				cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
				So(cr, ShouldNotBeNil)
				So(err, ShouldBeNil)
				So(cr.getPeer().IP, ShouldEqual, "127.0.0.1")
			})
			Convey("Error if if Bad request IP is passed", func() {
				values.Set("ip", "bozo")
				config.ExternalIp = false
				cr, err := NewClientRequest(config, values, "bozo:8000")
				So(cr, ShouldBeNil)
				So(err, ShouldNotBeNil)
				So(err.Error(), ShouldEqual, "d14:failure reason31:invalid ip, dotted decimal onlye")
			})
			Convey("Error if if no ip is passed and request ip is empty", func() {
				values.Del("ip")
				config.ExternalIp = true
				cr, err := NewClientRequest(config, values, "")
				So(cr, ShouldBeNil)
				So(err, ShouldNotBeNil)
				So(err.Error(), ShouldEqual, "d14:failure reason27:could not locate clients ipe")
			})
		})
		Convey("Numwant", func() {
			Convey("Get numwant from query string if below the limit", func() {
				values.Set("numwant", "2")
				config.MaxPeers = 3
				cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
				So(cr, ShouldNotBeNil)
				So(err, ShouldBeNil)
				So(cr.numWant, ShouldEqual, 2)
			})
			Convey("Get max numwant if above the limit", func() {
				values.Set("numwant", "4")
				config.MaxPeers = 3
				config.DefaultPeers = 1
				cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
				So(cr, ShouldNotBeNil)
				So(err, ShouldBeNil)
				So(cr.numWant, ShouldEqual, 3)
			})
			Convey("Get default numwant not set", func() {
				values.Del("numwant")
				config.MaxPeers = 3
				config.DefaultPeers = 2
				cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
				So(cr, ShouldNotBeNil)
				So(err, ShouldBeNil)
				So(cr.numWant, ShouldEqual, 2)
			})
		})
		Convey("Get event", func() {
			values.Set("event", "start")
			config.MaxPeers = 3
			config.DefaultPeers = 1
			cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
			So(cr, ShouldNotBeNil)
			So(err, ShouldBeNil)
			So(cr.event, ShouldEqual, "start")
		})
		Convey("Differs returns true", func() {
			cr, err := NewClientRequest(config, values, "127.0.0.1:8000")
			So(cr, ShouldNotBeNil)
			So(err, ShouldBeNil)
			result := cr.differs(&Peer{
				IP:    "124.0.0.1",
				Port:  1,
				State: stateDownloading,
			})
			So(result, ShouldBeTrue)
		})
	})
}


text:
exports.config = {
  seleniumServerJar: __dirname +
    '/../node_modules/gulp-protractor' +
    '/node_modules/protractor/selenium/selenium-server-standalone-2.47.1.jar',

  browserName: 'chrome',

  // Exclude patterns are relative to this directory.
  // exclude: [],

  // run tests in parallel
  //capabilities: {
  //  browserName: 'chrome',
  //  shardTestFiles: true,
  //  maxInstances: 10
  //},

  jasmineNodeOpts: {
    showColors: true,
    isVerbose: true, // List all tests in the console
    includeStackTrace: true,
    defaultTimeoutInterval: 10000
  },

  plugins: [{
    path: __dirname + '/../node_modules/gulp-protractor/node_modules/' +
      'protractor/node_modules/jasmine/node_modules/jasmine-core/lib/console/console.js',
    failOnWarning: true,
    failOnError: true
  }],

  baseUrl: 'http://localhost:8888'
};


text:
<!DOCTYPE html>

  <title>Bootstrap Example</title>
  <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
  <link href="css/bootstrap.min.css" rel="stylesheet">

Well Size

Small Well

Normal Well

Large Well

   <button type="button" class="btn btn-default">Back</button>


text:
package com.fromnibly.jinq.enumerable;

import static org.junit.Assert.assertEquals;

import java.util.ArrayList;

import org.junit.Test;

import com.fromnibly.jinq.Gen;
import com.fromnibly.jinq.Jinq;

public class Count_Test
{

  @Test
  public void getsSize()
  {
    ArrayList<Integer> list = Gen.arrayListOf10();
    assertEquals(10, Jinq.from(list).count());
  }

  @Test
  public void getsSize_WithPredicate()
  {
    ArrayList<Integer> list = Gen.arrayListOf10();
    assertEquals(5, Jinq.from(list).count(x -> x > 4));
  }

  @Test
  public void emptyList_Returns0()
  {
    ArrayList<Integer> list = new ArrayList<Integer>();
    assertEquals(0, Jinq.from(list).count());
  }

  @Test
  public void emptyList_WithPredicate_Returns0()
  {
    ArrayList<Integer> list = Gen.arrayListOf10();
    assertEquals(0, Jinq.from(list).count(x -> x < 0));
  }

}


text:
double desktop_idle_get_time(void) {
  LASTINPUTINFO lif;
  lif.cbSize = sizeof(lif);
  if (!GetLastInputInfo(&lif)) return -1;
  uint64_t tickCount = GetTickCount64();
  uint32_t IdleTime = (uint32_t)((tickCount - (uint64_t)lif.dwTime));
  return static_cast<double>(IdleTime / 1000);
}


text:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Data;
using System.Windows.Media;
using uEN.Core;
using uEN.UI;

namespace uEN.UI.Controls
{
    /// <summary>
    ///
    /// </summary>
    [VisualElements(typeof(BrandColorView))]
    public class BrandColorViewModel : BizViewModel
    {
        public override string Description { get { return Properties.Resources.BrandColor; } }

        public override void ApplyView()
        {
            ColorCollection = new ListCollectionView(ColorToStringConverter.ListColors().ToList());
            ColorCollection.MoveCurrentTo(Singleton<ThemeManager>.Value.BrandColor);
            ColorCollection.CurrentChanged -= ColorCollection_CurrentChanged;
            ColorCollection.CurrentChanged += ColorCollection_CurrentChanged;
        }

        void ColorCollection_CurrentChanged(object sender, EventArgs e)
        {
            Singleton<ThemeManager>.Value.BrandColor = ColorCollection.CurrentItem as Color?;
        }
        public ListCollectionView ColorCollection { get; set; }

    }
}


text:
    {{ filing.date }}

      {{ property }}
      {{ summary.filings[filing.date][statement][property] }}


text:
from django.conf.urls import url

urlpatterns = [
    url(r'^$','contact.views.contact', name='contact')
]


text:
{-# LANGUAGE TypeFamilies, OverloadedStrings #-}

module Data.Recorder (
    Recorder,
    SqlType(..),
    HaskellType(..),
    Record(..),
    Column(..),
    Field(..),
    Table(..),
    haskFromSql,
    typeToStr
) where

import Database.MySQL.Simple
import Data.Text

data SqlType = SqlText | SqlInt | SqlLong | SqlBool | SqlNumeric | SqlByteArray | SqlDate | SqlTime | SqlOther | SqlMaybe SqlType

data HaskellType = HaskText | HaskInt | HaskLong | HaskBool | HaskNumeric | HaskByteArray | HaskDate | HaskTime | HaskUnknown | HaskMaybe HaskellType

data Table = Table {
    tname :: Text,
    cols  :: [Column]
}

data Column = Column {
    cname :: Text,
    ctyp  :: SqlType,
    dflt  :: Maybe Text,
    iskey :: Bool
}

data Record = Record {
    hname  :: Text,
    fields :: [Field]
}

data Field = Field {
    fname :: Text,
    ftyp  :: HaskellType,
    fiskey :: Bool
}

haskFromSql SqlText = HaskText
haskFromSql SqlInt = HaskInt
haskFromSql SqlLong = HaskLong
haskFromSql SqlBool = HaskBool
haskFromSql SqlNumeric = HaskNumeric
haskFromSql SqlByteArray = HaskByteArray
haskFromSql SqlDate = HaskDate
haskFromSql SqlTime = HaskTime
haskFromSql SqlOther = HaskUnknown
haskFromSql (SqlMaybe t) = HaskMaybe $ haskFromSql t

typeToStr HaskText = "!Text"
typeToStr HaskInt = "!Int"
typeToStr HaskLong = "!Integer"
typeToStr HaskBool = "!Bool"
typeToStr HaskNumeric = "!Decimal"
typeToStr HaskByteArray = "!ByteArray"
typeToStr HaskDate = "!Day"
typeToStr HaskTime = "!UTCTime"
typeToStr HaskUnknown = "!Unknown"
typeToStr (HaskMaybe t) = append "!Maybe " $ typeToStr t

class Recorder a where
    type Key a
    key :: a -> Key a
    find :: ConnectInfo -> Key a -> IO (Maybe a)
    findAll :: ConnectInfo -> IO [a]


text:
package pipe

func pipe(max int, rsync, wsync bool) (*Reader, *Writer) {
	r := &Reader{}
	w := &Writer{}
	r.init(max, rsync)
	w.initFrom(&r.ringbuf, wsync)
	return r, w
}

func Pipe(max int) (*Reader, *Writer) {
	return pipe(max, false, false)
}

func SyncWritePipe(max int) (*Reader, *Writer) {
	return pipe(max, false, true)
}

func SyncPipe(max int) (*Reader, *Writer) {
	return pipe(max, true, true)
}


text:
copyright:
  years: 2015, 2017

lastupdated: "2017-03-01"

---

{:shortdesc: .shortdesc}
{:new_window: target="_blank"}
{:codeblock: .codeblock}
{:screen: .screen}

# Filtrado de los registros de una app CF por origen
{:#k4_filter_logs_by_source}

Vea y filtre por tipo de origen los registros de una aplicación Cloud Foundry mediante el panel de control.
{:shortdesc}

Siga estos pasos para buscar entradas que incluyan un determinado origen de registro:

1. Examine la página Descubrir de Kibana para ver el subconjunto de datos que muestra. Para obtener más información, consulte [Identificación de los datos que se muestran en la página Descubrir de Kibana](logging_kibana_analize_logs_interactively.html#k4_identify_data).

2. En la *Lista de campos*, seleccione el campo **source_id**.

    ![Lista de filtros que muestra el campo source_id](images/k4_filter_sourceid_F1.jpg "Lista de filtros que muestra el campo source_id")

3. Para añadir un filtro que busque las entradas que incluyan un determinado source_id, seleccione el botón de lupa ![Botón de lupa en modalidad de inclusión](images/k4_include_field_icon.jpg "Botón de lupa en modalidad de inclusión") para dicho valor.

    Para obtener una lista de orígenes de registro que están disponibles para apps CF, consulte [Orígenes de registro para apps CF](../logging_cf_apps.html#logging_bluemix_cf_apps_log_sources).

    Por ejemplo, para añadir un filtro que incluya las entradas de registro sobre el inicio, detención o caída de una aplicación CF, seleccione el botón de lupa ![Botón de lupa en modalidad de inclusión](images/k4_include_field_icon.jpg "Botón de lupa en modalidad de inclusión") que está disponible para el valor *CELL* en la sección *Lista de campos*. En la figura siguiente se muestra el filtro para el valor de source_id *CELL* habilitado.

    ![Filtro que incluye el valor de campo](images/k4_filter_sourceid_F2.jpg "Filtro que incluye el valor de campo")

    Para añadir un filtro que busque las entradas que no incluyan un determinado source_id, elija el botón de lupa ![Botón de lupa en modalidad de exclusión](images/k4_exclude_field_icon.jpg "Botón de lupa en modalidad de exclusión") para el valor.

    Por ejemplo, para añadir un filtro que excluya las entradas de registro sobre el inicio, detención o caída de una aplicación CF, seleccione el botón de lupa ![Botón de lupa en modalidad de inclusión](images/k4_exclude_field_icon.jpg "Botón de lupa en modalidad de inclusión") que está disponible para el valor *CELL* en la sección *Lista de campos*. La figura siguiente muestra el filtro que excluye entradas para el valor de source_id *CELL*.

    ![Filtro que excluye el valor de campo](images/k4_filter_sourceid_F3.jpg "Filtro que excluye el valor de campo")


text:
<!DOCTYPE html>

  <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />

  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
  <title>XID+ Example Run Script — XID+ 3.0.0 documentation</title>
      <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
      <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />

    <link rel="index" title="Index" href="../../genindex.html" />
    <link rel="search" title="Search" href="../../search.html" />

             XID+

                3.0.0

  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>

User Guide:

A Generative Probabilistic Framework

Installation

Docker

Tutorials

Large Fields

Beyond Positional Priors

API documentation

          XID+

 »

XID+ Example Run Script

             View page source

XID+ Example Run Script¶

(This is based on a Jupyter notebook, available in the XID+ package and can be interactively run and edited)

XID+ is a probababilistic deblender for confusion dominated maps. It is designed to:

Use a MCMC based approach to get FULL posterior probability distribution on flux

Provide a natural framework to introduce additional prior information

Allows more representative estimation of source flux density uncertainties

Provides a platform for doing science with the maps (e.g XID+ Hierarchical stacking, Luminosity function from the map etc)

Cross-identification tends to be done with catalogues, then science with the matched catalogues.

XID+ takes a different philosophy. Catalogues are a form of data compression. OK in some cases, not so much in others, i.e. confused images: catalogue compression loses correlation information. Ideally, science should be done without compression.

XID+ provides a framework to cross identify galaxies we know about in different maps, with the idea that it can be extended to do science with the maps!!

Philosophy:

build a probabilistic generative model for the SPIRE maps

Infer model on SPIRE maps

Bayes Theorem

\(p(\mathbf{f}|\mathbf{d}) \propto p(\mathbf{d}|\mathbf{f}) \times p(\mathbf{f})\)

In order to carry out Bayesian inference, we need a model to carry out inference on.

For the SPIRE maps, our model is quite simple, with likelihood defined as: \(L = p(\mathbf{d}|\mathbf{f}) \propto |\mathbf{N_d}|^{-1/2} \exp\big\{ -\frac{1}{2}(\mathbf{d}-\mathbf{Af})^T\mathbf{N_d}^{-1}(\mathbf{d}-\mathbf{Af})\big\}\)

where: \(\mathbf{N_{d,ii}} =\sigma_{inst.,ii}^2+\sigma_{conf.}^2\)

Simplest model for XID+ assumes following:

All sources are known and have positive flux (fi)

A global background (B) contributes to all pixels

PRF is fixed and known

Confusion noise is constant and not correlated across pixels

Because we are getting the joint probability distribution, our model is generative i.e. given parameters, we generate data and vica-versa

Compared to discriminative model (i.e. neural network), which only obtains conditional probability distribution i.e. Neural network, give inputs, get output. Can't go other way'

Generative model is full probabilistic model. Allows more complex relationships between observed and target variables

[2]:

import numpy as np
from astropy.io import fits
from astropy import wcs
import pickle
import dill
import sys
import os
import xidplus
import copy
from xidplus import moc_routines, catalogue
from xidplus import posterior_maps as postmaps
from builtins import input

Work out what small tiles are in the test large tile file for PACS

[38]:

from healpy import pixelfunc

order_large=6
order_small=10
tile_large=21875
output_folder='../../../test_files/'
outfile=output_folder+'Tile_'+str(tile_large)+'_'+str(order_large)+'.pkl'
with open(outfile, 'rb') as f:
    obj=pickle.load(f)
priors=obj['priors']

theta, phi =pixelfunc.pix2ang(2**order_large, tile_large, nest=True)
tile_small = pixelfunc.ang2pix(2**order_small, theta, phi, nest=True)

[39]:

priors[0].moc.write(output_folder+'Tile_'+str(tile_large)+'_'+str(order_large)+'_moc_.fits')

---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
<ipython-input-39-05cdf7b4fd86> in <module>
----> 1 priors[0].moc.write(output_folder+'Tile_'+str(tile_large)+'_'+str(order_large)+'_moc_.fits')

~/anaconda3/envs/xidplus/lib/python3.6/site-packages/pymoc/moc.py in write(self, filename, filetype, **kwargs)
    749         if filetype == 'fits':
    750             from .io.fits import write_moc_fits
--> 751             write_moc_fits(self, filename, **kwargs)
    752
    753         elif filetype == 'json':

~/anaconda3/envs/xidplus/lib/python3.6/site-packages/pymoc/io/fits.py in write_moc_fits(moc, filename, **kwargs)
    111     prihdu = fits.PrimaryHDU(header=prihdr)
    112     hdulist = fits.HDUList([prihdu, tbhdu])
--> 113     hdulist.writeto(filename, **kwargs)
    114
    115

~/anaconda3/envs/xidplus/lib/python3.6/site-packages/astropy/utils/decorators.py in wrapper(*args, **kwargs)
    533                     warnings.warn(message, warning_type, stacklevel=2)
    534
--> 535             return function(*args, **kwargs)
    536
    537         return wrapper

~/anaconda3/envs/xidplus/lib/python3.6/site-packages/astropy/io/fits/hdu/hdulist.py in writeto(self, fileobj, output_verify, overwrite, checksum)
    930         # This can accept an open file object that's open to write only, or in
    931         # append/update modes but only if the file doesn't exist.
--> 932         fileobj = _File(fileobj, mode=mode, overwrite=overwrite)
    933         hdulist = self.fromfile(fileobj)
    934         try:

~/anaconda3/envs/xidplus/lib/python3.6/site-packages/astropy/utils/decorators.py in wrapper(*args, **kwargs)
    533                     warnings.warn(message, warning_type, stacklevel=2)
    534
--> 535             return function(*args, **kwargs)
    536
    537         return wrapper

~/anaconda3/envs/xidplus/lib/python3.6/site-packages/astropy/io/fits/file.py in __init__(self, fileobj, mode, memmap, overwrite, cache)
    173             self._open_fileobj(fileobj, mode, overwrite)
    174         elif isinstance(fileobj, (str, bytes)):
--> 175             self._open_filename(fileobj, mode, overwrite)
    176         else:
    177             self._open_filelike(fileobj, mode, overwrite)

~/anaconda3/envs/xidplus/lib/python3.6/site-packages/astropy/io/fits/file.py in _open_filename(self, filename, mode, overwrite)
    551
    552         if mode == 'ostream':
--> 553             self._overwrite_existing(overwrite, None, True)
    554
    555         if os.path.exists(self.name):

~/anaconda3/envs/xidplus/lib/python3.6/site-packages/astropy/io/fits/file.py in _overwrite_existing(self, overwrite, fileobj, closed)
    441                     os.remove(self.name)
    442             else:
--> 443                 raise OSError(f"File {self.name!r} already exists.")
    444
    445     def _try_read_compressed(self, obj_or_name, magic, mode, ext=''):

OSError: File '../../../test_files/Tile_21875_6_moc_.fits' already exists.

[33]:

from astropy.table import Table, join, vstack,hstack
Table([priors[0].sra,priors[0].sdec]).write(output_folder+'Tile_'+str(tile_large)+'_'+str(order_large)+'_table.fits')

[40]:

moc=moc_routines.get_fitting_region(order_small,tile_small)
for p in priors:
    p.moc=moc
    p.cut_down_prior()
    p.prior_bkg(0.0,1)
    p.get_pointing_matrix()

[41]:

moc.write(output_folder+'Tile_'+str(tile_small)+'_'+str(order_small)+'_moc_.fits')

[42]:

print('fitting '+ str(priors[0].nsrc)+' sources \n')
print('there are '+ str(priors[0].snpix)+' pixels')

fitting 216 sources

there are 10314 pixels

[43]:

%%time
from xidplus.stan_fit import PACS

#priors[0].upper_lim_map()

#priors[0].prior_flux_upper=(priors[0].prior_flux_upper-10.0+0.02)/np.max(priors[0].prf)

fit=PACS.all_bands(priors[0],priors[1],iter=1000)

INFO:pystan:COMPILING THE C++ CODE FOR MODEL anon_model_36b659b8dec7ac05627af3c419328786 NOW.

/XID+PACS not found or wrong version. Compiling

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<timed exec> in <module>

NameError: name 'tiles' is not defined

[ ]:

Took  13205.7 seconds (3.6 hours)

[45]:

outfile=output_folder+'Tile_'+str(tile_small)+'_'+str(order_small)

posterior=xidplus.posterior_stan(fit,priors)
xidplus.save(priors,posterior,outfile)

[47]:

post_rep_map = postmaps.replicated_maps(priors, posterior, nrep=2000)
band = ['PACS_100', 'PACS_160']
for i, p in enumerate(priors):
    Bayesian_Pval = postmaps.make_Bayesian_pval_maps(priors[i], post_rep_map[i])
    wcs_temp = wcs.WCS(priors[i].imhdu)
    ra, dec = wcs_temp.wcs_pix2world(priors[i].sx_pix, priors[i].sy_pix, 0)
    kept_pixels = np.array(moc_routines.sources_in_tile([tile_small], order_small, ra, dec))
    Bayesian_Pval[np.invert(kept_pixels)] = np.nan
    Bayes_map = postmaps.make_fits_image(priors[i], Bayesian_Pval)
    Bayes_map.writeto(outfile + '_' + band[i] + '_Bayes_Pval.fits', overwrite=True)

cat = catalogue.create_PACS_cat(posterior, priors[0], priors[1])
kept_sources = moc_routines.sources_in_tile([tile_small], order_small, priors[0].sra, priors[0].sdec)
kept_sources = np.array(kept_sources)
cat[1].data = cat[1].data[kept_sources]

cat.writeto(outfile + '_PACS_cat.fits', overwrite=True)

You can fit with the numpyro backend.

[49]:

%%time
from xidplus.numpyro_fit import PACS
fit_numpyro=PACS.all_bands(priors)

CPU times: user 2h 15min 49s, sys: 38.5 s, total: 2h 16min 27s
Wall time: 40min 12s

[50]:

outfile=output_folder+'Tile_'+str(tile_small)+'_'+str(order_small)+'_numpyro'

posterior_numpyro=xidplus.posterior_numpyro(fit_numpyro,priors)
xidplus.save(priors,posterior_numpyro,outfile)

Number of divergences: 0

[52]:

post_rep_map = postmaps.replicated_maps(priors, posterior_numpyro, nrep=2000)
band = ['PACS_100', 'PACS_160']
for i, p in enumerate(priors):
    Bayesian_Pval = postmaps.make_Bayesian_pval_maps(priors[i], post_rep_map[i])
    wcs_temp = wcs.WCS(priors[i].imhdu)
    ra, dec = wcs_temp.wcs_pix2world(priors[i].sx_pix, priors[i].sy_pix, 0)
    kept_pixels = np.array(moc_routines.sources_in_tile([tile_small], order_small, ra, dec))
    Bayesian_Pval[np.invert(kept_pixels)] = np.nan
    Bayes_map = postmaps.make_fits_image(priors[i], Bayesian_Pval)
    Bayes_map.writeto(outfile + '_' + band[i] + '_Bayes_Pval_numpyro.fits', overwrite=True)

cat = catalogue.create_PACS_cat(posterior_numpyro, priors[0], priors[1])
kept_sources = moc_routines.sources_in_tile([tile_small], order_small, priors[0].sra, priors[0].sdec)
kept_sources = np.array(kept_sources)
cat[1].data = cat[1].data[kept_sources]

cat.writeto(outfile + '_PACS_cat_numpyro.fits', overwrite=True)

[54]:

moc.area_sq_deg

[54]:

0.007376604303613771

[59]:

100.0*(20.0*np.pi*(1.0/3600.0)**2)/moc.area_sq_deg

[59]:

0.06572315135190694

[ ]:

© Copyright 2022, Peter Hurley.

  Built with Sphinx using a
    theme
    provided by Read the Docs.