text:
#include <tbb/mutex.h>
#include <boost/bind.hpp>
#include <boost/asio.hpp>
#include <boost/tuple/tuple.hpp>
#include "base/util.h"
#include "base/logging.h"
#include <tbb/atomic.h>
#include <cstdlib>
#include <utility>
#include "hiredis/hiredis.h"
#include "hiredis/boostasio.hpp"
#include <list>
#include "../analytics/redis_connection.h"
#include "base/work_pipeline.h"
#include "QEOpServerProxy.h"
#include "rapidjson/document.h"
#include "rapidjson/stringbuffer.h"
#include "rapidjson/writer.h"
#include "query.h"
#include "analytics_types.h"
#include "stats_select.h"
#include <base/connection_info.h>

using std::list;
using std::string;
using std::vector;
using std::map;
using boost::assign::list_of;
using boost::ptr_map;
using boost::nullable;
using boost::tuple;
using boost::shared_ptr;
using boost::scoped_ptr;
using std::pair;
using std::auto_ptr;
using std::make_pair;
using process::ConnectionState;
using process::ConnectionType;
using process::ConnectionStatus;

extern RedisAsyncConnection * rac_alloc(EventManager *, const std::string & ,unsigned short,
RedisAsyncConnection::ClientConnectCbFn ,
RedisAsyncConnection::ClientDisconnectCbFn );

extern RedisAsyncConnection * rac_alloc_nocheck(EventManager *, const std::string & ,unsigned short,
RedisAsyncConnection::ClientConnectCbFn ,
RedisAsyncConnection::ClientDisconnectCbFn );

SandeshTraceBufferPtr QeTraceBuf(SandeshTraceBufferCreate(QE_TRACE_BUF, 10000));

typedef pair<QEOpServerProxy::QPerfInfo,
             pair<shared_ptr<QEOpServerProxy::BufferT>,
                  shared_ptr<QEOpServerProxy::OutRowMultimapT> > > RawResultT;
typedef pair<redisReply,vector<string> > RedisT;

bool RedisAsyncArgCommand(RedisAsyncConnection * rac,
            void *rpi, const vector<string>& args) {

    return rac->RedisAsyncArgCmd(rpi, args);
}

class QEOpServerProxy::QEOpServerImpl {
public:
    typedef std::vector<std::string> QEOutputT;

    struct Input {
        int cnum;
        string hostname;
        QueryEngine::QueryParams qp;
        vector<uint64_t> chunk_size;
        bool need_merge;
        bool map_output;
        string where;
        string select;
        string post;
        uint64_t time_period;
        string table;
        tbb::atomic<uint32_t> chunk_q;
    };

    void JsonInsert(std::vector<query_column> &columns,
            rapidjson::Document& dd,
            std::pair<const string,string> * map_it) {

        bool found = false;
        for (size_t j = 0; j < columns.size(); j++)
        {
            if (0 == map_it->first.compare(0,5,string("COUNT"))) {
                rapidjson::Value val(rapidjson::kNumberType);
                unsigned long num = 0;
                stringToInteger(map_it->second, num);
                val.SetUint64(num);
                dd.AddMember(map_it->first.c_str(), val, dd.GetAllocator());
                found = true;
            } else if (columns[j].name == map_it->first) {
                // find out type and convert
                if (columns[j].datatype == "string" ||
                    columns[j].datatype == "uuid")
                {
                    rapidjson::Value val(rapidjson::kStringType);
                    val.SetString(map_it->second.c_str());
                    dd.AddMember(map_it->first.c_str(), val, dd.GetAllocator());
                } else if (columns[j].datatype == "ipv4") {
                    rapidjson::Value val(rapidjson::kStringType);
                    char str[INET_ADDRSTRLEN];
                    uint32_t ipaddr = 0;

                    stringToInteger(map_it->second, ipaddr);
                    ipaddr = htonl(ipaddr);
                    inet_ntop(AF_INET, &(ipaddr), str, INET_ADDRSTRLEN);
                    map_it->second = str;

                    val.SetString(map_it->second.c_str(), map_it->second.size());
                    dd.AddMember(map_it->first.c_str(), val, dd.GetAllocator());

                } else if (columns[j].datatype == "double") {
                    rapidjson::Value val(rapidjson::kNumberType);
                    double dval = (double) strtod(map_it->second.c_str(), NULL);
                    val.SetDouble(dval);
                    dd.AddMember(map_it->first.c_str(), val, dd.GetAllocator());
                } else {
                    rapidjson::Value val(rapidjson::kNumberType);
                    unsigned long num = 0;
                    stringToInteger(map_it->second, num);
                    val.SetUint64(num);
                    dd.AddMember(map_it->first.c_str(), val, dd.GetAllocator());
                }
                found = true;
            }
        }
        assert(found);
    }

    void QueryJsonify(const string& table, bool map_output,
        const BufferT* raw_res, const OutRowMultimapT* raw_mres, QEOutputT* raw_json) {

        vector<OutRowT>::iterator res_it;

        std::vector<query_column>  columns;

        if (!table.size()) return;
        bool found = false;
        for(size_t i = 0; i < g_viz_constants._TABLES.size(); i++)
        {
            if (g_viz_constants._TABLES[i].name == table) {
                found = true;
                columns = g_viz_constants._TABLES[i].schema.columns;
            }
        }
        if (!found) {
            if (g_viz_constants.OBJECT_VALUE_TABLE == table) {
                found = true;
                columns = g_viz_constants._OBJECT_TABLE_SCHEMA.columns;
            }
        }
        if (!found) {
            for (std::map<std::string, objtable_info>::const_iterator it =
                    g_viz_constants._OBJECT_TABLES.begin();
                    it != g_viz_constants._OBJECT_TABLES.end(); it++) {
                if (it->first == table) {
                    found = true;
                    columns = g_viz_constants._OBJECT_TABLE_SCHEMA.columns;
                }
            }
        }
        assert(found || map_output);

        if (map_output) {
            OutRowMultimapT::const_iterator mres_it;
            for (mres_it = raw_mres->begin(); mres_it != raw_mres->end(); ++mres_it) {
                string jstr;
                StatsSelect::Jsonify(mres_it->second.first, mres_it->second.second, jstr);
                raw_json->push_back(jstr);
            }
        } else {
            QEOpServerProxy::BufferT* raw_result =
                const_cast<QEOpServerProxy::BufferT*>(raw_res);
            QEOpServerProxy::BufferT::iterator res_it;
            for (res_it = raw_result->begin(); res_it != raw_result->end(); ++res_it) {
                std::map<std::string, std::string>::iterator map_it;
                rapidjson::Document dd;
                dd.SetObject();

                for (map_it = (*res_it).first.begin();
                     map_it != (*res_it).first.end(); ++map_it) {
                    // search for column name in the schema
                    JsonInsert(columns, dd, &(*map_it));
                }
                rapidjson::StringBuffer sb;
                rapidjson::Writer<rapidjson::StringBuffer> writer(sb);
                dd.Accept(writer);
                raw_json->push_back(sb.GetString());
            }
        }
    }

    void QECallback(void * qid, QPerfInfo qperf, auto_ptr<QEOpServerProxy::BufferT> res,
            auto_ptr<QEOpServerProxy::OutRowMultimapT> mres) {

        RawResultT* raw(new RawResultT);
        raw->first = qperf;
        raw->second.first = res;
        raw->second.second = mres;

        ExternalProcIf<RawResultT> * rpi = NULL;
        if (qid)
            rpi = reinterpret_cast<ExternalProcIf<RawResultT> *>(qid);

        if (rpi) {
            QE_LOG_NOQID(DEBUG,  " Rx data from QE for " <<
                    rpi->Key());
            auto_ptr<RawResultT> rp(raw);
            rpi->Response(rp);
        }
    }

    struct Stage0Out {
        Input inp;
        bool ret_code;
        vector<QPerfInfo> ret_info;
        vector<uint32_t> chunk_merge_time;
        shared_ptr<BufferT> result;
        shared_ptr<OutRowMultimapT> mresult;
    };

    ExternalBase::Efn QueryExec(uint32_t inst, const vector<RawResultT*> & exts,
            const Input & inp, Stage0Out & res) {
        uint32_t step = exts.size();

        if (!step) {
            res.inp = inp;
            res.ret_code = true;

            if (inp.map_output)
                res.mresult = shared_ptr<OutRowMultimapT>(new OutRowMultimapT());
            else
                res.result = shared_ptr<BufferT>(new BufferT());

            Input& cinp = const_cast<Input&>(inp);
            uint32_t chunknum = cinp.chunk_q.fetch_and_increment();
            if (chunknum < inp.chunk_size.size()) {

                string key = "QUERY:" + res.inp.qp.qid;

                // Update query status
                RedisAsyncConnection * rac = conns_[res.inp.cnum].get();
                string rkey = "REPLY:" + res.inp.qp.qid;
                char stat[40];
                uint prg = 10 + (chunknum * 75)/inp.chunk_size.size();
                QE_LOG_NOQID(DEBUG,  "QueryExec for inst " << inst <<
                    " step " << step << " PROGRESS " << prg);
                sprintf(stat,"{\"progress\":%d}", prg);
                RedisAsyncArgCommand(rac, NULL,
                    list_of(string("RPUSH"))(rkey)(stat));

                return boost::bind(&QueryEngine::QueryExec, qosp_->qe_,
                        _1,
                        inp.qp,
                        chunknum);
            } else {
                return NULL;
            }
        }

        res.ret_info.push_back(exts[step-1]->first);
        if (exts[step-1]->first.error) {
            res.ret_code =false;
        }
        if (res.ret_code) {
            if (inp.need_merge) {
                uint64_t then = UTCTimestampUsec();
                if (inp.map_output) {
                    // TODO: This interface should not be Stats-Specific
                    StatsSelect::Merge(*(exts[step-1]->second.second), *(res.mresult));
                } else {
                    res.ret_code =
                        qosp_->qe_->QueryAccumulate(inp.qp,
                            *(exts[step-1]->second.first), *(res.result));
                }
                res.chunk_merge_time.push_back(
                    static_cast<uint32_t>((UTCTimestampUsec() - then)/1000));

            } else {
                // TODO : When merge is not needed, we can just send
                //        a result upto redis at this point.

                if (inp.map_output) {
                    OutRowMultimapT::iterator jt = res.mresult->begin();
                    for (OutRowMultimapT::const_iterator it = exts[step-1]->second.second->begin();
                            it != exts[step-1]->second.second->end(); it++ ) {

                        jt = res.mresult->insert(jt,
                                std::make_pair(it->first, it->second));
                    }
                } else {
                    res.result->insert(res.result->begin(),
                        exts[step-1]->second.first->begin(),
                        exts[step-1]->second.first->end());
                }
            }
            Input& cinp = const_cast<Input&>(inp);
            uint32_t chunknum = cinp.chunk_q.fetch_and_increment();
            if (chunknum < inp.chunk_size.size()) {
                string key = "QUERY:" + res.inp.qp.qid;

                // Update query status
                RedisAsyncConnection * rac = conns_[res.inp.cnum].get();
                string rkey = "REPLY:" + res.inp.qp.qid;
                char stat[40];
                uint prg = 10 + (chunknum * 75)/inp.chunk_size.size();
                QE_LOG_NOQID(DEBUG,  "QueryExec for inst " << inst <<
                    " step " << step << " PROGRESS " << prg);
                sprintf(stat,"{\"progress\":%d}", prg);
                RedisAsyncArgCommand(rac, NULL,
                    list_of(string("RPUSH"))(rkey)(stat));
                return boost::bind(&QueryEngine::QueryExec, qosp_->qe_,
                        _1,
                        inp.qp,
                        chunknum);
            } else {
                return NULL;
            }
        }
        return NULL;
    }

    struct Stage0Merge {
        Input inp;
        bool ret_code;
        uint32_t fm_time;
        vector<vector<QPerfInfo> > ret_info;
        vector<vector<uint32_t> > chunk_merge_time;
        BufferT result;
        OutRowMultimapT mresult;
    };
    bool QueryMerge(const std::vector<boost::shared_ptr<Stage0Out> > & subs,
           const boost::shared_ptr<Input> & inp, Stage0Merge & res) {

        res.ret_code = true;
        res.inp = subs[0]->inp;
        res.fm_time = 0;

        std::vector<boost::shared_ptr<OutRowMultimapT> > mqsubs;
        std::vector<boost::shared_ptr<QEOpServerProxy::BufferT> > qsubs;
        for (vector<shared_ptr<Stage0Out> >::const_iterator it = subs.begin() ;
                it!=subs.end(); it++) {

            res.ret_info.push_back((*it)->ret_info);
            res.chunk_merge_time.push_back((*it)->chunk_merge_time);

            if ((*it)->ret_code == false) {
                res.ret_code = false;
            } else {
                if (res.inp.map_output)
                    mqsubs.push_back((*it)->mresult);
                else
                    qsubs.push_back((*it)->result);
            }
        }

        if (!res.ret_code) return true;

        if (res.inp.need_merge) {
            uint64_t then = UTCTimestampUsec();

            if (res.inp.map_output) {
                res.ret_code =
                    qosp_->qe_->QueryFinalMerge(res.inp.qp, mqsubs, res.mresult);

            } else {
                res.ret_code =
                    qosp_->qe_->QueryFinalMerge(res.inp.qp, qsubs, res.result);
            }

            uint64_t now = UTCTimestampUsec();
            res.fm_time = static_cast<uint32_t>((now - then)/1000);
        } else {
            // TODO : If a merge was not needed, results have been sent to
            //        redis already. The only thing still needed is the status
            for (vector<shared_ptr<Stage0Out> >::const_iterator it = subs.begin() ;
                    it!=subs.end(); it++) {

                if (res.inp.map_output) {
                    OutRowMultimapT::iterator jt = res.mresult.begin();
                    for (OutRowMultimapT::const_iterator kt = (*it)->mresult->begin();
                            kt != (*it)->mresult->end(); kt++) {
                        jt = res.mresult.insert(jt,
                                std::make_pair(kt->first, kt->second));
                    }
                } else {
                    res.result.insert(res.result.begin(),
                        (*it)->result->begin(),
                        (*it)->result->end());
                }
            }
        }
        return true;
    }

    struct Output {
        Input inp;
        uint32_t redis_time;
        bool ret_code;
    };
    ExternalBase::Efn QueryResp(uint32_t inst, const vector<RedisT*> & exts,
            const Stage0Merge & inp, Output & ret) {
        uint32_t step = exts.size();
        switch (inst) {
        case 0: {
                if (!step)  {

                    ret.inp = inp.inp;
                    RedisAsyncConnection * rac = conns_[ret.inp.cnum].get();
                    std::stringstream keystr;
                    auto_ptr<QEOutputT> jsonresult(new QEOutputT);

                    QE_LOG_NOQID(INFO,  "Will Jsonify #rows " <<
                        inp.result.size() + inp.mresult.size());
                    QueryJsonify(inp.inp.table, inp.inp.map_output,
                        &inp.result, &inp.mresult, jsonresult.get());

                    vector<string> const * const res = jsonresult.get();
                    vector<string>::size_type idx = 0;
                    uint32_t rownum = 0;

                    QE_LOG_NOQID(INFO,  "Did Jsonify #rows " << res->size());

                    uint64_t then = UTCTimestampUsec();
                    char stat[80];
                    string key = "REPLY:" + ret.inp.qp.qid;
                    if (!inp.ret_code) {
                        sprintf(stat,"{\"progress\":%d}", - 5);
                    } else {
                        while (idx < res->size()) {
                            uint32_t rowsize = 0;
                            keystr.str(string());
                            keystr << "RESULT:" << ret.inp.qp.qid << ":" << rownum;
                            vector<string> command = list_of(string("RPUSH"))(keystr.str());
                            while ((idx < res->size()) && (((int)rowsize) < kMaxRowThreshold)) {
                                command.push_back(res->at(idx));
                                rowsize += res->at(idx).size();
                                idx++;
                            }
                            RedisAsyncArgCommand(rac, NULL, command);
                            RedisAsyncArgCommand(rac, NULL,
                                list_of(string("EXPIRE"))(keystr.str())("300"));
                            sprintf(stat,"{\"progress\":90, \"lines\":%d}",
                                (int)rownum);
                            RedisAsyncArgCommand(rac, NULL,
                                list_of(string("RPUSH"))(key)(stat));
                            rownum++;
                        }
                        sprintf(stat,"{\"progress\":100, \"lines\":%d, \"count\":%d}",
                            (int)rownum, (int)res->size());
                    }
                    uint64_t now = UTCTimestampUsec();
                    ret.redis_time = static_cast<uint32_t>((now - then)/1000);
                    QE_LOG_NOQID(DEBUG,  "QE Query Result is " << stat);
                    return boost::bind(&RedisAsyncArgCommand,
                            conns_[ret.inp.cnum].get(), _1,
                            list_of(string("RPUSH"))(key)(stat));
                } else {
                    RedisAsyncConnection * rac = conns_[ret.inp.cnum].get();
                    string key = "REPLY:" + ret.inp.qp.qid;
                    RedisAsyncArgCommand(rac, NULL,
                        list_of(string("EXPIRE"))(key)("300"));

                    key = "QUERY:" + ret.inp.qp.qid;
                    RedisAsyncArgCommand(rac, NULL,
                        list_of(string("EXPIRE"))(key)("300"));

                    uint64_t now = UTCTimestampUsec();
                    uint32_t qtime = static_cast<uint32_t>(
                            (now - ret.inp.qp.query_starttm)/1000);

                    QueryPerfInfo qpi;
                    qpi.set_name(Sandesh::source());
                    qpi.set_table(inp.inp.table);

                    uint64_t enqtm = atol(ret.inp.qp.terms["enqueue_time"].c_str());
                    uint32_t enq_delay = static_cast<uint32_t>(
                            (ret.inp.qp.query_starttm - enqtm)/1000);
                    qpi.set_enq_delay(enq_delay);

                    QueryStats qs;
                    size_t outsize;

                    if (ret.inp.map_output)
                        outsize = inp.mresult.size();
                    else
                        outsize = inp.result.size();

                    qs.set_rows(static_cast<uint32_t>(outsize));

                    qs.set_time(qtime);
                    qs.set_qid(ret.inp.qp.qid);
                    qs.set_chunks(inp.inp.chunk_size.size());
                    std::ostringstream wherestr, selstr, poststr;
                    for (size_t i=0; i < inp.ret_info.size(); i++) {
                        for (size_t j=0; j < inp.ret_info[i].size(); j++) {
                            wherestr << inp.ret_info[i][j].chunk_where_time << ",";
                            selstr << inp.ret_info[i][j].chunk_select_time << ",";
                            poststr << inp.ret_info[i][j].chunk_postproc_time << ",";
                        }
                        wherestr << " ";
                        selstr << " ";
                        poststr << " ";
                    }
                    qs.set_chunk_where_time(wherestr.str());
                    qs.set_chunk_select_time(selstr.str());
                    qs.set_chunk_postproc_time(poststr.str());

                    std::ostringstream mergestr;
                    for (size_t i=0; i < inp.chunk_merge_time.size(); i++) {
                        for (size_t j=0; j < inp.chunk_merge_time[i].size(); j++) {
                            mergestr << inp.chunk_merge_time[i][j] << ",";
                        }
                        mergestr << " ";
                    }

                    qs.set_chunk_merge_time(mergestr.str());
                    qs.set_final_merge_time(inp.fm_time);
                    qs.set_where(inp.inp.where);
                    qs.set_select(inp.inp.select);
                    qs.set_post(inp.inp.post);
                    qs.set_time_span(inp.inp.time_period);
                    std::vector<QueryStats> vqs;
                    vqs.push_back(qs);
                    qpi.set_query_stats(vqs);
                    QueryPerfInfoTrace::Send(qpi);

                    QueryObjectData qo;
                    qo.set_qid(ret.inp.qp.qid);
                    qo.set_table(inp.inp.table);
                    qo.set_ops_start_ts(enqtm);
                    qo.set_qed_start_ts(ret.inp.qp.query_starttm);
                    qo.set_qed_end_ts(now);
                    qo.set_flow_query_rows(static_cast<uint32_t>(outsize));
        		    QUERY_OBJECT_SEND(qo);

                    //g_viz_constants.COLLECTOR_GLOBAL_TABLE
                    QE_LOG_NOQID(INFO, "Finished: QID " << ret.inp.qp.qid <<
                        " Table " << inp.inp.table <<
                        " Time(ms) " << qtime <<
                        " RedisTime(ms) " << ret.redis_time <<
                        " MergeTime(ms) " << inp.fm_time <<
                        " Rows " << outsize <<
                        " EnQ-delay" << enq_delay);

                    ret.ret_code = true;
                }
            }
            break;
        case 1: {
                if (!step)  {
                    string key = "ENGINE:" + inp.inp.hostname;
                    return boost::bind(&RedisAsyncArgCommand,
                            conns_[inp.inp.cnum].get(), _1,
                            list_of(string("LREM"))(key)("0")(inp.inp.qp.qid));
                } else {
                    ret.ret_code = true;
                }
            }
            break;
        }
        return NULL;
    }

    typedef WorkPipeline<Input, Stage0Merge, Output> QEPipeT;

    void QEPipeCb(QEPipeT *wp, bool ret_code) {
        tbb::mutex::scoped_lock lock(mutex_);

        boost::shared_ptr<Output> res = wp->Result();
        assert(pipes_.find(res->inp.qp.qid)->second == wp);
        pipes_.erase(res->inp.qp.qid);
        npipes_[res->inp.cnum-1]--;
        QE_LOG_NOQID(DEBUG,  " Result " << res->ret_code << " , " << res->inp.cnum << " conn");
        delete wp;
    }

    void ConnUpPostProcess(uint8_t cnum) {

        if (!connState_[cnum]) {
            QE_LOG_NOQID(DEBUG, "ConnUp SetCB" << cnum);
            cb_proc_fn_[cnum] = boost::bind(&QEOpServerImpl::CallbackProcess,
                    this, cnum, _1, _2, _3);
            conns_[cnum].get()->SetClientAsyncCmdCb(cb_proc_fn_[cnum]);
            connState_[cnum] = true;
        }

        bool isConnected = true;
        for(int i=0; i<kConnections+1; i++)
            if (!connState_[i]) isConnected = false;
        if (isConnected) {
            string key = "ENGINE:" + hostname_;
            conns_[0].get()->RedisAsyncArgCmd(0,
                    list_of(string("BRPOPLPUSH"))("QUERYQ")(key)("0"));
        }
    }

    int LeastLoadedConnection() {
        int minvalue = npipes_[0];
        int minindex = 0;
        for (int i=1; i<kConnections; i++) {
            if (npipes_[i] < minvalue) {
                minvalue = npipes_[i];
                minindex = i;
            }
        }
        return minindex;
    }

    void QueryError(string qid, int ret_code) {
        redisContext *c = redisConnect(redis_host_.c_str(), port_);
        if (c->err) {
            QE_LOG_NOQID(ERROR, "Cannot report query error for " << qid <<
                " . No Redis Connection");
            return;
        }
        //Authenticate the context with password
        if (!redis_password_.empty()) {
            redisReply * reply = (redisReply *) redisCommand(c, "AUTH %s",
                                                             redis_password_.c_str());
            if (reply->type == REDIS_REPLY_ERROR) {
                QE_LOG_NOQID(ERROR, "Authentication to redis error");
                freeReplyObject(reply);
                redisFree(c);
                return;
            }
            freeReplyObject(reply);
        }

        char stat[80];
        string key = "REPLY:" + qid;
        sprintf(stat,"{\"progress\":%d}", - ret_code);

        redisReply * reply = (redisReply *) redisCommand(c, "RPUSH %s %s",
            key.c_str(), stat);

        freeReplyObject(reply);
        redisFree(c);
    }

    void StartPipeline(const string qid) {
        QueryObjectData qo;
        qo.set_qid(qid);

        tbb::mutex::scoped_lock lock(mutex_);

        redisContext *c = redisConnect(redis_host_.c_str(), port_);

        if (c->err) {
            QE_LOG_NOQID(ERROR, "Cannot start Pipleline for " << qid <<
                " . No Redis Connection");
	    qo.set_error_string("No Redis Connection");
            QUERY_OBJECT_SEND(qo);
            return;
        }

        //Authenticate the context with password
        if ( !redis_password_.empty()) {
            redisReply * reply = (redisReply *) redisCommand(c, "AUTH %s",
                                                             redis_password_.c_str());
            if (reply->type == REDIS_REPLY_ERROR) {
                QE_LOG_NOQID(ERROR, "Authentication to redis error");
                freeReplyObject(reply);
                redisFree(c);
                return;
            }
            freeReplyObject(reply);
        }

        string key = "QUERY:" + qid;
        redisReply * reply = (redisReply *) redisCommand(c, "hgetall %s",
            key.c_str());

        map<string,string> terms;
        if (!(c->err) && (reply->type == REDIS_REPLY_ARRAY)) {
            for (uint32_t i=0; i<reply->elements; i+=2) {
                string idx(reply->element[i]->str);
                string val(reply->element[i+1]->str);
                terms[idx] = val;
            }
        } else {
            QE_LOG_NOQID(ERROR, "Cannot start Pipleline for " << qid <<
                ". Could not read query input");
            freeReplyObject(reply);
            redisFree(c);
            QueryError(qid, 5);
	    qo.set_error_string("Cannot read query input");
            QUERY_OBJECT_SEND(qo);
            return;
        }

        freeReplyObject(reply);
        redisFree(c);

        QueryEngine::QueryParams qp(qid, terms, max_tasks_,
            UTCTimestampUsec());

        vector<uint64_t> chunk_size;
        bool need_merge;
        bool map_output;
        string table;
        string where;
        string select;
        string post;
        uint64_t time_period;

        int ret = qosp_->qe_->QueryPrepare(qp, chunk_size, need_merge, map_output,
            where, select, post, time_period, table);

        if (ret!=0) {
            QueryError(qid, ret);
            QE_LOG_NOQID(ERROR, "Cannot start Pipleline for " << qid <<
                ". Query Parsing Error " << ret);
	    qo.set_error_string("Query parse error");
            QUERY_OBJECT_SEND(qo);
            return;
        } else {
            QE_LOG_NOQID(INFO, "Chunks: " << chunk_size.size() <<
                " Need Merge: " << need_merge);
        }

        shared_ptr<Input> inp(new Input());
        inp.get()->hostname = hostname_;
        inp.get()->qp = qp;
        inp.get()->map_output = map_output;
        inp.get()->need_merge = need_merge;
        inp.get()->chunk_size = chunk_size;
        inp.get()->where = where;
        inp.get()->select = select;
        inp.get()->post = post;
        inp.get()->time_period = time_period;
        inp.get()->table = table;
        inp.get()->chunk_q = 0;

        vector<pair<int,int> > tinfo;
        for (uint idx=0; idx<(uint)max_tasks_; idx++) {
            tinfo.push_back(make_pair(0, -1));
        }

        QEPipeT  * wp = new QEPipeT(
            new WorkStage<Input, Stage0Merge,
                    RawResultT, Stage0Out>(
                tinfo,
                boost::bind(&QEOpServerImpl::QueryExec, this, _1,_2,_3,_4),
                boost::bind(&QEOpServerImpl::QueryMerge, this, _1,_2,_3)),
            new WorkStage<Stage0Merge, Output,
                    RedisT>(
                list_of(make_pair(0,-1))(make_pair(0,-1)),
                boost::bind(&QEOpServerImpl::QueryResp, this, _1,_2,_3,_4)));

        pipes_.insert(make_pair(qid, wp));
        int conn = LeastLoadedConnection();
        npipes_[conn]++;

        // The cnum with index 0 is only used for receiving new queries
        inp.get()->cnum = conn+1;

        wp->Start(boost::bind(&QEOpServerImpl::QEPipeCb, this, wp, _1), inp);
        QE_LOG_NOQID(DEBUG, "Starting Pipeline for " << qid << " , " << conn+1 <<
            " conn, " << tinfo.size() << " tasks");

        // Update query status
        RedisAsyncConnection * rac = conns_[inp.get()->cnum].get();
        string rkey = "REPLY:" + qid;
        char stat[40];
        sprintf(stat,"{\"progress\":15}");
        RedisAsyncArgCommand(rac, NULL,
            list_of(string("RPUSH"))(rkey)(stat));

    }

    void ConnUpPrePostProcess(uint8_t cnum) {
        //Assign callback for AUTH command
        cb_proc_fn_[cnum] = boost::bind(&QEOpServerImpl::ConnectCallbackProcess,
                    this, cnum, _1, _2, _3);
        conns_[cnum].get()->SetClientAsyncCmdCb(cb_proc_fn_[cnum]);
        //Send AUTH command
        RedisAsyncConnection * rac = conns_[cnum].get();
        if (!redis_password_.empty()) {
            RedisAsyncArgCommand(rac, NULL,
                    list_of(string("AUTH"))(redis_password_.c_str()));
        } else {
            RedisAsyncArgCommand(rac, NULL,
                    list_of(string("PING")));
        }
    }

    void ConnUp(uint8_t cnum) {
        std::ostringstream ostr;
        ostr << "ConnUp.. UP " << cnum;
        QE_LOG_NOQID(DEBUG, ostr.str());
        qosp_->evm_->io_service()->post(
                    boost::bind(&QEOpServerImpl::ConnUpPrePostProcess,
                    this, cnum));
    }

    void ConnDown(uint8_t cnum) {
        QE_LOG_NOQID(DEBUG, "ConnDown.. DOWN.. Reconnect.." << cnum);
        connState_[cnum] = false;
        ConnectionState::GetInstance()->Update(ConnectionType::REDIS,
                "Query", ConnectionStatus::DOWN, conns_[cnum]->Endpoint(),
                std::string());
        qosp_->evm_->io_service()->post(boost::bind(&RedisAsyncConnection::RAC_Connect,
            conns_[cnum].get()));
    }

    void ConnectCallbackProcess(uint8_t cnum, const redisAsyncContext *c, void *r, void *privdata) {
        QE_LOG_NOQID(DEBUG,"In ConnectCallbackProcess..");
        redisReply reply = *reinterpret_cast<redisReply*>(r);
        if (reply.type != REDIS_REPLY_ERROR) {
             ConnectionState::GetInstance()->Update(ConnectionType::REDIS,
                "Query", ConnectionStatus::UP, conns_[cnum]->Endpoint(),
                std::string());
             qosp_->evm_->io_service()->post(
                     boost::bind(&QEOpServerImpl::ConnUpPostProcess,
                     this, cnum));
        }else {
            QE_LOG_NOQID(ERROR,"In connectCallbackProcess.. Error");
            QE_ASSERT(reply.type != REDIS_REPLY_ERROR);
        }
    }

    void CallbackProcess(uint8_t cnum, const redisAsyncContext *c, void *r, void *privdata) {

        //QE_TRACE_NOQID(DEBUG, "Redis CB" << cnum);
        if (0 == cnum) {
            if (r == NULL) {
                QE_LOG_NOQID(DEBUG,  __func__ << ": received NULL reply from redis");
                return;
            }

            redisReply reply = *reinterpret_cast<redisReply*>(r);
            QE_ASSERT(reply.type == REDIS_REPLY_STRING);
            string qid(reply.str);

            StartPipeline(qid);

            qosp_->evm_->io_service()->post(
                    boost::bind(&QEOpServerImpl::ConnUpPostProcess,
                    this, cnum));
            return;
        }

        auto_ptr<RedisT> fullReply;
        vector<string> elements;

        if (r == NULL) {
            //QE_TRACE_NOQID(DEBUG, "NULL Reply...\n");
        } else {
            redisReply reply = *reinterpret_cast<redisReply*>(r);
            fullReply.reset(new RedisT);
            fullReply.get()->first = reply;
            if (reply.type == REDIS_REPLY_ARRAY) {
                for (uint32_t i=0; i<reply.elements; i++) {
                    string element(reply.element[i]->str);
                    fullReply.get()->second.push_back(element);
                }
            } else if (reply.type == REDIS_REPLY_STRING) {
                fullReply.get()->second.push_back(string(reply.str));
            }
        }
        if (!privdata) {
            //QE_TRACE_NOQID(DEBUG, "Ignoring redis reply");
            return;
        }
        ExternalProcIf<RedisT> * rpi =
                reinterpret_cast<ExternalProcIf<RedisT> *>(privdata);
        QE_TRACE_NOQID(DEBUG,  " Rx data from REDIS for " << rpi->Key());

        rpi->Response(fullReply);

    }

    QEOpServerImpl(const string & redis_host, uint16_t port,
                   const string & redis_password, QEOpServerProxy * qosp,
                   int max_tasks) :
            hostname_(boost::asio::ip::host_name()),
            redis_host_(redis_host),
            port_(port),
            redis_password_(redis_password),
            qosp_(qosp),
            max_tasks_(max_tasks) {
        for (int i=0; i<kConnections+1; i++) {
            cb_proc_fn_[i] = boost::bind(&QEOpServerImpl::CallbackProcess,
                    this, i, _1, _2, _3);
            connState_[i] = false;
            if (i)
                conns_[i].reset(rac_alloc(qosp->evm_, redis_host_, port,
                        boost::bind(&QEOpServerImpl::ConnUp, this, i),
                        boost::bind(&QEOpServerImpl::ConnDown, this, i)));
            else
                conns_[i].reset(rac_alloc_nocheck(qosp->evm_, redis_host_, port,
                        boost::bind(&QEOpServerImpl::ConnUp, this, i),
                        boost::bind(&QEOpServerImpl::ConnDown, this, i)));

            // The cnum with index 0 is only used for receiving new queries
            // It does not host any pipelines
            if (i) npipes_[i-1] = 0;
        }
    }

    ~QEOpServerImpl() {
    }
private:

    static const int kMaxRowThreshold = 10000;

    // We always have one connection to receive new queries from OpServer
    // This is the number of addition connections, which will be
    // used to read query parameters and write query results
    static const uint8_t kConnections = 4;

    const string hostname_;
    const string redis_host_;
    const unsigned short port_;
    const string redis_password_;
    QEOpServerProxy * const qosp_;
    boost::shared_ptr<RedisAsyncConnection> conns_[kConnections+1];
    RedisAsyncConnection::ClientAsyncCmdCbFn cb_proc_fn_[kConnections+1];
    bool connState_[kConnections+1];

    tbb::mutex mutex_;
    map<string,QEPipeT*> pipes_;
    int npipes_[kConnections];
    int max_tasks_;

};

QEOpServerProxy::QEOpServerProxy(EventManager *evm, QueryEngine *qe,
            const string & hostname, uint16_t port, const string & redis_password,
            int max_tasks) :
        evm_(evm),
        qe_(qe),
        impl_(new QEOpServerImpl(hostname, port, redis_password, this,
            max_tasks)) {}

QEOpServerProxy::~QEOpServerProxy() {}

void
QEOpServerProxy::QueryResult(void * qid, QPerfInfo qperf,
        auto_ptr<BufferT> res, auto_ptr<OutRowMultimapT> mres) {

    impl_->QECallback(qid, qperf, res, mres);
}


text:
CREATE TABLE PERSON (
	PERSONID INT NOT NULL,
	PERSONNAME VARCHAR(20)
);

redpajama_set_name: RedPajamaGithub


text:
/* cranberry-server
 * https://github.com/jahnf/cranberry-server
 * For licensing see LICENSE file or
 * https://github.com/jahnf/cranberry-server/blob/master/LICENSE
 */

#ifndef LUASP_TYPES_H_
#define LUASP_TYPES_H_
#include "config.h"

#if LUA_SUPPORT

#define LUASP_GLOB_USERDATA_NAME "__luasp_args_"
#define LUASP_ENV_VAR_NAME "env"

/* Date in the past, that is sent to make cookies invalid
 * .. also, most important date in history of mankind ;) */
#define LUASP_DATE_IN_PAST "Fri, 01 Oct 1982 23:52:00 GMT"

/* Maximum cache entry age in seconds*/
#define LUASP_CACHE_MAX_AGE 3600

#include "webthread.h"
#include "websession.h"
#include "http_request.h"

/** Lua page state that is registered as global Lua userdata
 * variable, so it can be used as information by custom
 * Lua c functions like http_header and others */
typedef struct {
    thread_arg_t *args;
    http_req_info_t *ri;
    int headers_sent;
    int http_status_code;
    kv_item *headers;
    session_t *session;
    kv_item **sess_vars;
} luasp_page_state_t;

#endif
#endif


text: Migrated to https://gitlab.com/SomeWaffleGuy/debscript


text:
package org.apache.wicket.core.util.resource.locator;

import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Locale;

import org.apache.wicket.Application;
import org.apache.wicket.util.file.IResourceFinder;
import org.apache.wicket.util.resource.IResourceStream;
import org.apache.wicket.util.resource.ResourceUtils;
import org.apache.wicket.util.resource.ResourceUtils.PathLocale;
import org.apache.wicket.core.util.resource.UrlResourceStream;
import org.apache.wicket.util.string.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Locate Wicket resource.
 *

 * Contains the logic to locate a resource based on a path, a style (see
 * {@link org.apache.wicket.Session}), a locale and an extension string. The full filename will be
 * built like: <path>_<style>_<locale>.<extension>.
 *

 * Resource matches will be attempted in the following order:
 *

 *
1. <path>_<style>_<locale>.<extension>

 *
2. <path>_<locale>.<extension>

 *
3. <path>_<style>.<extension>

 *
4. <path>.<extension>

 *

 *

 * Locales may contain a language, a country and a region or variant. Combinations of these
 * components will be attempted in the following order:
 *

 *
locale.toString() see javadoc for Locale for more details

 *
<language>_<country>

 *
<language>

 *

 *
 * @author Juergen Donnerstag
 * @author Jonathan Locke
 */
public class ResourceStreamLocator implements IResourceStreamLocator
{
	/** Logging */
	private static final Logger log = LoggerFactory.getLogger(ResourceStreamLocator.class);

	private static final Iterable<String> NO_EXTENSIONS = new ArrayList<String>(0);

	/** If null, the application registered finder will be used */
	private IResourceFinder finder;

	/**
	 * Constructor
	 */
	public ResourceStreamLocator()
	{
	}

	/**
	 * Constructor
	 *
	 * @param finder
	 *            resource finder
	 */
	public ResourceStreamLocator(final IResourceFinder finder)
	{
		this.finder = finder;
	}

	/**
	 *
	 * @see org.apache.wicket.core.util.resource.locator.IResourceStreamLocator#locate(java.lang.Class,
	 *      java.lang.String)
	 */
	@Override
	public IResourceStream locate(final Class<?> clazz, final String path)
	{
		// First try with the resource finder registered with the application
		// (allows for markup reloading)
		IResourceStream stream = locateByResourceFinder(clazz, path);
		if (stream != null)
		{
			return stream;
		}

		// Then search the resource on the classpath
		stream = locateByClassLoader(clazz, path);
		if (stream != null)
		{
			return stream;
		}

		return null;
	}

	/**
	 *
	 * @see org.apache.wicket.core.util.resource.locator.IResourceStreamLocator#locate(java.lang.Class,
	 *      java.lang.String, java.lang.String, java.lang.String, java.util.Locale,
	 *      java.lang.String, boolean)
	 */
	@Override
	public IResourceStream locate(final Class<?> clazz, String path, final String style,
		final String variation, Locale locale, final String extension, final boolean strict)
	{
		// If path contains a locale, than it'll replace the locale provided to this method
		PathLocale data = ResourceUtils.getLocaleFromFilename(path);
		if ((data != null) && (data.locale != null))
		{
			path = data.path;
			locale = data.locale;
		}

		// Try the various combinations of style, locale and extension to find the resource.
		ResourceNameIterator iter = newResourceNameIterator(path, locale, style, variation,
			extension, strict);
		while (iter.hasNext())
		{
			String newPath = iter.next();

			IResourceStream stream = locate(clazz, newPath);
			if (stream != null)
			{
				stream.setLocale(iter.getLocale());
				stream.setStyle(iter.getStyle());
				stream.setVariation(iter.getVariation());
				return stream;
			}
		}

		return null;
	}

	/**
	 * Search the the resource my means of the various classloaders available
	 *
	 * @param clazz
	 * @param path
	 * @return resource stream
	 */
	protected IResourceStream locateByClassLoader(final Class<?> clazz, final String path)
	{
		IResourceStream resourceStream = null;

		if (clazz != null)
		{
			resourceStream = getResourceStream(clazz.getClassLoader(), path);
			if (resourceStream != null)
			{
				return resourceStream;
			}
		}

		// use context classloader when no specific classloader is set
		// (package resources for instance)
		resourceStream = getResourceStream(Thread.currentThread().getContextClassLoader(), path);
		if (resourceStream != null)
		{
			return resourceStream;
		}

		// use Wicket classloader when no specific classloader is set
		resourceStream = getResourceStream(getClass().getClassLoader(), path);
		if (resourceStream != null)
		{
			return resourceStream;
		}
		return null;
	}

	/**
	 * Get the resource
	 *
	 * @param classLoader
	 * @param path
	 * @return resource stream
	 */
	private IResourceStream getResourceStream(final ClassLoader classLoader, final String path)
	{
		if (classLoader == null)
		{
			return null;
		}

		if (log.isDebugEnabled())
		{
			log.debug("Attempting to locate resource '" + path + "' using classloader " +
				classLoader);
		}

		// Try loading path using classloader
		URL url = classLoader.getResource(path);
		if (url == null)
		{
			// maybe it is in the Servlet 3.0 like directory
			url = classLoader.getResource("META-INF/resources/" + path);
		}

		if (url != null)
		{
			return new UrlResourceStream(url);
		}
		return null;
	}

	/**
	 * Search the resource by means of the application registered resource finder
	 *
	 * @param clazz
	 * @param path
	 * @return resource stream
	 */
	protected IResourceStream locateByResourceFinder(final Class<?> clazz, final String path)
	{
		if (finder == null)
		{
			finder = Application.get().getResourceSettings().getResourceFinder();
		}

		// Log attempt
		if (log.isDebugEnabled())
		{
			log.debug("Attempting to locate resource '" + path + "' on path " + finder);
		}

		// Try to find file resource on the path supplied
		return finder.find(clazz, path);
	}

	/**
	 *
	 * @see org.apache.wicket.core.util.resource.locator.IResourceStreamLocator#newResourceNameIterator(java.lang.String,
	 *      java.util.Locale, java.lang.String, java.lang.String, java.lang.String, boolean)
	 */
	@Override
	public ResourceNameIterator newResourceNameIterator(final String path, final Locale locale,
		final String style, final String variation, final String extension, final boolean strict)
	{
		final Iterable<String> extensions = extension == null ? NO_EXTENSIONS : Arrays.asList(extension);

		final String realPath;
		final String realExtension;

		if ((extension == null) && (path != null) && (path.indexOf('.') != -1))
		{
			realPath = Strings.beforeLast(path, '.');
			// for extensions with separator take the first extension
			realExtension = Strings.afterLast(path, '.');
			if (realExtension.indexOf(',') > -1)
			{
				// multiple extensions are not allowed in the path parameter
				return new EmptyResourceNameIterator();
			}
		}
		else
		{
			realPath = path;
			realExtension = extension;
		}

		return new ResourceNameIterator(path, style, variation, locale, extensions, strict);
	}
}


text:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Noodles
{
    public static class Configuration
    {
        static Configuration()
        {
             //use Harden rules to control method access
            NodeMethodsRuleRegistry.ShowMethodRules.Insert(0, (o, info) => Harden.Allow.Call(o, info) ? null as bool? : false);
            NodeMethodsRuleRegistry.ShowMethodRules.Insert(0, (o, m) => m.Name.StartsWith("Allow") ? false : null as bool?);

            NodePropertiesRuleRegistry.ShowPropertyRules.Insert(0, (o, info) => Harden.Allow.Get(o, info) ? null as bool? : false);

        }

        public static void Initialise()
        {
        }
    }
}


text:
class AdminScaffoldGenerator < Rails::Generator::NamedBase
  default_options :skip_timestamps => false,
                  :skip_migration => false,
                  :force_plural => false,
                  :authenticated => false,
                  :admin_authenticated => false,
                  :include_activation => false

  attr_reader   :controller_name,
                :controller_class_path,
                :controller_file_path,
                :controller_class_nesting,
                :controller_class_nesting_depth,
                :controller_class_name,
                :controller_underscore_name,
                :controller_singular_name,
                :controller_plural_name,
                :controller_routing_name,
                :controller_routing_path,
                :controller_controller_name,
                :controller_file_name
  alias_method  :controller_file_name,  :controller_underscore_name
  alias_method  :controller_table_name, :controller_plural_name
  attr_reader   :sessions_controller_name,
                :sessions_controller_class_path,
                :sessions_controller_file_path,
                :sessions_controller_class_nesting,
                :sessions_controller_class_nesting_depth,
                :sessions_controller_class_name,
                :sessions_controller_singular_name,
                :sessions_controller_plural_name,
                :sessions_controller_routing_name,
                :sessions_controller_routing_path,
                :sessions_controller_controller_name
  alias_method  :sessions_controller_file_name,  :sessions_controller_singular_name
  alias_method  :sessions_controller_table_name, :sessions_controller_plural_name

  def initialize(runtime_args, runtime_options = {})
    super

    if @name == @name.pluralize && !options[:force_plural]
      logger.warning "Plural version of the model detected, using singularized version.  Override with --force-plural."
      @name = @name.singularize
      assign_names!(@name)
    end

    @controller_name = 'admin/' + @name.pluralize

    base_name, @controller_class_path, @controller_file_path, @controller_class_nesting, @controller_class_nesting_depth = extract_modules(@controller_name)
    @controller_class_name_without_nesting, @controller_underscore_name, @controller_plural_name = inflect_names(base_name)
    @controller_singular_name = base_name.singularize
    if @controller_class_nesting.empty?
      @controller_class_name = @controller_class_name_without_nesting
    else
      @controller_class_name = "#{@controller_class_nesting}::#{@controller_class_name_without_nesting}"
    end
    @controller_routing_name  = @controller_singular_name
    @controller_routing_path  = @controller_file_path.singularize
    @controller_controller_name = @controller_plural_name

    if options[:admin_authenticated] || options[:authenticated]
      require File.expand_path(File.dirname(__FILE__) + "/lib/insert_routes.rb")
      require 'digest/sha1'
      initialize_sessions_controller_name
      load_or_initialize_site_keys
    end
  end

  def initialize_sessions_controller_name
    @sessions_controller_name = "#{'admin/' if options[:admin_authenticated]}#{@name.pluralize}_sessions"
    base_name, @sessions_controller_class_path, @sessions_controller_file_path, @sessions_controller_class_nesting, @sessions_controller_class_nesting_depth = extract_modules(@sessions_controller_name)
    @sessions_controller_class_name_without_nesting, @sessions_controller_file_name, @sessions_controller_plural_name = inflect_names(base_name)
    @sessions_controller_singular_name = @sessions_controller_file_name.singularize
    if @sessions_controller_class_nesting.empty?
      @sessions_controller_class_name = @sessions_controller_class_name_without_nesting
    else
      @sessions_controller_class_name = "#{@sessions_controller_class_nesting}::#{@sessions_controller_class_name_without_nesting}"
    end
    @sessions_controller_routing_name  = @sessions_controller_singular_name
    @sessions_controller_routing_path  = @sessions_controller_file_path.singularize
    @sessions_controller_controller_name = @sessions_controller_plural_name
  end

  def manifest
    record do |m|
      # Check for class naming collisions.
      m.class_collisions("#{controller_class_name}Controller", "#{controller_class_name}Helper")
      m.class_collisions(class_name)

      # Controller, helper, views, test and stylesheets directories.
      m.directory(File.join('app/models', class_path))
      m.directory(File.join('app/controllers', controller_class_path))
      m.directory(File.join('app/helpers', controller_class_path))
      m.directory(File.join('app/views', controller_class_path, controller_file_name))
      m.directory(File.join('app/views', controller_class_path, "shared"))
      m.directory(File.join('test/functional', controller_class_path))
      m.directory(File.join('test/unit', class_path))
      m.directory(File.join('test/unit/helpers', controller_class_path))
      m.directory(File.join('public/stylesheets', class_path))
      m.directory(File.join('public/javascripts', class_path))

      m.template 'controller.rb', File.join('app/controllers', controller_class_path, "#{controller_file_name}_controller.rb")
      m.template 'helper.rb',     File.join('app/helpers',     controller_class_path, "#{controller_file_name}_helper.rb")

      # Views and Builders
      m.template "partial_form.html.erb", File.join('app/views', controller_class_path, controller_file_name, "_form.html.erb")
      m.template "partial_list.html.erb", File.join('app/views', controller_class_path, controller_file_name, "_list.html.erb")
      m.template "partial_show.html.erb", File.join('app/views', controller_class_path, controller_file_name, "_show.html.erb")
      m.template "partial_edit.html.erb", File.join('app/views', controller_class_path, controller_file_name, "_edit.html.erb")
      m.template "partial_bulk.html.erb", File.join('app/views', controller_class_path, controller_file_name, "_bulk.html.erb")
      m.template "view_index.html.erb",   File.join('app/views', controller_class_path, controller_file_name, "index.html.erb")
      m.template "view_new.html.erb",     File.join('app/views', controller_class_path, controller_file_name, "new.html.erb")
      m.template "view_show.html.erb",    File.join('app/views', controller_class_path, controller_file_name, "show.html.erb")
      m.template "view_edit.html.erb",    File.join('app/views', controller_class_path, controller_file_name, "edit.html.erb")
      m.template "builder_index.xml.builder", File.join('app/views', controller_class_path, controller_file_name, "index.xml.builder")
      m.template "builder_index.xls.builder", File.join('app/views', controller_class_path, controller_file_name, "index.xls.builder")
      m.template "builder_index.pdf.prawn",   File.join('app/views', controller_class_path, controller_file_name, "index.pdf.prawn")

      # Locales templates
      %w( en zh-TW ).each do |locale|
        m.template "locales_#{locale}.yml", File.join('config/locales', "#{controller_file_name}_#{locale}.yml")
      end

      # Application, Layout and Stylesheet and Javascript.
      # m.template_without_destroy 'layout.html.erb', File.join('app/views/layouts', controller_class_path, "admin.html.erb"), :collision => :skip
      # m.template_without_destroy 'application_helper.rb', File.join('app/helpers', controller_class_path, "admin_helper.rb"), :collision => :skip
      # m.template_without_destroy 'partial_menu.html.erb', File.join('app/views', controller_class_path, "shared", "_menu.html.erb"), :collision => :skip
      m.header_menu(controller_file_name) unless options[:no_header_menu]
      # m.template_without_destroy 'context_menu.js', 'public/javascripts/context_menu.js', :collision => :skip
      # m.template_without_destroy 'select_list_move.js', 'public/javascripts/select_list_move.js', :collision => :skip
      # m.template('style.css', 'public/stylesheets/scaffold.css')

      m.template('functional_test.rb', File.join('test/functional', controller_class_path, "#{controller_file_name}_controller_test.rb"))
      m.template('helper_test.rb',     File.join('test/unit/helpers', controller_class_path, "#{controller_file_name}_helper_test.rb"))

      m.admin_route_resources controller_file_name

      if options[:admin_authenticated] || options[:authenticated]
        generate_sessions_controller(m)
      else
        m.dependency 'model', [name] + @args, :collision => :skip
      end

    end
  end

  #
  # !! These must match the corresponding routines in by_password.rb !!
  #
  def secure_digest(*args)
    Digest::SHA1.hexdigest(args.flatten.join('--'))
  end
  def make_token
    secure_digest(Time.now, (1..10).map{ rand.to_s })
  end
  def password_digest(password, salt)
    digest = $rest_auth_site_key_from_generator
    $rest_auth_digest_stretches_from_generator.times do
      digest = secure_digest(digest, salt, password, $rest_auth_site_key_from_generator)
    end
    digest
  end

  #
  # Try to be idempotent:
  # pull in the existing site key if any,
  # seed it with reasonable defaults otherwise
  #
  def load_or_initialize_site_keys
    case
    when defined? REST_AUTH_SITE_KEY
      if (options[:old_passwords]) && ((! REST_AUTH_SITE_KEY.blank?) || (REST_AUTH_DIGEST_STRETCHES != 1))
        raise "You have a site key, but --old-passwords will overwrite it.  If this is really what you want, move the file #{site_keys_file} and re-run."
      end
      $rest_auth_site_key_from_generator         = REST_AUTH_SITE_KEY
      $rest_auth_digest_stretches_from_generator = REST_AUTH_DIGEST_STRETCHES
    when options[:old_passwords]
      $rest_auth_site_key_from_generator         = nil
      $rest_auth_digest_stretches_from_generator = 1
      $rest_auth_keys_are_new                    = true
    else
      $rest_auth_site_key_from_generator         = make_token
      $rest_auth_digest_stretches_from_generator = 10
      $rest_auth_keys_are_new                    = true
    end
  end
  def site_keys_file
    File.join("config", "initializers", "site_keys.rb")
  end

  protected
    # Override with your own usage banner.
    def banner
      "Usage: #{$0} admin_scaffold ModelName [field:type, field:type]"
    end

    def add_options!(opt)
      opt.separator ''
      opt.separator 'Options:'
      opt.on("--skip-timestamps",
             "Don't add timestamps to the migration file for this model") { |v| options[:skip_timestamps] = v }
      opt.on("--skip-migration",
             "Don't generate a migration file for this model") { |v| options[:skip_migration] = v }
      opt.on("--force-plural",
             "Forces the generation of a plural ModelName") { |v| options[:force_plural] = v }
      opt.on("--admin-authenticated",
             "Generate authenticated model") { |v| options[:admin_authenticated] = true }
      opt.on("--authenticated",
            "Generate authenticated model") { |v| options[:authenticated] = true }
      opt.on("--include-activation",
             "Generate signup 'activation code' confirmation via email") { |v| options[:include_activation] = true }
      opt.on("--no-header",
             "Will Not Generate admin header menu") {|v| options[:no_header_menu] = true}
    end

    def model_name
      class_name.demodulize
    end

  private

    def generate_sessions_controller(m)
      # Check for class naming collisions.
      m.class_collisions sessions_controller_class_path, "#{sessions_controller_class_name}Controller", # Sessions Controller
                                                         "#{sessions_controller_class_name}Helper"
      m.class_collisions                                 "#{class_name}Mailer", "#{class_name}MailerTest", "#{class_name}Observer"
      m.class_collisions [], "#{class_name}AuthenticatedSystem", "#{class_name}AuthenticatedTestHelper"

      # Controller, helper, views, and test directories.
      m.directory File.join('app/controllers', sessions_controller_class_path)
      m.directory File.join('app/helpers', sessions_controller_class_path)
      m.directory File.join('app/views', sessions_controller_class_path, @sessions_controller_file_name)
      m.directory File.join('app/views', class_path, "#{file_name}_mailer") if options[:include_activation]
      m.directory File.join('config/initializers')
      m.directory File.join('test/functional', sessions_controller_class_path)
      m.directory File.join('test/fixtures', class_path)

      m.template 'authenticated/model.rb', File.join('app/models', class_path, "#{file_name}.rb")
      if options[:include_activation]
        %w( mailer observer ).each do |model_type|
          m.template "authenticated/#{model_type}.rb", File.join('app/models', class_path, "#{file_name}_#{model_type}.rb")
        end
      end

      m.template 'authenticated/sessions_controller.rb', File.join('app/controllers', sessions_controller_class_path, "#{@sessions_controller_file_name}_controller.rb")

      m.template 'authenticated/authenticated_system.rb', File.join('lib', "#{file_name}_authenticated_system.rb")
      m.template 'authenticated/authenticated_test_helper.rb', File.join('lib', "#{file_name}_authenticated_test_helper.rb")
      m.template_without_destroy 'authenticated/site_keys.rb', site_keys_file

      m.template 'authenticated/test/sessions_functional_test.rb', File.join('test/functional', sessions_controller_class_path, "#{@sessions_controller_file_name}_controller_test.rb")
      m.template 'authenticated/test/mailer_test.rb', File.join('test/unit', class_path, "#{file_name}_mailer_test.rb") if options[:include_activation]
      m.template 'authenticated/test/unit_test.rb', File.join('test/unit', class_path, "#{file_name}_test.rb")

      m.template 'authenticated/test/users.yml', File.join('test/fixtures', class_path, "#{table_name}.yml")
      m.template 'authenticated/session_helper.rb', File.join('app/helpers', sessions_controller_class_path, "#{@sessions_controller_file_name}_helper.rb")

      # View templates
      m.template 'authenticated/login.html.erb',  File.join('app/views', sessions_controller_class_path, @sessions_controller_file_name, "new.html.erb")
      # m.template 'authenticated/signup.html.erb', File.join('app/views', controller_class_path, controller_file_name, "signup.html.erb")
      # m.template 'authenticated/_model_partial.html.erb', File.join('app/views', controller_class_path, controller_file_name, "_#{file_name}_bar.html.erb")

      if options[:include_activation]
        # Mailer templates
        %w( activation signup_notification ).each do |action|
          m.template "authenticated/#{action}.erb", File.join('app/views', "#{file_name}_mailer", "#{action}.erb")
        end
      end

      unless options[:skip_migration]
        m.migration_template 'authenticated/migration.rb', 'db/migrate', :assigns => {
          :migration_name => "Create#{class_name.pluralize.gsub(/::/, '')}"
        }, :migration_file_name => "create_#{file_path.gsub(/\//, '_').pluralize}"
      end

      unless options[:skip_routes]
        if options[:admin_authenticated]
          # Note that this fails for nested classes -- you're on your own with setting up the routes.
          m.admin_route_resource  sessions_controller_singular_name
          m.admin_route_name("#{file_name}_signup",   "/#{controller_plural_name}/signup",   {:controller => controller_plural_name, :action => 'signup'})
          m.admin_route_name("#{file_name}_register", "/#{controller_plural_name}/register", {:controller => controller_plural_name, :action => 'register'})
          m.admin_route_name("#{file_name}_login",    "/#{controller_plural_name}/login",    {:controller => sessions_controller_controller_name, :action => 'new'})
          m.admin_route_name("#{file_name}_logout",   "/#{controller_plural_name}/logout",   {:controller => sessions_controller_controller_name, :action => 'destroy'})
        else
          # Note that this fails for nested classes -- you're on your own with setting up the routes.
          m.template 'authenticated/controller.rb', File.join('app/controllers', "#{@controller_plural_name}_controller.rb")
          m.template 'authenticated/helper.rb', File.join('app/helpers', "#{@controller_plural_name}_helper.rb")
          m.directory File.join('app/views', controller_file_name)
          m.template 'authenticated/signup.html.erb', File.join('app/views', controller_file_name, "new.html.erb")
          m.template 'authenticated/_model_partial.html.erb', File.join('app/views', controller_file_name, "_#{file_name}_bar.html.erb")
          m.route_resource  sessions_controller_singular_name
          m.route_name("#{file_name}_signup",   "/#{controller_plural_name}/signup",    {:controller => controller_plural_name, :action => 'new'})
          m.route_name("#{file_name}_register", "/#{controller_plural_name}/register",  {:controller => controller_plural_name, :action => 'create'})
          m.route_name("#{file_name}_login",    "/#{controller_plural_name}/login",    {:controller => sessions_controller_controller_name, :action => 'new'})
          m.route_name("#{file_name}_logout",   "/#{controller_plural_name}/logout",    {:controller => sessions_controller_controller_name, :action => 'destroy'})
        end
      end
    end
end

class Rails::Generator::Commands::Create
  def admin_route_resource(*resources)
    resource_list = resources.map { |r| r.to_sym.inspect }.join(', ')
    sentinel = 'map.namespace(:admin) do |admin|'

    logger.route "admin.resource #{resource_list}"
    unless options[:pretend]
      gsub_file 'config/routes.rb', /(#{Regexp.escape(sentinel)})/mi do |match|
        "#{match}\n    admin.resource #{resource_list}"
      end
    end
  end

  def admin_route_resources(*resources)
    resource_list = resources.map { |r| r.to_sym.inspect }.join(', ')
    sentinel = 'map.namespace(:admin) do |admin|'

    logger.route "admin.resources #{resource_list}"
    unless options[:pretend]
      gsub_file 'config/routes.rb', /(#{Regexp.escape(sentinel)})/mi do |match|
        "#{match}\n    admin.resources #{resource_list}, :collection => {:bulk => :post}"
      end
    end
  end

  def admin_route_name(name, path, route_options = {})
    sentinel = 'map.namespace(:admin) do |admin|'

    logger.route "admin.#{name} '#{path}', :controller => '#{route_options[:controller]}', :action => '#{route_options[:action]}'"
    unless options[:pretend]
      gsub_file 'config/routes.rb', /(#{Regexp.escape(sentinel)})/mi do |match|
        "#{match}\n    admin.#{name} '#{path}', :controller => '#{route_options[:controller]}', :action => '#{route_options[:action]}'"
      end
    end
  end

  def route_namespaced_resources(namespace, *resources)
    resource_list = resources.map { |r| r.to_sym.inspect }.join(', ')
    sentinel = 'ActionController::Routing::Routes.draw do |map|'
    logger.route "#{namespace}.resources #{resource_list}"
    unless options[:pretend]
      gsub_file 'config/routes.rb', /(#{Regexp.escape(sentinel)})/mi do |match|
        "#{match}\n  map.namespace(:#{namespace}) do |#{namespace}|\n    #{namespace}.resources #{resource_list}\n  end\n"
      end
    end
  end

  def header_menu(resource)
    gsub_file File.join('app/views/admin/shared', "_menu.html.erb"), /\z/mi do |match|
      "
<%= link_to #{class_name}.human_name, '/admin/#{resource}', :class => (match_controller?('#{controller_file_name}'))  ? 'selected' : ''%>
\n"
    end
  end

  alias_method  :template_without_destroy,  :template
end

class Rails::Generator::Commands::Destroy
  def admin_route_resource(*resources)
    resource_list = resources.map { |r| r.to_sym.inspect }.join(', ')
    look_for = "\n    admin.resource #{resource_list}"
    logger.route "admin.resource #{resource_list}"
    unless options[:pretend]
      gsub_file 'config/routes.rb', /(Regexp.escape(look_for))/mi, ''
    end
  end

  def admin_route_resources(*resources)
    resource_list = resources.map { |r| r.to_sym.inspect }.join(', ')
    look_for = "\n    admin.resources #{resource_list}, :collection => {:bulk => :post}"
    logger.route "admin.resources #{resource_list}"
    unless options[:pretend]
      gsub_file 'config/routes.rb', /(Regexp.escape(look_for))/mi, ''
    end
  end

  def admin_route_name(name, path, route_options = {})
    look_for =   "\n    admin.#{name} '#{path}', :controller => '#{route_options[:controller]}', :action => '#{route_options[:action]}'"
    logger.route "admin.#{name} '#{path}',     :controller => '#{route_options[:controller]}', :action => '#{route_options[:action]}'"
    unless options[:pretend]
      gsub_file    'config/routes.rb', /(Regexp.escape(look_for))/mi, ''
    end
  end

  def route_namespaced_resources(namespace, *resources)
    resource_list = resources.map { |r| r.to_sym.inspect }.join(', ')
    look_for = "\n  map.namespace(:#{namespace}) do |#{namespace}|\n    #{namespace}.resources #{resource_list}\n  end\n"
    logger.route "#{namespace}.resources #{resource_list}"
    unless options[:pretend]
      gsub_file 'config/routes.rb', /(#{Regexp.escape(look_for)})/mi, ''
    end
  end

  def header_menu(resource)
    # resource_list = resources.map { |r| r.to_sym.inspect }.join(', ')
    look_for = "
<%= link_to #{class_name}.human_name, '/admin/#{resource}', :class => (match_controller?('#{controller_file_name}'))  ? 'selected' : ''%>
\n"
    gsub_file File.join('app/views/admin/shared', "_menu.html.erb"), /(#{Regexp.escape(look_for)})/mi, ''
  end

  def template_without_destroy(relative_source, relative_destination, file_options = {})
  end
end

class Rails::Generator::Commands::List
  def admin_route_resource(*resources)
    resource_list = resources.map { |r| r.to_sym.inspect }.join(', ')
    logger.route "map.resource #{resource_list}"
  end

  def admin_route_name(name, path, options = {})
    logger.route "map.#{name} '#{path}', :controller => '{options[:controller]}', :action => '#{options[:action]}'"
  end
end


text:
from doajtest.fixtures.v2.applications import *
from doajtest.fixtures.v2.journals import *


text:
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Buffers;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text.Unicode;
using Xunit;

namespace System.Text.Encodings.Web.Tests
{
    public partial class JavaScriptStringEncoderTests
    {
        [Fact]
        public void TestSurrogate()
        {
            // Encode(string)
            Assert.Equal("\\uD83D\\uDCA9", System.Text.Encodings.Web.JavaScriptEncoder.Default.Encode("\U0001f4a9"));

            // Encode(writer, string)
            using (var writer = new StringWriter())
            {
                System.Text.Encodings.Web.JavaScriptEncoder.Default.Encode(writer, "\U0001f4a9");
                Assert.Equal("\\uD83D\\uDCA9", writer.GetStringBuilder().ToString());
            }

            // Encode(Span, ...)
            Span<char> destination = new char[12];
            OperationStatus status = System.Text.Encodings.Web.JavaScriptEncoder.Default.Encode(
                "\U0001f4a9".AsSpan(), destination, out int charsConsumed, out int charsWritten, isFinalBlock: true);

            Assert.Equal(OperationStatus.Done, status);
            Assert.Equal(2, charsConsumed);
            Assert.Equal(12, charsWritten);
            Assert.Equal("\\uD83D\\uDCA9", new string(destination.Slice(0, charsWritten).ToArray()));
        }

        [Fact]
        public void TestSurrogateBufferDoesNotUnderOrOverWrite()
        {
            Span<char> destination = new char[212];
            destination[99] = 'x';
            destination[112] = 'x';

            // Pass in destination + 100 to check for underwrite.
            OperationStatus status = System.Text.Encodings.Web.JavaScriptEncoder.Default.Encode(
                "\U0001f4a9".AsSpan(), destination.Slice(100, 12), out int charsConsumed, out int charsWritten, isFinalBlock: true);

            Assert.Equal(OperationStatus.Done, status);
            Assert.Equal(2, charsConsumed);
            Assert.Equal(12, charsWritten);
            Assert.Equal('x', destination[99]);
            Assert.Equal('x', destination[112]);
        }

        [Fact]
        public void TestSurrogateBufferOverlaps()
        {
            Span<char> destination = new char[100];
            "\U0001f4a9".AsSpan().CopyTo(destination);

            // Overlap behavior is undefined but documented that it is not valid. Here we don't expect any issues.
            OperationStatus status = System.Text.Encodings.Web.JavaScriptEncoder.Default.Encode(
                destination.Slice(0, 2), destination, out int charsConsumed, out int charsWritten, isFinalBlock: true);

            Assert.Equal(OperationStatus.Done, status);
            Assert.Equal(2, charsConsumed);
            Assert.Equal(12, charsWritten);
        }

        [Fact]
        public void TestSurrogateBufferTooSmall()
        {
            Span<char> destination = new char[11];
            OperationStatus status = System.Text.Encodings.Web.JavaScriptEncoder.Default.Encode(
                "\U0001f4a9".AsSpan(), destination, out int charsConsumed, out int charsWritten, isFinalBlock: true);

            Assert.Equal(OperationStatus.DestinationTooSmall, status);
            Assert.Equal(0, charsConsumed);
            Assert.Equal(0, charsWritten);
        }

        [Fact]
        public void JavaScriptStringEncoder_NonEmptySource_EmptyDest_Throws()
        {
            OperationStatus status = System.Text.Encodings.Web.JavaScriptEncoder.Default.Encode(
                "\U0001f4a9".AsSpan(), destination: null, out int _, out int _, isFinalBlock: true);

            Assert.Equal(OperationStatus.DestinationTooSmall, status);
        }

        [Fact]
        public void JavaScriptStringEncoder_EmptySource_EmptyDest()
        {
            OperationStatus status = System.Text.Encodings.Web.JavaScriptEncoder.Default.Encode(
                "".AsSpan(), destination: null, out int _, out int _, isFinalBlock: true);

            Assert.Equal(OperationStatus.Done, status);
        }

        [Fact]
        public void TestEmptySourceEncode()
        {
            // Encode(string)
            Assert.Equal("", System.Text.Encodings.Web.JavaScriptEncoder.Default.Encode(""));

            // Encode(writer, string)
            using (var writer = new StringWriter())
            {
                System.Text.Encodings.Web.JavaScriptEncoder.Default.Encode(writer, "");
                Assert.Equal("", writer.GetStringBuilder().ToString());
            }

            // Encode(Span, ...)
            Span<char> destination = new char[12];
            OperationStatus status = System.Text.Encodings.Web.JavaScriptEncoder.Default.Encode(
                "".AsSpan(), destination, out int charsConsumed, out int charsWritten, isFinalBlock: true);

            Assert.Equal(OperationStatus.Done, status);
            Assert.Equal(0, charsConsumed);
            Assert.Equal(0, charsWritten);
            Assert.Equal("", new string(destination.Slice(0, charsWritten).ToArray()));

            destination = null; // null doesn't throw is no characters to encode
            status = System.Text.Encodings.Web.JavaScriptEncoder.Default.Encode(
                "".AsSpan(), destination, out charsConsumed, out charsWritten, isFinalBlock: true);

            Assert.Equal(OperationStatus.Done, status);
            Assert.Equal(0, charsConsumed);
            Assert.Equal(0, charsWritten);
            Assert.Equal("", new string(destination.Slice(0, charsWritten).ToArray()));
        }

        [Fact]
        public void Ctor_WithTextEncoderSettings()
        {
            // Arrange
            var filter = new TextEncoderSettings();
            filter.AllowCharacters('a', 'b');
            filter.AllowCharacters('\0', '&', '\uFFFF', 'd');

            JavaScriptStringEncoder encoder = new JavaScriptStringEncoder(filter);

            // Act & assert
            Assert.Equal("a", encoder.JavaScriptStringEncode("a"));
            Assert.Equal("b", encoder.JavaScriptStringEncode("b"));
            Assert.Equal(@"\u0063", encoder.JavaScriptStringEncode("c"));
            Assert.Equal("d", encoder.JavaScriptStringEncode("d"));
            Assert.Equal(@"\u0000", encoder.JavaScriptStringEncode("\0")); // we still always encode control chars
            Assert.Equal(@"\u0026", encoder.JavaScriptStringEncode("&")); // we still always encode HTML-special chars
            Assert.Equal(@"\uFFFF", encoder.JavaScriptStringEncode("\uFFFF")); // we still always encode non-chars and other forbidden chars
        }

        [Fact]
        public void Ctor_WithUnicodeRanges()
        {
            // Arrange
            JavaScriptStringEncoder encoder = new JavaScriptStringEncoder(UnicodeRanges.Latin1Supplement, UnicodeRanges.MiscellaneousSymbols);

            // Act & assert
            Assert.Equal(@"\u0061", encoder.JavaScriptStringEncode("a"));
            Assert.Equal("\u00E9", encoder.JavaScriptStringEncode("\u00E9" /* LATIN SMALL LETTER E WITH ACUTE */));
            Assert.Equal("\u2601", encoder.JavaScriptStringEncode("\u2601" /* CLOUD */));
        }

        [Fact]
        public void Ctor_WithNoParameters_DefaultsToBasicLatin()
        {
            // Arrange
            JavaScriptStringEncoder encoder = new JavaScriptStringEncoder();

            // Act & assert
            Assert.Equal("a", encoder.JavaScriptStringEncode("a"));
            Assert.Equal(@"\u00E9", encoder.JavaScriptStringEncode("\u00E9" /* LATIN SMALL LETTER E WITH ACUTE */));
            Assert.Equal(@"\u2601", encoder.JavaScriptStringEncode("\u2601" /* CLOUD */));
        }

        [Fact]
        public void Default_EquivalentToBasicLatin()
        {
            // Arrange
            JavaScriptStringEncoder controlEncoder = new JavaScriptStringEncoder(UnicodeRanges.BasicLatin);
            JavaScriptStringEncoder testEncoder = JavaScriptStringEncoder.Default;

            // Act & assert
            for (int i = 0; i <= char.MaxValue; i++)
            {
                if (!IsSurrogateCodePoint(i))
                {
                    string input = new string((char)i, 1);
                    Assert.Equal(controlEncoder.JavaScriptStringEncode(input), testEncoder.JavaScriptStringEncode(input));
                }
            }
        }

        [Fact]
        public void JavaScriptStringEncode_AllRangesAllowed_StillEncodesForbiddenChars_Simple_Escaping() {
            // The following two calls could be simply InlineData to the Theory below
            // Unfortunately, the xUnit logger fails to escape the inputs when logging the test results,
            // and so the suite fails despite all tests passing.
            // TODO: I will try to fix it in xUnit, but for now this is a workaround to enable these tests.
            JavaScriptStringEncode_AllRangesAllowed_StillEncodesForbiddenChars_Simple("\b", @"\b");
            JavaScriptStringEncode_AllRangesAllowed_StillEncodesForbiddenChars_Simple("\f", @"\f");
        }

        [Theory]
        [InlineData("<", @"\u003C")]
        [InlineData(">", @"\u003E")]
        [InlineData("&", @"\u0026")]
        [InlineData("'", @"\u0027")]
        [InlineData("\"", @"\u0022")]
        [InlineData("+", @"\u002B")]
        [InlineData("\\", @"\\")]
        [InlineData("\n", @"\n")]
        [InlineData("\t", @"\t")]
        [InlineData("\r", @"\r")]
        public void JavaScriptStringEncode_AllRangesAllowed_StillEncodesForbiddenChars_Simple(string input, string expected)
        {
            // Arrange
            JavaScriptStringEncoder encoder = new JavaScriptStringEncoder(UnicodeRanges.All);

            // Act
            string retVal = encoder.JavaScriptStringEncode(input);

            // Assert
            Assert.Equal(expected, retVal);
        }

        [Fact]
        public void JavaScriptStringEncode_AllRangesAllowed_StillEncodesForbiddenChars_Extended()
        {
            // Arrange
            JavaScriptStringEncoder encoder = new JavaScriptStringEncoder(UnicodeRanges.All);

            // Act & assert - BMP chars
            for (int i = 0; i <= 0xFFFF; i++)
            {
                string input = new string((char)i, 1);
                string expected;
                if (IsSurrogateCodePoint(i))
                {
                    expected = "\uFFFD"; // unpaired surrogate -> Unicode replacement char
                }
                else
                {
                    if (input == "\b") { expected = @"\b"; }
                    else if (input == "\t") { expected = @"\t"; }
                    else if (input == "\n") { expected = @"\n"; }
                    else if (input == "\f") { expected = @"\f"; }
                    else if (input == "\r") { expected = @"\r"; }
                    else if (input == "\\") { expected = @"\\"; }
                    else if (input == "`") { expected = @"\u0060"; }
                    else
                    {
                        bool mustEncode = false;
                        switch (i)
                        {
                            case '<':
                            case '>':
                            case '&':
                            case '\"':
                            case '\'':
                            case '+':
                                mustEncode = true;
                                break;
                        }

                        if (i <= 0x001F || (0x007F <= i && i <= 0x9F))
                        {
                            mustEncode = true; // control char
                        }
                        else if (!UnicodeHelpers.IsCharacterDefined((char)i))
                        {
                            mustEncode = true; // undefined (or otherwise disallowed) char
                        }

                        if (mustEncode)
                        {
                            expected = string.Format(CultureInfo.InvariantCulture, @"\u{0:X4}", i);
                        }
                        else
                        {
                            expected = input; // no encoding
                        }
                    }
                }

                string retVal = encoder.JavaScriptStringEncode(input);
                Assert.Equal(expected, retVal);
            }

            // Act & assert - astral chars
            for (int i = 0x10000; i <= 0x10FFFF; i++)
            {
                string input = char.ConvertFromUtf32(i);
                string expected = string.Format(CultureInfo.InvariantCulture, @"\u{0:X4}\u{1:X4}", (uint)input[0], (uint)input[1]);
                string retVal = encoder.JavaScriptStringEncode(input);
                Assert.Equal(expected, retVal);
            }
        }

        [Fact]
        public void JavaScriptStringEncode_NoRangesAllowed_EmitsShortFormForCertainCodePoints()
        {
            // This test ensures that when we're encoding, we always emit the "\uXXXX" form of the
            // code point except for very specific code points where we allow a shorter representation.

            // Arrange
            JavaScriptStringEncoder encoder = new JavaScriptStringEncoder(UnicodeRanges.None); // allow no codepoints

            // "[U+0000][U+0001]...[U+007F]"
            string input = new string(Enumerable.Range(0, 128).Select(i => (char)i).ToArray());

            // @"\u0000\u0001..\u007F", then replace certain specific code points
            string expected = string.Concat(Enumerable.Range(0, 128).Select(i => FormattableString.Invariant($@"\u{i:X4}")));

            expected = expected.Replace(@"\u0008", @"\b"); // U+0008 BACKSPACE -> "\b"
            expected = expected.Replace(@"\u0009", @"\t"); // U+0009 CHARACTER TABULATION -> "\t"
            expected = expected.Replace(@"\u000A", @"\n"); // U+000A LINE FEED -> "\n"
            expected = expected.Replace(@"\u000C", @"\f"); // U+000C FORM FEED -> "\f"
            expected = expected.Replace(@"\u000D", @"\r"); // U+000D CARRIAGE RETURN -> "\n"
            expected = expected.Replace(@"\u005C", @"\\"); // U+005C REVERSE SOLIDUS -> "\\"

            // Act
            string retVal = encoder.JavaScriptStringEncode(input);

            // Assert
            Assert.Equal(expected, retVal);
        }

        [Fact]
        public void JavaScriptStringEncode_BadSurrogates_ReturnsUnicodeReplacementChar()
        {
            // Arrange
            JavaScriptStringEncoder encoder = new JavaScriptStringEncoder(UnicodeRanges.All); // allow all codepoints

            // "a<unpaired leading>b<unpaired trailing>c<trailing before leading>d<unpaired trailing><valid>e<high at end of string>"
            const string Input = "a\uD800b\uDFFFc\uDFFF\uD800d\uDFFF\uD800\uDFFFe\uD800";
            const string Expected = "a\uFFFDb\uFFFDc\uFFFD\uFFFDd\uFFFD\\uD800\\uDFFFe\uFFFD"; // 'D800' 'DFFF' was preserved since it's valid

            // Act
            string retVal = encoder.JavaScriptStringEncode(Input);

            // Assert
            Assert.Equal(Expected, retVal);
        }

        [Fact]
        public void JavaScriptEncoder_BadSurrogates_ReturnsUnicodeReplacementChar()
        {
            // Arrange
            JavaScriptEncoder encoder = JavaScriptEncoder.Create(UnicodeRanges.All); // allow all codepoints

            // "a<unpaired leading>b<unpaired trailing>c<trailing before leading>d<unpaired trailing><valid>e<high at end of string>"
            const string Input = "a\uD800b\uDFFFc\uDFFF\uD800d\uDFFF\uD800\uDFFFe\uD800";
            const string Expected = "a\uFFFDb\uFFFDc\uFFFD\uFFFDd\uFFFD\\uD800\\uDFFFe\uFFFD"; // 'D800' 'DFFF' was preserved since it's valid

            // String-based Encode()
            string retVal = encoder.Encode(Input);
            Assert.Equal(Expected, retVal);

            // OperationStatus-based Encode()
            Span<char> destination = new char[23];
            OperationStatus status = encoder.Encode(Input.AsSpan(), destination, out int charsConsumed, out int charsWritten, isFinalBlock: true);
            Assert.Equal(OperationStatus.Done, status);
            Assert.Equal(13, charsConsumed);
            Assert.Equal(13, Input.Length);
            Assert.Equal(23, charsWritten);
            Assert.Equal(Expected, new string(destination.Slice(0, charsWritten).ToArray()));
        }

        [Fact]
        public void JavaScriptEncoder_UnpairedSurrogatesReplaced()
        {
            // Arrange
            JavaScriptEncoder encoder = JavaScriptEncoder.Create(UnicodeRanges.All); // allow all codepoints

            // "a<unpaired leading low><unpaired leading high><unpaired leading high>"
            const string Input = "a\uDFFF\uD800\uD800";
            const string Expected = "a\uFFFD\uFFFD\uFFFD";

            Assert.Equal(4, Input.Length);

            // String-based Encode()
            string retVal = encoder.Encode(Input);
            Assert.Equal(Expected, retVal);

            // OperationStatus-based Encode()
            OperationStatus status;
            Span<char> destination = new char[100];
            status = encoder.Encode(Input.AsSpan(), destination, out int charsConsumed, out int charsWritten, isFinalBlock: true);
            Assert.Equal(OperationStatus.Done, status);
            Assert.Equal(4, charsConsumed);
            Assert.Equal(4, charsWritten);
            Assert.Equal(Expected, new string(destination.Slice(0, charsWritten).ToArray()));
        }

        [Fact]
        public void JavaScriptEncoder_NeedsMoreData()
        {
            // "a<paired leading><paired trailing>"
            const string Input = "a\uD800\uDFFF";
            const string Expected = "a\\uD800\\uDFFF";

            Assert.Equal(3, Input.Length);

            JavaScriptEncoder encoder = JavaScriptEncoder.Create(UnicodeRanges.All); // allow all codepoints
            Span<char> destination = new char[100];

            OperationStatus status;

            // Just pass in the first two characters, making uD800 an unpaired high surrogate. Set isFinalBlock=false so we get NeedMoreData.
            status = encoder.Encode(Input.AsSpan(0, 2), destination, out int charsConsumed1, out int charsWritten1, isFinalBlock: false);
            Assert.Equal(OperationStatus.NeedMoreData, status);
            Assert.Equal(1, charsConsumed1);
            Assert.Equal(1, charsWritten1);
            Assert.Equal("a", new string(destination.Slice(0, charsWritten1).ToArray()));

            // Append additional data; keep IsFinalBlock=false
            status = encoder.Encode(Input.AsSpan(charsConsumed1, 2), destination.Slice(charsWritten1), out int charsConsumed2, out int charsWritten2, isFinalBlock: false);
            Assert.Equal(OperationStatus.Done, status);
            Assert.Equal(2, charsConsumed2);
            Assert.Equal(12, charsWritten2);
            Assert.Equal(Expected, new string(destination.Slice(0, charsWritten1 + charsWritten2).ToArray()));

            // Ensure isFinalBlock=true has the same result since there is no longer a trailing unpaired high surrogate.
            status = encoder.Encode(Input.AsSpan(charsConsumed1, 2), destination.Slice(charsWritten1), out charsConsumed2, out charsWritten2, isFinalBlock: true);
            Assert.Equal(OperationStatus.Done, status);
            Assert.Equal(2, charsConsumed2);
            Assert.Equal(12, charsWritten2);
            Assert.Equal(Expected, new string(destination.Slice(0, charsWritten1 + charsWritten2).ToArray()));
        }

        [Fact]
        public void JavaScriptStringEncode_EmptyStringInput_ReturnsEmptyString()
        {
            // Arrange
            JavaScriptStringEncoder encoder = new JavaScriptStringEncoder();

            // Act & assert
            Assert.Equal("", encoder.JavaScriptStringEncode(""));
        }

        [Fact]
        public void JavaScriptStringEncode_InputDoesNotRequireEncoding_ReturnsOriginalStringInstance()
        {
            // Arrange
            JavaScriptStringEncoder encoder = new JavaScriptStringEncoder();
            string input = "Hello, there!";

            // Act & assert
            Assert.Same(input, encoder.JavaScriptStringEncode(input));
        }

        [Fact]
        public void JavaScriptStringEncode_NullInput_Throws()
        {
            // Arrange
            JavaScriptStringEncoder encoder = new JavaScriptStringEncoder();

            Assert.Throws<ArgumentNullException>(() => { encoder.JavaScriptStringEncode(null); });
        }

        [Fact]
        public void JavaScriptStringEncode_WithCharsRequiringEncodingAtBeginning()
        {
            Assert.Equal(@"\u0026Hello, there!", new JavaScriptStringEncoder().JavaScriptStringEncode("&Hello, there!"));
        }

        [Fact]
        public void JavaScriptStringEncode_WithCharsRequiringEncodingAtEnd()
        {
            Assert.Equal(@"Hello, there!\u0026", new JavaScriptStringEncoder().JavaScriptStringEncode("Hello, there!&"));
        }

        [Fact]
        public void JavaScriptStringEncode_WithCharsRequiringEncodingInMiddle()
        {
            Assert.Equal(@"Hello, \u0026there!", new JavaScriptStringEncoder().JavaScriptStringEncode("Hello, &there!"));
        }

        [Fact]
        public void JavaScriptStringEncode_WithCharsRequiringEncodingInterspersed()
        {
            Assert.Equal(@"Hello, \u003Cthere\u003E!", new JavaScriptStringEncoder().JavaScriptStringEncode("Hello, <there>!"));
        }

        [Fact]
        public void JavaScriptStringEncode_CharArray()
        {
            // Arrange
            JavaScriptStringEncoder encoder = new JavaScriptStringEncoder();
            var output = new StringWriter();

            // Act
            encoder.JavaScriptStringEncode("Hello+world!".ToCharArray(), 3, 5, output);

            // Assert
            Assert.Equal(@"lo\u002Bwo", output.ToString());
        }

        [Fact]
        public void JavaScriptStringEncode_StringSubstring()
        {
            // Arrange
            JavaScriptStringEncoder encoder = new JavaScriptStringEncoder();
            var output = new StringWriter();

            // Act
            encoder.JavaScriptStringEncode("Hello+world!", 3, 5, output);

            // Assert
            Assert.Equal(@"lo\u002Bwo", output.ToString());
        }

        [Theory]
        [InlineData("\"", @"\u0022")]
        [InlineData("'", @"\u0027")]
        public void JavaScriptStringEncode_Quotes(string input, string expected)
        {
            // Per the design document, we provide additional defense-in-depth
            // against breaking out of HTML attributes by having the encoders
            // never emit the ' or " characters. This means that we want to
            // \u-escape these characters instead of using \' and \".

            // Arrange
            JavaScriptStringEncoder encoder = new JavaScriptStringEncoder(UnicodeRanges.All);

            // Act
            string retVal = encoder.JavaScriptStringEncode(input);

            // Assert
            Assert.Equal(expected, retVal);
        }

        [Fact]
        public void JavaScriptStringEncode_DoesNotOutputHtmlSensitiveCharacters()
        {
            // Per the design document, we provide additional defense-in-depth
            // by never emitting HTML-sensitive characters unescaped.

            // Arrange
            JavaScriptStringEncoder javaScriptStringEncoder = new JavaScriptStringEncoder(UnicodeRanges.All);
            HtmlEncoder htmlEncoder = new HtmlEncoder(UnicodeRanges.All);

            // Act & assert
            for (int i = 0; i <= 0x10FFFF; i++)
            {
                if (IsSurrogateCodePoint(i))
                {
                    continue; // surrogates don't matter here
                }

                string javaScriptStringEncoded = javaScriptStringEncoder.JavaScriptStringEncode(char.ConvertFromUtf32(i));
                string thenHtmlEncoded = htmlEncoder.HtmlEncode(javaScriptStringEncoded);
                Assert.Equal(javaScriptStringEncoded, thenHtmlEncoded); // should have contained no HTML-sensitive characters
            }
        }

        private static bool IsSurrogateCodePoint(int codePoint)
        {
            return (0xD800 <= codePoint && codePoint <= 0xDFFF);
        }
    }
}


text:
package com.devicehive.services;

import com.datastax.driver.core.querybuilder.Delete;
import com.datastax.driver.core.querybuilder.QueryBuilder;
import com.datastax.driver.core.querybuilder.Select;
import com.datastax.driver.core.querybuilder.Update;
import com.devicehive.domain.DeviceCommand;
import com.devicehive.repository.DeviceCommandRepository;
import com.devicehive.utils.MessageUtils;
import com.devicehive.utils.mapper.CommandRowMapper;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cassandra.core.CqlOperations;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;

import java.util.Date;
import java.util.List;

/**
 * Created by tmatvienko on 2/13/15.
 */
@Service
public class DeviceCommandService {
    private static final Logger LOGGER = LoggerFactory.getLogger(DeviceCommandService.class);

    @Autowired
    private CqlOperations cqlTemplate;
    @Autowired
    private DeviceCommandRepository commandRepository;
    @Autowired
    private MessageUtils messageUtils;

    public List<DeviceCommand> get(final int count, final String commandId, final String deviceGuids, final String commandNames, final Date date) {
        Select.Where select = QueryBuilder.select().from("device_command").where();
        if (StringUtils.isNotBlank(deviceGuids)) {
            select.and(QueryBuilder.in("device_guid", messageUtils.getDeviceGuids(deviceGuids)));
        }
        if (StringUtils.isNotBlank(commandId)) {
            select.and(QueryBuilder.in("id", commandId));
        }
        List<DeviceCommand> commands = cqlTemplate.query(select.limit(count).allowFiltering(), new CommandRowMapper());
        if (!commands.isEmpty()) {
            LOGGER.warn("Select query: {}, result: {}, timestamp: {}", select.getQueryString(), commands.size(), date);
            if (date != null) {
                CollectionUtils.filter(commands, new Predicate() {
                    @Override
                    public boolean evaluate(Object o) {
                        return date.compareTo(((DeviceCommand) o).getTimestamp()) < 0;
                    }
                });
            }
            LOGGER.warn("Filter result: {}", commands);
            if (StringUtils.isNotEmpty(commandNames)) {
                CollectionUtils.filter(commands, new Predicate() {
                    @Override
                    public boolean evaluate(Object o) {
                        return commandNames.contains(((DeviceCommand) o).getCommand());
                    }
                });
            }
        }
        return commands;
    }

    public Long getCommandsCount() {
        return commandRepository.count();
    }

    @Async
    public void updateDeviceCommmand(final DeviceCommand command) {
        Update.Assignments update = QueryBuilder.update("device_command")
                .with(QueryBuilder.set("timestamp", command.getTimestamp()));
        if (command.getCommand() != null) {
            update.and(QueryBuilder.set("command", command.getCommand()));
        }
        if (command.getParameters() != null) {
            update.and(QueryBuilder.set("parameters", command.getParameters()));
        }
        if (command.getLifetime() != null) {
            update.and(QueryBuilder.set("lifetime", command.getLifetime()));
        }
        if (command.getStatus() != null) {
            update.and(QueryBuilder.set("status", command.getStatus()));
        }
        if (command.getResult() != null) {
            update.and(QueryBuilder.set("result", command.getResult()));
        }
        update.and(QueryBuilder.set("updated", true));
        update.where(QueryBuilder.eq("id", command.getId())).and(QueryBuilder.eq("device_guid", command.getDeviceGuid()));
        cqlTemplate.executeAsynchronously(update);
    }

    @Async
    public void delete(String deviceGuids) {
        Delete.Where delete = QueryBuilder.delete().from("device_command").where();
        if (StringUtils.isNotEmpty(deviceGuids)) {
            delete.and(QueryBuilder.in("device_guid", messageUtils.getDeviceGuids(deviceGuids)));
        }
        cqlTemplate.execute(delete);
    }
}