text:
#include "sendPrdcSetDlg.h"
#include "selUserDlg.h"
#include "ICtkPluginManager.h"
#include "IRadarPathAdapter.h"
#include "IFileDistribute.h"
#include "IDataCollection.h"
#include "commontypedef.h"
#include "ctkLog.h"
#include "CollectManager.h"
#include "event_macro.h"
#include <QMessageBox>

const int colNum = 3;

SendPrdcSetDlg::SendPrdcSetDlg(ICtkPluginManager *pManager, QWidget *parent/*=0*/)
    : m_nTaskFlag(0)
    , m_pManager(pManager)
    , m_pDataCollectionSev(NULL)
    , m_pFileTransfer(NULL)
{
    m_pParam = TransParam::getInstance();
    ui.setupUi(this);
    connect(ui.listWidget_ID, SIGNAL(currentRowChanged(int)), this, SLOT(onChangeTask(int)));
    //connect(ui.btn_apply, SIGNAL(clicked()), this, SLOT(onApplyTask()));
    connect(ui.tBtn_selUser, SIGNAL(clicked()), this, SLOT(onSelUser()));
    connect(ui.btn_add, SIGNAL(clicked()), this, SLOT(onBtnAddTask()));
    connect(ui.btn_del, SIGNAL(clicked()), this, SLOT(onBtnDelTask()));
    m_pThread = QSharedPointer<QThread>(new QThread);

    QObject *op = m_pManager->getService("IRadarPathAdapter");
    m_pMakePath = (qobject_cast<IRadarPathAdapter *>(op))->createRadarProductPath();

    op = m_pManager->getService("IFileDistribute");
    m_pFileTransfer = qobject_cast<IFileDistribute *>(op);
    m_pDistributer = m_pFileTransfer->createDistributeManager();
}

void SendPrdcSetDlg::initDlg()
{
    ui.listWidget_ID->clear();
    m_vecCheckBox.clear();
    m_vecCheckBox.append(ui.checkBox_1);
    m_vecCheckBox.append(ui.checkBox_2);
    m_vecCheckBox.append(ui.checkBox_3);
    m_vecCheckBox.append(ui.checkBox_4);
    m_vecCheckBox.append(ui.checkBox_5);
    m_vecCheckBox.append(ui.checkBox_6);
    m_vecCheckBox.append(ui.checkBox_7);
    m_vecCheckBox.append(ui.checkBox_8);

    // 禁止编辑
    ui.tb_users->setEditTriggers(QAbstractItemView::NoEditTriggers);
    ui.tb_users->setColumnCount(colNum);
    int nWidth = ui.tb_users->width() - colNum;
    for (int i=0; i<colNum; ++i)
    {
        ui.tb_users->setColumnWidth(i, nWidth / colNum);
    }

    const list<PrdcTransferSet> &allSet = m_pParam->m_prdcTransetList.lst;
    if (allSet.size() == 0)
    {
        m_nTaskFlag = 1;
        return;
    }
    list<PrdcTransferSet>::const_iterator it = allSet.begin();
    QString strTmp("");
    for (; it != allSet.end(); it++)
    {
        strTmp = QString::fromStdString(it->strID);
        ui.listWidget_ID->addItem(strTmp);
    }

    ui.listWidget_ID->setCurrentRow(0);
    it = allSet.begin();
    showInfo(*it);
}

void SendPrdcSetDlg::showInfo(const PrdcTransferSet &set)
{
    if (set.strID.empty())
    {
        return;
    }
    ui.le_taskName->setText(QString::fromStdString(set.strID));
    ui.le_taskName->setEnabled(false);
    ui.checkBox_1->setChecked(set.bRadarData);
    ui.checkBox_2->setChecked(set.bMosaicData);
    ui.checkBox_3->setChecked(set.bPupData);
    ui.checkBox_4->setChecked(set.bHighData);
    ui.checkBox_5->setChecked(set.bRadarPic);
    ui.checkBox_6->setChecked(set.bMosaicPic);
    ui.checkBox_7->setChecked(set.bPupPic);
    ui.checkBox_8->setChecked(set.bHighPic);
    ui.tb_users->setRowCount(0);
    ui.tb_users->clearContents();
    for (uint i=0; i<set.vecRecvID.size(); ++i)
    {
        int row = i / colNum;
        int col = i % colNum;
        if (col == 0)
        {
            ui.tb_users->insertRow(row);
        }
        ui.tb_users->setItem(row, col, new QTableWidgetItem(QString::fromStdString(set.vecRecvID.at(i))));
    }
    m_nTaskFlag = 2;
}

void SendPrdcSetDlg::onChangeTask(int rowNo)
{
    const list<PrdcTransferSet> &allSet = m_pParam->m_prdcTransetList.lst;
    if (rowNo < 0 || rowNo >= (int)allSet.size())
    {
        clearAllInfo();
        return;
    }

    list<PrdcTransferSet>::const_iterator it = allSet.begin();
    for (int i=0; i<rowNo; i++)
    {
        it++;
    }

    showInfo(*it);

    m_nTaskFlag = 2;
}

void SendPrdcSetDlg::onApplyTask()
{
    if (m_nTaskFlag == 0)
    {
        return;
    }

    QString strTaskId = ui.le_taskName->text();
    // 判断是否合法
    if (m_nTaskFlag == 1)
    {
        if (strTaskId.trimmed().isEmpty())
        {
            QMessageBox::warning(this, QStringLiteral("提示"), QStringLiteral("任务名不能为空!"));
            return;
        }

        list<PrdcTransferSet>::iterator it = m_pParam->m_prdcTransetList.lst.begin();
        while (it != m_pParam->m_prdcTransetList.lst.end())
        {
            if (it->strID == strTaskId.toStdString())
            {
                QMessageBox::critical(this, QStringLiteral("提示"), QStringLiteral("任务名已存在!"));
                return;
            }
            it++;
        }
    }

    PrdcTransferSet set;
    set.strID = strTaskId.toStdString();
    set.bRadarData = ui.checkBox_1->isChecked();
    set.bMosaicData = ui.checkBox_2->isChecked();
    set.bPupData = ui.checkBox_3->isChecked();
    set.bHighData = ui.checkBox_4->isChecked();
    set.bRadarPic = ui.checkBox_5->isChecked();
    set.bMosaicPic = ui.checkBox_6->isChecked();
    set.bPupPic = ui.checkBox_7->isChecked();
    set.bHighPic = ui.checkBox_8->isChecked();
    string userId("");
    for (int i=0; i<ui.tb_users->rowCount(); ++i)
    {
        for (int j=0; j<ui.tb_users->columnCount(); ++j)
        {
            QTableWidgetItem *pItem = ui.tb_users->item(i, j);
            if (pItem != 0)
            {
                userId = pItem->text().toStdString();
                set.vecRecvID.push_back(userId);
            }
        }
    }

    // 记录到配置文件
    if (m_nTaskFlag == 1)	// 新增
    {
        m_pParam->m_prdcTransetList.lst.push_back(set);
        ui.listWidget_ID->addItem(strTaskId);
        ui.listWidget_ID->setCurrentRow(ui.listWidget_ID->count() - 1);
    }
    else					// 编辑
    {
        list<PrdcTransferSet>::iterator it = m_pParam->m_prdcTransetList.lst.begin();
        while (it != m_pParam->m_prdcTransetList.lst.end())
        {
            if (it->strID == strTaskId.toStdString())
            {
                *it = set;
                break;
            }
            it++;
        }
    }

    m_tranSet = set;
    m_nTaskFlag = 0;

    m_pParam->savePrdcTranSet();

    // 启动收集分发
    startTask(set);
}

void SendPrdcSetDlg::onBtnDelTask()
{
    list<PrdcTransferSet> &allSet = m_pParam->m_prdcTransetList.lst;
    int nRowNo = ui.listWidget_ID->currentRow();
    if (nRowNo < 0 || nRowNo >= (int)allSet.size())
    {
        QSLOG_DEBUG("nRowNo is out of range.");
        return;
    }
    if (QMessageBox::No == QMessageBox::warning(this, QStringLiteral("系统提示"), QStringLiteral("你确定要删除该任务吗?"), QMessageBox::Yes | QMessageBox::No, QMessageBox::Yes))
    {
        return;
    }
    list<PrdcTransferSet>::iterator it = allSet.begin();
    for (int i=0; i<nRowNo; ++i)
    {
        it++;
    }
    QString strID = ui.listWidget_ID->currentItem()->text();
    stopTask(strID);
    allSet.erase(it);
    m_pParam->savePrdcTranSet();
    ui.listWidget_ID->takeItem(nRowNo);
    ui.listWidget_ID->setCurrentRow(0);
}

void SendPrdcSetDlg::onBtnAddTask()
{
    clearAllInfo();
    m_nTaskFlag = 1;
    ui.listWidget_ID->setCurrentRow(-1);
}

void SendPrdcSetDlg::onSelUser()
{
    getUserID();
    SelUserDlg dlg(ui.le_taskName->text(), m_lstUserID, this);
    if (QDialog::Accepted == dlg.exec())
    {
        m_lstUserID = dlg.m_lstUserID;

        ui.tb_users->setRowCount(0);
        ui.tb_users->clearContents();
        for (int i=0; i<m_lstUserID.size(); ++i)
        {
            int row = i / colNum;
            int col = i % colNum;
            if (col == 0)
            {
                ui.tb_users->insertRow(row);
            }
            ui.tb_users->setItem(row, col, new QTableWidgetItem(m_lstUserID.at(i)));
        }
    }
}

void SendPrdcSetDlg::getUserID()
{
    m_lstUserID.clear();
    for (int i=0; i<ui.tb_users->rowCount(); ++i)
    {
        for (int j=0; j<ui.tb_users->columnCount(); ++j)
        {
            QTableWidgetItem *pItem = ui.tb_users->item(i, j);
            if (pItem != 0)
            {
                QString strTmp = pItem->text().remove(";");
                m_lstUserID.append( strTmp);
            }
        }
    }
}

void SendPrdcSetDlg::showEvent(QShowEvent *e)
{
    initDlg();
    return QWidget::showEvent(e);
}

bool SendPrdcSetDlg::event(QEvent *e)
{
    if (e->type() == ApplyEvent)
    {
        onApplyTask();
        return true;
    }
    else if (e->type() == OkEvent)
    {
        onApplyTask();
        emit ok();
        return true;
    }
    else if (e->type() == CancelEvent)
    {
        cancel();
        return true;
    }

    return QWidget::event(e);
}

void SendPrdcSetDlg::clearAllInfo()
{
    ui.le_taskName->clear();
    ui.le_taskName->setEnabled(true);
    for (int i=0; i<m_vecCheckBox.size(); ++i)
    {
        m_vecCheckBox[i]->setChecked(false);
    }
    ui.tb_users->setRowCount(0);
    ui.tb_users->clearContents();
}

void SendPrdcSetDlg::startTask(const PrdcTransferSet &set)
{
    if (m_pDataCollectionSev == NULL)
    {
        QObject *obj = m_pManager->getService("IDataCollection");
        if (NULL != obj)
        {
            m_pDataCollectionSev = qobject_cast<IDataCollection*>(obj);
            if (m_pDataCollectionSev != NULL)
            {
                m_pCollector = m_pDataCollectionSev->createCollector();
                QObject::connect(m_pCollector.data(), SIGNAL(newTransFile(const TransferSet &)), this, SLOT(TransFile(const TransferSet &)));
            }
        }
    }

    if (!m_pCollector.isNull())
    {
        QVector<string> vecNewID;
        // 转换为TransferSet,ID也不同
        QVector<TransferSet> tSets = changeToTransferSet(set);
        for (int i=0; i<tSets.size(); ++i)
        {
            const TransferSet &tSet = tSets.at(i);
            m_pCollector->addTransCollect(tSet);
            vecNewID.append(tSet.strName);
        }
        m_mapCollectID.insert(QString::fromStdString(set.strID), vecNewID);
    }
}

void SendPrdcSetDlg::stopTask(const QString &strID)
{
    if (m_mapCollectID.contains(strID))
    {
        QVector<string> vecCollects = m_mapCollectID.take(strID);
        for (int i=0; i<vecCollects.size(); i++)
        {
            m_pCollector->delTransCollect(vecCollects.at(i));
        }
    }
}

QVector<TransferSet> SendPrdcSetDlg::changeToTransferSet(const PrdcTransferSet &set)
{
    QVector<TransferSet> vecRes;
    QStringList lstUrlPath;
    QString strTmp;
    if (set.bPupData)
    {
        strTmp = m_pMakePath->getFileRootPath(PupType, false);
        lstUrlPath.append(strTmp);
    }
    if (set.bPupPic)
    {
        strTmp = m_pMakePath->getFileRootPath(PupType, true);
        lstUrlPath.append(strTmp);
    }
    if (set.bMosaicData)
    {
        strTmp = m_pMakePath->getFileRootPath(MosaciType, false);
        lstUrlPath.append(strTmp);
    }
    if (set.bMosaicPic)
    {
        strTmp = m_pMakePath->getFileRootPath(MosaciType, true);
        lstUrlPath.append(strTmp);
    }
    if (set.bHighData)
    {
        strTmp = m_pMakePath->getFileRootPath(HighType, false);
        lstUrlPath.append(strTmp);
    }
    if (set.bHighPic)
    {
        strTmp = m_pMakePath->getFileRootPath(HighType, true);
        lstUrlPath.append(strTmp);
    }

    vector<SendSet> userInfo;
    for (uint i=0; i<set.vecRecvID.size(); ++i)
    {
        SendSet user;
        if (m_pParam->getUserInfo(set.vecRecvID.at(i), user))
        {
            userInfo.push_back(user);
        }
    }

    for (int i=0; i<lstUrlPath.size(); ++i)
    {
        QString strPath = lstUrlPath.at(i).trimmed();
        if (strPath.isEmpty())
        {
            continue;
        }
        TransferSet tset;
        tset.strName = set.strID + QString("_%1").arg(i).toStdString();
        tset.strPath = strPath.toStdString();
        tset.nCollectWay = 0;
        tset.nWatchWay = 1;
        tset.vecRecvID = set.vecRecvID;
        tset.vecRcvers = userInfo;
        tset.vecRegs.push_back(".*");
        vecRes.append(tset);
    }

    return vecRes;
}

void SendPrdcSetDlg::TransFile(const TransferSet &tset)
{
    if (!m_pDistributer.isNull())
    {
        m_pDistributer->AddSendTask(tset);
    }
}


text:
package skadistats.clarity.io.decoder.factory.s1;

import skadistats.clarity.io.s1.SendProp;
import skadistats.clarity.io.decoder.*;
import skadistats.clarity.model.s1.PropFlag;

public class IntDecoderFactory implements DecoderFactory<Integer> {

    public static Decoder<Integer> createDecoderStatic(SendProp prop) {
        int flags = prop.getFlags();
        if ((flags & PropFlag.ENCODED_AS_VARINT) != 0) {
            if ((flags & PropFlag.UNSIGNED) != 0) {
                return new IntVarUnsignedDecoder();
            } else {
                return new IntVarSignedDecoder();
            }
        } else {
            if ((flags & PropFlag.UNSIGNED) != 0) {
                return new IntUnsignedDecoder(prop.getNumBits());
            } else {
                return new IntSignedDecoder(prop.getNumBits());
            }
        }
    }

    @Override
    public Decoder<Integer> createDecoder(SendProp prop) {
        return createDecoderStatic(prop);
    }
}

redpajama_set_name: RedPajamaGithub


text:
<?php

namespace backend\models\search;

use Yii;
use yii\base\Model;
use yii\data\ActiveDataProvider;
use backend\models\TipoUsuario;

/**
 * TipoUsuarioSearch represents the model behind the search form about `backend\models\TipoUsuario`.
 */
class TipoUsuarioSearch extends TipoUsuario
{
    /**
     * @inheritdoc
     */
    public function rules()
    {
        return [
            [['id', 'tipo_usuario_valor'], 'integer'],
            [['tipo_usuario_nombre'], 'safe'],
        ];
    }

    /**
     * @inheritdoc
     */
    public function scenarios()
    {
        // bypass scenarios() implementation in the parent class
        return Model::scenarios();
    }

    /**
     * Creates data provider instance with search query applied
     *
     * @param array $params
     *
     * @return ActiveDataProvider
     */
    public function search($params)
    {
        $query = TipoUsuario::find();

        $dataProvider = new ActiveDataProvider([
            'query' => $query,
        ]);

        $this->load($params);

        if (!$this->validate()) {
            // uncomment the following line if you do not want to return any records when validation fails
            // $query->where('0=1');
            return $dataProvider;
        }

        $query->andFilterWhere([
            'id' => $this->id,
            'tipo_usuario_valor' => $this->tipo_usuario_valor,
        ]);

        $query->andFilterWhere(['like', 'tipo_usuario_nombre', $this->tipo_usuario_nombre]);

        return $dataProvider;
    }
}


text:
import psycopg2

def connect():
        """Connect to the PostgreSQL database.
           Returns a database connection."""
        try:
            return psycopg2.connect("dbname=tournament")
        except psycopg2.Error, e:
            print e

def execute_query(query, values=None):
    """Executes a query in the database.
       Returns fetched results (None if no results)

    Args:
            query: the sql query string.
            values: a tuple containing values to insert into the database."""
    db = connect()
    c = db.cursor()
    if values is not None:
        c.execute(query, values)
    else:
        c.execute(query)
    try:
        results = c.fetchall()
    except:
        results = None
        db.commit()
    db.close()
    return results

def deleteMatches():
        """Remove all the match records from the database."""
        execute_query("delete from match;")

def deletePlayers():
        """Remove all the player records from the database."""
        execute_query("delete from players;")

def countPlayers():
        """Returns the number of players currently registered."""
        players = execute_query("select count(id) as num from players;")
        return int(players[0][0])

def registerPlayer(name):
        """Adds a player to the tournament database.

        The database assigns a unique serial id number for the player.
        (This should be handled by your SQL database schema,
        not in your Python code.)

        Args:
            name: the player's full name (need not be unique).
        """

        execute_query("insert into players (name) values (%s);", (name,))

def playerStandings():
        """Returns a list of the players and their win records, sorted by wins.

        The first entry in the list should be the player in first place,
        or a player tied for first place if there is currently a tie.

        Returns:
            A list of tuples, each of which contains (id, name, wins, matches):
                id: the player's unique id (assigned by the database)
                name: the player's full name (as registered)
                wins: the number of matches the player has won
                matches: the number of matches the player has played
        """

        return execute_query("""
            select players.id, players.name,
                (select count(match.id) from match
                    where match.winner=players.id) as wins,
                (select count(match.id) from match
                    where match.winner = players.id
                    or match.loser = players.id) as matches
            from players
            order by wins desc;""")

def reportMatch(winner, loser):
        """Records the outcome of a single match between two players.

        Args:
            winner:  the id number of the player who won
            loser:  the id number of the player who lost
        """

        execute_query("""insert into match (winner, loser)
                     values (%s, %s);""", (winner, loser))

def swissPairings():
        """Returns a list of pairs of players for the next round of a match.

        Assuming that there are an even number of players registered,
        each player appears exactly once in the pairings.
        Each player is paired with another player with an equal
        or nearly-equal win record, that is, a player adjacent
        to him or her in the standings.

        Returns:
            A list of tuples, each of which contains (id1, name1, id2, name2)
                id1: the first player's unique id
                name1: the first player's name
                id2: the second player's unique id
                name2: the second player's name
        """

        standings = playerStandings()
        pairings = []
        i = 0
        for i in range(0, len(standings), 2):
            pairings.append((standings[i][0],
                             standings[i][1],
                             standings[i+1][0],
                             standings[i+1][1]))
        return pairings


text:
int main(int argc, char *argv[])
{
    // Command line options
    Options options{};
    DeviceOptions deviceOptions{};
    DataOptions dataOptions{};
    GeneratorOptions generatorOptions{};
    // Memory
    HostMemory hostMemory{};
    DeviceMemory deviceMemory{};
    HostMemoryDumpFiles hostMemoryDumpFiles{};
    // OpenCL kernels
    isa::OpenCL::OpenCLRunTime openclRunTime{};
    KernelConfigurations kernelConfigurations{};
    Kernels kernels{};
    KernelRunTimeConfigurations kernelRunTimeConfigurations{};
    // Timers
    Timers timers{};
    // Observation
    AstroData::Observation observation;

    // Process command line arguments
    isa::utils::ArgumentList args(argc, argv);
    timers.commandLine.start();
    try
    {
        processCommandLineOptions(args, options, deviceOptions, dataOptions, hostMemoryDumpFiles, kernelConfigurations, generatorOptions, observation);
    }
    catch (std::exception & err)
    {
        return 1;
    }
    timers.commandLine.stop();
    if ( options.debug )
    {
        std::cout << "Command line processing time: " << timers.commandLine.getTotalTime() << " seconds." << std::endl;
    }

    // Load or generate input data
    try
    {
        hostMemory.input.resize(observation.getNrBeams());
        if (dataOptions.dataLOFAR || dataOptions.dataSIGPROC || dataOptions.dataPSRDADA)
        {
            loadInput(observation, deviceOptions, dataOptions, hostMemory, timers);
        }
        else
        {
            for (unsigned int beam = 0; beam < observation.getNrBeams(); beam++)
            {
                // TODO: if there are multiple synthesized beams, the generated data should take this into account
                hostMemory.input.at(beam) = new std::vector<std::vector<inputDataType> *>(observation.getNrBatches());
                AstroData::generateSinglePulse(generatorOptions.width, generatorOptions.DM, observation, deviceOptions.padding.at(deviceOptions.deviceName), *(hostMemory.input.at(beam)), inputBits, generatorOptions.random);
            }
        }
        try
        {
            hostMemory.zappedChannels.resize(observation.getNrChannels(deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(unsigned int)));
        }
        catch (std::out_of_range & err)
        {
            std::cerr << "No padding specified for " << deviceOptions.deviceName << "." << std::endl;
            return 1;
        }
        try
        {
            AstroData::readZappedChannels(observation, dataOptions.channelsFile, hostMemory.zappedChannels);
            if ( dataOptions.integrationFile.size() > 1 )
            {
                AstroData::readIntegrationSteps(observation, dataOptions.integrationFile, hostMemory.integrationSteps);
            }
            if ( options.rfimOptions.timeDomainSigmaCut )
            {
                RFIm::readSigmaSteps(options.rfimOptions.timeDomainSigmaCutStepsFile, hostMemory.timeDomainSigmaCutSteps);
            }
            if ( options.rfimOptions.frequencyDomainSigmaCut )
            {
                RFIm::readSigmaSteps(options.rfimOptions.frequencyDomainSigmaCutStepsFile, hostMemory.frequencyDomainSigmaCutSteps);
            }
        }
        catch (AstroData::FileError & err)
        {
            std::cerr << err.what() << std::endl;
            return 1;
        }
    }
    catch (std::exception & err)
    {
        std::cerr << err.what() << std::endl;
        return 1;
    }
    // Compute the amount of samples necessary for dedispersion
    if ( options.subbandDedispersion )
    {
        float maxShiftStepOne = 0;
        float maxShiftStepTwo = 0;
        float inverseHighFreq = 1.0f / std::pow(observation.getSubbandMaxFreq(), 2.0f);
        float inverseFreq = 1.0f / std::pow(observation.getSubbandMinFreq(), 2.0f);
        maxShiftStepTwo = 4148.808f * (inverseFreq - inverseHighFreq) * observation.getNrSamplesPerBatch();
        maxShiftStepTwo /= observation.getNrSamplesPerBatch() * observation.getSamplingTime();
        observation.setNrSamplesPerBatch(static_cast<unsigned int>(std::ceil(observation.getNrSamplesPerBatch() + (maxShiftStepTwo * (observation.getFirstDM() + ((observation.getNrDMs() - 1) * observation.getDMStep()))))), true);
        inverseHighFreq = 1.0f / std::pow(observation.getMaxFreq(), 2.0f);
        inverseFreq = 1.0f / std::pow(observation.getMinFreq(), 2.0f);
        maxShiftStepOne = 4148.808f * (inverseFreq - inverseHighFreq) * observation.getNrSamplesPerBatch();
        maxShiftStepOne /= observation.getNrSamplesPerBatch() * observation.getSamplingTime();
        observation.setNrSamplesPerDispersedBatch(static_cast<unsigned int>(std::ceil(observation.getNrSamplesPerBatch(true) + (maxShiftStepOne * (observation.getFirstDM(true) + ((observation.getNrDMs(true) - 1) * observation.getDMStep(true)))))), true);
        if ( options.downsampling )
        {
            observation.setNrSamplesPerBatch(observation.getNrSamplesPerBatch(true, observation.getDownsampling()), true);
            observation.setNrSamplesPerDispersedBatch(observation.getNrSamplesPerDispersedBatch(true, observation.getDownsampling()), true);
        }
        observation.setNrDelayBatches(static_cast<unsigned int>(std::ceil(static_cast<double>(observation.getNrSamplesPerDispersedBatch(true)) / observation.getNrSamplesPerBatch(true))), true);
    }
    else
    {
        float maxShiftSingleStep = 0;
        float inverseHighFreq = 1.0f / std::pow(observation.getMaxFreq(), 2.0f);
        float inverseFreq = 1.0f / std::pow(observation.getMinFreq(), 2.0f);
        maxShiftSingleStep = 4148.808f * (inverseFreq - inverseHighFreq) * observation.getNrSamplesPerBatch();
        maxShiftSingleStep /= observation.getNrSamplesPerBatch() * observation.getSamplingTime();
        observation.setNrSamplesPerDispersedBatch(static_cast<unsigned int>(std::ceil(observation.getNrSamplesPerBatch() + (maxShiftSingleStep * (observation.getFirstDM() + ((observation.getNrDMs() - 1) * observation.getDMStep()))))));
        if ( options.downsampling )
        {
            observation.setNrSamplesPerDispersedBatch(observation.getNrSamplesPerDispersedBatch(false, observation.getDownsampling()));
        }
        observation.setNrDelayBatches(static_cast<unsigned int>(std::ceil(static_cast<double>(observation.getNrSamplesPerDispersedBatch()) / observation.getNrSamplesPerBatch())));
    }
    // Computing dedispersion shifts and mapping of synthesized beams
    if ( options.subbandDedispersion )
    {
        hostMemory.shiftsStepOne = Dedispersion::getShifts(observation, deviceOptions.padding.at(deviceOptions.deviceName));
        hostMemory.shiftsStepTwo = Dedispersion::getShiftsStepTwo(observation, deviceOptions.padding.at(deviceOptions.deviceName));
        hostMemory.beamMapping.resize(observation.getNrSynthesizedBeams() * observation.getNrSubbands(deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(unsigned int)));
    }
    else
    {
        hostMemory.shiftsSingleStep = Dedispersion::getShifts(observation, deviceOptions.padding.at(deviceOptions.deviceName));
        hostMemory.beamMapping.resize(observation.getNrSynthesizedBeams() * observation.getNrChannels(deviceOptions.padding.at(deviceOptions.deviceName) / sizeof(unsigned int)));
    }
    if ( dataOptions.synthesizedBeamsFile.empty() )
    {
        AstroData::generateBeamMapping(observation, hostMemory.beamMapping, deviceOptions.padding.at(deviceOptions.deviceName), options.subbandDedispersion);
    }
    else
    {
        AstroData::readBeamMapping(observation, dataOptions.synthesizedBeamsFile, hostMemory.beamMapping, deviceOptions.padding.at(deviceOptions.deviceName), options.subbandDedispersion);
    }
    // Print message with observation and search information
    if (options.print)
    {
        std::cout << "Device: " << deviceOptions.deviceName << " (" + std::to_string(deviceOptions.platformID) + ", ";
        std::cout << std::to_string(deviceOptions.deviceID) + ")" << std::endl;
        std::cout << "Padding: " << deviceOptions.padding[deviceOptions.deviceName] << " bytes" << std::endl;
        std::cout << "Beams: " << observation.getNrBeams() << std::endl;
        std::cout << "Synthesized Beams: " << observation.getNrSynthesizedBeams() << std::endl;
        std::cout << "Synthesized Beams (per chunk): " << options.nrSynthesizedBeamsPerChunk << std::endl;
        std::cout << "Batches: " << observation.getNrBatches() << std::endl;
        if ( options.rfimOptions.enable )
        {
            std::cout << "RFIm enabled" << std::endl;
            if ( options.rfimOptions.timeDomainSigmaCut )
            {
                std::cout << "Time Domain Sigma Cut steps: " << hostMemory.timeDomainSigmaCutSteps.size() << std::endl;
            }
            if ( options.rfimOptions.frequencyDomainSigmaCut )
            {
                std::cout << "Frequency Domain Sigma Cut steps: " << hostMemory.frequencyDomainSigmaCutSteps.size() << std::endl;
            }
        }
        else
        {
            std::cout << "RFIm disabled" << std::endl;
        }
        if ( options.subbandDedispersion )
        {
            std::cout << "Max Delay (in Batches): " << observation.getNrDelayBatches(true) << std::endl;
            std::cout << "Samples per Batch (input): " << observation.getNrSamplesPerBatch() << std::endl;
            std::cout << "Samples per Dispersed Batch: " << observation.getNrSamplesPerDispersedBatch(true) / observation.getDownsampling() << std::endl;
            std::cout << "Samples per Batch (subband): " << observation.getNrSamplesPerBatch(true) / observation.getDownsampling() << std::endl;
            std::cout << "Samples per Batch (dedispersed): " << observation.getNrSamplesPerBatch() / observation.getDownsampling() << std::endl;
        }
        else
        {
            std::cout << "Max Delay (in Batches): " << observation.getNrDelayBatches() << std::endl;
            std::cout << "Samples per Batch (input): " << observation.getNrSamplesPerBatch() << std::endl;
            std::cout << "Samples per Dispersed Batch: " << observation.getNrSamplesPerDispersedBatch() / observation.getDownsampling() << std::endl;
        }
        std::cout << "Sampling time: " << observation.getSamplingTime() * observation.getDownsampling() << std::endl;
        if ( options.downsampling )
        {
            std::cout << "Downsampling factor: " << observation.getDownsampling() << std::endl;
        }
        std::cout << "Frequency range: " << observation.getMinFreq() << " MHz, " << observation.getMaxFreq() << " MHz";
        std::cout << std::endl;
        std::cout << "Subbands: " << observation.getNrSubbands() << " (" << observation.getSubbandBandwidth() << " MHz)";
        std::cout << std::endl;
        std::cout << "Channels: " << observation.getNrChannels() << " (" << observation.getChannelBandwidth() << " MHz)";
        std::cout << std::endl;
        std::cout << "Zapped Channels: " << observation.getNrZappedChannels() << std::endl;
        std::cout << "Integration steps: " << hostMemory.integrationSteps.size() << std::endl;
        if (options.subbandDedispersion)
        {
            std::cout << "Subbanding DMs: " << observation.getNrDMs(true) << " (" << observation.getFirstDM(true) << ", ";
            std::cout << observation.getFirstDM(true) + ((observation.getNrDMs(true) - 1) * observation.getDMStep(true));
            std::cout << ")" << std::endl;
        }
        std::cout << "DMs: " << observation.getNrDMs() << " (" << observation.getFirstDM() << ", ";
        std::cout << observation.getFirstDM() + ((observation.getNrDMs() - 1) * observation.getDMStep()) << ")";
        std::cout << std::endl;
        std::cout << std::endl;
    }

    // Initialize OpenCL
    timers.openclInitialization.start();
    for ( unsigned int attempt = 0; attempt < deviceOptions.initializationAttempts; attempt++ )
    {
        try
        {
            isa::OpenCL::initializeOpenCL(deviceOptions.platformID, 1, openclRunTime);
            break;
        }
        catch (isa::OpenCL::OpenCLError & err)
        {
            std::cerr << err.what() << std::endl;
            if ( attempt == deviceOptions.initializationAttempts - 1 )
            {
                return 1;
            }
        }
        std::this_thread::sleep_for(std::chrono::seconds(3));
    }
    timers.openclInitialization.stop();
    if ( options.debug )
    {
        std::cout << "OpenCL initialization time: " << timers.openclInitialization.getTotalTime() << " seconds." << std::endl;
    }

    // Memory allocation
    timers.memoryAllocation.start();
    allocateHostMemory(observation, options, deviceOptions, dataOptions, kernelConfigurations, hostMemory);
    if (observation.getNrDelayBatches() > observation.getNrBatches())
    {
        std::cerr << "Not enough input batches for the specified search." << std::endl;
        return 1;
    }
    try
    {
        allocateDeviceMemory(observation, openclRunTime, options, deviceOptions, hostMemory, deviceMemory);
    }
    catch (cl::Error & err)
    {
        std::cerr << "Memory error: " << err.what() << " " << err.err() << std::endl;
        return 1;
    }
    if ( options.debug )
    {
        std::uint64_t hostMemorySize = 0;
        hostMemorySize += hostMemory.zappedChannels.size() * sizeof(unsigned int);
        hostMemorySize += hostMemory.beamMapping.size() * sizeof(unsigned int);
        hostMemorySize += hostMemory.dispersedData.size() * sizeof(inputDataType);
        hostMemorySize += hostMemory.subbandedData.size() * sizeof(outputDataType);
        hostMemorySize += hostMemory.dedispersedData.size() * sizeof(outputDataType);
        hostMemorySize += hostMemory.integratedData.size() * sizeof(outputDataType);
        if ( options.snrMode == SNRMode::Standard || options.snrMode == SNRMode::SigmaCut )
        {
            hostMemorySize += hostMemory.snrData.size() * sizeof(float);
            hostMemorySize += hostMemory.snrSamples.size() * sizeof(unsigned int);
        }
        else if ( options.snrMode == SNRMode::Momad )
        {
            hostMemorySize += hostMemory.maxValues.size() * sizeof(outputDataType);
            hostMemorySize += hostMemory.maxIndices.size() * sizeof(unsigned int);
            hostMemorySize += hostMemory.stdevs.size() * sizeof(outputDataType);
            hostMemorySize += hostMemory.medianOfMediansStepOne.size() * sizeof(outputDataType);
            hostMemorySize += hostMemory.medianOfMedians.size() * sizeof(outputDataType);
            hostMemorySize += hostMemory.medianOfMediansAbsoluteDeviation.size() * sizeof(outputDataType);
        }
        if ( options.subbandDedispersion )
        {
            hostMemorySize += hostMemory.shiftsStepOne->size() * sizeof(float);
            hostMemorySize += hostMemory.shiftsStepTwo->size() * sizeof(float);
        }
        else
        {
            hostMemorySize += hostMemory.shiftsSingleStep->size() * sizeof(float);
        }
        std::cout << "Allocated memory: " << isa::utils::giga(hostMemorySize) << " GB" << std::endl;
        std::cout << std::endl;
    }
    timers.memoryAllocation.stop();
    if ( options.debug )
    {
        std::cout << "Memory allocation time: " << timers.memoryAllocation.getTotalTime() << " seconds." << std::endl;
    }

    // Generate OpenCL kernels
    timers.kernelGeneration.start();
    try
    {
        generateOpenCLKernels(openclRunTime, observation, options, deviceOptions, kernelConfigurations, hostMemory, deviceMemory, kernels);
    }
    catch (std::exception & err)
    {
        std::cerr << "OpenCL code generation error: " << err.what() << std::endl;
        return 1;
    }
    timers.kernelGeneration.stop();
    if ( options.debug )
    {
        std::cout << "OpenCL kernel generation time: " << timers.kernelGeneration.getTotalTime() << " seconds." << std::endl;
    }

    // Generate run time configurations for the OpenCL kernels
    generateOpenCLRunTimeConfigurations(observation, options, deviceOptions, kernelConfigurations, hostMemory, kernelRunTimeConfigurations);

    // Search loop
    pipeline(openclRunTime, observation, options, deviceOptions, dataOptions, timers, kernels, kernelRunTimeConfigurations, hostMemory, deviceMemory, hostMemoryDumpFiles);

    // Store performance statistics before shutting down
    std::ofstream outputStats;
    outputStats.open(dataOptions.outputFile + ".stats");
    outputStats << std::fixed << std::setprecision(6);
    outputStats << "# nrDMs" << std::endl;
    if (options.subbandDedispersion)
    {
        outputStats << observation.getNrDMs(true) * observation.getNrDMs() << std::endl;
    }
    else
    {
        outputStats << observation.getNrDMs() << std::endl;
    }
    outputStats << "# timers.inputLoad" << std::endl;
    outputStats << timers.inputLoad.getTotalTime() << std::endl;
    outputStats << "# timers.search" << std::endl;
    outputStats << timers.search.getTotalTime() << std::endl;
    outputStats << "# inputHandlingTotal inputHandlingAvg err" << std::endl;
    outputStats << timers.inputHandling.getTotalTime() << " " << timers.inputHandling.getAverageTime() << " ";
    outputStats << timers.inputHandling.getStandardDeviation() << std::endl;
    outputStats << "# inputCopyTotal inputCopyAvg err" << std::endl;
    outputStats << timers.inputCopy.getTotalTime() << " " << timers.inputCopy.getAverageTime() << " ";
    outputStats << timers.inputCopy.getStandardDeviation() << std::endl;
    if ( options.rfimOptions.enable )
    {
        if ( options.rfimOptions.timeDomainSigmaCut )
        {
            outputStats << "# timeDomainSigmaCutTotal timeDomainSigmaCutAvg err" << std::endl;
            outputStats << timers.timeDomainSigmaCut.getTotalTime() << " " << timers.timeDomainSigmaCut.getAverageTime() << " ";
            outputStats << timers.timeDomainSigmaCut.getStandardDeviation() << std::endl;
        }
        if ( options.rfimOptions.frequencyDomainSigmaCut )
        {
            outputStats << "# frequencyDomainSigmaCutTotal frequencyDomainSigmaCutAvg err" << std::endl;
            outputStats << timers.frequencyDomainSigmaCut.getTotalTime() << " " << timers.frequencyDomainSigmaCut.getAverageTime() << " ";
            outputStats << timers.frequencyDomainSigmaCut.getStandardDeviation() << std::endl;
        }
    }
    if ( options.downsampling )
    {
        outputStats << "# downsamplingTotal downsamplingAvg err" << std::endl;
        outputStats << timers.downsampling.getTotalTime() << " " << timers.downsampling.getAverageTime() << " ";
        outputStats << timers.downsampling.getStandardDeviation() << std::endl;
    }
    if (!options.subbandDedispersion)
    {
        outputStats << "# dedispersionSingleStepTotal dedispersionSingleStepAvg err" << std::endl;
        outputStats << timers.dedispersionSingleStep.getTotalTime() << " ";
        outputStats << timers.dedispersionSingleStep.getAverageTime() << " ";
        outputStats << timers.dedispersionSingleStep.getStandardDeviation() << std::endl;
    }
    else
    {
        outputStats << "# dedispersionStepOneTotal dedispersionStepOneAvg err" << std::endl;
        outputStats << timers.dedispersionStepOne.getTotalTime() << " ";
        outputStats << timers.dedispersionStepOne.getAverageTime() << " ";
        outputStats << timers.dedispersionStepOne.getStandardDeviation() << std::endl;
        outputStats << "# dedispersionStepTwoTotal dedispersionStepTwoAvg err" << std::endl;
        outputStats << timers.dedispersionStepTwo.getTotalTime() << " ";
        outputStats << timers.dedispersionStepTwo.getAverageTime() << " ";
        outputStats << timers.dedispersionStepTwo.getStandardDeviation() << std::endl;
    }
    outputStats << "# integrationTotal integrationAvg err" << std::endl;
    outputStats << timers.integration.getTotalTime() << " " << timers.integration.getAverageTime() << " ";
    outputStats << timers.integration.getStandardDeviation() << std::endl;
    if ( options.snrMode == SNRMode::Standard || options.snrMode == SNRMode::SigmaCut )
    {
        outputStats << "# snrTotal snrAvg err" << std::endl;
        outputStats << timers.snr.getTotalTime() << " " << timers.snr.getAverageTime() << " ";
        outputStats << timers.snr.getStandardDeviation() << std::endl;
    }
    else if (options.snrMode == SNRMode::Momad)
    {
        outputStats << "# maxTotal maxAvg err" << std::endl;
        outputStats << timers.max.getTotalTime() << " " << timers.max.getAverageTime() << " ";
        outputStats << timers.max.getStandardDeviation() << std::endl;
        outputStats << "# medianOfMediansStepOneTotal medianOfMediansStepOneAvg err" << std::endl;
        outputStats << timers.medianOfMediansStepOne.getTotalTime() << " " << timers.medianOfMediansStepOne.getAverageTime() << " ";
        outputStats << timers.medianOfMediansStepOne.getStandardDeviation() << std::endl;
        outputStats << "# medianOfMediansStepTwoTotal medianOfMediansStepTwoAvg err" << std::endl;
        outputStats << timers.medianOfMediansStepTwo.getTotalTime() << " " << timers.medianOfMediansStepTwo.getAverageTime() << " ";
        outputStats << timers.medianOfMediansStepTwo.getStandardDeviation() << std::endl;
        outputStats << "# medianOfMediansAbsoluteDeviationStepOneTotal medianOfMediansAbsoluteDeviationStepOneAvg err" << std::endl;
        outputStats << timers.medianOfMediansAbsoluteDeviationStepOne.getTotalTime() << " " << timers.medianOfMediansAbsoluteDeviationStepOne.getAverageTime() << " ";
        outputStats << timers.medianOfMediansAbsoluteDeviationStepOne.getStandardDeviation() << std::endl;
        outputStats << "# medianOfMediansAbsoluteDeviationStepTwoTotal medianOfMediansAbsoluteDeviationStepTwoAvg err" << std::endl;
        outputStats << timers.medianOfMediansAbsoluteDeviationStepTwo.getTotalTime() << " " << timers.medianOfMediansAbsoluteDeviationStepTwo.getAverageTime() << " ";
        outputStats << timers.medianOfMediansAbsoluteDeviationStepTwo.getStandardDeviation() << std::endl;
    }
    else if (options.snrMode == SNRMode::MomSigmaCut)
    {
        outputStats << "# maxStdSigmaCutTotal maxAvg err" << std::endl;
        outputStats << timers.max.getTotalTime() << " " << timers.max.getAverageTime() << " ";
        outputStats << timers.max.getStandardDeviation() << std::endl;
        outputStats << "# medianOfMediansStepOneTotal medianOfMediansStepOneAvg err" << std::endl;
        outputStats << timers.medianOfMediansStepOne.getTotalTime() << " " << timers.medianOfMediansStepOne.getAverageTime() << " ";
        outputStats << timers.medianOfMediansStepOne.getStandardDeviation() << std::endl;
        outputStats << "# medianOfMediansStepTwoTotal medianOfMediansStepTwoAvg err" << std::endl;
        outputStats << timers.medianOfMediansStepTwo.getTotalTime() << " " << timers.medianOfMediansStepTwo.getAverageTime() << " ";
        outputStats << timers.medianOfMediansStepTwo.getStandardDeviation() << std::endl;
    }
    outputStats << "# outputCopyTotal outputCopyAvg err" << std::endl;
    outputStats << timers.outputCopy.getTotalTime() << " " << timers.outputCopy.getAverageTime() << " ";
    outputStats << timers.outputCopy.getStandardDeviation() << std::endl;
    outputStats << "# triggerTimeTotal triggerTimeAvg err" << std::endl;
    outputStats << timers.trigger.getTotalTime() << " " << timers.trigger.getAverageTime() << " ";
    outputStats << timers.trigger.getStandardDeviation() << std::endl;
    outputStats.close();

    return 0;
}


text:
module ImageSubmitHelper

  def wizardly_submit
    @@wizardly_submit ||= {}
    step_id = "#{controller_name}_#{@step}".to_sym
    unless @@wizardly_submit[step_id]
      buttons = @wizard.pages[@step].buttons
      @@wizardly_submit[step_id] = buttons.inject(StringIO.new) do |io, button|
        io << submit_tag(button.name)
      end.string
    end
    @@wizardly_submit[step_id]
  end

  def wizardly_image_submit(asset_dir = nil, opts = {})
    @@wizardly_image_submit ||={}
    step_id = "#{controller_name}_#{@step}".to_sym
    unless @@wizardly_image_submit[step_id]
      asset_dir = asset_dir ? "#{asset_dir}/".squeeze("/").sub(/^\//,'') : "wizardly/"
      buttons = @wizard.pages[@step].buttons
      opts[:name] = 'commit'
      @@wizardly_image_submit[step_id] = buttons.inject(StringIO.new) do |io, button|
        opts[:value] = button.name
        io << image_submit_tag("#{asset_dir}#{button.id}.png", opts)
      end.string
    end
    @@wizardly_image_submit[step_id]
  end

end


text:
import warnings
from typing import Any, Callable, List, Optional, Union, Sequence

import numpy as np
import tensorflow as tf
from odin.bay.layers import DistributionDense
from odin.bay.random_variable import RVconf
from odin.bay.vi.utils import permute_dims
from odin.networks import SequentialNetwork, dense_network
from odin.utils import as_tuple
from tensorflow_probability.python.distributions import (Distribution,
                                                         Independent)
from typing_extensions import Literal

class FactorDiscriminator(SequentialNetwork):
  r""" The main goal is minimizing the total correlation (the mutual information
  which quantifies the redundancy or dependency among latent variables).

  We use a discriminator to estimate total-correlation

  This class also support Semi-supervised factor discriminator, a combination
  of supervised objective and total correlation estimation using density-ratio.

    - 0: real sample for q(z) (or last unit in case n_outputs > 2) and
    - 1: fake sample from q(z-)

  If `n_outputs` > 2, suppose the number of classes is `K` then:

    - 0 to K: is the classes' logits for real sample from q(z)
    - K + 1: fake sample from q(z-)

  This class is also extended to handle supervised loss for semi-supervised
  systems.

  Paramters
  -----------
  units : a list of Integer, the number of hidden units for each hidden layer.
  n_outputs : an Integer or instance of `RVmeta`,
    the number of output units and its distribution
  ss_strategy : {'sum', 'logsumexp', 'mean', 'max', 'min'}.
    Strategy for combining the outputs semi-supervised learning into the
    logit for real sample from q(z):
    - 'logsumexp' : used for semi-supervised GAN in (Salimans T. 2016)

  Example
  --------
  ```
  # for FactorVAE
  FactorDiscriminator(
    observation=RVmeta(1, 'bernoulli', projection=True, name="ind_factors"))

  # for classifier of ConditionalVAE
  FactorDiscriminator(
    observation=RVmeta(ds.shape, 'bernoulli', projection=True, name='image'))
  ```

  References
  ------------
  Kim, H., Mnih, A., 2018. "Disentangling by Factorising".
    arXiv:1802.05983 [cs, stat].
  Salimans, T., Goodfellow, I., Zaremba, W., et al 2016.
    "Improved Techniques for Training GANs". arXiv:1606.03498 [cs.LG].
  """

  def __init__(
      self,
      batchnorm: bool = False,
      input_dropout: float = 0.,
      dropout: float = 0.,
      units: Sequence[int] = (1000, 1000, 1000, 1000, 1000),
      observation: Union[RVconf, Sequence[RVconf]] = RVconf(1,
                                                            'bernoulli',
                                                            projection=True,
                                                            name="discriminator"),
      activation: Union[
        str, Callable[[tf.Tensor], tf.Tensor]] = tf.nn.leaky_relu,
      ss_strategy: Literal['sum', 'logsumexp', 'mean', 'max',
                           'min'] = 'logsumexp',
      name: str = "FactorDiscriminator",
  ):
    if not isinstance(observation, (tuple, list)):
      observation = [observation]
    assert len(observation) > 0, "No output is given for FactorDiscriminator"
    assert all(
      isinstance(o, (RVconf, DistributionDense)) for o in observation), (
      f"outputs must be instance of RVmeta, but given:{observation}")
    n_outputs = 0
    for o in observation:
      if not o.projection:
        warnings.warn(f'Projection turn off for observation {o}!')
      o.event_shape = (int(np.prod(o.event_shape)),)
      n_outputs += o.event_shape[0]
    layers = dense_network(units=units,
                           batchnorm=batchnorm,
                           dropout=dropout,
                           flatten_inputs=True,
                           input_dropout=input_dropout,
                           activation=activation,
                           prefix=name)
    super().__init__(layers, name=name)
    self.ss_strategy = str(ss_strategy)
    self.observation = observation
    self.n_outputs = n_outputs
    self._distributions = []
    assert self.ss_strategy in {'sum', 'logsumexp', 'mean', 'max', 'min'}

  def build(self, input_shape=None):
    super().build(input_shape)
    shape = self.output_shape[1:]
    self._distributions = [
      o.create_posterior(shape) if isinstance(o, RVconf) else o
      for o in self.observation
    ]
    self.input_ndim = len(self.input_shape) - 1
    return self

  def call(self, inputs, **kwargs):
    if isinstance(inputs, (tuple, list)) and len(inputs) == 1:
      inputs = inputs[0]
    outputs = super().call(inputs, **kwargs)
    # project into different output distributions
    distributions = [d(outputs, **kwargs) for d in self.distributions]
    return distributions[0] if len(distributions) == 1 else tuple(distributions)

  def _to_samples(self, qz_x, mean=False, stop_grad=False):
    qz_x = tf.nest.flatten(qz_x)
    if mean:
      z = tf.concat([q.mean() for q in qz_x], axis=-1)
    else:
      z = tf.concat([tf.convert_to_tensor(q) for q in qz_x], axis=-1)
    z = tf.reshape(z, tf.concat([(-1,), z.shape[-self.input_ndim:]], axis=0))
    if stop_grad:
      z = tf.stop_gradient(z)
    return z

  def _tc_logits(self, logits):
    # use ss_strategy to infer appropriate logits value for
    # total-correlation estimator (logits for q(z)) in case of n_outputs > 1
    Xs = []
    for x in tf.nest.flatten(logits):
      if isinstance(x, Distribution):
        if isinstance(x, Independent):
          x = x.distribution
        if hasattr(x, 'logits'):
          x = x.logits
        elif hasattr(x, 'concentration'):
          x = x.concentration
        else:
          raise RuntimeError(
            f"Distribution {x} doesn't has 'logits' or 'concentration' "
            "attributes, cannot not be used for estimating total correlation."
          )
      Xs.append(x)
    # concatenate the outputs
    if len(Xs) == 0:
      raise RuntimeError(
        f"No logits values found for total correlation: {logits}")
    elif len(Xs) == 1:
      Xs = Xs[0]
    else:
      Xs = tf.concat(Xs, axis=-1)
    # only 1 unit, only estimate TC
    if self.n_outputs == 1:
      return Xs[..., 0]
    # multiple units, reduce
    return getattr(tf, 'reduce_%s' % self.ss_strategy)(Xs, axis=-1)

  def total_correlation(self,
                        qz_x: Distribution,
                        training: Optional[bool] = None) -> tf.Tensor:
    r""" Total correlation Eq(3)
    ```
    TC(z) = KL(q(z)||q(z-)) = E_q(z)[log(q(z) / q(z-))]
          ~ E_q(z)[ log(D(z)) - log(1 - D(z)) ]
    ```

    We want to minimize the total correlation to achieve factorized latent units

    Note:
      In many implementation, `log(q(z-)) - log(q(z))` is referred as `total
      correlation loss`, here, we return `log(q(z)) - log(q(z-))` as the total
      correlation for the construction of the ELBO in Eq(2)

    Arguments:
      qz_x : a Tensor, [batch_dim, latent_dim] or Distribution

    Return:
      TC(z) : a scalar, approximation of the density-ratio that arises in the
        KL-term.
    """
    z = self._to_samples(qz_x, stop_grad=False)
    logits = self(z, training=training)
    logits = self._tc_logits(logits)
    # in case using sigmoid, other implementation use -logits here but it
    # should be logits.
    # if it is negative here, TC is reduce, but reasonably, it must be positive (?)
    return tf.reduce_mean(logits)

  def dtc_loss(self,
               qz_x: Distribution,
               qz_xprime: Optional[Distribution] = None,
               training: Optional[bool] = None) -> tf.Tensor:
    r""" Discriminated total correlation loss Algorithm(2)

    Minimize the probability of:
     - `q(z)` misclassified as `D(z)[:, 0]`
     - `q(z')` misclassified as `D(z')[:, 1]`

    Arguments:
      qz_x : `Tensor` or `Distribution`.
        Samples of the latents from first batch
      qz_xprime : `Tensor` or `Distribution` (optional).
        Samples of the latents from second batch, this will be permuted.
        If not given, then reuse `qz_x`.

    Return:
      scalar - loss value for training the discriminator
    """
    # we don't want the gradient to be propagated to the encoder
    z = self._to_samples(qz_x, stop_grad=True)
    z_logits = self._tc_logits(self(z, training=training))
    # using log_softmax function give more numerical stabalized results than
    # logsumexp yourself.
    d_z = -tf.math.log_sigmoid(z_logits)  # must be negative here
    # for X_prime
    if qz_xprime is not None:
      z = self._to_samples(qz_xprime, stop_grad=True)
    z_perm = permute_dims(z)
    zperm_logits = self._tc_logits(self(z_perm, training=training))
    d_zperm = -tf.math.log_sigmoid(zperm_logits)  # also negative here
    # reduce the negative of d_z, and the positive of d_zperm
    # this equal to cross_entropy(d_z, zeros) + cross_entropy(d_zperm, ones)
    loss = 0.5 * (tf.reduce_mean(d_z) + tf.reduce_mean(zperm_logits + d_zperm))
    return loss

  def supervised_loss(self,
                      labels: Union[tf.Tensor, List[tf.Tensor]],
                      qz_x: Distribution,
                      mean: bool = False,
                      mask: Optional[tf.Tensor] = None,
                      training: Optional[bool] = None) -> tf.Tensor:
    labels = as_tuple(labels)
    z = self._to_samples(qz_x, mean=mean, stop_grad=True)
    distributions = as_tuple(self(z, training=training))
    ## applying the mask (1-labelled, 0-unlabelled)
    if mask is not None:
      mask = tf.reshape(mask, (-1,))
      # labels = [tf.boolean_mask(y, mask, axis=0) for y in labels]
      # z_logits = tf.boolean_mask(z_logits, mask, axis=0)
    ## calculate the loss
    loss = 0.
    for dist, y_true in zip(distributions, labels):
      llk = dist.log_prob(y_true)
      # check the mask careful here
      # if no data for labels, just return 0
      if mask is not None:
        llk = tf.cond(tf.reduce_all(tf.logical_not(mask)), lambda: 0.,
                      lambda: tf.boolean_mask(llk, mask, axis=0))
      # negative log-likelihood here
      loss += -llk
    # check non-zero, if zero the gradient must be stop or NaN gradient happen
    loss = tf.reduce_mean(loss)
    loss = tf.cond(
      tf.abs(loss) < 1e-8, lambda: tf.stop_gradient(loss), lambda: loss)
    return loss

  @property
  def n_observation(self) -> int:
    return len(self.observation)

  @property
  def distributions(self) -> List[DistributionDense]:
    return self._distributions

  @property
  def prior(self) -> List[Distribution]:
    return [d.prior for d in self._distributions]

  def __str__(self):
    s = super().__str__()
    s1 = ['\n Outputs:']
    for i, d in enumerate(self.distributions):
      s1.append(f'  [{i}]{d}')
    return s + '\n'.join(s1)


text:
package org.grouplens.mooc.cbf;

import org.grouplens.lenskit.ItemRecommender;
import org.grouplens.lenskit.ItemScorer;
import org.grouplens.lenskit.Recommender;
import org.grouplens.lenskit.RecommenderBuildException;
import org.grouplens.lenskit.core.LenskitConfiguration;
import org.grouplens.lenskit.core.LenskitRecommender;
import org.grouplens.lenskit.data.dao.EventDAO;
import org.grouplens.lenskit.data.dao.ItemDAO;
import org.grouplens.lenskit.data.dao.UserDAO;
import org.grouplens.lenskit.scored.ScoredId;
import org.grouplens.mooc.cbf.dao.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.util.List;

/**
 * Simple hello-world program.
 * @author GroupLens Research
 */
public class CBFMain {
    private static final Logger logger = LoggerFactory.getLogger(CBFMain.class);

    public static void main(String[] args) throws RecommenderBuildException {
        LenskitConfiguration config = configureRecommender();

        logger.info("building recommender");
        Recommender rec = LenskitRecommender.build(config);

        if (args.length == 0) {
            logger.error("No users specified; provide user IDs as command line arguments");
        }

        // we automatically get a useful recommender since we have a scorer
        ItemRecommender irec = rec.getItemRecommender();
        assert irec != null;
        try {
            // Generate 5 recommendations for each user
            for (String user: args) {
                long uid;
                try {
                    uid = Long.parseLong(user);
                } catch (NumberFormatException e) {
                    logger.error("cannot parse user {}", user);
                    continue;
                }
                logger.info("searching for recommendations for user {}", user);
                List<ScoredId> recs = irec.recommend(uid, 5);
                if (recs.isEmpty()) {
                    logger.warn("no recommendations for user {}, do they exist?", uid);
                }
                System.out.format("recommendations for user %d:\n", uid);
                for (ScoredId id: recs) {
                    System.out.format("  %d: %.4f\n", id.getId(), id.getScore());
                }
            }
        } catch (UnsupportedOperationException e) {
            if (e.getMessage().equals("stub implementation")) {
                System.out.println("Congratulations, the stub builds and runs!");
            }
        }
    }

    /**
     * Create the LensKit recommender configuration.
     * @return The LensKit recommender configuration.
     */
    // LensKit configuration API generates some unchecked warnings, turn them off
    @SuppressWarnings("unchecked")
    private static LenskitConfiguration configureRecommender() {
        LenskitConfiguration config = new LenskitConfiguration();
        // configure the rating data source
        config.bind(EventDAO.class)
              .to(MOOCRatingDAO.class);
        config.set(RatingFile.class)
              .to(new File("data/ratings.csv"));

        // use custom item and user DAOs
        // specify item DAO implementation with tags
        config.bind(ItemDAO.class)
              .to(CSVItemTagDAO.class);
        // specify tag file
        config.set(TagFile.class)
              .to(new File("data/movie-tags.csv"));
        // and title file
        config.set(TitleFile.class)
              .to(new File("data/movie-titles.csv"));

        // our user DAO can look up by user name
        config.bind(UserDAO.class)
              .to(MOOCUserDAO.class);
        config.set(UserFile.class)
              .to(new File("data/users.csv"));

        // use the TF-IDF scorer you will implement to score items
        config.bind(ItemScorer.class)
              .to(TFIDFItemScorer.class);
        return config;
    }
}


text:
echo "Downloading node modules..."
npm install

echo "Starting grunt tasks..."
grunt -v continuous


text:
<!DOCTYPE html>

  <meta http-equiv="content-type" content="text/html; charset=utf-8" />
  <title>404 - Stella Blog</title>

  <meta name="author" content="Delano Mandelbaum" />
  <meta name="description" content="The blog for the web monitoring service of the future: BlameStella." />

  <link rel="alternate" type="application/rss+xml" href="/atom.xml" />

  <link rel="stylesheet" href="/css/base.css" type="text/css" media="screen, projection" />
  <link rel="stylesheet" href="/css/pygments.css" type="text/css" />
  <link media="only screen and (max-device-width: 480px)" href="/css/iphone.css" type="text/css" rel="stylesheet" />
  <link media="only screen and (device-width: 768px)" href="/css/iphone.css" type="text/css" rel="stylesheet" />
  <link href='http://fonts.googleapis.com/css?family=Yanone+Kaffeesatz' rel='stylesheet' type='text/css'>
  <link rel="apple-touch-icon" href="/apple-touch-icon.png" />

        The
Blame
Stella
Blog

blog home

about

posts

Not Found

Oops! Couldn't find that page.

About BlameStella

BlameStella.com is a web monitoring service built with
       Ruby,
       Redis,
       PostgreSQL, and
       PhantomJS. It checks your site every few minutes and notifies you when there's a problem. Since we render entire pages we can tell when there's problems with your javascript too.

We just launched a new version on January 1st, 2013. There's a lot more we want to do so signup today (for free) and you can enjoy the ride along with us!


text:
package com.hazelcast.jet.core;

import com.hazelcast.function.FunctionEx;
import com.hazelcast.function.SupplierEx;
import com.hazelcast.jet.JetInstance;
import com.hazelcast.jet.Job;
import com.hazelcast.jet.JobAlreadyExistsException;
import com.hazelcast.jet.SimpleTestInClusterSupport;
import com.hazelcast.jet.config.JetConfig;
import com.hazelcast.jet.config.JobConfig;
import com.hazelcast.jet.core.TestProcessors.Identity;
import com.hazelcast.jet.core.TestProcessors.ListSource;
import com.hazelcast.jet.core.TestProcessors.MockP;
import com.hazelcast.jet.core.TestProcessors.MockPS;
import com.hazelcast.jet.core.TestProcessors.NoOutputSourceP;
import com.hazelcast.jet.core.processor.DiagnosticProcessors;
import com.hazelcast.nio.ObjectDataInput;
import com.hazelcast.nio.ObjectDataOutput;
import com.hazelcast.nio.serialization.StreamSerializer;
import com.hazelcast.test.ExpectedRuntimeException;
import com.hazelcast.test.HazelcastSerialClassRunner;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;

import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;

import static com.hazelcast.jet.core.Edge.between;
import static com.hazelcast.jet.core.JobStatus.COMPLETED;
import static com.hazelcast.jet.core.JobStatus.FAILED;
import static com.hazelcast.jet.core.JobStatus.NOT_RUNNING;
import static com.hazelcast.jet.core.JobStatus.RUNNING;
import static com.hazelcast.jet.core.JobStatus.STARTING;
import static com.hazelcast.jet.core.JobStatus.SUSPENDED;
import static java.util.stream.Collectors.toList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

@RunWith(HazelcastSerialClassRunner.class)
public class JobTest extends SimpleTestInClusterSupport {

    private static final int NODE_COUNT = 2;
    private static final int LOCAL_PARALLELISM = 1;
    private static final int TOTAL_PARALLELISM = NODE_COUNT * LOCAL_PARALLELISM;

    @Rule
    public ExpectedException expectedException = ExpectedException.none();

    @BeforeClass
    public static void beforeClass() {
        JetConfig config = new JetConfig();
        config.getInstanceConfig().setCooperativeThreadCount(LOCAL_PARALLELISM);
        initializeWithClient(NODE_COUNT, config, null);
    }

    @Before
    public void setup() {
        TestProcessors.reset(TOTAL_PARALLELISM);
    }

    @Test
    public void when_jobIsSubmittedFromNonMaster_then_jobStatusShouldBeStarting() {
        testJobStatusDuringStart(instances()[1]);
    }

    @Test
    public void when_jobIsSubmittedFromClient_then_jobStatusShouldBeStarting() {
        testJobStatusDuringStart(client());
    }

    private void testJobStatusDuringStart(JetInstance submitter) {
        PSThatWaitsOnInit.initLatch = new CountDownLatch(1);
        DAG dag = new DAG().vertex(new Vertex("test", new PSThatWaitsOnInit(Identity::new)));

        // When
        Job job = submitter.newJob(dag);
        JobStatus status = job.getStatus();

        assertTrue(status == NOT_RUNNING || status == STARTING);

        PSThatWaitsOnInit.initLatch.countDown();

        // Then
        assertJobStatusEventually(job, COMPLETED);
    }

    @Test
    public void when_jobSubmittedWithNewJobIfAbsent_then_jobStatusIsStarting_fromNonMaster() {
        when_jobSubmittedWithNewJobIfAbsent_then_jobStatusIsStarting(instances()[1]);
    }

    @Test
    public void when_jobSubmittedWithNewJobIfAbsent_then_jobStatusIsStarting_fromClient() {
        when_jobSubmittedWithNewJobIfAbsent_then_jobStatusIsStarting(client());
    }

    private void when_jobSubmittedWithNewJobIfAbsent_then_jobStatusIsStarting(JetInstance submitter) {
        PSThatWaitsOnInit.initLatch = new CountDownLatch(1);
        DAG dag = new DAG().vertex(new Vertex("test", new PSThatWaitsOnInit(Identity::new)));

        // When
        Job job = submitter.newJobIfAbsent(dag, new JobConfig());
        JobStatus status = job.getStatus();

        assertTrue(status == NOT_RUNNING || status == STARTING);

        PSThatWaitsOnInit.initLatch.countDown();

        // Then
        assertJobStatusEventually(job, COMPLETED);
    }

    @Test
    public void when_jobIsCancelled_then_jobStatusIsCompletedEventually() throws InterruptedException {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));

        // When
        Job job = instance().newJob(dag);
        NoOutputSourceP.executionStarted.await();

        // Then
        cancelAndJoin(job);

        NoOutputSourceP.proceedLatch.countDown();
        assertJobStatusEventually(job, FAILED);
    }

    @Test
    public void when_jobIsFailed_then_jobStatusIsCompletedEventually() throws InterruptedException {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS((SupplierEx<Processor>)
                () -> new MockP().setCompleteError(new ExpectedRuntimeException()), NODE_COUNT)));

        // When
        Job job = instance().newJob(dag);

        // Then
        try {
            job.getFuture().get();
            fail();
        } catch (ExecutionException expected) {
            assertEquals(FAILED, job.getStatus());
        }
    }

    @Test
    public void when_jobIsSubmitted_then_trackedJobCanQueryJobStatus() throws InterruptedException {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));

        // When
        Job submittedJob = instance().newJob(dag);
        NoOutputSourceP.executionStarted.await();

        Collection<Job> trackedJobs = instances()[1].getJobs();
        Job trackedJob = trackedJobs.stream().filter(j -> j.getId() == submittedJob.getId()).findFirst().get();

        // Then
        assertJobStatusEventually(trackedJob, RUNNING);

        cancelAndJoin(submittedJob);
    }

    @Test
    public void when_jobIsSubmittedWithNewJobIfAbsent_then_trackedJobCanQueryJobStatus() throws InterruptedException {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));

        // When
        Job submittedJob = instance().newJobIfAbsent(dag, new JobConfig());
        NoOutputSourceP.executionStarted.await();

        Collection<Job> trackedJobs = instances()[1].getJobs();
        Job trackedJob = trackedJobs.stream().filter(j -> j.getId() == submittedJob.getId()).findFirst().get();

        // Then
        assertJobStatusEventually(trackedJob, RUNNING);

        cancelAndJoin(submittedJob);
    }

    @Test
    public void when_namedJobIsSubmittedWithNewJobIfAbsent_then_trackedJobCanQueryJobStatus() throws InterruptedException {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));
        JobConfig config = new JobConfig()
                .setName(randomName());

        // When
        Job submittedJob = instance().newJobIfAbsent(dag, config);
        NoOutputSourceP.executionStarted.await();

        Collection<Job> trackedJobs = instances()[1].getJobs();
        Job trackedJob = trackedJobs.stream().filter(j -> j.getId() == submittedJob.getId()).findFirst().get();

        // Then
        assertJobStatusEventually(trackedJob, RUNNING);

        cancelAndJoin(submittedJob);
    }

    @Test
    public void when_jobIsCompleted_then_trackedJobCanQueryJobResult() throws InterruptedException {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));

        // When
        Job submittedJob = instance().newJob(dag);
        NoOutputSourceP.executionStarted.await();

        Collection<Job> trackedJobs = instances()[1].getJobs();
        Job trackedJob = trackedJobs.stream().filter(j -> j.getId() == submittedJob.getId()).findFirst().get();

        NoOutputSourceP.proceedLatch.countDown();

        // Then
        trackedJob.join();

        assertEquals(COMPLETED, trackedJob.getStatus());
    }

    @Test
    public void when_jobIsCancelled_then_trackedJobCanQueryJobResult() throws InterruptedException {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));

        // When
        Job submittedJob = instance().newJob(dag);
        NoOutputSourceP.executionStarted.await();

        Collection<Job> trackedJobs = instances()[1].getJobs();
        Job trackedJob = trackedJobs.stream().filter(j -> j.getId() == submittedJob.getId()).findFirst().get();

        submittedJob.cancel();

        // Then
        joinAndExpectCancellation(trackedJob);

        NoOutputSourceP.proceedLatch.countDown();
        assertJobStatusEventually(trackedJob, FAILED);
    }

    @Test
    public void when_jobIsFailed_then_trackedJobCanQueryJobResult() throws InterruptedException {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS((SupplierEx<Processor>)
                () -> new MockP().setCompleteError(new ExpectedRuntimeException()), NODE_COUNT)));

        // When
        Job submittedJob = instance().newJob(dag);

        Collection<Job> trackedJobs = instances()[1].getJobs();
        Job trackedJob = trackedJobs.stream().filter(j -> j.getId() == submittedJob.getId()).findFirst().get();

        // Then
        try {
            trackedJob.getFuture().get();
            fail();
        } catch (ExecutionException expected) {
            assertEquals(FAILED, trackedJob.getStatus());
        }
    }

    @Test
    public void when_trackedJobCancels_then_jobCompletes() {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));

        Job submittedJob = instance().newJob(dag);

        Collection<Job> trackedJobs = instances()[1].getJobs();
        Job trackedJob = trackedJobs.stream().filter(j -> j.getId() == submittedJob.getId()).findFirst().get();

        // When
        trackedJob.cancel();

        // Then
        joinAndExpectCancellation(trackedJob);
        joinAndExpectCancellation(submittedJob);

        NoOutputSourceP.proceedLatch.countDown();

        assertJobStatusEventually(trackedJob, FAILED);
        assertJobStatusEventually(submittedJob, FAILED);
    }

    @Test
    public void when_jobIsCompleted_then_trackedJobCanQueryJobResultFromClient() throws InterruptedException {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));

        // When
        Job submittedJob = instance().newJob(dag);
        NoOutputSourceP.executionStarted.await();

        Collection<Job> trackedJobs = client().getJobs();
        Job trackedJob = trackedJobs.stream().filter(j -> j.getId() == submittedJob.getId()).findFirst().get();

        NoOutputSourceP.proceedLatch.countDown();

        // Then
        trackedJob.join();

        assertEquals(COMPLETED, trackedJob.getStatus());
    }

    @Test
    public void when_jobIsRunning_then_itIsQueriedByName() throws InterruptedException {
        testGetJobByNameWhenJobIsRunning(instances()[1]);
    }

    @Test
    public void when_jobIsRunning_then_itIsQueriedByNameFromClient() throws InterruptedException {
        testGetJobByNameWhenJobIsRunning(client());
    }

    private void testGetJobByNameWhenJobIsRunning(JetInstance instance) throws InterruptedException {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));
        JobConfig config = new JobConfig();
        String jobName = randomName();
        config.setName(jobName);

        // When
        Job job = instance().newJob(dag, config);
        assertEquals(jobName, job.getName());
        NoOutputSourceP.executionStarted.await();

        // Then
        Job trackedJob = instance.getJob(jobName);

        assertNotNull(trackedJob);
        assertEquals(jobName, trackedJob.getName());
        assertEquals(job.getId(), trackedJob.getId());
        assertJobStatusEventually(trackedJob, RUNNING);

        NoOutputSourceP.proceedLatch.countDown();
    }

    @Test
    public void when_jobIsRunning_then_itIsQueriedById() throws InterruptedException {
        testGetJobByIdWhenJobIsRunning(instance());
    }

    @Test
    public void when_jobIsRunning_then_itIsQueriedByIdFromClient() throws InterruptedException {
        testGetJobByIdWhenJobIsRunning(client());
    }

    private void testGetJobByIdWhenJobIsRunning(JetInstance instance) throws InterruptedException {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));

        // When
        Job job = instance().newJob(dag);
        NoOutputSourceP.executionStarted.await();

        // Then
        Job trackedJob = instance.getJob(job.getId());

        assertNotNull(trackedJob);
        assertEquals(job.getId(), trackedJob.getId());
        assertJobStatusEventually(trackedJob, RUNNING);

        NoOutputSourceP.proceedLatch.countDown();
    }

    @Test
    public void when_jobIsCompleted_then_itIsQueriedByName() {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));
        JobConfig config = new JobConfig();
        String jobName = randomName();
        config.setName(jobName);

        // When
        Job job = instance().newJob(dag, config);
        NoOutputSourceP.proceedLatch.countDown();
        job.join();

        // Then
        Job trackedJob = instance().getJob(jobName);

        assertNotNull(trackedJob);
        assertEquals(jobName, trackedJob.getName());
        assertEquals(job.getId(), trackedJob.getId());
        assertEquals(COMPLETED, trackedJob.getStatus());
    }

    @Test
    public void when_jobIsCompleted_then_itIsQueriedById() {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));

        // When
        Job job = instance().newJob(dag);
        NoOutputSourceP.proceedLatch.countDown();
        job.join();

        // Then
        Job trackedJob = instance().getJob(job.getId());

        assertNotNull(trackedJob);
        assertEquals(job.getId(), trackedJob.getId());
        assertEquals(COMPLETED, trackedJob.getStatus());
    }

    @Test
    public void when_jobIsQueriedByInvalidId_then_noJobIsReturned() {
        assertNull(instance().getJob(0));
    }

    @Test
    public void when_jobIsQueriedByInvalidIdFromClient_then_noJobIsReturned() {
        assertNull(client().getJob(0));
    }

    @Test
    public void when_namedJobIsRunning_then_newNamedSubmitJoinsToExistingJob_member() {
        when_namedJobIsRunning_then_newNamedSubmitJoinsToExistingJob(instance());
    }

    @Test
    public void when_namedJobIsRunning_then_newNamedSubmitJoinsToExistingJob_client() {
        when_namedJobIsRunning_then_newNamedSubmitJoinsToExistingJob(client());
    }

    private void when_namedJobIsRunning_then_newNamedSubmitJoinsToExistingJob(JetInstance instance) {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT * 2)));
        JobConfig config = new JobConfig()
                .setName(randomName());
        Job job1 = instance.newJob(dag, config);
        assertTrueEventually(() -> assertEquals(RUNNING, job1.getStatus()));

        // When
        Job job2 = instance.newJobIfAbsent(dag, config);

        // Then
        assertEquals(job1.getId(), job2.getId());
        NoOutputSourceP.proceedLatch.countDown();
    }

    @Test
    public void stressTest_parallelNamedJobSubmission_member() throws Exception {
        stressTest_parallelNamedJobSubmission(instance());
    }

    @Test
    public void stressTest_parallelNamedJobSubmission_client() throws Exception {
        stressTest_parallelNamedJobSubmission(client());
    }

    private void stressTest_parallelNamedJobSubmission(JetInstance instance) throws Exception {
        final int nThreads = 3;
        ExecutorService executor = Executors.newFixedThreadPool(nThreads);
        String randomPrefix = randomName();
        try {
            for (int round = 0; round < 10; round++) {
                DAG dag = new DAG().vertex(new Vertex("test" + round, new MockPS(NoOutputSourceP::new, NODE_COUNT * 2)));
                System.out.println("Starting round " + round);
                JobConfig config = new JobConfig().setName(randomPrefix + round);
                List<Future<Job>> futures = new ArrayList<>();
                for (int i = 0; i < nThreads; i++) {
                    futures.add(executor.submit(() -> instance.newJobIfAbsent(dag, config)));
                }
                for (int i = 1; i < nThreads; i++) {
                    assertEquals(futures.get(0).get().getId(), futures.get(i).get().getId());
                }
            }
        } finally {
            executor.shutdownNow();
            executor.awaitTermination(1, TimeUnit.MINUTES);
        }
    }

    @Test
    public void when_namedJobIsRunning_then_newNamedJobFails_member() {
        when_namedJobIsRunning_then_newNamedJobFails(instance());
    }

    @Test
    public void when_namedJobIsRunning_then_newNamedJobFails_client() {
        when_namedJobIsRunning_then_newNamedJobFails(client());
    }

    private void when_namedJobIsRunning_then_newNamedJobFails(JetInstance instance) {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT * 2)));
        JobConfig config = new JobConfig()
                .setName(randomName());

        // When
        Job job1 = instance.newJob(dag, config);
        assertTrueEventually(() -> assertEquals(RUNNING, job1.getStatus()));

        // Then
        expectedException.expect(JobAlreadyExistsException.class);
        instance.newJob(dag, config);
    }

    @Test
    public void when_namedJobHasCompletedAndAnotherWasSubmitted_then_runningOneIsQueriedByName() {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT * 2)));
        JobConfig config = new JobConfig();
        String jobName = randomName();
        config.setName(jobName);

        // When
        Job job1 = instance().newJob(dag, config);
        NoOutputSourceP.proceedLatch.countDown();
        job1.join();

        NoOutputSourceP.proceedLatch = new CountDownLatch(1);
        Job job2 = instance().newJob(dag, config);
        assertTrueEventually(() -> assertEquals(RUNNING, job2.getStatus()));

        // Then
        Job trackedJob = instance().getJob(jobName);

        assertNotNull(trackedJob);
        assertEquals(jobName, trackedJob.getName());
        assertNotEquals(job1.getId(), trackedJob.getId());
        assertEquals(job2.getId(), trackedJob.getId());
        assertEquals(RUNNING, trackedJob.getStatus());
    }

    @Test
    public void when_namedJobHasCompletedAndAnotherWasSubmitted_then_bothAreQueriedByName() {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT * 2)));
        JobConfig config = new JobConfig();
        String jobName = randomName();
        config.setName(jobName);

        // When
        Job job1 = instance().newJob(dag, config);
        NoOutputSourceP.proceedLatch.countDown();
        job1.join();

        NoOutputSourceP.proceedLatch = new CountDownLatch(1);
        Job job2 = instance().newJob(dag, config);
        assertTrueEventually(() -> assertEquals(RUNNING, job2.getStatus()));

        // Then
        List<Job> trackedJobs = instance().getJobs(jobName);

        assertEquals(2, trackedJobs.size());

        Job trackedJob1 = trackedJobs.get(0);
        Job trackedJob2 = trackedJobs.get(1);

        assertEquals(job2.getId(), trackedJob1.getId());
        assertEquals(RUNNING, trackedJob1.getStatus());

        assertEquals(job1.getId(), trackedJob2.getId());
        assertEquals(COMPLETED, trackedJob2.getStatus());
    }

    @Test
    public void when_jobConfigChanged_then_doesNotAffectSubmittedJob() {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT * 2)));
        JobConfig config = new JobConfig()
                .setName(randomName());

        Job job1 = instance().newJob(dag, config);
        assertJobStatusEventually(job1, RUNNING);

        // When
        config.setName(randomName());

        // Then
        Job job2 = instance().newJob(dag, config);
        assertJobStatusEventually(job2, RUNNING);
    }

    @Test
    public void when_jobsAreCompleted_then_theyAreQueriedByName() {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT * 2)));
        JobConfig config = new JobConfig()
                .setName(randomName());

        // When
        Job job1 = instance().newJob(dag, config);
        NoOutputSourceP.proceedLatch.countDown();
        job1.join();

        sleepAtLeastMillis(1);

        NoOutputSourceP.proceedLatch = new CountDownLatch(1);
        Job job2 = instance().newJob(dag, config);
        NoOutputSourceP.proceedLatch.countDown();
        job2.join();

        // Then
        List<Job> jobs = instance().getJobs(config.getName());
        assertEquals(2, jobs.size());

        Job trackedJob1 = jobs.get(0);
        Job trackedJob2 = jobs.get(1);

        assertEquals(job2.getId(), trackedJob1.getId());
        assertEquals(config.getName(), trackedJob1.getName());
        assertEquals(COMPLETED, trackedJob1.getStatus());
        assertEquals(job1.getId(), trackedJob2.getId());
        assertEquals(config.getName(), trackedJob2.getName());
        assertEquals(COMPLETED, trackedJob2.getStatus());
    }

    @Test
    public void when_suspendedNamedJob_then_newJobIfAbsentWithEqualNameJoinsIt() {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT * 2)));
        JobConfig config = new JobConfig()
                .setName(randomName());

        // When
        Job job1 = instance().newJob(dag, config);
        assertJobStatusEventually(job1, RUNNING);
        job1.suspend();
        assertJobStatusEventually(job1, SUSPENDED);

        // Then
        Job job2 = instances()[1].newJobIfAbsent(dag, config);
        assertEquals(job1.getId(), job2.getId());
        assertEquals(job2.getStatus(), SUSPENDED);
    }

    @Test
    public void when_suspendedNamedJob_then_newJobWithEqualNameFails() {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT * 2)));
        JobConfig config = new JobConfig()
                .setName(randomName());

        // When
        Job job1 = instance().newJob(dag, config);
        assertJobStatusEventually(job1, RUNNING);
        job1.suspend();
        assertJobStatusEventually(job1, SUSPENDED);

        // Then
        expectedException.expect(JobAlreadyExistsException.class);
        instances()[1].newJob(dag, config);
    }

    @Test
    public void when_namedJobCancelledAndJoined_then_newJobWithSameNameCanBeSubmitted() {
        DAG dag = new DAG();
        dag.newVertex("v", () -> new TestProcessors.MockP().streaming());
        for (int i = 0; i < 10; i++) {
            JobConfig config = new JobConfig()
                    .setName("job");
            Job job = instance().newJob(dag, config);
            assertJobStatusEventually(job, RUNNING);
            job.cancel();
            try {
                job.join();
            } catch (CancellationException ignored) {
            }
        }
    }

    @Test
    public void when_jobIsSubmitted_then_jobSubmissionTimeIsQueried() throws InterruptedException {
        testJobSubmissionTimeWhenJobIsRunning(instance());
    }

    @Test
    public void when_jobIsRunning_then_jobSubmissionTimeIsQueriedFromClient() throws InterruptedException {
        testJobSubmissionTimeWhenJobIsRunning(client());
    }

    private void testJobSubmissionTimeWhenJobIsRunning(JetInstance instance) throws InterruptedException {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));
        JobConfig config = new JobConfig();
        String jobName = randomName();
        config.setName(jobName);

        // When
        Job job = instance().newJob(dag, config);
        NoOutputSourceP.executionStarted.await();
        Job trackedJob = instance.getJob(jobName);

        // Then
        assertNotNull(trackedJob);
        assertNotEquals(0, job.getSubmissionTime());
        assertNotEquals(0, trackedJob.getSubmissionTime());
        NoOutputSourceP.proceedLatch.countDown();
    }

    @Test
    public void when_jobIsCompleted_then_jobSubmissionTimeIsQueried() {
        // Given
        DAG dag = new DAG().vertex(new Vertex("test", new MockPS(NoOutputSourceP::new, NODE_COUNT)));
        JobConfig config = new JobConfig();
        String jobName = randomName();
        config.setName(jobName);

        // When
        Job job = instance().newJob(dag, config);
        NoOutputSourceP.proceedLatch.countDown();
        job.join();
        Job trackedJob = instance().getJob(jobName);

        // Then
        assertNotNull(trackedJob);
        assertNotEquals(0, job.getSubmissionTime());
        assertNotEquals(0, trackedJob.getSubmissionTime());
    }

    @Test
    public void when_serializerIsRegistered_then_itIsAvailableForTheJob() {
        // Given
        DAG dag = new DAG();
        Vertex source = dag.newVertex("source", () -> new ListSource(new Value(1), new Value(2)));
        Vertex sink = dag.newVertex("sink", DiagnosticProcessors.writeLoggerP());
        dag.edge(between(source, sink).distributed().partitioned(FunctionEx.identity()));

        JobConfig config = new JobConfig()
                .registerSerializer(Value.class, ValueSerializer.class);

        // When
        Job job = instance().newJob(dag, config);

        // Then
        assertJobStatusEventually(job, COMPLETED);
    }

    private void joinAndExpectCancellation(Job job) {
        try {
            job.join();
            fail();
        } catch (CancellationException ignored) {
        }
    }

    private static final class PSThatWaitsOnInit implements ProcessorSupplier {

        public static volatile CountDownLatch initLatch;
        private final SupplierEx<Processor> supplier;

        PSThatWaitsOnInit(SupplierEx<Processor> supplier) {
            this.supplier = supplier;
        }

        @Override
        public void init(@Nonnull Context context) throws Exception {
            initLatch.await();
        }

        @Nonnull
        @Override
        public Collection<? extends Processor> get(int count) {
            return Stream.generate(supplier).limit(count).collect(toList());

        }
    }

    private static final class Value {

        private final int value;

        private Value(int value) {
            this.value = value;
        }
    }

    private static class ValueSerializer implements StreamSerializer<Value> {

        @Override
        public int getTypeId() {
            return 1;
        }

        @Override
        public void write(ObjectDataOutput output, Value value) throws IOException {
            output.writeInt(value.value);
        }

        @Override
        public Value read(ObjectDataInput input) throws IOException {
            return new Value(input.readInt());
        }
    }
}


text: /*# sourceMappingURL=meals.css.map */


text:
cryptiles
=========

General purpose crypto utilities

[![Build Status](https://secure.travis-ci.org/hapijs/cryptiles.png)](http://travis-ci.org/hapijs/cryptiles)

Lead Maintainer - [Eran Hammer](https://github.com/hueniverse)

## Methods

### `randomString(<Number> size)`
Returns a cryptographically strong pseudo-random data string. Takes a size argument for the length of the string.

### `randomDigits(<Number> size)`
Returns a cryptographically strong pseudo-random data string consisting of only numerical digits (0-9). Takes a size argument for the length of the string.


text:
require 'spec_helper'

describe UsersWithPendingTodosFinder do
  describe '#execute' do
    it 'returns the users for all pending todos of a target' do
      issue = create(:issue)
      note = create(:note)
      todo = create(:todo, :pending, target: issue)

      create(:todo, :pending, target: note)

      users = described_class.new(issue).execute

      expect(users).to eq([todo.user])
    end
  end
end


text:
* Term frequency—The more times the words you're looking for appear in a document, the higher the score.
* Inverse document frequency—The weight of each word is higher if the word is uncommon across other documents.

# 2 Diving into the functionality

## Understanding the logical layout: documents, types, and indices

### document

* It's self-contained. A document contains both the fields (name) and their values (Elasticsearch Denver).
* It can be hierarchical. Think of this as documents within documents. A value of a field can be simple, like the value of the location field can be a string. It can also contain other fields and values. For example, the location field might contain both a city and a street address within it.
* It has a flexible structure. Your documents don't depend on a predefined schema.  For example, not all events need description values, so that field can be omitted altogether. But it might require new fields, such as the latitude and longitude of the location.

### types

### indices

An Elasticsearch index is an independent chunk of documents, much like a database is in the relational world: each index is **stored on the disk** in the same set of files; it stores all the fields from all the mapping types in there, and it has its own settings. For example, each index has a setting called *refresh_interval*, which defines the interval at which newly indexed documents are made available for searches.

## Understanding the physical layout: nodes and shards

![nodes_and_shards](./pic/nodes_and_shards.png)

### Understanding primary and replica shards

 A shard is a Lucene index: a directory of files containing an inverted index. An inverted index is a structure that enables Elasticsearch to tell you which document contains a term (a word) without having to look at all the documents.

 > **Elasticsearch index vs. Lucene index**
 > An Elasticsearch index is broken down into chunks: shards. A shard is a Lucene index, so an Elasticsearch index is made up of multiple Lucene indices. This makes sense because Elasticsearch uses Apache Lucene as its core library to index your data and search through it.

### Distributed indexing and searching

* indexing

The Elasticsearch node that receives your indexing request first selects the shard to index the document to. By default, documents are distributed evenly between shards: for each document, the shard is determined by **hashing its ID string**. Each shard has an equal **hash range**, with equal chances of receiving the new document. Once the target shard is determined, the current node forwards the document to the node holding that shard. Subsequently, that indexing operation is replayed by all the replicas of that shard. The indexing command successfully returns after all the available replicas finish indexing the document.

![distribute_indexing_searching](./pic/distribute_indexing_searching.png)

* searching

With searching, the node that receives the request forwards it to a set of shards containing all your data. Using a round-robin(轮询调度算法), Elasticsearch selects an available shard (which can be primary or replica) and forwards the search request to it. Elasticsearch then gathers results from those shards, aggregates them into a single reply, and forwards the reply back to the client application.
By default, primary and replica shards get hit by searches in round-robin, assuming all nodes in your cluster are equally fast (identical hardware and software configurations). If that's not the case, you can organize your data or configure your shards to prevent the slower nodes from becoming a bottleneck.


text:
<messagebundle>
    <msg name="go_to_app_composer">Go to App Composer</msg>
    <msg name="you_must_be_logged_first">You must be logged in Graasp</msg>
    <msg name="click_on_top_right_corner">Click on the "Sign in" button in the top right corner</msg>
</messagebundle>


text:
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

    <title>DetailForm 详细表单</title>
    <meta http-equiv="content-type" content="text/html; charset=UTF-8" /><link href="../demo.css" rel="stylesheet" type="text/css" />

DetailForm 详细表单

    New Row

#

员工帐号

性别

年龄

出生日期

创建日期

    <fieldset style="width:700px;border:solid 1px #aaa;position:relative;">
        <legend>员工详细信息</legend>

            <input class="mini-hidden" name="id"/>

                    员工帐号:
                    <input name="loginname" class="mini-textbox" />
                    姓名:
                    <input name="name" class="mini-textbox" />
                    薪资:
                    <input name="salary" class="mini-textbox" />

                    性别:
                    <input name="gender" class="mini-combobox" data="Genders" shownullitem="true"  multiselect="false"/>
                    年龄:
                    <input name="age" class="mini-spinner" minValue="0" maxValue="200" value="25"  />
                    出生日期:
                    <input name="birthday" class="mini-datepicker" />

                        Update
                        Cancel

    </fieldset>

Description


text:
import os

from setuptools import setup

with open("README") as f:
    readme = f.read()

with open(os.path.join(os.path.dirname(__file__), 'requirements.txt')) as f:
    required = f.read().splitlines()

with open(os.path.join(os.path.dirname(__file__), 'test_requirements.txt')) as f:
    test_required = f.read().splitlines() + required

setup(
    name='asyncio_hn',
    version='0.4.0',
    description=" Simple asyncio wrapper to download hackernews",
    long_description=readme + '\n',
    author="Itiel Shwartz",
    author_email='itiel@etlsh.com',
    url='https://github.com/itielshwartz/asyncio-hn',
    packages=[
        'asyncio_hn',
    ],
    include_package_data=True,
    install_requires=required,
    license="MIT license",
    zip_safe=False,
    keywords=['asyncio', 'aiohttp', 'hackernews'],
    classifiers=[
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Natural Language :: English',
        'Programming Language :: Python :: 3.6',
    ],
    test_suite='tests',
    tests_require=test_required,

)


text:
// ==========================================================================
//  Squidex Headless CMS
// ==========================================================================
//  Copyright (c) Squidex UG (haftungsbeschränkt)
//  All rights reserved. Licensed under the MIT license.
// ==========================================================================

using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Xunit;

namespace Squidex.Infrastructure
{
    public class InMemoryPubSubTests
    {
        private readonly InMemoryPubSub sut = new InMemoryPubSub();

        private sealed class MessageA
        {
            public string Text { get; set; }
        }

        private sealed class MessageB
        {
            public string Text { get; set; }
        }

        [Fact]
        public void Should_publish_to_handlers()
        {
            var channel1Events = new List<string>();
            var channel2Events = new List<string>();

            sut.Subscribe<MessageA>(m =>
            {
                channel1Events.Add(m.Text);
            });

            sut.Subscribe<MessageA>(m =>
            {
                channel1Events.Add(m.Text);
            });

            sut.Subscribe<MessageB>(m =>
            {
                channel2Events.Add(m.Text);
            });

            sut.Publish(new MessageA { Text = "1" }, true);
            sut.Publish(new MessageA { Text = "2" }, true);
            sut.Publish(new MessageA { Text = "3" }, false);

            sut.Publish(new MessageB { Text = "a" }, true);
            sut.Publish(new MessageB { Text = "b" }, true);

            Assert.Equal(new[] { "1", "1", "2", "2" }, channel1Events.ToArray());
            Assert.Equal(new[] { "a", "b" }, channel2Events.ToArray());
        }

        [Fact]
        public async Task Should_make_request_reply_requests()
        {
            sut.ReceiveAsync<int, int>(x =>
            {
                return Task.FromResult(x + x);
            }, true);

            var response = await sut.RequestAsync<int, int>(2, TimeSpan.FromSeconds(2), true);

            Assert.Equal(4, response);
        }

        [Fact]
        public async Task Should_timeout_when_response_is_too_slow()
        {
            sut.ReceiveAsync<int, int>(async x =>
            {
                await Task.Delay(1000);

                return x + x;
            }, true);

            await Assert.ThrowsAsync<TaskCanceledException>(() => sut.RequestAsync<int, int>(1, TimeSpan.FromSeconds(0.5), true));
        }

        [Fact]
        public async Task Should_timeout_when_nobody_responds()
        {
            await Assert.ThrowsAsync<TaskCanceledException>(() => sut.RequestAsync<int, int>(2, TimeSpan.FromSeconds(0.5), true));
        }
    }
}