text:
#include "cryptoTools/Common/Timer.h"
#include "DualExActor.h"
#include <future>
#include "cryptoTools/Crypto/Commit.h"
#include <algorithm>
#include "PSI/PSIReceiver.h"
#include "PSI/PSISender.h"
#include "cryptoTools/Common/Log.h"
#include <mutex>
#include <fstream>
#include "cryptoTools/Network/Endpoint.h"
#include "DualEx/Bucket.h"
#include "libOTe/Base/naor-pinkas.h"

#define PARALLEL_PSI

namespace osuCrypto
{

    void GetRandSubset(u64 subsetSize, u64 setSize, PRNG& prng, std::vector<u64>& subset, std::vector<u64>& difference)
    {
        subset.clear();
        difference.clear();
        subset.resize(subsetSize);
        difference.resize(setSize - subsetSize);

        for (u64 i = 0; i < subsetSize; ++i)
            subset[i] = i;

        if (subsetSize)
        {
            for (u64 i = subsetSize, j = 0; i < setSize; ++i, ++j)
            {
                difference[j] = i;

                u64 randIdx = prng.get<u64>() % i;
                if (randIdx < subsetSize)
                    std::swap(subset[randIdx], difference[j]);
            }
        }
        else
        {
            for (u64 i = 0; i < setSize; ++i)
                difference[i] = i;
        }
    }

    DualExActor::DualExActor(
        const Circuit& cir,
        const Role role,
        const u64 numExes,
        const u64 bucketSize,
        const u64 numOpened,
        const u64 psiSecParam,
        Endpoint & netMgr)
        :
        mCircuit(cir),
        mRole(role),
        mNetMgr(netMgr),
        mNumExe(numExes),
        mBucketSize(bucketSize),
        mNumOpened(numOpened),
        mNumCircuits(numOpened + bucketSize * numExes),
        mPsiSecParam(psiSecParam),
        //mEvalIdx(0),
        PRINT_SETUP_TIMES(0),
        PRINT_EVAL_TIMES(0),
        mCnCCommitRecvDone(false),
        mBuckets(numExes),
        mOnlineFuture(mOnlineProm.get_future())
    {
        if (cir.Inputs()[0] == 0 || cir.Inputs()[1] == 0)
            throw std::runtime_error("There are better protocols for such things...");
    }

    void DualExActor::getRecvOTs(block prngSeed, u64 numInit, u64 numOTExtPer)
    {

        PRNG prng(prngSeed);
        auto chl = mNetMgr.addChannel("OTRecv", "OTSend");
        NaorPinkas baseOTs;// (chl, OTRole::Sender);

        std::array<std::array < block, 2>, gOtExtBaseOtCount> baseOTsSender_inputs;
        baseOTs.send(baseOTsSender_inputs, prng, chl);

        std::vector<std::thread> thrds(numInit - 1);
        //std::vector<PRNG> prngs(numInit - 1);

        std::vector<std::array< std::array<block, 2>, gOtExtBaseOtCount>> bases(numInit);
        std::array< std::array<PRNG, 2>, gOtExtBaseOtCount> extenders;

        for (u64 i = 0; i < gOtExtBaseOtCount; ++i)
        {
            extenders[i][0].SetSeed(baseOTsSender_inputs[i][0]);
            extenders[i][1].SetSeed(baseOTsSender_inputs[i][1]);
        }

        for (u64 t = 0; t < thrds.size(); ++t)
        {
            //prngs[t].SetSeed(prng.get<block>());

            auto& idx = mOTRecvDoneIdx[t];
            assert(idx == 0);

            for (u64 i = 0; i < gOtExtBaseOtCount; ++i)
            {
                bases[t][i][0] = extenders[i][0].get<block>();
                bases[t][i][1] = extenders[i][1].get<block>();
            }
            block seed = prng.get<block>();

            thrds[t] = std::thread([&, t, seed]() {
                PRNG prng2(seed);
                Channel chl = mNetMgr.addChannel("OTRecv" + ToString(t), "OTSend" + ToString(t));
                mOTRecv[t].Extend(bases[t], numOTExtPer, prng2, chl, mOTRecvDoneIdx[t]);
                chl.close();
            });
        }

        for (u64 i = 0; i < gOtExtBaseOtCount; ++i)
        {
            bases[numInit - 1][i][0] = extenders[i][0].get<block>();
            bases[numInit - 1][i][1] = extenders[i][1].get<block>();
        }
        mOTRecv[numInit - 1].Extend(bases[numInit - 1], numOTExtPer, prng, chl, mOTRecvDoneIdx[numInit - 1]);

        chl.close();

        for (auto& thrd : thrds)
            thrd.join();
    }

    void DualExActor::getSendOTs(block prngSeed, u64 numInit, u64 numOTExtPer)
    {
        auto chl = mNetMgr.addChannel("OTSend", "OTRecv");
        NaorPinkas baseOTs;// (chl, OTRole::Receiver);
        PRNG prng(prngSeed);

        BitVector baseOTsReceiver_inputs(gOtExtBaseOtCount);
        baseOTsReceiver_inputs.randomize(prng);
        std::array < block, gOtExtBaseOtCount> baseOTsReceiver_outputs;
        baseOTs.receive(baseOTsReceiver_inputs, baseOTsReceiver_outputs, prng, chl);

        std::vector<std::thread> thrds(numInit - 1);
        //std::vector<PRNG> prngs(numInit - 1);

        std::vector<std::array< block, gOtExtBaseOtCount>> bases(numInit);
        std::array<PRNG, gOtExtBaseOtCount> extenders;

        for (u64 i = 0; i < gOtExtBaseOtCount; ++i)
        {
            extenders[i].SetSeed(baseOTsReceiver_outputs[i]);
        }

        for (u64 t = 0; t < thrds.size(); ++t)
        {
            auto& idx = mOTSendDoneIdx[t];
            assert(idx == 0);
            //prngs[t].SetSeed(prng.get<block>());
            block seed = prng.get<block>();

            for (u64 i = 0; i < gOtExtBaseOtCount; ++i)
            {
                bases[t][i] = extenders[i].get<block>();
            }
            thrds[t] = std::thread([&, t, seed]() {
                PRNG prng2(seed);
                Channel chl = mNetMgr.addChannel("OTSend" + ToString(t), "OTRecv" + ToString(t));
                mOTSend[t].Extend(bases[t], baseOTsReceiver_inputs, numOTExtPer, prng2, chl, mOTSendDoneIdx[t]);
                chl.close();
            });
        }

        for (u64 i = 0; i < gOtExtBaseOtCount; ++i)
        {
            bases[numInit - 1][i] = extenders[i].get<block>();
        }
        mOTSend[numInit - 1].Extend(bases[numInit - 1], baseOTsReceiver_inputs, numOTExtPer, prng, chl, mOTSendDoneIdx[numInit - 1]);
        chl.close();

        for (auto& thrd : thrds)
            thrd.join();
    }

    void DualExActor::close()
    {
        mRecvMainChls.clear();

        for (auto& sendThrd : mSendMainThreads)
            sendThrd.join();
        mSendMainThreads.clear();

        for (auto& sendThrd : mSendSubThreads)
            sendThrd.join();
        mSendSubThreads.clear();

        mSendSubChls.clear();

        for (auto& thrd : mEvalThreads)
            thrd.join();
        mEvalThreads.clear();

        mRecvSubChls.clear();
    }

    /// This function starts the process of generating and receiving circuits and related things. It will spin off
    /// numParallelInit * 4 threads. Two for generating and receiving OTs, Two for generating and receiving Circuits.
    /// numParallelEval determines how many parallel evaluations can be performed at once. default it 1.
    /// numThreadsPerEval determines how many threads should be used for each evaluation. Typically bucket size...
    void DualExActor::init(PRNG& prng, u64 numParallelInit, u64 numParallelEval, u64 numThreadsPerEval, Timer& timer)
    {
        Channel chl = mNetMgr.addChannel("init");

        // send cut n choose commitments
        block myCnCSeed = prng.get<block>();
        auto CnCom = Commit(myCnCSeed);
        chl.asyncSend(&CnCom, sizeof(Commit));

        // compute my cut n choose sets
        PRNG mySetsPrng(myCnCSeed);
        GetRandSubset(mNumOpened, mNumCircuits, mySetsPrng, mMyCnCSets[0], mMyCnCSets[1]);

        // set up the k-probe resistant matrices in both directions
        u64 kProbeSecParam = mPsiSecParam;
        block seeds[2] = { _mm_set_epi64x(23456543, 3456544765) ,_mm_set_epi64x(654387654567, 6789544) };
        PRNG myKPRrobePrng(seeds[mRole]), theirKProbePrng(seeds[1 - mRole]);

        mMyKProbe.loadOrBuild(mCircuit.Inputs()[mRole], kProbeSecParam, myKPRrobePrng);
        mTheirKProbe.loadOrBuild(mCircuit.Inputs()[1 - mRole], kProbeSecParam, theirKProbePrng);

        // Lets Send over the parameters we are using
        std::unique_ptr<ByteStream> paramBuff(new ByteStream());
        paramBuff->append((u8*)&numParallelInit, sizeof(numParallelInit));
        paramBuff->append((u8*)&numParallelEval, sizeof(numParallelEval));
        paramBuff->append((u8*)&numThreadsPerEval, sizeof(numThreadsPerEval));
        paramBuff->append((u8*)&mNumExe, sizeof(mNumExe));
        paramBuff->append((u8*)&mBucketSize, sizeof(mBucketSize));
        paramBuff->append((u8*)&mNumCircuits, sizeof(mNumCircuits));
        paramBuff->append((u8*)&mPsiSecParam, sizeof(mPsiSecParam));
        paramBuff->append((u8*)&mMyKProbe.mSignature, sizeof(u64));
        paramBuff->append((u8*)&mTheirKProbe.mSignature, sizeof(u64));
        chl.asyncSend(std::move(paramBuff));

        // Compute the required size of the index array. Used for many things when doing AES fixed key or seed based PRNG.
        u64 size = std::max(std::max(std::max(mMyKProbe.encodingSize(), mTheirKProbe.encodingSize()), mCircuit.WireCount()), mCircuit.Gates().size() * 2);
        // Generate the Index Array, simply as 1,2,3,4,..., size
        mIndexArray.resize(size);
        for (u64 i = 0; i < size; ++i)
            mIndexArray[i] = _mm_set_epi64x(0, i);

        // the init phase will have 4 concurrent pipes that things will be divided between...
        mNumInitThreads = std::min((u64)numParallelInit, mNumExe);

#ifdef ASYNC_PSI
        u64 otsPerPSI = AsyncPsiSender::PsiOTCount(mBucketSize, mPsiSecParam);
#else
        u64 otsPerPSI = PsiSender::PsiOTCount(mBucketSize, mPsiSecParam);
#endif // ASYNC_PSI

        // compute how many OTs each pipe will need
        u64 numSendOTsPer = mTheirKProbe.encodingSize() * (mNumCircuits + mNumInitThreads - 1) / mNumInitThreads
            + otsPerPSI * (mNumExe + mNumInitThreads - 1) / mNumInitThreads;
        u64 numRecvOTsPer = mMyKProbe.encodingSize() * (mNumCircuits + mNumInitThreads - 1) / mNumInitThreads
            + otsPerPSI * (mNumExe + mNumInitThreads - 1) / mNumInitThreads;

        // each one of these will hold numSendOTsPer OTs and will be used by the garbling threads
        mOTRecv.resize(mNumInitThreads);
        mOTSend.resize(mNumInitThreads);

        // these are used to synchronize the OT generator threads with the garbling threads.
        mOTSendDoneIdx.reset(new std::atomic<u64>[mNumInitThreads]());
        mOTRecvDoneIdx.reset(new std::atomic<u64>[mNumInitThreads]());
        for (u64 i = 0; i < mNumInitThreads; ++i) { mOTSendDoneIdx[i] = mOTRecvDoneIdx[i] = 0; }

        // spin off two threads that will start generating the OTs, these will fork into numInitThreads threads
        block sendOTSeed(prng.get<block>());
        block recvOTSeed(prng.get<block>());

        // Spin off the OT threads. There two threads will generate mNumInitThreads-1 other threads, all doing OTs
        mOTSendThrd = std::thread([&, sendOTSeed]() { getSendOTs(sendOTSeed, mNumInitThreads, numSendOTsPer); });
        mOTRecvThrd = std::thread([&, recvOTSeed]() { getRecvOTs(recvOTSeed, mNumInitThreads, numRecvOTsPer); });

        //mOTSendThrd.join();
        //mOTRecvThrd.join();
        // allocate threads for generating and receiving circuits
        std::vector<std::thread> sendThrds(mNumInitThreads);
        std::vector<std::thread> recvThrds(mNumInitThreads);

        // allocate circuits
        mMyCircuits.resize(mNumCircuits);
        mTheirCircuits.resize(mNumCircuits);

        // set up some synchronizing vars for when we have received all their circuits
        mCirRecvProm.resize(mNumInitThreads);
        mCirRecvFutr.resize(mNumInitThreads);
        for (u64 i = 0; i < mNumInitThreads; ++i) mCirRecvFutr[i] = mCirRecvProm[i].get_future().share();

        // set up a shared future for when we know their cut n choose selection
        mTheirSetsFutr = mOpenEvalSetsProm.get_future().share();

        // get their cut n choose commitment
        Commit theirCnCCommit;
        chl.recv(&theirCnCCommit, sizeof(Commit));

        // Lets check that the parameters they are using match ours...
        ByteStream paramBuffRecv(sizeof(u64) * 9);
        chl.recv(paramBuffRecv.data(), paramBuffRecv.size());

        u64 * params = (u64*)paramBuffRecv.data();
        if (*params++ != numParallelInit) { std::cout << "parameter mismatch: numParallelInit us " << numParallelInit << "  them " << *--params << std::endl;  throw std::runtime_error("parameter mismatch: numParallelInit"); }
        if (*params++ != numParallelEval) { std::cout << "parameter mismatch: numParallelEval us " << numParallelEval << "  them " << *--params << std::endl;  throw std::runtime_error("parameter mismatch: numParallelEval"); }
        if (*params++ != numThreadsPerEval) { std::cout << "parameter mismatch: numThreadsPerEval us " << numThreadsPerEval << "  them " << *--params << std::endl;  throw std::runtime_error("parameter mismatch: numThreadsPerEval"); }
        if (*params++ != mNumExe) { std::cout << "parameter mismatch: mNumExe us " << mNumExe << "  them " << *--params << std::endl;  throw std::runtime_error("parameter mismatch: mNumExe"); }
        if (*params++ != mBucketSize) { std::cout << "parameter mismatch: mBucketSize us " << mBucketSize << "  them " << *--params << std::endl;  throw std::runtime_error("parameter mismatch: mBucketSize"); }
        if (*params++ != mNumCircuits) { std::cout << "parameter mismatch: mNumCircuits us " << mNumCircuits << "  them " << *--params << std::endl;  throw std::runtime_error("parameter mismatch: mNumCircuits"); }
        if (*params++ != mPsiSecParam) { std::cout << "parameter mismatch: mPsiSecParam us " << mPsiSecParam << "  them " << *--params << std::endl;  throw std::runtime_error("parameter mismatch: mPsiSecParam"); }
        if (*params++ != mTheirKProbe.mSignature) { std::cout << "parameter mismatch: theirKProbe us " << mTheirKProbe.mSignature << "  them " << *--params << std::endl;  throw std::runtime_error("parameter mismatch: theirKProbe"); }
        if (*params++ != mMyKProbe.mSignature) { std::cout << "parameter mismatch: myKProbe us " << mMyKProbe.mSignature << "  them " << *--params << std::endl;  throw std::runtime_error("parameter mismatch: myKProbe"); }

        // now spin off the send threads what will start sending and receiving circuits
        for (u64 i = 0; i < mNumInitThreads; ++i)
        {
            block sendSeed(prng.get<block>());
            block recvSeed(prng.get<block>());

            sendThrds[i] = std::thread([&, i, sendSeed]()
            {
                setThreadName("initSend_" + std::to_string(mRole) + "_" + std::to_string(i));
                initSend(i, sendSeed, mCirRecvFutr[i], mTheirSetsFutr);
            });
            recvThrds[i] = std::thread([&, i, recvSeed]()
            {
                setThreadName("initRecv_" + std::to_string(mRole) + "_" + std::to_string(i));
                initRecv(i, recvSeed, mCirRecvProm[i], mMyCnCSets);
            });
        }

        // in the mean time, lets init some stuff used in the online phase
        initOnlinePhase(numParallelEval, numThreadsPerEval, prng);

        // wait for us to receive all their circuits
        for (u64 i = 0; i < mNumInitThreads; ++i)
            mCirRecvFutr[i].get();

        timer.setTimePoint("circuits received");

        // decommit the cut n choose selection
        std::unique_ptr<ByteStream> sBuff(new ByteStream());
        sBuff->append(myCnCSeed);
        chl.asyncSend(std::move(sBuff));

        // get their cut n choose decommit
        block cncSeed;
        chl.recv(&cncSeed, sizeof(block));

        // check the decommit
        if (Commit(cncSeed) != theirCnCCommit)
            throw std::runtime_error(LOCATION);

        // compute the open and eval sets
        PRNG theirSetsPrng(cncSeed);
        GetRandSubset(mNumOpened, mNumCircuits, theirSetsPrng, mTheirCncSets[0], mTheirCncSets[1]);

        // pass the send threads the sets that need to be opened and bucketed
        mOpenEvalSetsProm.set_value(&mTheirCncSets);

        // join all the threads
#ifdef ASYNC_EVAL
        mOTSendThrd.detach();
        mOTRecvThrd.detach();
        for (u64 i = 0; i < mNumInitThreads; ++i)
        {
            sendThrds[i].detach();
            recvThrds[i].detach();
        }
        timer.setTimePoint("bucketing started");
#else
        mOTSendThrd.join();
        mOTRecvThrd.join();
        for (u64 i = 0; i < mNumInitThreads; ++i)
        {
            sendThrds[i].join();
            recvThrds[i].join();
        }
        timer.setTimePoint("bucketing done");
#endif

        mOnlineProm.set_value();

        chl.close();

    }

    /// This function create the channels and threads that will be used in the online phase.
    void DualExActor::initOnlinePhase(u64 numParallelEval, u64 numThreadsPerEval, PRNG& prng)
    {

        mLabels.resize(numParallelEval);
        mOutLabels.resize(numParallelEval);
        mSendMainThreads.resize(numParallelEval);

        // set up all the online channels at once (faster unless i implement async connect...)
        //TODO("fix addChannels, There are bugs when too many channels are connected at once. like more than 6.");
        //std::cout << "adding " << (numParallelEval * (1)) << " online channels" << std::endl;

        //std::cout << "added  " << (numParallelEval * (1)) << " online channels" << std::endl;

        mRecvMainChls.resize(numParallelEval);
        //mSendMainChls.resize(numParallelEval);

        numThreadsPerEval = std::min((u64)numThreadsPerEval, mBucketSize);
        mEvalThreads.resize(numThreadsPerEval * numParallelEval);
        mSendSubThreads.resize(numThreadsPerEval * numParallelEval);
        auto evalThrdIter = mEvalThreads.begin();
        auto sendSubThrdIter = mSendSubThreads.begin();
        mTimes.resize(numParallelEval *  numThreadsPerEval);

        // not create the threads used in the online phase.
        for (u64 j = 0; j < numParallelEval; ++j) {

            // allocate buffers for there wire labels will be stored.
            mLabels[j].resize(mBucketSize);

            // this buffer is used to store the complete set of output labels used by the other parties buckets.
            // i.e. then they send the open translation seed, we use this buffer to get the actual labels.
            mOutLabels[j].resize(mCircuit.OutputCount() * 2);

            for (u64 i = 0; i < numThreadsPerEval; ++i)
            {
                // make enough room for all the wire labels.
                mLabels[j][i].resize(mCircuit.WireCount());

                // spin off the thread that will evaluate every i + numThreadsPerEval * j circuit in a bucket for j=0,1,2,...
                block evalSeed = prng.get<block>();
                *evalThrdIter++ = std::thread([this, j, i, numParallelEval, numThreadsPerEval, evalSeed]()
                {
                    // set this threads name. used for debugging in visual studio.
                    setThreadName("EvalThread_" + ToString(mRole) + "_" + ToString(j) + "_" + ToString(i));

                    // create a PRNG used by this thread
                    PRNG prng(evalSeed);

                    // connect to the other party's send input/PSI thread.
                    Channel chl = mNetMgr.addChannel("onlineEval" + ToString(j) + "_" + ToString(i), "onlineSend" + ToString(j) + "_" + ToString(i));

                    // now start evaluating threads (once the offline is done and inputs are sent).
                    evalThreadLoop(i, numThreadsPerEval, j, numParallelEval, prng, chl);// , mPSIRecvChls[j], mPsiChannelLocks[0][j].get());

                    // all done :)
                    chl.close();
                });

                *sendSubThrdIter++ = std::thread([this, j, i, numParallelEval, numThreadsPerEval]()
                {
                    // set this threads name. used for debugging in visual studio.
                    setThreadName("SendSubThread_" + ToString(mRole) + "_" + ToString(j) + "_" + ToString(i));

                    // connect to the other party's Eval/PSI thread.
                    Channel chl = mNetMgr.addChannel("onlineSend" + ToString(j) + "_" + ToString(i), "onlineEval" + ToString(j) + "_" + ToString(i));

                    // now wait for inputs and the send the labels and do the PSI.
                    sendCircuitInputLoop(i, numThreadsPerEval, j, numParallelEval, chl);

                    // all done :)
                    chl.close();
                });
            }

            // This thread is used as the place the input correction string is send by the receiving party.
            // Really it doesn't do too much.
            block seed = prng.get<block>();
            mSendMainThreads[j] = std::thread([this, j, numParallelEval, seed]() {

                // set this threads name. used for debugging in visual studio.
                setThreadName("SendInputLoop_" + ToString(mRole) + "_" + ToString(j));

                // create a PRNG used by this thread
                PRNG prng(seed);
                auto recvName = "onlineMainRecv_" + std::to_string(j);
                auto sendName = "onlineMainSend_" + std::to_string(j);

                Channel sendChlPtr;
                if (mRole)
                {
                    mRecvMainChls[j] = mNetMgr.addChannel(recvName, sendName);
                    sendChlPtr = mNetMgr.addChannel(sendName, recvName);
                }
                else
                {
                    sendChlPtr = mNetMgr.addChannel(sendName, recvName);
                    mRecvMainChls[j] = mNetMgr.addChannel(recvName, sendName);
                }

                // This function waits for input correction string and then gives it to the sendCircuitInputLoop.
                // Also is capable of performing the PSI. But normally its performed in parallel and not by this thread.
                sendLoop(j, numParallelEval, sendChlPtr, prng);

            });

        }

    }

    void DualExActor::initRecv(
        u64 initThrdIdx,
        block prngSeed,
        std::promise<void>& allCirReceived,
        std::array<std::vector<u64>, 2>& cutnChoose)
    {
        // Circuits are split between several threads. This thread will receive the circuits in the following range.
        u64 startIdx = initThrdIdx* mNumCircuits / mNumInitThreads;
        u64 endIdx = (initThrdIdx + 1) * mNumCircuits / mNumInitThreads;
        Timer timer;

        // connect to the other party's circuit generating thread
        Channel chl = mNetMgr.addChannel("initRecv" + ToString(initThrdIdx), "initSend" + ToString(initThrdIdx));

        // get their circuits, my K-prone input wire label commitments and output label commitments.
        for (u64 i = startIdx; i < endIdx; i++)
        {
            mTheirCircuits[i].init(mCircuit, mMyKProbe, chl, i);
        }

        u64 dummyMessage;
        chl.recv(&dummyMessage, sizeof(u64));
        if (dummyMessage != 0)
            throw std::runtime_error("");

        timer.setTimePoint("Circuits_generated");

        // this index is updated by the OT generating thread and indicates how many completed OT messages there are.
        auto& otDoneIdx = mOTSendDoneIdx[initThrdIdx];

        // this is the index of the OTs that have been used. The functions that use OTs will update it. i.e. initOT, Bucket.initRecv.
        u64 otIdx = 0;

        // get their input wire label commitments.
        for (u64 i = startIdx; i < endIdx; i++)
        {
            while (otDoneIdx < otIdx + mTheirKProbe.encodingSize())std::this_thread::sleep_for(std::chrono::milliseconds(1));

            //u64 tt = otIdx;

            mTheirCircuits[i].initOT(chl, mTheirKProbe, mOTSend[initThrdIdx], otIdx, mCircuit.Inputs()[1 ^ mRole]);

            //if (otIdx != tt + mMyKProbe.encodingSize())
            //	throw std::runtime_error("");
        }

        allCirReceived.set_value();

        chl.recv(&dummyMessage, sizeof(u64));
        if (dummyMessage != 0)
            throw std::runtime_error("");

        timer.setTimePoint("Circuits_OTInit_done");

        // a buffer that is used to unmask the circuit and
        std::vector<block> blockBuff(std::max(std::max(mCircuit.NonXorGateCount() * 2, mTheirKProbe.encodingSize()), mMyKProbe.encodingSize()));

        block theirOTMsgXORSum = ZeroBlock;

        //std::cout << "receive  " + ToString(initThrdIdx) + "  otIdx "  + ToString(otIdx) << std::endl;

        // check the open circuits
        for (u64 cirIdx : cutnChoose[0])
        {
            if (cirIdx >= startIdx && cirIdx < endIdx)
            {
#ifdef ADAPTIVE_SECURE
                //std::cout << "receive " + ToString(initThrdIdx) + " Opening " + ToString(cirIdx) << std::endl;

                mTheirCircuits[cirIdx].open(mCircuit, mTheirKProbe, mMyKProbe, mRole, chl, theirOTMsgXORSum, mIndexArray, blockBuff);
#else
                mTheirCircuits[cirIdx].open(mCircuit, mTheirKProbe, mRole, chl);
#endif
                mTheirCircuits[cirIdx].clear();
            }
        }

        //std::cout << "receive " + ToString(initThrdIdx) + "  Open Done" << std::endl;

        chl.recv(&dummyMessage, sizeof(u64));
        if (dummyMessage != 0)
            throw std::runtime_error("");

        block theirOTMsgProof;
        chl.recv(&theirOTMsgProof, sizeof(block));

        if (neq(theirOTMsgProof, theirOTMsgXORSum))
        {
            std::cout << "receive " + ToString(initThrdIdx) + "  proof my " << theirOTMsgXORSum << " thr " << theirOTMsgProof << std::endl;
            throw std::runtime_error(LOCATION);
        }

        timer.setTimePoint("circuits_opened");
        PRNG prng(prngSeed);

        // we will skip this many circuits between each of out buckets since other threads are bucketing them.
        u64 numSkip = (mNumInitThreads - 1)* mBucketSize;

        // the circuit we should start bucketing with
        auto evalIter = cutnChoose[1].begin() + initThrdIdx * mBucketSize;

#ifdef ASYNC_PSI
        u64 otsPerPSI = AsyncPsiSender::PsiOTCount(mBucketSize, mPsiSecParam);
#else
        u64 otsPerPSI = PsiSender::PsiOTCount(mBucketSize, mPsiSecParam);
#endif // ASYNC_PSI

        // put the remaining circuits into buckets and get their OT aggregation values
        for (u64 bcktIdx = initThrdIdx; bcktIdx < mBuckets.size(); bcktIdx += mNumInitThreads)
        {
            // make sure we have completed the required number of OTs
            while (otDoneIdx < otIdx + otsPerPSI)
                std::this_thread::sleep_for(std::chrono::milliseconds(1));

            // group together the circuits by having the aggregate OTs. Also set up some stuff used in the online phase.
            mBuckets[bcktIdx].initRecv(mCircuit, chl, mTheirKProbe, mBucketSize, mPsiSecParam, evalIter, mTheirCircuits, mOTSend[initThrdIdx], prng, otIdx, mRole);

            // since we have many threads doing the bucketing, lets skip the circuits that they are bucketing...
            if (bcktIdx + mNumInitThreads < mBuckets.size())
                evalIter += numSkip;
        }

        chl.recv(&dummyMessage, sizeof(u64));
        if (dummyMessage != 0)
            throw std::runtime_error("");

        //std::cout << "receive " + ToString(initThrdIdx) + " bucketing Done" << std::endl;

        // no that the bucketing is done, lets get our K-probe inputs. These are random inputs
        // done in the offline phase to speed up the online phase. This is done at the end because
        // it requires the receive OTs to be done. Which are performed by the other threads.
        for (u64 bcktIdx = initThrdIdx; bcktIdx < mBuckets.size(); bcktIdx += mNumInitThreads)
        {
            mBuckets[bcktIdx].initKProbeInputRecv(mCircuit, chl, mMyKProbe, prng, mRole);
        }

        //std::cout << "receive " + ToString(initThrdIdx) + " kprobe Done" << std::endl;

        // all done :)
        chl.close();

        timer.setTimePoint("bucketing_done");
#ifdef ASYNC_PSI
        std::string psi("Async_");
#else
        std::string psi("Sync_");
#endif

        if (PRINT_SETUP_TIMES)
        {
            std::string filename = "timeFile_setup_" + psi + ToString(mNumExe) + "_b" + ToString(mBucketSize) + "_" + ToString(initThrdIdx) + ".txt";
            std::fstream file;
            file.open(filename, std::ios::out);

            if (file.is_open() == false)
                std::cout << "couldnt open file " << filename << std::endl;

            file << timer;
        }

    }

    void DualExActor::initSend(
        u64 initThrdIdx,
        block prngSeed,
        std::shared_future<void>& allCirReceived,
        std::shared_future<std::array<std::vector<u64>, 2>*>& cutnChooseSets)
    {
        PRNG prng(prngSeed);
        Channel chl = mNetMgr.addChannel("initSend" + ToString(initThrdIdx), "initRecv" + ToString(initThrdIdx));

        u64 startIdx = initThrdIdx* mNumCircuits / mNumInitThreads;
        u64 endIdx = (initThrdIdx + 1) * mNumCircuits / mNumInitThreads;

        // allocate these buffers once. They are used during gabling.
        std::vector<block> wireBuff(mIndexArray.size());
#ifdef ADAPTIVE_SECURE
        std::vector<block> adaptiveSecureTableMasks(mCircuit.NonXorGateCount() * 2);
#endif

        // generate and send the circuits
        for (u64 i = startIdx; i < endIdx; i++)
        {
#ifdef ADAPTIVE_SECURE
            mMyCircuits[i].init(mCircuit, mRole, prng, chl, i, mTheirKProbe, wireBuff, mIndexArray, adaptiveSecureTableMasks);
#else
            mMyCircuits[i].init(mCircuit, mRole, prng, chl, i, wireBuff);
#endif

        }

        // this message blocks until everything before has been sent.
        //Temp fixed for sending too much stuff and overflowing the OS TCP buffer
        //TODO("come up with something better inside the network manager");
        u64 dummyMessage = 0;
        chl.send(&dummyMessage, sizeof(dummyMessage));

        u64 otIdx = 0;
        auto& otDoneIdx = mOTRecvDoneIdx[initThrdIdx];
        // generate and send the circuits
        for (u64 i = startIdx; i < endIdx; i++)
        {
            while (otDoneIdx < otIdx + mMyKProbe.encodingSize())std::this_thread::sleep_for(std::chrono::milliseconds(1));

            mMyCircuits[i].initOT(mCircuit, mRole, chl, mMyKProbe, mOTRecv[initThrdIdx], otIdx);
        }

        // this message blocks until everything before has been sent.
        //Temp fixed for sending too much stuff and overflowing the OS TCP buffer
        //TODO("come up with something better inside the network manager");
        chl.send(&dummyMessage, sizeof(dummyMessage));

        //std::cout << "Sender " + ToString(initThrdIdx) + " otIdx " + ToString(otIdx) << std::endl;

        auto& cutnChoose = *cutnChooseSets.get();
        block myOTMsgProof = ZeroBlock;

        //std::cout << "Sender " + ToString(initThrdIdx) + " opening" << std::endl;

        // send over all the open circuits
        for (u64 cirIdx : cutnChoose[0])
        {
            if (cirIdx >= startIdx && cirIdx < endIdx)
            {
                //std::cout << "Sender " + ToString(initThrdIdx) + "  opening " + ToString(cirIdx) << std::endl;

                mMyCircuits[cirIdx].open(mCircuit, chl, mRole, myOTMsgProof);
                mMyCircuits[cirIdx].clear();
            }
        }

        // this message blocks until everything before has been sent.
        //Temp fixed for sending too much stuff and overflowing the OS TCP buffer
        //TODO("come up with something better inside the network manager");
        chl.send(&dummyMessage, sizeof(dummyMessage));

        //std::cout << "Sender " + ToString(initThrdIdx) + "  opened w/ prf "<< myOTMsgProof << std::endl;

        chl.asyncSendCopy(&myOTMsgProof, sizeof(block));

#ifdef ASYNC_PSI
        u64 otsPerPSI = AsyncPsiSender::PsiOTCount(mBucketSize, mPsiSecParam);
#else
        u64 otsPerPSI = PsiSender::PsiOTCount(mBucketSize, mPsiSecParam);
#endif // ASYNC_PSI

        u64 numSkip = (mNumInitThreads - 1) * mBucketSize;
        auto evalIter = cutnChoose[1].begin() + initThrdIdx * mBucketSize;
        for (u64 bcktIdx = initThrdIdx; bcktIdx < mBuckets.size(); bcktIdx += mNumInitThreads)
        {
            while (otDoneIdx < otIdx + otsPerPSI)std::this_thread::sleep_for(std::chrono::milliseconds(1));

            mBuckets[bcktIdx].initSend(mCircuit, chl, mBucketSize, mPsiSecParam, mRole, mMyCircuits,
                evalIter, prng, mTheirKProbe, mMyKProbe, mOTRecv[initThrdIdx], otIdx, mIndexArray);

            if (bcktIdx + mNumInitThreads < mBuckets.size())
                evalIter += numSkip;
        }
        //std::cout << "Sender " + ToString(initThrdIdx) + " kprobe" << std::endl;

        // this message blocks until everything before has been sent.
        //Temp fixed for sending too much stuff and overflowing the OS TCP buffer
        //TODO("come up with something better inside the network manager");
        chl.send(&dummyMessage, sizeof(dummyMessage));

        for (u64 bcktIdx = initThrdIdx; bcktIdx < mBuckets.size(); bcktIdx += mNumInitThreads)
        {
            mBuckets[bcktIdx].initKProbeInputSend(mCircuit, chl, mTheirKProbe, prng, mRole, mIndexArray);
        }

        chl.close();
    }

    BitVector DualExActor::execute(u64 evalIdx, PRNG& prng, const BitVector & input, Timer& timer)
    {
        //u64 evalIdx = mEvalIdx++;
        u64 bufferOffset = evalIdx % mLabels.size();
        //std::lock_guard<std::mutex>lock(mMtxs[bufferOffset]);

        //std::cout << "recv start " << evalIdx << "  " << bufferOffset << " " << (int)mRole << std::endl;

        if (input.size() != mCircuit.Inputs()[(int)mRole]) throw std::invalid_argument("input size");
        Bucket& bucket = mBuckets[evalIdx];

        // pass our inputs to the thread that sends them.
        bucket.mInputPromise.set_value(&input);

        auto& chl = mRecvMainChls[bufferOffset];
        //chl.asyncSend(&evalIdx, sizeof(evalIdx));

        // evaluate the circuits that we have received
        bucket.evaluate(mCircuit, mTheirKProbe, mMyKProbe, prng, chl, mRole, input, mLabels[bufferOffset], timer);

#ifndef PARALLEL_PSI
        PSIReceiver& psiRecv = bucket.mPsiRecv;
        for (u64 i = 0; i < mBucketSize; ++i)
        {
            block psiInput = bucket.mPsiInputFuture[i].get();
            psiRecv.CommitSend(psiInput, chl, i);
        }
        for (u64 i = 0; i < mBucketSize; ++i)
            psiRecv.CommitRecv(chl, i);
#endif

        block theirOutLabelSeed;
        chl.recv(&theirOutLabelSeed, sizeof(block));

        AES outGen(theirOutLabelSeed);
        outGen.ecbEncBlocks(mIndexArray.data(), mOutLabels[bufferOffset].size(), mOutLabels[bufferOffset].data());

        bucket.mTheirOutputLabelsProm.set_value(&mOutLabels[bufferOffset]);

#ifndef PARALLEL_PSI

        u64 idx = (u64)-1;
        bucket.mTranslationCheckDoneFuture.get();
        for (u64 i = 0; i < mBucketSize; ++i)
        {
            psiRecv.open(chl, i, mRole);

            for (u64 j = 0; j < mBucketSize; ++j)
            {
                if (bucket.mPSIInputPermutes[j] == i)
                    idx = j;
                //return bucket.mOutputs[i];
            }
        }
        return bucket.mOutputs[idx];

#else

        return *bucket.mOutputFuture.get();
#endif

    }

    void DualExActor::sendLoop(u64 bucketOffset, u64 bucketStep, Channel& chl, PRNG& prng)
    {
        mOnlineFuture.get();
        //auto& chl = *mSendMainChls[bucketOffset];

        for (u64 bucketIdx = bucketOffset; bucketIdx < mNumExe; bucketIdx += bucketStep)
        {

            auto& bucket = mBuckets[bucketIdx];

            // receive their input correction and place it in the bucket
            chl.recv(bucket.mTheirInputCorrection);

            // notify the other threads that their input correction has arrived.
            bucket.mTheirInputCorrectionPromise.set_value();

#ifdef PARALLEL_PSI
            bucket.mPsiSend.mCommitedFuture.get();
            bucket.mPsiRecv.mCommitedFuture.get();
#else
            this is not tested.breaking changes have been made.Tho it should / could work in theory
                PSISender& psiSend = bucket.mPsiSend;
#ifdef ASYNC_PSI
            for (u64 i = 0; i < mBucketSize; ++i)
            {
                block psiInput = bucket.mPsiInputFuture[i].get();
                psiSend.AsyncCommitSend(psiInput, chl, i);
            }

            serial AsyncCommitRecv is broken.There is a std::future.get() that blocks.disable that and fill the openBuffs **only** after all receive commits have arrived...
                for (u64 i = 0; i < mBucketSize; ++i)
                    psiSend.AsyncCommitRecv(chl, i);
#else
            for (u64 i = 0; i < mBucketSize; ++i)
                psiSend.CommitRecv(chl, i);
            for (u64 i = 0; i < mBucketSize; ++i)
            {
                block psiInput = bucket.mPsiInputFuture[i].get();
                psiSend.CommitSend(psiInput, chl, i);
            }

#endif
#endif

            bucket.openTranslation(mCircuit, chl);

#ifndef PARALLEL_PSI
            bucket.mTranslationCheckDoneFuture.get();
            for (u64 i = 0; i < mBucketSize; ++i)
                psiSend.open(chl, i);
#endif
        }
    }

    void DualExActor::sendCircuitInputLoop(u64 circuitOffset, u64 circuitStep, u64  bucketOffset, u64 bucketStep, Channel& chl)
    {

#ifdef PARALLEL_PSI
        assert(circuitStep == mBucketSize && "psi channels are not concurrent safe");
#endif

        u64 psiPermuteIdx(circuitOffset);

        ////u64 bucketIdx = bucketOffset;
        for (u64 bucketIdx = bucketOffset; bucketIdx < mNumExe; bucketIdx += bucketStep)
        {

            auto& bucket = mBuckets[bucketIdx];
            const BitVector& input = *bucket.mInputFuture.get();
            bucket.sendCircuitInputs(mCircuit, input, mRole, chl, circuitOffset, circuitStep);

#ifdef PARALLEL_PSI
            psiPermuteIdx = circuitOffset;

            auto& psiChl = chl;

#ifdef ASYNC_PSI
            block psiInput = bucket.mPsiInputFuture[psiPermuteIdx].get();

            bucket.mPsiSend.AsyncCommitSend(psiInput, psiChl, psiPermuteIdx);

            bucket.mPsiSend.AsyncCommitRecv(psiChl, psiPermuteIdx);
#else
            bucket.mPsiSend.CommitRecv(psiChl, psiPermuteIdx);

            block psiInput = bucket.mPsiInputFuture[psiPermuteIdx].get();

            bucket.mPsiSend.CommitSend(psiInput, psiChl, psiPermuteIdx);
#endif
            //bucket.mPsiSend.mCommitedFuture.get();

            bucket.mTranslationCheckDoneFuture.get();

            bucket.mPsiSend.open(psiChl, psiPermuteIdx);
#endif
        }

    }

    void DualExActor::evalThreadLoop(u64 circuitOffset, u64 circuitStep, u64  bucketOffset, u64 bucketStep, PRNG& prng, Channel& chl)
    {
#ifdef PARALLEL_PSI
        assert(circuitStep == mBucketSize && "psi channels are not concurrent safe");
#endif

        //std::vector<u64> psiIdxs(mBucketSize);
        std::vector<block> wireBuff(mCircuit.OutputCount() * 2);

#ifdef ADAPTIVE_SECURE
        std::vector<block> adaptiveSecureTableMasks(mCircuit.NonXorGateCount() * 2);
#endif
        mOnlineFuture.get();
        Timer timer;

        u64 psiPermuteIdx = circuitOffset;
        for (u64 bucketIdx = bucketOffset; bucketIdx < mNumExe; bucketIdx += bucketStep)
        {
            auto& bucket = mBuckets[bucketIdx];

            u64 circuitIdx = circuitOffset;
            //for (u64 circuitIdx = circuitOffset, i=0; circuitIdx < mBucketSize; circuitIdx += circuitStep, ++i)
            //{
                //std::cout << i << std::endl;
            block psiInput = bucket.evalCircuit(
                circuitIdx,
                mCircuit,
                prng,
                mMyKProbe,
                mLabels[bucketOffset][circuitIdx],
                chl,
#ifdef ADAPTIVE_SECURE
                adaptiveSecureTableMasks,
                mIndexArray,
#endif
                mRole,
                timer);

            {
                std::lock_guard<std::mutex> lock(bucket.mPsiInputMtx);

                // atomic increment the head pointer of there the next psi value should be stored
                i32 psiIdx = bucket.mPSIIdxHead++;
                psiPermuteIdx = bucket.mPSIInputPermutes[psiIdx];

                for (u64 i = 0; i < psiIdx; ++i)
                    if (eq(psiInput, bucket.mPSIInputBlocks[bucket.mPSIInputPermutes[i]]))
                    {
                        psiInput = prng.get<block>();
                    }

                // store our psi value.
                bucket.mPSIInputBlocks[psiPermuteIdx] = psiInput;
            }
            bucket.mPsiInputPromise[psiPermuteIdx].set_value(psiInput);
            psiInput = bucket.mPsiInputFuture[circuitIdx].get();

            //std::cout << psiInput << std::endl;

#ifdef PARALLEL_PSI

            //std::lock_guard<std::mutex> lock(psiChannelLocks[circuitIdx]);

            bucket.mPsiRecv.CommitSend(psiInput, chl, circuitIdx);
            //}

            ///for (u64 circuitIdx = circuitOffset, i = 0; circuitIdx < mBucketSize; circuitIdx += circuitStep, ++i)
            //{
            //u64 circuitIdx = bucket.mPSIInputPermutes[psiIdx];

            bucket.mPsiRecv.CommitRecv(chl, circuitIdx);
            timer.setTimePoint("psiCommited");
            //}

            //for (u64 circuitIdx = circuitOffset, i = 0; circuitIdx < mBucketSize; circuitIdx += circuitStep, ++i)
            //{
#endif
            bucket.checkTranslation(mCircuit, circuitIdx, wireBuff, mRole);
            timer.setTimePoint("TranslationOK");

#ifdef PARALLEL_PSI
            //}
            //for (u64 circuitIdx = circuitOffset, i = 0; circuitIdx < mBucketSize; circuitIdx += circuitStep, ++i)
            //{
            //u64 circuitIdx = bucket.mPSIInputPermutes[psiIdx];

            bool match = bucket.mPsiRecv.open(chl, circuitIdx, mRole);

            timer.setTimePoint("PSIOpen");

			if (match)
			{
				bucket.mOutputProm.set_value(&bucket.mOutputs[circuitIdx]);
			}
			else
			{
				u32 missCount = ++bucket.mOutputMissCount;
				if (missCount == bucket.mTheirCircuits.size())
				{
					try {
						std::cout << "No items found in the intersection " << std::endl;
						throw std::runtime_error("No items found in the intersection " LOCATION);
					}
					catch (...)
					{
						bucket.mOutputProm.set_exception(std::current_exception());
					}
				}
			}
			//psiRecv.CommitSend()
			//bucket.checkTranslation(mCircuit, circuitIdx, wireBuff, mRole);
		//}
#endif

        }

        if (PRINT_EVAL_TIMES)
        {

#ifdef ASYNC_PSI
            std::string psi("Async_");
#else
            std::string psi("Sync_");
#endif
            std::string filename = "timeFile_" + psi + ToString(mNumExe) + "-" + ToString(mNumOpened) + "_b" + ToString(mBucketSize) + "_" + ToString(bucketOffset) + "_" + ToString(circuitOffset) + ".txt";
            std::fstream file;
            file.open(filename, std::ios::out);

            if (file.is_open() == false)
                std::cout << "couldnt open file " << filename << std::endl;

            file << timer;
        }
    }

    void DualExActor::printTimes(std::string filename)
    {
        //		u64 numParallelEvals = mRecvMainChls.size();
        //		u64 threadsPer = mEvalThreads.size() / numParallelEvals;
        //
        //		std::cout << "open file range " << (filename + "_" + ToString(numParallelEvals) + "_" + ToString(threadsPer)) << std::endl;
        //
        //
        //		for (u64 i = 0; i <numParallelEvals; ++i)
        //		{
        //
        //			for (u64 j = 0; j < threadsPer; ++j)
        //			{
        //
        //				std::fstream file;
        //				file.open(filename + "_" + ToString(i) + "_" + ToString(j), std::ios::out);
        //
        //				if (file.is_open() == false)
        //					std::cout << "couldnt open file " << (filename + "_" + ToString(i) + "_" + ToString(j)) << std::endl;
        //
        //				file << mTimes[i*numParallelEvals + j];
        //			}
        //
        ////		}
        /////*
        //		circuitOffset + bucketOffset * bucketStep
        //		out << mTimes[0] << std::endl;
        //*/
    }

}


text:
package org.teavm.runtime;

import org.teavm.interop.Address;
import org.teavm.interop.Export;
import org.teavm.interop.StaticInit;
import org.teavm.interop.Structure;
import org.teavm.interop.Unmanaged;

@Unmanaged
@StaticInit
public final class ExceptionHandling {
    private ExceptionHandling() {
    }

    public static native CallSite findCallSiteById(int id);

    private static Throwable thrownException;

    @Export(name = "sys$catchException")
    public static Throwable catchException() {
        Throwable exception = thrownException;
        thrownException = null;
        return exception;
    }

    public static void throwException(Throwable exception) {
        thrownException = exception;

        RuntimeObject exceptionPtr = Address.ofObject(exception).toStructure();
        RuntimeClass exceptionClass = RuntimeClass.getClass(exceptionPtr);
        IsSupertypeFunction isExceptionSupertype = exceptionClass.isSupertypeOf;

        Address stackFrame = ShadowStack.getStackTop();
        stackLoop: while (stackFrame != null) {
            int callSiteId = ShadowStack.getCallSiteId(stackFrame);
            CallSite callSite = findCallSiteById(callSiteId);
            ExceptionHandler handler = callSite.firstHandler;

            for (int i = 0; i < callSite.handlerCount; ++i) {
                if (handler.exceptionClass == null || isExceptionSupertype.apply(handler.exceptionClass)) {
                    ShadowStack.setExceptionHandlerId(stackFrame, handler.id);
                    break stackLoop;
                }

                handler = Structure.add(ExceptionHandler.class, handler, 1);
            }

            ShadowStack.setExceptionHandlerId(stackFrame, callSiteId - 1);
            stackFrame = ShadowStack.getNextStackFrame(stackFrame);
        }
    }
}

redpajama_set_name: RedPajamaGithub


text:
package de.iritgo.aktera.scheduler.ui;

import de.iritgo.aktera.model.ModelException;
import de.iritgo.aktera.model.ModelRequest;
import de.iritgo.aktera.persist.PersistenceException;
import de.iritgo.aktera.ui.form.FormularDescriptor;
import de.iritgo.aktera.ui.form.FormularHandler;
import de.iritgo.aktera.ui.form.PersistentDescriptor;

/**
 *
 */
public class HolidayFormularHandler extends FormularHandler
{
	/**
	 * @see de.iritgo.aktera.ui.form.FormularHandler#adjustFormular(de.iritgo.aktera.model.ModelRequest, de.iritgo.aktera.ui.form.FormularDescriptor, de.iritgo.aktera.ui.form.PersistentDescriptor)
	 */
	@Override
	public void adjustFormular(ModelRequest request, FormularDescriptor formular, PersistentDescriptor persistents)
		throws ModelException, PersistenceException
	{
		persistents.putAttribute("holiday.provinceCountry", persistents.getPersistent("holiday").getFieldString(
						"country"));
	}
}


text:
function AudioController(){

  //if(
 /* try {
    window.AudioContext = window.AudioContext;//||window.webkitAudioContext;
  }catch(e) {
    alert( 'WEB AUDIO API NOT SUPPORTED' );
  }*/

  this.ctx      = new AudioContext();

  this.mute     = this.ctx.createGain();
  this.gain     = this.ctx.createGain();
  this.analyzer = this.ctx.createAnalyser();

  this.analyzer.frequencyBinCount = 1024;
  this.analyzer.array = new Uint8Array( this.analyzer.frequencyBinCount );

  this.freqByteData = new Uint8Array( this.analyzer.frequencyBinCount );

  this.audioTexture  = new AudioTexture( this );

  this.texture = this.audioTexture.texture;

  this.gain.connect( this.analyzer );
  this.analyzer.connect( this.mute );
  this.mute.connect( this.ctx.destination );

  this.updateArray = [];

  this.notes = [];

}

AudioController.prototype.update = function(){

  this.analyzer.getByteFrequencyData( this.analyzer.array );
  this.analyzer.getByteFrequencyData( this.freqByteData );

  this.audioTexture.update();

  for( var i = 0; i < this.notes.length; i++ ){

    this.notes[i].update();

  }

  for( var i = 0; i < this.updateArray.length; i++ ){

    this.updateArray[i]();

  }

}

AudioController.prototype.addToUpdateArray = function( callback ){

  this.updateArray.push( callback );

}

AudioController.prototype.removeFromUpdateArray = function( callback ){

  for( var i = 0; i< this.updateArray.length; i++ ){

    if( this.updateArray[i] === callback ){

      this.updateArray.splice( i , 1 );
      console.log( 'SPLICED' );

    }

  }

}


text:
package com.dyuproject.jetg;

import java.io.StringWriter;
import java.util.Map;
import java.util.Properties;
import com.dyuproject.jetg.resource.loader.ClasspathResourceLoader;
import org.junit.Assert;
import org.junit.Test;
import testcase.TagUtils;

public class AutoCompiledTestCase {

    @Test
    public void autoCompile() throws Exception {
        Properties config = new Properties();
        config.put(JetConfig.COMPILE_STRATEGY, "auto");
        config.put(JetConfig.IMPORT_PACKAGES, "testcase.model.*");
        config.put(JetConfig.IMPORT_VARIABLES, "User user, Book book");
        config.put(JetConfig.IMPORT_TAGS, TagUtils.class.getName());
        config.put(JetConfig.TEMPLATE_LOADER, ClasspathResourceLoader.class.getName());
        config.put(JetConfig.TEMPLATE_PATH, "/");
        config.put(JetConfig.COMPILE_DEBUG, "true");
        JetEngine engine = JetEngine.create(config);

        JetTemplate template = engine.getTemplate("/template/text-plain.jetx");
        StringWriter out = new StringWriter();
        template.render((Map<String, Object>) null, out);
        Assert.assertEquals("1234567890", out.toString());
    }
}


text:
"""
:mod:`pandas.io.html` is a module containing functionality for dealing with
HTML IO.

"""

from __future__ import annotations

from collections import abc
import numbers
import os
import re
from typing import (
    Pattern,
    Sequence,
)

from pandas._typing import FilePathOrBuffer
from pandas.compat._optional import import_optional_dependency
from pandas.errors import (
    AbstractMethodError,
    EmptyDataError,
)
from pandas.util._decorators import deprecate_nonkeyword_arguments

from pandas.core.dtypes.common import is_list_like

from pandas.core.construction import create_series_with_explicit_dtype
from pandas.core.frame import DataFrame

from pandas.io.common import (
    is_url,
    stringify_path,
    urlopen,
    validate_header_arg,
)
from pandas.io.formats.printing import pprint_thing
from pandas.io.parsers import TextParser

_IMPORTS = False
_HAS_BS4 = False
_HAS_LXML = False
_HAS_HTML5LIB = False

def _importers():
    # import things we need
    # but make this done on a first use basis

    global _IMPORTS
    if _IMPORTS:
        return

    global _HAS_BS4, _HAS_LXML, _HAS_HTML5LIB
    bs4 = import_optional_dependency("bs4", errors="ignore")
    _HAS_BS4 = bs4 is not None

    lxml = import_optional_dependency("lxml.etree", errors="ignore")
    _HAS_LXML = lxml is not None

    html5lib = import_optional_dependency("html5lib", errors="ignore")
    _HAS_HTML5LIB = html5lib is not None

    _IMPORTS = True

#############
# READ HTML #
#############
_RE_WHITESPACE = re.compile(r"[\r\n]+|\s{2,}")

def _remove_whitespace(s: str, regex=_RE_WHITESPACE) -> str:
    """
    Replace extra whitespace inside of a string with a single space.

    Parameters
    ----------
    s : str or unicode
        The string from which to remove extra whitespace.
    regex : re.Pattern
        The regular expression to use to remove extra whitespace.

    Returns
    -------
    subd : str or unicode
        `s` with all extra whitespace replaced with a single space.
    """
    return regex.sub(" ", s.strip())

def _get_skiprows(skiprows):
    """
    Get an iterator given an integer, slice or container.

    Parameters
    ----------
    skiprows : int, slice, container
        The iterator to use to skip rows; can also be a slice.

    Raises
    ------
    TypeError
        * If `skiprows` is not a slice, integer, or Container

    Returns
    -------
    it : iterable
        A proper iterator to use to skip rows of a DataFrame.
    """
    if isinstance(skiprows, slice):
        start, step = skiprows.start or 0, skiprows.step or 1
        return list(range(start, skiprows.stop, step))
    elif isinstance(skiprows, numbers.Integral) or is_list_like(skiprows):
        return skiprows
    elif skiprows is None:
        return 0
    raise TypeError(f"{type(skiprows).__name__} is not a valid type for skipping rows")

def _read(obj):
    """
    Try to read from a url, file or string.

    Parameters
    ----------
    obj : str, unicode, or file-like

    Returns
    -------
    raw_text : str
    """
    if is_url(obj):
        with urlopen(obj) as url:
            text = url.read()
    elif hasattr(obj, "read"):
        text = obj.read()
    elif isinstance(obj, (str, bytes)):
        text = obj
        try:
            if os.path.isfile(text):
                with open(text, "rb") as f:
                    return f.read()
        except (TypeError, ValueError):
            pass
    else:
        raise TypeError(f"Cannot read object of type '{type(obj).__name__}'")
    return text

class _HtmlFrameParser:
    """
    Base class for parsers that parse HTML into DataFrames.

    Parameters
    ----------
    io : str or file-like
        This can be either a string of raw HTML, a valid URL using the HTTP,
        FTP, or FILE protocols or a file-like object.

    match : str or regex
        The text to match in the document.

    attrs : dict
        List of HTML
 element attributes to match.

    encoding : str
        Encoding to be used by parser

    displayed_only : bool
        Whether or not items with "display:none" should be ignored

    Attributes
    ----------
    io : str or file-like
        raw HTML, URL, or file-like object

    match : regex
        The text to match in the raw HTML

    attrs : dict-like
        A dictionary of valid table attributes to use to search for table
        elements.

    encoding : str
        Encoding to be used by parser

    displayed_only : bool
        Whether or not items with "display:none" should be ignored

    Notes
    -----
    To subclass this class effectively you must override the following methods:
        * :func:`_build_doc`
        * :func:`_attr_getter`
        * :func:`_text_getter`
        * :func:`_parse_td`
        * :func:`_parse_thead_tr`
        * :func:`_parse_tbody_tr`
        * :func:`_parse_tfoot_tr`
        * :func:`_parse_tables`
        * :func:`_equals_tag`
    See each method's respective documentation for details on their
    functionality.
    """

    def __init__(self, io, match, attrs, encoding, displayed_only):
        self.io = io
        self.match = match
        self.attrs = attrs
        self.encoding = encoding
        self.displayed_only = displayed_only

    def parse_tables(self):
        """
        Parse and return all tables from the DOM.

        Returns
        -------
        list of parsed (header, body, footer) tuples from tables.
        """
        tables = self._parse_tables(self._build_doc(), self.match, self.attrs)
        return (self._parse_thead_tbody_tfoot(table) for table in tables)

    def _attr_getter(self, obj, attr):
        """
        Return the attribute value of an individual DOM node.

        Parameters
        ----------
        obj : node-like
            A DOM node.

        attr : str or unicode
            The attribute, such as "colspan"

        Returns
        -------
        str or unicode
            The attribute value.
        """
        # Both lxml and BeautifulSoup have the same implementation:
        return obj.get(attr)

    def _text_getter(self, obj):
        """
        Return the text of an individual DOM node.

        Parameters
        ----------
        obj : node-like
            A DOM node.

        Returns
        -------
        text : str or unicode
            The text from an individual DOM node.
        """
        raise AbstractMethodError(self)

    def _parse_td(self, obj):
        """
        Return the td elements from a row element.

        Parameters
        ----------
        obj : node-like
            A DOM
 node.

        Returns
        -------
        list of node-like
            These are the elements of each row, i.e., the columns.
        """
        raise AbstractMethodError(self)

    def _parse_thead_tr(self, table):
        """
        Return the list of thead row elements from the parsed table element.

        Parameters
        ----------
        table : a table element that contains zero or more thead elements.

        Returns
        -------
        list of node-like
            These are the
 row elements of a table.
        """
        raise AbstractMethodError(self)

    def _parse_tbody_tr(self, table):
        """
        Return the list of tbody row elements from the parsed table element.

        HTML5 table bodies consist of either 0 or more
 elements (which
        only contain
 elements) or 0 or more
 elements. This method
        checks for both structures.

        Parameters
        ----------
        table : a table element that contains row elements.

        Returns
        -------
        list of node-like
            These are the
 row elements of a table.
        """
        raise AbstractMethodError(self)

    def _parse_tfoot_tr(self, table):
        """
        Return the list of tfoot row elements from the parsed table element.

        Parameters
        ----------
        table : a table element that contains row elements.

        Returns
        -------
        list of node-like
            These are the
 row elements of a table.
        """
        raise AbstractMethodError(self)

    def _parse_tables(self, doc, match, attrs):
        """
        Return all tables from the parsed DOM.

        Parameters
        ----------
        doc : the DOM from which to parse the table element.

        match : str or regular expression
            The text to search for in the DOM tree.

        attrs : dict
            A dictionary of table attributes that can be used to disambiguate
            multiple tables on a page.

        Raises
        ------
        ValueError : `match` does not match any text in the document.

        Returns
        -------
        list of node-like
            HTML
 elements to be parsed into raw data.
        """
        raise AbstractMethodError(self)

    def _equals_tag(self, obj, tag):
        """
        Return whether an individual DOM node matches a tag

        Parameters
        ----------
        obj : node-like
            A DOM node.

        tag : str
            Tag name to be checked for equality.

        Returns
        -------
        boolean
            Whether `obj`'s tag name is `tag`
        """
        raise AbstractMethodError(self)

    def _build_doc(self):
        """
        Return a tree-like object that can be used to iterate over the DOM.

        Returns
        -------
        node-like
            The DOM from which to parse the table element.
        """
        raise AbstractMethodError(self)

    def _parse_thead_tbody_tfoot(self, table_html):
        """
        Given a table, return parsed header, body, and foot.

        Parameters
        ----------
        table_html : node-like

        Returns
        -------
        tuple of (header, body, footer), each a list of list-of-text rows.

        Notes
        -----
        Header and body are lists-of-lists. Top level list is a list of
        rows. Each row is a list of str text.

        Logic: Use
,
,
 elements to identify
               header, body, and footer, otherwise:
               - Put all rows into body
               - Move rows from top of body to header only if
                 all elements inside row are
               - Move rows from bottom of body to footer only if
                 all elements inside row are
        """
        header_rows = self._parse_thead_tr(table_html)
        body_rows = self._parse_tbody_tr(table_html)
        footer_rows = self._parse_tfoot_tr(table_html)

        def row_is_all_th(row):
            return all(self._equals_tag(t, "th") for t in self._parse_td(row))

        if not header_rows:
            # The table has no
. Move the top all- rows from
            # body_rows to header_rows. (This is a common case because many
            # tables in the wild have no
 or

            while body_rows and row_is_all_th(body_rows[0]):
                header_rows.append(body_rows.pop(0))

        header = self._expand_colspan_rowspan(header_rows)
        body = self._expand_colspan_rowspan(body_rows)
        footer = self._expand_colspan_rowspan(footer_rows)

        return header, body, footer

    def _expand_colspan_rowspan(self, rows):
        """
        Given a list of
s, return a list of text rows.

        Parameters
        ----------
        rows : list of node-like
            List of
s

        Returns
        -------
        list of list
            Each returned row is a list of str text.

        Notes
        -----
        Any cell with ``rowspan`` or ``colspan`` will have its contents copied
        to subsequent cells.
        """
        all_texts = []  # list of rows, each a list of str
        remainder: list[tuple[int, str, int]] = []  # list of (index, text, nrows)

        for tr in rows:
            texts = []  # the output for this row
            next_remainder = []

            index = 0
            tds = self._parse_td(tr)
            for td in tds:
                # Append texts from previous rows with rowspan>1 that come
                # before this
                while remainder and remainder[0][0] <= index:
                    prev_i, prev_text, prev_rowspan = remainder.pop(0)
                    texts.append(prev_text)
                    if prev_rowspan > 1:
                        next_remainder.append((prev_i, prev_text, prev_rowspan - 1))
                    index += 1

                # Append the text from this , colspan times
                text = _remove_whitespace(self._text_getter(td))
                rowspan = int(self._attr_getter(td, "rowspan") or 1)
                colspan = int(self._attr_getter(td, "colspan") or 1)

                for _ in range(colspan):
                    texts.append(text)
                    if rowspan > 1:
                        next_remainder.append((index, text, rowspan - 1))
                    index += 1

            # Append texts from previous rows at the final position
            for prev_i, prev_text, prev_rowspan in remainder:
                texts.append(prev_text)
                if prev_rowspan > 1:
                    next_remainder.append((prev_i, prev_text, prev_rowspan - 1))

            all_texts.append(texts)
            remainder = next_remainder

        # Append rows that only appear because the previous row had non-1
        # rowspan
        while remainder:
            next_remainder = []
            texts = []
            for prev_i, prev_text, prev_rowspan in remainder:
                texts.append(prev_text)
                if prev_rowspan > 1:
                    next_remainder.append((prev_i, prev_text, prev_rowspan - 1))
            all_texts.append(texts)
            remainder = next_remainder

        return all_texts

    def _handle_hidden_tables(self, tbl_list, attr_name):
        """
        Return list of tables, potentially removing hidden elements

        Parameters
        ----------
        tbl_list : list of node-like
            Type of list elements will vary depending upon parser used
        attr_name : str
            Name of the accessor for retrieving HTML attributes

        Returns
        -------
        list of node-like
            Return type matches `tbl_list`
        """
        if not self.displayed_only:
            return tbl_list

        return [
            x
            for x in tbl_list
            if "display:none"
            not in getattr(x, attr_name).get("style", "").replace(" ", "")
        ]

class _BeautifulSoupHtml5LibFrameParser(_HtmlFrameParser):
    """
    HTML to DataFrame parser that uses BeautifulSoup under the hood.

    See Also
    --------
    pandas.io.html._HtmlFrameParser
    pandas.io.html._LxmlFrameParser

    Notes
    -----
    Documentation strings for this class are in the base class
    :class:`pandas.io.html._HtmlFrameParser`.
    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        from bs4 import SoupStrainer

        self._strainer = SoupStrainer("table")

    def _parse_tables(self, doc, match, attrs):
        element_name = self._strainer.name
        tables = doc.find_all(element_name, attrs=attrs)

        if not tables:
            raise ValueError("No tables found")

        result = []
        unique_tables = set()
        tables = self._handle_hidden_tables(tables, "attrs")

        for table in tables:
            if self.displayed_only:
                for elem in table.find_all(style=re.compile(r"display:\s*none")):
                    elem.decompose()

            if table not in unique_tables and table.find(text=match) is not None:
                result.append(table)
            unique_tables.add(table)

        if not result:
            raise ValueError(f"No tables found matching pattern {repr(match.pattern)}")
        return result

    def _text_getter(self, obj):
        return obj.text

    def _equals_tag(self, obj, tag):
        return obj.name == tag

    def _parse_td(self, row):
        return row.find_all(("td", "th"), recursive=False)

    def _parse_thead_tr(self, table):
        return table.select("thead tr")

    def _parse_tbody_tr(self, table):
        from_tbody = table.select("tbody tr")
        from_root = table.find_all("tr", recursive=False)
        # HTML spec: at most one of these lists has content
        return from_tbody + from_root

    def _parse_tfoot_tr(self, table):
        return table.select("tfoot tr")

    def _setup_build_doc(self):
        raw_text = _read(self.io)
        if not raw_text:
            raise ValueError(f"No text parsed from document: {self.io}")
        return raw_text

    def _build_doc(self):
        from bs4 import BeautifulSoup

        bdoc = self._setup_build_doc()
        if isinstance(bdoc, bytes) and self.encoding is not None:
            udoc = bdoc.decode(self.encoding)
            from_encoding = None
        else:
            udoc = bdoc
            from_encoding = self.encoding
        return BeautifulSoup(udoc, features="html5lib", from_encoding=from_encoding)

def _build_xpath_expr(attrs) -> str:
    """
    Build an xpath expression to simulate bs4's ability to pass in kwargs to
    search for attributes when using the lxml parser.

    Parameters
    ----------
    attrs : dict
        A dict of HTML attributes. These are NOT checked for validity.

    Returns
    -------
    expr : unicode
        An XPath expression that checks for the given HTML attributes.
    """
    # give class attribute as class_ because class is a python keyword
    if "class_" in attrs:
        attrs["class"] = attrs.pop("class_")

    s = " and ".join([f"@{k}={repr(v)}" for k, v in attrs.items()])
    return f"[{s}]"

_re_namespace = {"re": "http://exslt.org/regular-expressions"}
_valid_schemes = "http", "file", "ftp"

class _LxmlFrameParser(_HtmlFrameParser):
    """
    HTML to DataFrame parser that uses lxml under the hood.

    Warning
    -------
    This parser can only handle HTTP, FTP, and FILE urls.

    See Also
    --------
    _HtmlFrameParser
    _BeautifulSoupLxmlFrameParser

    Notes
    -----
    Documentation strings for this class are in the base class
    :class:`_HtmlFrameParser`.
    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def _text_getter(self, obj):
        return obj.text_content()

    def _parse_td(self, row):
        # Look for direct children only: the "row" element here may be a
        #
 or
 (see _parse_thead_tr).
        return row.xpath("./td|./th")

    def _parse_tables(self, doc, match, kwargs):
        pattern = match.pattern

        # 1. check all descendants for the given pattern and only search tables
        # 2. go up the tree until we find a table
        xpath_expr = f"//table//*[re:test(text(), {repr(pattern)})]/ancestor::table"

        # if any table attributes were given build an xpath expression to
        # search for them
        if kwargs:
            xpath_expr += _build_xpath_expr(kwargs)

        tables = doc.xpath(xpath_expr, namespaces=_re_namespace)

        tables = self._handle_hidden_tables(tables, "attrib")
        if self.displayed_only:
            for table in tables:
                # lxml utilizes XPATH 1.0 which does not have regex
                # support. As a result, we find all elements with a style
                # attribute and iterate them to check for display:none
                for elem in table.xpath(".//*[@style]"):
                    if "display:none" in elem.attrib.get("style", "").replace(" ", ""):
                        elem.getparent().remove(elem)

        if not tables:
            raise ValueError(f"No tables found matching regex {repr(pattern)}")
        return tables

    def _equals_tag(self, obj, tag):
        return obj.tag == tag

    def _build_doc(self):
        """
        Raises
        ------
        ValueError
            * If a URL that lxml cannot parse is passed.

        Exception
            * Any other ``Exception`` thrown. For example, trying to parse a
              URL that is syntactically correct on a machine with no internet
              connection will fail.

        See Also
        --------
        pandas.io.html._HtmlFrameParser._build_doc
        """
        from lxml.etree import XMLSyntaxError
        from lxml.html import (
            HTMLParser,
            fromstring,
            parse,
        )

        parser = HTMLParser(recover=True, encoding=self.encoding)

        try:
            if is_url(self.io):
                with urlopen(self.io) as f:
                    r = parse(f, parser=parser)
            else:
                # try to parse the input in the simplest way
                r = parse(self.io, parser=parser)
            try:
                r = r.getroot()
            except AttributeError:
                pass
        except (UnicodeDecodeError, OSError) as e:
            # if the input is a blob of html goop
            if not is_url(self.io):
                r = fromstring(self.io, parser=parser)

                try:
                    r = r.getroot()
                except AttributeError:
                    pass
            else:
                raise e
        else:
            if not hasattr(r, "text_content"):
                raise XMLSyntaxError("no text parsed from document", 0, 0, 0)
        return r

    def _parse_thead_tr(self, table):
        rows = []

        for thead in table.xpath(".//thead"):
            rows.extend(thead.xpath("./tr"))

            # HACK: lxml does not clean up the clearly-erroneous
            #
foo	bar
. (Missing
). Add
            # the
 and _pretend_ it's a
; _parse_td() will find its
            # children as though it's a
.
            #
            # Better solution would be to use html5lib.
            elements_at_root = thead.xpath("./td|./th")
            if elements_at_root:
                rows.append(thead)

        return rows

    def _parse_tbody_tr(self, table):
        from_tbody = table.xpath(".//tbody//tr")
        from_root = table.xpath("./tr")
        # HTML spec: at most one of these lists has content
        return from_tbody + from_root

    def _parse_tfoot_tr(self, table):
        return table.xpath(".//tfoot//tr")

def _expand_elements(body):
    data = [len(elem) for elem in body]
    lens = create_series_with_explicit_dtype(data, dtype_if_empty=object)
    lens_max = lens.max()
    not_max = lens[lens != lens_max]

    empty = [""]
    for ind, length in not_max.items():
        body[ind] += empty * (lens_max - length)

def _data_to_frame(**kwargs):
    head, body, foot = kwargs.pop("data")
    header = kwargs.pop("header")
    kwargs["skiprows"] = _get_skiprows(kwargs["skiprows"])
    if head:
        body = head + body

        # Infer header when there is a
 or top -only rows
        if header is None:
            if len(head) == 1:
                header = 0
            else:
                # ignore all-empty-text rows
                header = [i for i, row in enumerate(head) if any(text for text in row)]

    if foot:
        body += foot

    # fill out elements of body that are "ragged"
    _expand_elements(body)
    with TextParser(body, header=header, **kwargs) as tp:
        return tp.read()

_valid_parsers = {
    "lxml": _LxmlFrameParser,
    None: _LxmlFrameParser,
    "html5lib": _BeautifulSoupHtml5LibFrameParser,
    "bs4": _BeautifulSoupHtml5LibFrameParser,
}

def _parser_dispatch(flavor):
    """
    Choose the parser based on the input flavor.

    Parameters
    ----------
    flavor : str
        The type of parser to use. This must be a valid backend.

    Returns
    -------
    cls : _HtmlFrameParser subclass
        The parser class based on the requested input flavor.

    Raises
    ------
    ValueError
        * If `flavor` is not a valid backend.
    ImportError
        * If you do not have the requested `flavor`
    """
    valid_parsers = list(_valid_parsers.keys())
    if flavor not in valid_parsers:
        raise ValueError(
            f"{repr(flavor)} is not a valid flavor, valid flavors are {valid_parsers}"
        )

    if flavor in ("bs4", "html5lib"):
        if not _HAS_HTML5LIB:
            raise ImportError("html5lib not found, please install it")
        if not _HAS_BS4:
            raise ImportError("BeautifulSoup4 (bs4) not found, please install it")
        # Although we call this above, we want to raise here right before use.
        bs4 = import_optional_dependency("bs4")  # noqa:F841

    else:
        if not _HAS_LXML:
            raise ImportError("lxml not found, please install it")
    return _valid_parsers[flavor]

def _print_as_set(s) -> str:
    arg = ", ".join([pprint_thing(el) for el in s])
    return f"{{{arg}}}"

def _validate_flavor(flavor):
    if flavor is None:
        flavor = "lxml", "bs4"
    elif isinstance(flavor, str):
        flavor = (flavor,)
    elif isinstance(flavor, abc.Iterable):
        if not all(isinstance(flav, str) for flav in flavor):
            raise TypeError(
                f"Object of type {repr(type(flavor).__name__)} "
                f"is not an iterable of strings"
            )
    else:
        msg = repr(flavor) if isinstance(flavor, str) else str(flavor)
        msg += " is not a valid flavor"
        raise ValueError(msg)

    flavor = tuple(flavor)
    valid_flavors = set(_valid_parsers)
    flavor_set = set(flavor)

    if not flavor_set & valid_flavors:
        raise ValueError(
            f"{_print_as_set(flavor_set)} is not a valid set of flavors, valid "
            f"flavors are {_print_as_set(valid_flavors)}"
        )
    return flavor

def _parse(flavor, io, match, attrs, encoding, displayed_only, **kwargs):
    flavor = _validate_flavor(flavor)
    compiled_match = re.compile(match)  # you can pass a compiled regex here

    retained = None
    for flav in flavor:
        parser = _parser_dispatch(flav)
        p = parser(io, compiled_match, attrs, encoding, displayed_only)

        try:
            tables = p.parse_tables()
        except ValueError as caught:
            # if `io` is an io-like object, check if it's seekable
            # and try to rewind it before trying the next parser
            if hasattr(io, "seekable") and io.seekable():
                io.seek(0)
            elif hasattr(io, "seekable") and not io.seekable():
                # if we couldn't rewind it, let the user know
                raise ValueError(
                    f"The flavor {flav} failed to parse your input. "
                    "Since you passed a non-rewindable file "
                    "object, we can't rewind it to try "
                    "another parser. Try read_html() with a different flavor."
                ) from caught

            retained = caught
        else:
            break
    else:
        assert retained is not None  # for mypy
        raise retained

    ret = []
    for table in tables:
        try:
            ret.append(_data_to_frame(data=table, **kwargs))
        except EmptyDataError:  # empty table
            continue
    return ret

@deprecate_nonkeyword_arguments(version="2.0")
def read_html(
    io: FilePathOrBuffer,
    match: str | Pattern = ".+",
    flavor: str | None = None,
    header: int | Sequence[int] | None = None,
    index_col: int | Sequence[int] | None = None,
    skiprows: int | Sequence[int] | slice | None = None,
    attrs: dict[str, str] | None = None,
    parse_dates: bool = False,
    thousands: str | None = ",",
    encoding: str | None = None,
    decimal: str = ".",
    converters: dict | None = None,
    na_values=None,
    keep_default_na: bool = True,
    displayed_only: bool = True,
) -> list[DataFrame]:
    r"""
    Read HTML tables into a ``list`` of ``DataFrame`` objects.

    Parameters
    ----------
    io : str, path object or file-like object
        A URL, a file-like object, or a raw string containing HTML. Note that
        lxml only accepts the http, ftp and file url protocols. If you have a
        URL that starts with ``'https'`` you might try removing the ``'s'``.

    match : str or compiled regular expression, optional
        The set of tables containing text matching this regex or string will be
        returned. Unless the HTML is extremely simple you will probably need to
        pass a non-empty string here. Defaults to '.+' (match any non-empty
        string). The default value will return all tables contained on a page.
        This value is converted to a regular expression so that there is
        consistent behavior between Beautiful Soup and lxml.

    flavor : str, optional
        The parsing engine to use. 'bs4' and 'html5lib' are synonymous with
        each other, they are both there for backwards compatibility. The
        default of ``None`` tries to use ``lxml`` to parse and if that fails it
        falls back on ``bs4`` + ``html5lib``.

    header : int or list-like, optional
        The row (or list of rows for a :class:`~pandas.MultiIndex`) to use to
        make the columns headers.

    index_col : int or list-like, optional
        The column (or list of columns) to use to create the index.

    skiprows : int, list-like or slice, optional
        Number of rows to skip after parsing the column integer. 0-based. If a
        sequence of integers or a slice is given, will skip the rows indexed by
        that sequence.  Note that a single element sequence means 'skip the nth
        row' whereas an integer means 'skip n rows'.

    attrs : dict, optional
        This is a dictionary of attributes that you can pass to use to identify
        the table in the HTML. These are not checked for validity before being
        passed to lxml or Beautiful Soup. However, these attributes must be
        valid HTML table attributes to work correctly. For example, ::

            attrs = {'id': 'table'}

        is a valid attribute dictionary because the 'id' HTML tag attribute is
        a valid HTML attribute for *any* HTML tag as per `this document
        <https://html.spec.whatwg.org/multipage/dom.html#global-attributes>`__. ::

            attrs = {'asdf': 'table'}

        is *not* a valid attribute dictionary because 'asdf' is not a valid
        HTML attribute even if it is a valid XML attribute.  Valid HTML 4.01
        table attributes can be found `here
        <http://www.w3.org/TR/REC-html40/struct/tables.html#h-11.2>`__. A
        working draft of the HTML 5 spec can be found `here
        <https://html.spec.whatwg.org/multipage/tables.html>`__. It contains the
        latest information on table attributes for the modern web.

    parse_dates : bool, optional
        See :func:`~read_csv` for more details.

    thousands : str, optional
        Separator to use to parse thousands. Defaults to ``','``.

    encoding : str, optional
        The encoding used to decode the web page. Defaults to ``None``.``None``
        preserves the previous encoding behavior, which depends on the
        underlying parser library (e.g., the parser library will try to use
        the encoding provided by the document).

    decimal : str, default '.'
        Character to recognize as decimal point (e.g. use ',' for European
        data).

    converters : dict, default None
        Dict of functions for converting values in certain columns. Keys can
        either be integers or column labels, values are functions that take one
        input argument, the cell (not column) content, and return the
        transformed content.

    na_values : iterable, default None
        Custom NA values.

    keep_default_na : bool, default True
        If na_values are specified and keep_default_na is False the default NaN
        values are overridden, otherwise they're appended to.

    displayed_only : bool, default True
        Whether elements with "display: none" should be parsed.

    Returns
    -------
    dfs
        A list of DataFrames.

    See Also
    --------
    read_csv : Read a comma-separated values (csv) file into DataFrame.

    Notes
    -----
    Before using this function you should read the :ref:`gotchas about the
    HTML parsing libraries <io.html.gotchas>`.

    Expect to do some cleanup after you call this function. For example, you
    might need to manually assign column names if the column names are
    converted to NaN when you pass the `header=0` argument. We try to assume as
    little as possible about the structure of the table and push the
    idiosyncrasies of the HTML contained in the table to the user.

    This function searches for ``
`` elements and only for ``
``
    and ```` rows and ```` elements within each ``
`` or ````
    element in the table. ```` stands for "table data". This function
    attempts to properly handle ``colspan`` and ``rowspan`` attributes.
    If the function has a ``
`` argument, it is used to construct
    the header, otherwise the function attempts to find the header within
    the body (by putting rows with only ```` elements into the header).

    Similar to :func:`~read_csv` the `header` argument is applied
    **after** `skiprows` is applied.

    This function will *always* return a list of :class:`DataFrame` *or*
    it will fail, e.g., it will *not* return an empty list.

    Examples
    --------
    See the :ref:`read_html documentation in the IO section of the docs
    <io.read_html>` for some examples of reading in HTML tables.
    """
    _importers()

    # Type check here. We don't want to parse only to fail because of an
    # invalid value of an integer skiprows.
    if isinstance(skiprows, numbers.Integral) and skiprows < 0:
        raise ValueError(
            "cannot skip rows starting from the end of the "
            "data (you passed a negative value)"
        )
    validate_header_arg(header)

    io = stringify_path(io)

    return _parse(
        flavor=flavor,
        io=io,
        match=match,
        header=header,
        index_col=index_col,
        skiprows=skiprows,
        parse_dates=parse_dates,
        thousands=thousands,
        attrs=attrs,
        encoding=encoding,
        decimal=decimal,
        converters=converters,
        na_values=na_values,
        keep_default_na=keep_default_na,
        displayed_only=displayed_only,
    )


text:
namespace Dune {
namespace HDD {
namespace LinearElliptic {
namespace TestCases {
namespace OS2014 {
namespace internal {

template< class GridType >
class ParametricConvergenceBase
{
  static_assert(GridType::dimension == 2, "This test case is only available in 2d!");
public:
  typedef typename GridType::template Codim< 0 >::Entity EntityType;
  typedef typename GridType::ctype                       DomainFieldType;
  static const unsigned int                              dimDomain = GridType::dimension;
  typedef double            RangeFieldType;
  static const unsigned int dimRange = 1;
public:
  typedef Problems::OS2014::ParametricESV2007
      < EntityType, DomainFieldType, dimDomain, RangeFieldType, dimRange > ProblemType;
  typedef Stuff::Functions::Constant
      < EntityType, DomainFieldType, dimDomain, RangeFieldType, dimRange > ExactSolutionType;
  typedef typename ProblemType::FunctionType::NonparametricType            FunctionType;

  typedef std::map< std::string, Pymor::ParameterType > ParameterTypesMapType;
  typedef std::map< std::string, Pymor::Parameter >     ParametersMapType;

protected:
  static const size_t default_num_refinements_ = 3;

  static int initial_refinements()
  {
    int ret = 1;
#if HAVE_ALUGRID
    if (std::is_same< GridType, ALUConformGrid< 2, 2 > >::value
        || std::is_same< GridType, ALUGrid< 2, 2, simplex, conforming > >::value)
      ret += 1;
#endif // HAVE_ALUGRID
    return ret;
  } // ... initial_refinements()

public:
  static ParameterTypesMapType required_parameters()
  {
    return ParameterTypesMapType({{"mu",            Pymor::ParameterType("mu", 1)},
                                  {"mu_bar",        Pymor::ParameterType("mu", 1)},
                                  {"mu_hat",        Pymor::ParameterType("mu", 1)},
                                  {"mu_minimizing", Pymor::ParameterType("mu", 1)}});
  }

  ParametricConvergenceBase(const ParametersMapType parameters)
    : parameters_(parameters)
    , boundary_info_cfg_(Stuff::Grid::BoundaryInfoConfigs::AllDirichlet::default_config())
    , problem_(3)
    , exact_solution_(0)
  {}

  void print_header(std::ostream& out = std::cout) const
  {
    out << "+==================================================================+\n"
        << "|+================================================================+|\n"
        << "||  Testcase OS2014: (parametric) estimator convergence study     ||\n"
        << "||  (see Ohlberger, Schindler, 2014)                              ||\n"
        << "|+----------------------------------------------------------------+|\n"
        << "||  domain = [-1, 1] x [-1, 1]                                    ||\n"
        << "||  diffusion = 1 + (1 - mu) 3/4 sin(4 pi (x + 1/2 y))            ||\n"
        << "||  force     = 1/2 pi^2 cos(1/2 pi x) cos(1/2 pi y)              ||\n"
        << "||  dirichlet = 0                                                 ||\n"
        << "||  reference solution: discrete solution on finest grid          ||\n"
        << "||  parameter: mu in [0.1, 1]                                     ||\n"
        << "|+================================================================+|\n"
        << "+==================================================================+" << std::endl;
  } // ... print_header(...)

  const Stuff::Common::Configuration& boundary_info() const
  {
    return boundary_info_cfg_;
  }

  const ProblemType& problem() const
  {
    return problem_;
  }

  bool provides_exact_solution() const
  {
    return false;
  }

  const ExactSolutionType& exact_solution() const
  {
    DUNE_THROW(Stuff::Exceptions::you_are_using_this_wrong,
               "Do not call exact_solution() if provides_exact_solution() is false!");
  }

  const ParametersMapType& parameters() const
  {
    return parameters_;
  }

protected:
  const ParametersMapType parameters_;
  const Stuff::Common::Configuration boundary_info_cfg_;
  const ProblemType problem_;
  const ExactSolutionType exact_solution_;
}; // class ParametricConvergenceBase

} // namespace internal

template< class GridType >
class ParametricConvergence
  : public internal::ParametricConvergenceBase< GridType >
  , public Base< GridType >
{
  typedef internal::ParametricConvergenceBase< GridType > ParametricConvergenceBaseType;
  typedef Base< GridType >                 TestCaseBaseType;

  static std::shared_ptr< GridType > create_initial_grid(const int refinements)
  {
    Stuff::Grid::Providers::Cube< GridType > grid_provider(-1, 1, 4);
    auto grid = grid_provider.grid_ptr();
    grid->globalRefine(refinements);
    return grid;
  } // ... create_initial_grid(...)

public:
  typedef typename TestCaseBaseType::ParametersMapType ParametersMapType;

  using ParametricConvergenceBaseType::required_parameters;
  using ParametricConvergenceBaseType::parameters;

  ParametricConvergence(const ParametersMapType parameters,
                        const size_t num_refinements = ParametricConvergenceBaseType::default_num_refinements_)
    : ParametricConvergenceBaseType(parameters)
    , TestCaseBaseType(create_initial_grid(ParametricConvergenceBaseType::initial_refinements()), num_refinements)
  {
    this->check_parameters(ParametricConvergenceBaseType::required_parameters(), parameters);
    this->inherit_parameter_type(this->problem_, "problem");
  }
}; // class ParametricConvergence

#if HAVE_DUNE_GRID_MULTISCALE

template< class GridType >
class ParametricBlockConvergence
  : public internal::ParametricConvergenceBase< GridType >
  , public MultiscaleCubeBase< GridType >
{
  typedef internal::ParametricConvergenceBase< GridType > ParametricConvergenceBaseType;
  typedef MultiscaleCubeBase< GridType >   TestCaseBaseType;

  static Stuff::Common::Configuration initial_grid_cfg(const std::string num_partitions)
  {
    Stuff::Common::Configuration grid_cfg = Stuff::Grid::Providers::Cube< GridType >::default_config();
    grid_cfg["lower_left"] = "-1";
    grid_cfg["upper_right"] = "1";
    grid_cfg["num_elements"] = "4";
    grid_cfg["num_partitions"] = num_partitions;
    return grid_cfg;
  } // ... initial_grid_cfg(...)

public:
  typedef typename TestCaseBaseType::ParametersMapType ParametersMapType;

  using ParametricConvergenceBaseType::required_parameters;
  using ParametricConvergenceBaseType::parameters;

  ParametricBlockConvergence(const ParametersMapType parameters,
                             const std::string num_partitions = "[1 1 1]",
                             const size_t num_refinements = ParametricConvergenceBaseType::default_num_refinements_)
    : ParametricConvergenceBaseType(parameters)
    , TestCaseBaseType(initial_grid_cfg(num_partitions), ParametricConvergenceBaseType::initial_refinements(), num_refinements)
  {
    this->check_parameters(ParametricConvergenceBaseType::required_parameters(), parameters);
    this->inherit_parameter_type(this->problem_, "problem");
  }
}; // class ParametricBlockConvergence

#endif // HAVE_DUNE_GRID_MULTISCALE

} // namespace OS2014
} // namespace TestCases
} // namespace LinearElliptic
} // namespace HDD
} // namespace Dune

#endif // DUNE_HDD_LINEARELLIPTIC_TESTCASES_OS2014_HH