text:
// Disk class utilities

#include "SAMdisk.h"
#include "DiskUtil.h"
#include "SpecialFormat.h"
#include "TrackDataParser.h"

static const int MIN_DIFF_BLOCK = 16;
static const int DEFAULT_MAX_SPLICE = 72;   // limit of bits treated as splice noise between recognised gap patterns

static void item_separator(int items)
{
    if (!items)
        util::cout << colour::grey << '[' << colour::none;
    else
        util::cout << ',';
}

void DumpTrack(const CylHead& cylhead, const Track& track, const ScanContext& context, int flags)
{
    if (opt.hex != 1)
        util::cout << util::fmt(" %2u.%u  ", cylhead.cyl, cylhead.head);
    else
        util::cout << util::fmt(" %s.%u  ", CylStr(cylhead.cyl), cylhead.head);

    if (track.empty())
        util::cout << colour::grey << "<blank>" << colour::none;
    else
    {
        for (auto& sector : track.sectors())
        {
            util::cout << RecordStr(sector.header.sector);

            // Use 'd' suffix for deleted sectors, or 'a' for alt-DAM
            if (sector.is_deleted())
                util::cout << colour::green << "d" << colour::none;
            else if (sector.is_altdam())
                util::cout << colour::YELLOW << "a" << colour::none;
            else if (sector.is_rx02dam() && sector.encoding != Encoding::RX02)
                util::cout << colour::YELLOW << "x" << colour::none;

            auto items = 0;

            if (sector.header.cyl != context.sector.header.cyl)
            {
                item_separator(items++);
                util::cout << colour::yellow << "c" << std::string(CylStr(sector.header.cyl)) << colour::none;
            }

            if (sector.header.head != context.sector.header.head)
            {
                item_separator(items++);
                util::cout << colour::yellow << "h" << std::string(HeadStr(sector.header.head)) << colour::none;
            }

            if (sector.header.size != context.sector.header.size)
            {
                item_separator(items++);
                util::cout << colour::yellow << "n" << std::string(SizeStr(sector.header.size)) << colour::none;
            }

            if (sector.copies() > 1)
            {
                item_separator(items++);
                util::cout << "m" << sector.copies();
            }

            if (sector.has_badidcrc())
            {
                item_separator(items++);
                util::cout << colour::RED << "ic" << colour::none;  // ID header CRC error
            }

            if (!sector.has_badidcrc() && !sector.has_data())       // No data field
            {
                item_separator(items++);
                util::cout << colour::YELLOW << "nd" << colour::none;
            }

            if (sector.has_baddatacrc())
            {
                item_separator(items++);
                util::cout << colour::RED << "dc" << colour::none;  // Data field CRC error
            }

            if (track.is_repeated(sector))
            {
                item_separator(items++);
                util::cout << colour::YELLOW << "r" << colour::none;    // Repeated sector
            }

            if (sector.has_data() && sector.data_size() == 0)
            {

                item_separator(items++);
                util::cout << colour::YELLOW << "z" << colour::none;    // Zero data stored
            }
            else if (sector.has_data() && !sector.has_baddatacrc() && track.data_overlap(sector))
            {
                item_separator(items++);
                util::cout << colour::YELLOW << "o" << colour::none;    // Overlaps next sector
            }
            else if (sector.has_data() && sector.has_shortdata() && !sector.has_baddatacrc())
            {
                item_separator(items++);
                util::cout << colour::RED << "-" << (sector.size() - sector.data_size()) << colour::none;   // Less data than sector size
            }

            if (sector.encoding != context.sector.encoding)
            {
                item_separator(items++);
                util::cout << colour::CYAN << short_name(sector.encoding) << colour::none;  // FM sector on MFM track
            }

            if (sector.has_gapdata())
            {
                item_separator(items++);
                util::cout << colour::CYAN << "+" << (sector.data_size() - sector.size()) << colour::none;  // More data than sector size (crc, gaps, ...)
            }

            if (opt.verbose > 1 && track.is_8k_sector())
            {
                auto& data = track[0].data_copy();
                auto str_checksum = ChecksumNameShort(ChecksumMethods(data.data(), data.size()));
                if (!str_checksum.empty())
                {
                    item_separator(items++);
                    util::cout << colour::GREEN << str_checksum;
                }
            }

            if (items)
                util::cout << colour::grey << ']' << colour::none;

            util::cout << ' ';
        }
        //      p--;

        // If we've got a track length and sector data, show the sector offsets too
        if ((flags & DUMP_OFFSETS) && !track.empty() && track.tracklen)
        {
            auto prevoffset = 0;
            auto shift = (context.sector.encoding == Encoding::FM) ? 5 : 4;

            util::cout << util::fmt("\n         %s: ", WordStr(track.tracklen >> shift));

            for (const auto& sector : track.sectors())
            {
                auto offset = sector.offset;

                // Invalid or missing offset?
                if (offset < prevoffset)
                {
                    // Show a red question mark instead of the offset
                    util::cout << colour::RED << '?' << colour::none;
                }
                else
                {
                    // Show offset from previous sector
                    if (offset >= prevoffset)
                        util::cout << util::fmt("%s ", WordStr((offset - prevoffset) >> shift));
                    else
                        util::cout << util::fmt("-%s ", WordStr((prevoffset - offset) >> shift));

                    if (!opt.absoffsets)
                        prevoffset = offset;
                }
            }

            // Show the gap until end of track
            auto last_offset = track.sectors()[track.size() - 1].offset;
            if (track.tracklen > last_offset)
                util::cout << util::fmt("[%s]", WordStr((track.tracklen - last_offset) >> shift));
            else
                util::cout << util::fmt("[-%s]", WordStr((last_offset - track.tracklen) >> shift));
        }
    }

    util::cout << "\n";

    if (flags & DUMP_DIFF)
    {
        for (const auto& sector : track.sectors())
        {
            // If we have more than 1 copy, compare them
            if (sector.copies() > 1)
            {
                util::cout << "        diff (" << RecordStr(sector.header.sector) << "): ";

                auto i = 0;
                for (auto& diff : DiffSectorCopies(sector))
                {
                    util::cout << diff.first << diff.second << ' ';
                    if (++i > 12)
                    {
                        util::cout << "...";
                        break;
                    }
                }
                util::cout << '\n';
            }
        }
    }
}

// Normalise track contents, performing overrides and applying fixes as requested.
bool NormaliseTrack(const CylHead& cylhead, Track& track)
{
    bool changed = false;
    int i;

    // Clear the track length if offsets are disabled (cosmetic: no changed flag).
    if (opt.offsets == 0)
        track.tracklen = 0;

    // Pass 1
    for (i = 0; i < track.size(); ++i)
    {
        auto& sector = track[i];

        // Remove sectors with duplicate CHRN?
        if (opt.nodups)
        {
            // Remove duplicates found later on the track.
            for (int j = i + 1; j < track.size(); ++j)
            {
                auto& s = track[j];
                if (s.header.compare_chrn(sector.header) && s.encoding == sector.encoding)
                {
                    track.remove(j--);
                    changed = true;
                }
            }
        }

        // Clear all data, for privacy during diagnostics?
        if (opt.nodata && sector.has_data())
        {
            sector.remove_data();
            sector.add(Data());     // empty data rather than no data
            changed = true;
        }

        // Track offsets disabled? (cosmetic: no changed flag)
        if (opt.offsets == 0)
            sector.offset = 0;
#if 0
        // ToDo: move this to fdrawcmd disk type
        // Build ASRock corruption check block
        if (i < (int)arraysize(ah))
        {
            ah[i].cyl = sector.header.cyl;
            ah[i].head = sector.header.head;
            ah[i].sector = sector.header.sector;
            ah[i].size = sector.header.size;
        }
#endif

        if (sector.has_gapdata())
        {
            // Remove gap data if disabled, or the gap mask doesn't allow it
            if (opt.gaps == GAPS_NONE || !(opt.gapmask & (1 << i)))
            {
                sector.remove_gapdata();
                changed = true;
            }
            // Remove normal gaps unless we're asked to keep them.
            else if (opt.gaps == GAPS_CLEAN && sector.encoding == Encoding::MFM)
            {
                int gap3 = 0;
                if (test_remove_gap3(sector.data_copy(), sector.size(), gap3))
                {
                    sector.remove_gapdata(true);
                    changed = true;

                    if (!sector.gap3)
                        sector.gap3 = gap3;
                }
            }
        }
#if 0
        // ToDo: move this to affected disk types
        // Check for short sector wrapping the track end (includes logoprof.dmk and Les Justiciers 2 [1B].scp [CPC])
        // Also check it doesn't trigger in cyl 18 sector 5 of Super Sprint (KryoFlux).
        if (ps->uData > 0 && tracklen > 0 && ps->offset > 0 &&
            ps->uData < uSize &&
            GetDataExtent(i) > uSize &&
            (ps->offset + GetSectorOverhead(encrate) + uSize) > tracklen)
        {
            Message(msgWarning, "%s truncated at end of track data", CHSR(cyl, head, i, ps->sector));
        }
#endif
    }

    // Pass 2
    for (i = 0; i < track.size(); ++i)
    {
        Sector& sector = track[i];

        // Remove only the final gap if --no-gap4b was used
        if (i == (track.size() - 1) && opt.gap4b == 0 && sector.has_gapdata())
        {
            sector.remove_gapdata(true);
            changed = true;
        }

        // Allow override for sector datarate.
        if (opt.datarate != DataRate::Unknown)
        {
            sector.datarate = opt.datarate;
            changed = true;
        }

        // Allow override for sector encoding.
        if (opt.encoding != Encoding::Unknown)
        {
            sector.encoding = opt.encoding;
            changed = true;
        }

        // Allow overrides for track gap3 (cosmetic: no changed flag)
        if (opt.gap3 != -1)
            sector.gap3 = static_cast<uint8_t>(opt.gap3);
#if 0
        // ToDo: move this to fdrawcmd disk type
        // Check for the ASRock FDC problem that corrupts sector data with format headers (unless comparison block is all one byte)
        int nn = sizeof(ah) - sizeof(ah[0]) - 1;
        if (sectors > 1 && ps->apbData[0] && memcmp(ps->apbData[0], ps->apbData[0] + 1, sizeof(ah) - 1))
        {
            // Ignore the first sector and size in case of a placeholder sector
            if (!memcmp(&ah[1], ps->apbData[0] + sizeof(ah[0]), sizeof(ah) - sizeof(ah[0])))
                Message(msgWarning, "possible FDC data corruption on %s", CHSR(cyl, head, i, ps->sector));
        }
#endif
    }

    int weak_offset, weak_size;

    // Check for Speedlock weak sector (either +3 or CPC)
    if (opt.fix != 0 && cylhead.cyl == 0 && track.size() == 9)
    {
        auto& sector1 = track[1];
        if (sector1.copies() == 1 && IsSpectrumSpeedlockTrack(track, weak_offset, weak_size))
        {
            // Are we to add the missing weak sector?
            if (opt.fix == 1)
            {
                // Add a copy with differences matching the typical weak sector
                auto data = sector1.data_copy();
                for (i = weak_offset; i < data.size(); ++i)
                    data[i] = ~data[i];
                sector1.add(std::move(data), true);

                Message(msgFix, "added suitable second copy of +3 Speedlock weak sector");
                changed = true;
            }
            else
                Message(msgWarning, "missing multiple copies of +3 Speedlock weak sector");
        }

        auto& sector7 = track[7];
        if (sector7.copies() == 1 && IsCpcSpeedlockTrack(track, weak_offset, weak_size))
        {
            // Are we to add the missing weak sector?
            if (opt.fix == 1)
            {
                // Add a second data copy with differences matching the typical weak sector
                auto data = sector7.data_copy();
                for (i = weak_offset; i < data.size(); ++i)
                    data[i] = ~data[i];
                sector7.add(std::move(data), true);

                Message(msgFix, "added suitable second copy of CPC Speedlock weak sector");
                changed = true;
            }
            else
                Message(msgWarning, "missing multiple copies of CPC Speedlock weak sector");
        }
    }

    // Check for Rainbow Arts weak sector missing copies
    if (opt.fix != 0 && cylhead.cyl == 40 && track.size() == 9)
    {
        auto& sector1 = track[1];
        if (sector1.copies() == 1 && IsRainbowArtsTrack(track, weak_offset, weak_size))
        {
            // Are we to add the missing weak sector?
            if (opt.fix == 1)
            {
                // Ensure the weak sector has a data CRC error, to fix broken CPCDiskXP images
                if (!sector1.has_baddatacrc())
                {
                    auto data = sector1.data_copy();
                    sector1.remove_data();
                    sector1.add(std::move(data), true);
                    if (opt.debug) util::cout << "added missing data CRC error to Rainbow Arts track\n";
                }

                // Add a second data copy with differences matching the typical weak sector
                auto data = sector1.data_copy();
                for (i = weak_offset; i < data.size(); ++i)
                    data[i] = ~data[i];
                sector1.add(std::move(data), true);

                Message(msgFix, "added suitable second copy of Rainbow Arts weak sector");
                changed = true;
            }
            else
                Message(msgWarning, "missing multiple copies of Rainbow Arts weak sector");
        }
    }

    // Check for missing OperaSoft 32K sector (CPDRead dumps).
    if (opt.fix != 0 && cylhead.cyl == 40 && track.size() == 9)
    {
        auto& sector7 = track[7];
        auto& sector8 = track[8];
        if (sector7.has_data() && sector8.data_size() == 0 && IsOperaSoftTrack(track))
        {
            if (opt.fix == 1)
            {
                const auto& data7 = sector7.data_copy();

                // Add 0x55 data and correct CRC for 256 bytes
                auto data8{ Data(256, 0x55) };
                data8.push_back(0xe8);
                data8.push_back(0x9f);

                // Fill up to the protection check with gap filler
                auto fill8{ Data(0x512 - data8.size(), 0x4e) };
                data8.insert(data8.end(), fill8.begin(), fill8.end());

                // Append sector 7 data to offset 0x512 to pass the protection check.
                data8.insert(data8.end(), data7.begin(), data7.end());

                // Replace any existing sector 8 data with out hand-crafted version.
                sector8.remove_data();
                sector8.add(std::move(data8), true);

                Message(msgFix, "added missing data to OperaSoft 32K sector");
                changed = true;
            }
            else
                Message(msgWarning, "missing data in OperaSoft 32K sector");
        }
    }

    // Check for Prehistorik track followed by unused KBI-19 sectors (mastering error?).
    if (opt.fix != 0 && track.size() == 13)
    {
        if (track[6].header.sector == 12 && IsPrehistorikTrack(track))
        {
            if (opt.fix == 1)
            {
                // Trim the unwanted sectors.
                while (track.size() > 7)
                {
                    track.remove(7);
                    changed = true;
                }

                Message(msgFix, "removed unused KBI-19 sectors from end of Prehistorik track");
                changed = true;
            }
            else
                Message(msgWarning, "6 junk KBI-19 sectors found (use --fix to remove)");
        }
    }

    // Check for the problematic Reussir protection (CPC).
    if (opt.fix != 0 && track.size() == 10)
    {
        for (auto& s : track)
        {
            if (s.size() != 512 || !s.has_good_data())
                continue;

            auto& data = s.data_copy();
            if (std::memcmp(data.data(), "\0LANCE", 6))
                continue;

            // LD A,(IX+0); CP (HL); JR NZ,e
            static const uint8_t prot_check[]{ 0xdd, 0x7e, 0x00, 0xbe, 0x20 };

            for (i = 0; i < data.size() - static_cast<int>(sizeof(prot_check)); ++i)
            {
                if (data[i] == 0xdd && !memcmp(data.data() + i, prot_check, sizeof(prot_check)))
                {
                    // Are we to fix (disable) the protection?
                    if (opt.fix == 1)
                    {
                        data[i + 3] = 0xaf; // XOR A
                        Message(msgFix, "disabled problematic Reussir protection");
                    }
                    else
                        Message(msgWarning, "detected problematic Reussir protection (use --fix to disable)");
                    break;
                }
            }
        }
    }

    // Single copy of an 8K sector?
    if (opt.check8k != 0 && track.is_8k_sector() && track[0].copies() == 1 && track[0].data_size() >= 0x1801)
    {
        const auto& sector = track[0];
        const auto& data = sector.data_copy();

        static std::mutex map_mutex;
        std::lock_guard<std::mutex> lock(map_mutex);

        // The checksum method can change within a disk, but usually corresponds
        // to a change in sector id or DAM type. Fun Radio [2B] (CPC) does this.
        auto key = std::make_pair(sector.header.sector, sector.dam);
        static std::map<decltype(key), std::set<ChecksumType>> methods_map;
        auto& disk_methods = methods_map[key];

        // Determine the potential 8K checksum methods, if any.
        auto sector_methods = ChecksumMethods(data.data(), data.size());

        if (disk_methods.empty())
        {
            // First of key type follows first sector, or None.
            if (!sector_methods.empty())
                disk_methods = sector_methods;
            else
                disk_methods = { ChecksumType::None };
        }

        // Determine which common methods between disk and sector.
        std::set<ChecksumType> common_methods{};
        std::set_intersection(
            sector_methods.begin(), sector_methods.end(),
            disk_methods.begin(), disk_methods.end(),
            std::inserter(common_methods, common_methods.begin()));

        if (sector_methods.find(ChecksumType::None) != sector_methods.end())
        {
            // If None is an option, there's no checksum.
        }
        else if (common_methods.size() == 1)
        {
            // A single match means a good sector, so have the disk follow it.
            if (disk_methods.size() > 1)
                disk_methods = common_methods;
        }
        else if (disk_methods.empty())
        {
            // Unrecognised 8K method, probably the first on track containing one.
            // Ignore two matching bytes, or a single zero, as likely junk.
            if (data.size() >= 0x1802 && (data[0x1800] != data[0x1801]))
            {
                Message(msgWarning, "unknown or invalid 6K checksum [%02X %02X] on %s",
                    data[0x1800], data[0x1801], CH(cylhead.cyl, cylhead.head));
            }
            else if (data.size() >= 0x1801 && data[0x1800])
            {
                Message(msgWarning, "unknown or invalid 6K checksum [%02X] on %s",
                    data[0x1800], CH(cylhead.cyl, cylhead.head));
            }
        }
        else if (disk_methods.size() == 1 && common_methods.empty() &&
            disk_methods.find(ChecksumType::None) == disk_methods.end())
        {
            // The disk has a method, which the sector lacks. Probably bad.
            auto str_method = ChecksumName(disk_methods);
            auto checksum_len = ChecksumLength(*disk_methods.begin());

            if (checksum_len == 1 || data.size() < 0x1802)
            {
                Message(msgWarning, "invalid %s checksum [%02X] on %s",
                    str_method.c_str(), data[0x1800], CH(cylhead.cyl, cylhead.head));
            }
            else if (checksum_len == 2)
            {
                Message(msgWarning, "invalid %s checksum [%02X %02X] on %s",
                    str_method.c_str(), data[0x1800], data[0x1801],
                    CH(cylhead.cyl, cylhead.head));
            }
        }
    }

    // Return whether the supplied track was modified.
    return changed;
}

bool NormaliseBitstream(BitBuffer& bitbuf)
{
    bool modified = false;

    // Align sync marks to byte boundaries?
    if (opt.align)
        modified |= bitbuf.align();

    return modified;
}

// Attempt to repair a track, given another copy of the same track.
bool RepairTrack(const CylHead& cylhead, Track& track, const Track& src_track)
{
    bool changed = false;

    // Loop over all source sectors available.
    for (auto src_sector : src_track)
    {
        // Skip repeated source sectors, as the data source is ambiguous.
        if (src_track.is_repeated(src_sector))
            continue;

        // In real-world use 250Kbps/300Kbps are interchangable due to 300rpm/360rpm.
        if (!track.empty() &&
            (track[0].datarate == DataRate::_250K || track[0].datarate == DataRate::_300K) &&
            (src_sector.datarate == DataRate::_250K || src_sector.datarate == DataRate::_300K))
        {
            // Convert source to target data rate.
            src_sector.datarate = track[0].datarate;
        }

        // Find a target sector with the same CHRN, datarate, and encoding.
        auto it = track.find(src_sector.header, src_sector.datarate, src_sector.encoding);
        if (it != track.end())
        {
            // Skip repeated target sectors, as the repair target is ambiguous.
            if (track.is_repeated(*it))
                continue;

            // Merge the two sectors to give the best version.
            if (it->merge(std::move(src_sector)) == Sector::Merge::Improved)
            {
                if (it->has_good_data())
                    Message(msgFix, "repaired %s", CHR(cylhead.cyl, cylhead.head, it->header.sector));
                else
                    Message(msgFix, "improved %s", CHR(cylhead.cyl, cylhead.head, it->header.sector));
                changed = true;
            }
        }
        else
        {
            // Default to adding to the end of the track.
            auto insert_idx = track.size();

            // Loop over sectors appearing after the current sector on the source track.
            auto idx_src = src_track.index_of(src_sector);
            for (int i = idx_src + 1; i < src_track.size(); ++i)
            {
                auto& s = src_track[i];

                // Attempt to find the same sector on the target track.
                it = track.find(s.header, s.datarate, s.encoding);
                if (it != track.end())
                {
                    // The missing sector must appear before the match we just found.
                    insert_idx = track.index_of(*it);
                    break;
                }
            }

            Message(msgFix, "added missing %s", CHR(cylhead.cyl, cylhead.head, src_sector.header.sector));
            track.insert(insert_idx, std::move(src_sector));
            changed = true;
        }
    }

    return changed;
}

std::vector<std::pair<char, size_t>> DiffSectorCopies(const Sector& sector)
{
    assert(sector.copies() > 0);
    std::vector<std::pair<char, size_t>> diffs;

    auto& smallest = *std::min_element(
        sector.datas().begin(), sector.datas().end(),
        [](const Data& d1, const Data& d2) {
            return d1.size() < d2.size();
        }
    );
    auto it = smallest.begin();
    auto itEnd = smallest.end();

    //  int extent = pt_->GetDataExtent(nSector_);  // ToDo
    auto change_count = 0;

    auto diff = 0;
    while (it < itEnd)
    {
        auto offset = std::distance(smallest.begin(), it);
        auto same = std::distance(it, itEnd);

        for (const auto& data : sector.datas())
        {
            if (same)
            {
                // Find the prefix length where all data copies match
                auto pair = std::mismatch(it, it + same, data.begin() + offset);
                auto offset2 = std::distance(it, pair.first);
                same = std::min(same, offset2);
            }
        }
        it = it + same;

        // Show the matching block if big enough or if found
        // at the start of the data field. Other fragments will
        // be added to the diff block.
        if (same >= MIN_DIFF_BLOCK || (offset == 0 && same > 0))
        {
            if (diff)
            {
                ++change_count;
                diffs.push_back(std::make_pair('-', diff));
                diff = 0;
            }

            ++change_count;
            diffs.push_back(std::make_pair('=', same));
            same = 0;
        }

        offset = std::distance(smallest.begin(), it);
        auto match = std::distance(it, itEnd);

        for (const auto& data : sector.datas())
        {
            if (match)
            {
                // Find the prefix length where each copy has matching filler
                auto pair = std::mismatch(data.begin() + offset, data.begin() + offset + match - 1, data.begin() + offset + 1);
                auto offset2 = 1 + std::distance(data.begin() + offset, pair.first);
                match = std::min(match, offset2);
            }
        }
        it = it + match;

        // Show the filler block if big enough. Filler has matching
        // bytes in each copy, but a different value across copies.
        if (match >= MIN_DIFF_BLOCK)
        {
            if (diff)
            {
                ++change_count;
                diffs.push_back(std::make_pair('-', diff));
                diff = 0;
            }

            ++change_count;
            diffs.push_back(std::make_pair('+', match));
            match = 0;
        }

        diff += static_cast<int>(same + match);
    }

    if (diff)
    {
        ++change_count;
        diffs.push_back(std::make_pair('-', diff));
    }

    return diffs;
}

// Determine the common properties of sectors on a track.
// This is mostly used by scan track headers, to reduce the detail shown for each sector.
Sector GetTypicalSector(const CylHead& cylhead, const Track& track, const Sector& last)
{
    std::map<DataRate, int> datarates;
    std::map<Encoding, int> encodings;
    std::map<int, int> cyls, heads, sizes, gap3s;

    Sector typical = last;

    // Find the most common values
    for (const auto& sector : track)
    {
        if (++datarates[sector.datarate] > datarates[typical.datarate])
            typical.datarate = sector.datarate;

        if (++encodings[sector.encoding] > encodings[typical.encoding])
            typical.encoding = sector.encoding;

        if (++cyls[sector.header.cyl] > cyls[typical.header.cyl])
            typical.header.cyl = sector.header.cyl;

        if (++heads[sector.header.head] > heads[typical.header.head])
            typical.header.head = sector.header.head;

        if (++sizes[sector.header.size] > sizes[typical.header.size])
            typical.header.size = sector.header.size;

        if (++gap3s[sector.gap3] > gap3s[typical.gap3])
            typical.gap3 = sector.gap3;
    }

    // Use the previous values if the typical values are no better,
    // except for cyl and head where we favour the physical position
    // and prevent a single different sector changing the values.

    if (datarates[typical.datarate] == datarates[last.datarate])
        typical.datarate = last.datarate;

    if (encodings[typical.encoding] == encodings[last.encoding])
        typical.encoding = last.encoding;

    if (cyls[typical.header.cyl] == cyls[cylhead.cyl])
        typical.header.cyl = cylhead.cyl;
    else if (cyls[typical.header.cyl] == cyls[last.header.cyl])
        typical.header.cyl = last.header.cyl;
    else if (cyls[typical.header.cyl] == 1)
        typical.header.cyl = cylhead.cyl;

    if (heads[typical.header.head] == heads[cylhead.head])
        typical.header.head = cylhead.head;
    else if (heads[typical.header.head] == heads[last.header.head])
        typical.header.head = last.header.head;
    else if (heads[typical.header.head] == 1)
        typical.header.head = cylhead.head;

    if (sizes[typical.header.size] == sizes[last.header.size])
        typical.header.size = last.header.size;

    // Use previous gap3 if still present, or if no new gap3 was found.
    if (last.gap3 && (gap3s[last.gap3] || !typical.gap3))
        typical.gap3 = last.gap3;

    return typical;
}

bool WriteRegularDisk(FILE* f_, Disk& disk, const Format& fmt)
{
    auto missing = 0;

    fmt.range().each([&](const CylHead& cylhead) {
        const auto& track = disk.read_track(cylhead);
        Header header(cylhead, 0, fmt.size);

        for (header.sector = fmt.base; header.sector < fmt.base + fmt.sectors; ++header.sector)
        {
            Data buf(fmt.sector_size(), fmt.fill);

            auto it = track.find(header);
            if (it != track.end() && (*it).has_data())
            {
                const auto& data = (*it).data_copy();
                std::copy(data.begin(), data.begin() + std::min(data.size(), buf.size()), buf.begin());
            }
            else
            {
                missing++;
            }

            if (!fwrite(buf.data(), buf.size(), 1, f_))
                throw util::exception("write error, disk full?");
        }
        }, fmt.cyls_first);

    if (missing && !opt.minimal)
        Message(msgWarning, "source missing %u sectors from %u/%u/%u/%u regular format", missing, fmt.cyls, fmt.heads, fmt.sectors, fmt.sector_size());

    return true;
}

bool WriteAppleDODisk(FILE* f_, Disk& disk, const Format& fmt)
{
    auto missing = 0;
    static const int map[16] = { 0, 13, 11, 9, 7, 5, 3, 1, 14, 12, 10, 8, 6, 4, 2, 15 };

    fmt.range().each([&](const CylHead& cylhead) {
        const auto& track = disk.read_track(cylhead);
        Header header(cylhead, 0, fmt.size);

        for (int sector = fmt.base; sector < fmt.base + fmt.sectors; ++sector)
        {
            Data buf(fmt.sector_size(), fmt.fill);
            header.sector = map[sector];

            auto it = track.find(header);
            if (it != track.end() && (*it).has_data())
            {
                const auto& data = (*it).data_copy();
                std::copy(data.begin(), data.begin() + std::min(data.size(), buf.size()), buf.begin());
            }
            else
            {
                missing++;
            }

            if (!fwrite(buf.data(), buf.size(), 1, f_))
                throw util::exception("write error, disk full?");
        }
        }, fmt.cyls_first);

    if (missing && !opt.minimal)
        Message(msgWarning, "source missing %u sectors from %u/%u/%u/%u regular format", missing, fmt.cyls, fmt.heads, fmt.sectors, fmt.sector_size());

    return true;
}

bool test_remove_gap2(const Data& data, int offset)
{
    if (data.size() < offset)
        return false;

    TrackDataParser parser(data.data() + offset, data.size() - offset);
    auto max_splice = (opt.maxsplice == -1) ? DEFAULT_MAX_SPLICE : opt.maxsplice;
    auto splice = 0, len = 0;
    uint8_t fill;

    if (opt.debug) util::cout << "----gap2----:\n";

    parser.GetGapRun(len, fill);

    if (!len)
    {
        splice = 1;
        for (; parser.GetGapRun(len, fill) && !len; ++splice);

        if (opt.debug) util::cout << "found " << splice << " splice bits\n";
        if (splice > max_splice)
        {
            if (opt.debug) util::cout << "stopping due to too many splice bits\n";
            return false;
        }
    }

    if (len > 0 && fill == 0x4e)
    {
        if (opt.debug) util::cout << "found " << len << " bytes of " << fill << " filler\n";
        parser.GetGapRun(len, fill);
    }

    if (!len)
    {
        splice = 1;
        for (; parser.GetGapRun(len, fill) && !len; ++splice);

        if (opt.debug) util::cout << "found " << splice << " splice bits\n";
        if (splice > max_splice)
        {
            if (opt.debug) util::cout << "stopping due to too many splice bits\n";
            return false;
        }
    }

    if (len > 0 && fill == 0x00)
    {
        if (opt.debug) util::cout << "found " << len << " bytes of " << fill << " filler\n";
        parser.GetGapRun(len, fill);
    }

    if (!len)
    {
        splice = 1;
        for (; parser.GetGapRun(len, fill) && !len; ++splice);
        if (opt.debug) util::cout << "found " << splice << " splice bits\n";
        if (splice > max_splice)
            return false;
    }

    if (len > 0)
    {
        if (fill != 0x00)
        {
            if (opt.debug) util::cout << "stopping due to " << len << " bytes of " << fill << " filler\n";
            return false;
        }

        if (opt.debug) util::cout << "found " << len << " bytes of " << fill << " filler\n";
    }

    if (opt.debug) util::cout << "gap2 can be removed\n";
    return true;
}

bool test_remove_gap3(const Data& data, int offset, int& gap3)
{
    if (data.size() < offset)
        return false;

    TrackDataParser parser(data.data() + offset, data.size() - offset);
    auto max_splice = (opt.maxsplice == -1) ? DEFAULT_MAX_SPLICE : opt.maxsplice;
    auto splice = 0, len = 0;
    uint8_t fill = 0x00;
    bool unshifted = true;

    if (opt.debug) util::cout << "----gap3----:\n";
    while (!parser.IsWrapped())
    {
        parser.GetGapRun(len, fill, &unshifted);

        if (!len)
        {
            splice = 1;
            for (; parser.GetGapRun(len, fill) && !len; ++splice);
            if (opt.debug) util::cout << "found " << splice << " splice bits\n";
            if (splice > max_splice)
            {
                if (opt.debug) util::cout << "stopping due to too many splice bits\n";
                return false;
            }
        }

        if (len == 3 && fill == 0xa1)
        {
            auto am = parser.ReadByte();
            if (opt.debug) util::cout << "found AM (" << am << ")\n";
            break;
        }

        if (len > 0 && fill != 0x00 && fill != 0x4e)
        {
            if (opt.debug) util::cout << "stopping due to " << len << " bytes of " << fill << " filler\n";
            return false;
        }

        if (len > 0 && fill == 0x4e && !gap3)
        {
            gap3 = static_cast<uint8_t>(len);
            if (opt.debug) util::cout << "gap3 size is " << len << " bytes\n";
        }
        if (len > 0)
        {
            if (opt.debug) util::cout << "found " << len << " bytes of " << fill << " filler\n";
        }
    }

    // Ignore the detected gap3 if it's not naturally aligned in the bitstream.
    if (gap3 && !unshifted)
        gap3 = 0;

    if (opt.debug) util::cout << "gap3 can be removed\n";
    return true;
}

bool test_remove_gap4b(const Data& data, int offset)
{
    if (data.size() < offset)
        return false;

    TrackDataParser parser(data.data() + offset, data.size() - offset);
    auto splice = 0, len = 0;
    uint8_t fill;

    if (opt.debug) util::cout << "----gap4b----:\n";

    parser.GetGapRun(len, fill);

    if (!len)
    {
        splice = 1;
        for (; parser.GetGapRun(len, fill) && !len; ++splice);
        if (opt.debug) util::cout << "found " << splice << " splice bits\n";
        /*
        auto max_splice = (opt.maxsplice == -1) ? DEFAULT_MAX_SPLICE : opt.maxsplice;
        if (splice > max_splice)
        {
        if (opt.debug) util::cout << "stopping due to too many splice bits\n";
        return false;
        }
        */
    }

    if (len > 0 && (fill == 0x4e || fill == 0x00))
    {
        if (opt.debug) util::cout << "found " << len << " bytes of " << fill << " filler\n";
    }
    else if (len > 0)
    {
        if (opt.debug) util::cout << "stopping due to " << len << " bytes of " << fill << " filler\n";
        return false;
    }

    if (opt.debug) util::cout << "gap4b can be removed\n";
    return true;
}

// Attempt to determine the 8K checksum method for a given data block
std::set<ChecksumType> ChecksumMethods(const uint8_t* buf, int len)
{
    std::set<ChecksumType> methods;

    // If there's not enough data, there can be no checksum. Return an empty set.
    if (len <= 0x1800)
        return methods;

    // 2-byte CRC
    {
        // Check for CRC of 8C 15, which seems to be a strange fixed checksum found on some disks.
        // This is known to be used by The Cycles (+3), Vigilante (CPC), and Les Hits de Noel (CPC).
        if (len >= 0x1803 && buf[0x1800] == 0x8c && buf[0x1801] == 0x15)
            methods.insert(ChecksumType::Constant_8C15);

        // Calculate the CRC-16 of the first 0x1800 bytes, using a custom CRC init of D2F6.
        // This is known to be used by JPP's Goal Busters (CPC).
        if (len >= 0x1802)
        {
            CRC16 crc(buf, 0x1800 + 2, 0xd2f6); // include CRC bytes
            if (!crc)
                methods.insert(ChecksumType::CRC_D2F6_1800);
        }

        // Calculate the CRC-16 of the first 0x1802 bytes, using a custom CRC init of D2F6.
        // This is known to be used by Les Fous du Foot (CPC).
        if (len >= 0x1804)
        {
            CRC16 crc(buf, 0x1802 + 2, 0xd2f6); // include CRC bytes
            if (!crc)
                methods.insert(ChecksumType::CRC_D2F6_1802);
        }
    }

    // 1-byte checksum
    {
        // Sum the first 6K
        auto sum = static_cast<uint8_t>(std::accumulate(buf, buf + 0x1800, 0));
        if (buf[0x1800] == sum)
            methods.insert(ChecksumType::Sum_1800);

        // XOR together the first 6K
        auto xor_ = static_cast<uint8_t>(
            std::accumulate(buf, buf + 0x1800, 0, std::bit_xor<int>()));
        if (buf[0x1800] == xor_)
            methods.insert(ChecksumType::XOR_1800);

        if (len >= 0x18a0)
        {
            // Extend the XOR to 0x18a0 bytes, as needed for Coin-Op Hits
            xor_ = static_cast<uint8_t>(
                std::accumulate(buf + 0x1800, buf + 0x18a0, xor_, std::bit_xor<uint8_t>()));
            if (buf[0x18a0] == xor_)
                methods.insert(ChecksumType::XOR_18A0);
        }
    }

    // Check 6K of filler on unused tracks (Vigilante on CPC)
    // This is only done if we haven't yet matched another method.
    if (methods.empty() && !memcmp(buf, buf + 1, 0x17ff))
        methods.insert(ChecksumType::None);

    return methods;
}

std::string ChecksumName(std::set<ChecksumType> methods)
{
    std::stringstream ss;
    bool first = true;

    for (auto& m : methods)
    {
        if (!first)
            ss << "|";
        first = false;

        switch (m)
        {
        case ChecksumType::None:            ss << "None"; break;
        case ChecksumType::Constant_8C15:   ss << "Constant_8C15"; break;
        case ChecksumType::Sum_1800:        ss << "Sum"; break;
        case ChecksumType::XOR_1800:        ss << "XOR"; break;
        case ChecksumType::XOR_18A0:        ss << "XOR_18A0"; break;
        case ChecksumType::CRC_D2F6_1800:   ss << "CRC_D2F6"; break;
        case ChecksumType::CRC_D2F6_1802:   ss << "CRC_D2F6_1802"; break;
        }
    }

    return ss.str();
}

std::string ChecksumNameShort(std::set<ChecksumType> methods)
{
    std::stringstream ss;
    bool first = true;

    for (auto& m : methods)
    {
        if (!first)
            ss << "|";
        first = false;

        switch (m)
        {
        case ChecksumType::None:            ss << "none"; break;
        case ChecksumType::Constant_8C15:   ss << "8C15"; break;
        case ChecksumType::Sum_1800:        ss << "sum"; break;
        case ChecksumType::XOR_1800:        ss << "xor"; break;
        case ChecksumType::XOR_18A0:        ss << "xorA0"; break;
        case ChecksumType::CRC_D2F6_1800:   ss << "crc0"; break;
        case ChecksumType::CRC_D2F6_1802:   ss << "crc2"; break;
        }
    }

    return ss.str();
}

int ChecksumLength(ChecksumType method)
{
    switch (method)
    {
    case ChecksumType::None:
        break;
    case ChecksumType::Sum_1800:
    case ChecksumType::XOR_1800:
    case ChecksumType::XOR_18A0:
        return 1;
    case ChecksumType::Constant_8C15:
    case ChecksumType::CRC_D2F6_1800:
    case ChecksumType::CRC_D2F6_1802:
        return 2;
    }

    return 0;
}

void scale_flux(std::vector<uint32_t>& flux_rev, uint64_t numerator, uint64_t denominator)
{
    uint64_t old_total = 0, new_total = 0;
    for (auto& time : flux_rev)
    {
        old_total += time;
        auto new_target = old_total * numerator / denominator;
        time = static_cast<uint32_t>(new_target - new_total);
        new_total += time;
    }
}


text:
using System;
using System.Activities;
using System.ServiceModel;
using System.Globalization;
using System.Runtime.Serialization;
using System.IO;
using System.Text;
using System.Net;
using Microsoft.Xrm.Sdk;
using Microsoft.Xrm.Sdk.Workflow;
using System.Xml;
using System.Xml.XPath;

namespace DemoCrmWorkflowActivities
{
    public class GetLocation : CodeActivity
    {
        [Input("Bing key")]
        public InArgument<String> BingKey { get; set; }

        [Input("Country")]
        public InArgument<String> Country { get; set; }

        [Input("State province")]
        public InArgument<String> StateProvince { get; set; }

        [Input("City")]
        public InArgument<String> City { get; set; }

        [Input("Postal code")]
        public InArgument<String> PostalCode { get; set; }

        [Input("Address")]
        public InArgument<String> Address { get; set; }

        [Output("Latitude")]
        public OutArgument<String> Latitude { get; set; }

        [Output("Longitude")]
        public OutArgument<String> Longitude { get; set; }

        private string _processName = "GetBingLocation";
        private string _webAddress = "http://dev.virtualearth.net/REST/v1/Locations/";

        /// <summary>
        /// Executes the workflow activity.
        /// </summary>
        /// <param name="executionContext">The execution context.</param>
        protected override void Execute(CodeActivityContext executionContext)
        {
            string country = Country.Get(executionContext);
            string stateProvince = StateProvince.Get(executionContext);
            string address = Address.Get(executionContext);
            string postalCode = PostalCode.Get(executionContext);
            string city = City.Get(executionContext);
            string bingKey = BingKey.Get(executionContext);

            _webAddress += country + "/" + stateProvince + "/" + postalCode + "/" + city + "/" + address + "?o=xml&key=" + bingKey;

            // Create the tracing service
            ITracingService tracingService = executionContext.GetExtension<ITracingService>();

            if (tracingService == null)
            {
                throw new InvalidPluginExecutionException("Failed to retrieve tracing service.");
            }

            tracingService.Trace("Entered " + _processName + ".Execute(), Activity Instance Id: {0}, Workflow Instance Id: {1}",
                executionContext.ActivityInstanceId,
                executionContext.WorkflowInstanceId);

            // Create the context
            IWorkflowContext context = executionContext.GetExtension<IWorkflowContext>();

            if (context == null)
            {
                throw new InvalidPluginExecutionException("Failed to retrieve workflow context.");
            }

            tracingService.Trace(_processName + ".Execute(), Correlation Id: {0}, Initiating User: {1}",
                context.CorrelationId,
                context.InitiatingUserId);

            XmlDocument bingDoc = CallBing(_webAddress);

            //retrieve lat/long
            XmlNamespaceManager nsmgr = new XmlNamespaceManager(bingDoc.NameTable);
            nsmgr.AddNamespace("rest", "http://schemas.microsoft.com/search/local/ws/rest/v1");

            //set lat/long
            XmlNodeList latElements = bingDoc.SelectNodes("//rest:Latitude", nsmgr);
            XmlNodeList lonElements = bingDoc.SelectNodes("//rest:Longitude", nsmgr);

            Latitude.Set(executionContext, latElements[0].InnerText);
            Longitude.Set(executionContext, lonElements[0].InnerText);

            tracingService.Trace("Exiting " + _processName + ".Execute(), Correlation Id: {0}", context.CorrelationId);
        }

        /// <summary>
        /// method to call a RESTful Bing service using HTTP GET and return the XML response to the calling method
        /// </summary>
        /// <param name="svcUri">full URL</param>
        /// <returns>XML response</returns>
        XmlDocument CallBing(string svcUri)
        {
            try
            {
                WebRequest req = WebRequest.Create(svcUri);
                req.GetResponse().GetResponseStream();

                XmlDocument xmlDoc = new XmlDocument();
                xmlDoc.Load(req.GetResponse().GetResponseStream());

                return xmlDoc;
            }
            catch (WebException exception)
            {
                string str = string.Empty;
                if (exception.Response != null)
                {
                    using (StreamReader reader =
                        new StreamReader(exception.Response.GetResponseStream()))
                    {
                        str = reader.ReadToEnd();
                    }
                    exception.Response.Close();
                }
                if (exception.Status == WebExceptionStatus.Timeout)
                {
                    throw new InvalidPluginExecutionException("The timeout elapsed while attempting to issue the request.",
                        exception);
                }
                throw new InvalidPluginExecutionException(String.Format(CultureInfo.InvariantCulture,
                    "A Web exception ocurred while attempting to issue the request. {0}: {1}",
                    exception.Message, str), exception);
            }
            catch (Exception e)
            {
                throw new InvalidPluginExecutionException(String.Format(CultureInfo.InvariantCulture,
                    "Exception: {0}", e.ToString()));
            }
        }
    }
}

redpajama_set_name: RedPajamaGithub


text:
<?php

class Home extends MY_Controller {

    public function __construct()
    {
        parent::__construct();
    }

    public function index()
    {
        $data = array();
        $data['title'] = $this->lang->line('home');

        $this->load->view('templates/header', $data);
        $this->load->view('templates/nav', $data);
        $this->load->view('index', $data);
        $this->load->view('templates/footer', $data);
    }
}


text:
#include <gtest/gtest.h>
#include "tensorflow/contrib/lite/c/builtin_op_data.h"
#include "tensorflow/contrib/lite/interpreter.h"
#include "tensorflow/contrib/lite/kernels/register.h"
#include "tensorflow/contrib/lite/kernels/test_util.h"
#include "tensorflow/contrib/lite/model.h"

namespace tflite {
namespace {

using ::testing::ElementsAreArray;

class GatherOpModel : public SingleOpModel {
 public:
  GatherOpModel(std::initializer_list<int> input_shape, TensorType input_type,
                std::initializer_list<int> positions_shape) {
    input_ = AddInput(input_type);
    positions_ = AddInput(TensorType_INT32);
    output_ = AddOutput(input_type);
    SetBuiltinOp(BuiltinOperator_GATHER, BuiltinOptions_GatherOptions,
                 CreateGatherOptions(builder_, 0).Union());
    BuildInterpreter({input_shape, positions_shape});
  }

  void SetInputFloat(std::initializer_list<float> data) {
    PopulateTensor<float>(input_, data);
  }

  void SetInputUint8(std::initializer_list<uint8_t> data) {
    PopulateTensor<uint8_t>(input_, data);
  }

  void SetInput(std::initializer_list<string> data) {
    PopulateStringTensor(input_, data);
  }

  void SetPositions(std::initializer_list<int> data) {
    PopulateTensor<int>(positions_, data);
  }

  std::vector<float> GetOutputFloat() { return ExtractVector<float>(output_); }
  std::vector<uint8_t> GetOutputUint8() {
    return ExtractVector<uint8_t>(output_);
  }
  std::vector<string> GetOutputString() {
    return ExtractVector<string>(output_);
  }
  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }

 protected:
  int input_;
  int positions_;
  int output_;
};

TEST(GatherOpTest, Shuffle) {
  GatherOpModel m({2, 2}, TensorType_FLOAT32, {2});
  m.SetInputFloat({-2.0, 0.2, 0.7, 0.8});
  m.SetPositions({1, 0});
  m.Invoke();
  EXPECT_THAT(m.GetOutputFloat(),
              ElementsAreArray(ArrayFloatNear({0.7, 0.8, -2, 0.2})));
}

TEST(GatherOpTest, Test0DIndex) {
  GatherOpModel m({2, 2}, TensorType_FLOAT32, {});
  m.SetInputFloat({-2.0, 0.2, 0.7, 0.8});
  m.SetPositions({1});
  m.Invoke();
  EXPECT_THAT(m.GetOutputFloat(), ElementsAreArray(ArrayFloatNear({0.7, 0.8})));
  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2}));
}

TEST(GatherOpTest, Test0DIndexWith0DResult) {
  // 0D tensor is special case in current TFLite. Test it once to make sure
  // existing workarounds are fine with it.
  GatherOpModel m({3}, TensorType_FLOAT32, {});
  m.SetInputFloat({1.0, 2.0, 3.0});
  m.SetPositions({1});
  m.Invoke();
  EXPECT_THAT(m.GetOutputFloat(), ElementsAreArray(ArrayFloatNear({2.0})));
  EXPECT_TRUE(m.GetOutputShape().empty());
}

TEST(GatherOpTest, Test2DIndexWith2DResult) {
  GatherOpModel m({3}, TensorType_FLOAT32, {1, 2});
  m.SetInputFloat({1.0, 2.0, 3.0});
  m.SetPositions({1, 0});
  m.Invoke();
  EXPECT_THAT(m.GetOutputFloat(), ElementsAreArray(ArrayFloatNear({2.0, 1.0})));
  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
}

TEST(FloatGatherOpTest, Duplicate) {
  GatherOpModel m({1, 2, 2}, TensorType_FLOAT32, {2});
  m.SetInputFloat({-2.0, 0.2, 0.7, 0.8});
  m.SetPositions({0, 0});
  m.Invoke();
  EXPECT_THAT(
      m.GetOutputFloat(),
      ElementsAreArray(ArrayFloatNear({-2, 0.2, 0.7, 0.8, -2, 0.2, 0.7, 0.8})));
}

TEST(FloatGatherOpTest, Slice) {
  GatherOpModel m({4, 1}, TensorType_FLOAT32, {2});
  m.SetInputFloat({-2.0, 0.2, 0.7, 0.8});
  m.SetPositions({1, 3});
  m.Invoke();
  EXPECT_THAT(m.GetOutputFloat(), ElementsAreArray(ArrayFloatNear({0.2, 0.8})));
}

TEST(Uint8tGatherOpTest, Shuffle) {
  GatherOpModel m({2, 2}, TensorType_UINT8, {2});
  m.SetInputUint8({133, 134, 14, 15});
  m.SetPositions({1, 0});
  m.Invoke();

  EXPECT_THAT(m.GetOutputUint8(), ElementsAreArray({14, 15, 133, 134}));
}

TEST(GatherOpTest, SimpleString) {
  GatherOpModel m({3}, TensorType_STRING, {2});
  m.SetInput({"A", "B", "C"});
  m.SetPositions({0, 2});
  m.Invoke();
  ASSERT_THAT(m.GetOutputShape(), ElementsAreArray({2}));
  EXPECT_THAT(m.GetOutputString(), ElementsAreArray({"A", "C"}));
}
}  // namespace
}  // namespace tflite

int main(int argc, char** argv) {
  ::tflite::LogToStderr();
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}


text:
(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s})({1:[function(require,module,exports){
'use strict';

Object.defineProperty(exports, "__esModule", {
	value: true
});

var _welcome = require('./welcome');

var _welcome2 = _interopRequireDefault(_welcome);

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

exports.default = {
	Welcome: _welcome2.default
};

},{"./welcome":2}],2:[function(require,module,exports){
"use strict";

Object.defineProperty(exports, "__esModule", {
	value: true
});

var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();

function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }

var Welcome = function () {
	function Welcome(msg) {
		_classCallCheck(this, Welcome);

		this.message = msg;
	}

	_createClass(Welcome, [{
		key: "shout",
		value: function shout() {
			console.log(this.message);
		}
	}]);

	return Welcome;
}();

exports.default = Welcome;

},{}],3:[function(require,module,exports){
'use strict';

var _components = require('./components');

var HELLO_WORLD = new _components.Welcome('Hello from FrontBook!');

HELLO_WORLD.shout();

},{"./components":1}]},{},[3]);


text:
namespace content {
class NavigationHandle;
class NavigationThrottle;
}  // namespace content

namespace weblayer {
class NavigationImpl;
class TabImpl;

class NavigationControllerImpl : public NavigationController,
                                 public content::WebContentsObserver {
 public:
  explicit NavigationControllerImpl(TabImpl* tab);

  NavigationControllerImpl(const NavigationControllerImpl&) = delete;
  NavigationControllerImpl& operator=(const NavigationControllerImpl&) = delete;

  ~NavigationControllerImpl() override;

  // Creates the NavigationThrottle used to ensure WebContents::Stop() is called
  // at safe times. See NavigationControllerImpl for details.
  std::unique_ptr<content::NavigationThrottle> CreateNavigationThrottle(
      content::NavigationHandle* handle);

  // Returns the NavigationImpl for |handle|, or null if there isn't one.
  NavigationImpl* GetNavigationImplFromHandle(
      content::NavigationHandle* handle);

  // Returns the NavigationImpl for |navigation_id|, or null if there isn't one.
  NavigationImpl* GetNavigationImplFromId(int64_t navigation_id);

  // Called when the first contentful paint page load metric is available.
  // |navigation_start| is the navigation start time.
  // |first_contentful_paint_ms| is the duration to first contentful paint from
  // navigation start.
  void OnFirstContentfulPaint(const base::TimeTicks& navigation_start,
                              const base::TimeDelta& first_contentful_paint);

  // Called when the largest contentful paint page load metric is available.
  // |navigation_start| is the navigation start time.
  // |largest_contentful_paint| is the duration to largest contentful paint from
  // navigation start.
  void OnLargestContentfulPaint(
      const base::TimeTicks& navigation_start,
      const base::TimeDelta& largest_contentful_paint);

  void OnPageDestroyed(Page* page);
  void OnPageLanguageDetermined(Page* page, const std::string& language);

#if BUILDFLAG(IS_ANDROID)
  void SetNavigationControllerImpl(
      JNIEnv* env,
      const base::android::JavaParamRef<jobject>& java_controller);
  void Navigate(JNIEnv* env,
                const base::android::JavaParamRef<jstring>& url,
                jboolean should_replace_current_entry,
                jboolean disable_intent_processing,
                jboolean allow_intent_launches_in_background,
                jboolean disable_network_error_auto_reload,
                jboolean enable_auto_play,
                const base::android::JavaParamRef<jobject>& response);
  void GoBack(JNIEnv* env) { GoBack(); }
  void GoForward(JNIEnv* env) { GoForward(); }
  bool CanGoBack(JNIEnv* env) { return CanGoBack(); }
  bool CanGoForward(JNIEnv* env) { return CanGoForward(); }
  void GoToIndex(JNIEnv* env, int index) { return GoToIndex(index); }
  void Reload(JNIEnv* env) { Reload(); }
  void Stop(JNIEnv* env) { Stop(); }
  int GetNavigationListSize(JNIEnv* env) { return GetNavigationListSize(); }
  int GetNavigationListCurrentIndex(JNIEnv* env) {
    return GetNavigationListCurrentIndex();
  }
  base::android::ScopedJavaLocalRef<jstring> GetNavigationEntryDisplayUri(
      JNIEnv* env,
      int index);
  base::android::ScopedJavaLocalRef<jstring> GetNavigationEntryTitle(
      JNIEnv* env,
      int index);
  bool IsNavigationEntrySkippable(JNIEnv* env, int index);
  base::android::ScopedJavaGlobalRef<jobject> GetNavigationImplFromId(
      JNIEnv* env,
      int64_t id);
#endif

  bool should_delay_web_contents_deletion() {
    return should_delay_web_contents_deletion_;
  }

 private:
  class DelayDeletionHelper;

  class NavigationThrottleImpl;

  // Called from NavigationControllerImpl::WillRedirectRequest(). See
  // description of NavigationControllerImpl for details.
  void WillRedirectRequest(NavigationThrottleImpl* throttle,
                           content::NavigationHandle* navigation_handle);

  // NavigationController implementation:
  void AddObserver(NavigationObserver* observer) override;
  void RemoveObserver(NavigationObserver* observer) override;
  void Navigate(const GURL& url) override;
  void Navigate(const GURL& url, const NavigateParams& params) override;
  void GoBack() override;
  void GoForward() override;
  bool CanGoBack() override;
  bool CanGoForward() override;
  void GoToIndex(int index) override;
  void Reload() override;
  void Stop() override;
  int GetNavigationListSize() override;
  int GetNavigationListCurrentIndex() override;
  GURL GetNavigationEntryDisplayURL(int index) override;
  std::string GetNavigationEntryTitle(int index) override;
  bool IsNavigationEntrySkippable(int index) override;

  // content::WebContentsObserver implementation:
  void DidStartNavigation(
      content::NavigationHandle* navigation_handle) override;
  void DidRedirectNavigation(
      content::NavigationHandle* navigation_handle) override;
  void ReadyToCommitNavigation(
      content::NavigationHandle* navigation_handle) override;
  void DidFinishNavigation(
      content::NavigationHandle* navigation_handle) override;
  void DidStartLoading() override;
  void DidStopLoading() override;
  void LoadProgressChanged(double progress) override;
  void DidFirstVisuallyNonEmptyPaint() override;

  void OldPageNoLongerRendered(const GURL& url, bool success);
  void NotifyLoadStateChanged();

  void DoNavigate(
      std::unique_ptr<content::NavigationController::LoadURLParams> params);

  // Schedules a load to happen as soon as possible. This is used in cases
  // where it is not safe to call load. In particular, if a load was just
  // started. Content is generally not reentrant when starting a load and has
  // CHECKs to ensure it doesn't happen.
  void ScheduleDelayedLoad(
      std::unique_ptr<content::NavigationController::LoadURLParams> params);
  void CancelDelayedLoad();
  void ProcessDelayedLoad();

  base::ObserverList<NavigationObserver>::Unchecked observers_;
  std::map<content::NavigationHandle*, std::unique_ptr<NavigationImpl>>
      navigation_map_;

  // If non-null then processing is inside DidStartNavigation() and
  // |navigation_starting_| is the NavigationImpl that was created.
  NavigationImpl* navigation_starting_ = nullptr;

  // Set to non-null while in WillRedirectRequest().
  NavigationThrottleImpl* active_throttle_ = nullptr;

#if BUILDFLAG(IS_ANDROID)
  base::android::ScopedJavaGlobalRef<jobject> java_controller_;
#endif

  // Set to true while processing an observer/callback and it's unsafe to
  // delete the WebContents. This is not used for all callbacks, just the
  // ones that we need to allow deletion from (such as completed/failed).
  bool should_delay_web_contents_deletion_ = false;

  // See comment in ScheduleDelayedLoad() for details.
  std::unique_ptr<content::NavigationController::LoadURLParams>
      delayed_load_params_;

  base::WeakPtrFactory<NavigationControllerImpl> weak_ptr_factory_{this};
};

}  // namespace weblayer

#endif  // WEBLAYER_BROWSER_NAVIGATION_CONTROLLER_IMPL_H_


text:
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;

namespace Kooboo.CMS.Toolkit
{
    public static class ObjectExtensions
    {
        private static readonly string[] Booleans = new string[] { "true", "yes", "on", "1" };

        public static int AsInt(this object value)
        {
            return As<int>(value);
        }

        public static int AsInt(this object value, int defaultValue)
        {
            return As<int>(value, defaultValue);
        }

        public static float AsFloat(this object value)
        {
            return As<float>(value);
        }

        public static float AsFloat(this object value, float defaultValue)
        {
            return As<float>(value, defaultValue);
        }

        public static decimal AsDecimal(this object value)
        {
            return As<decimal>(value);
        }

        public static decimal AsDecimal(this object value, decimal defaultValue)
        {
            return As<decimal>(value, defaultValue);
        }

        public static double AsDouble(this object value)
        {
            return As<double>(value);
        }

        public static double AsDouble(this object value, double defaultValue)
        {
            return As<double>(value, defaultValue);
        }

        public static DateTime AsDateTime(this object value)
        {
            return As<DateTime>(value);
        }

        public static DateTime AsDateTime(this object value, DateTime defaultValue)
        {
            return As<DateTime>(value, defaultValue);
        }

        public static DateTime AsDateTime(this object value, string format)
        {
            return AsDateTime(value, format, default(DateTime));
        }

        public static DateTime AsDateTime(this object value, string format, DateTime defaultValue)
        {
            string date = value.AsString();
            if (!String.IsNullOrEmpty(date))
            {
                try
                {
                    return DateTime.ParseExact(date, format, CultureInfo.InvariantCulture);
                }
                catch { }
            }

            return defaultValue;
        }

        public static string AsString(this object value)
        {
            return AsString(value, String.Empty);
        }

        public static string AsString(this object value, string defaultValue)
        {
            return As<string>(value, defaultValue);
        }

        public static bool AsBool(this object value)
        {
            return AsBool(value, false);
        }

        public static bool AsBool(this object value, bool defaultValue)
        {
            if (value != null && value != DBNull.Value)
            {
                return Booleans.Contains(value.ToString().ToLower());
            }

            return defaultValue;
        }

        public static T As<T>(this object value)
        {
            return As<T>(value, default(T));
        }

        public static T As<T>(this object value, T defaultValue)
        {
            T convertedValue = defaultValue;
            if (value != null && value != DBNull.Value && value is IConvertible)
            {
                try
                {
                    convertedValue = (T)value;
                }
                catch
                {
                    try
                    {
                        convertedValue = (T)Convert.ChangeType(value, typeof(T));
                    }
                    catch
                    { }
                }
            }

            return convertedValue;
        }
    }
}


text:
package net.buycraft.plugin.bungeecord.command;

import net.buycraft.plugin.bungeecord.BuycraftPlugin;
import net.md_5.bungee.api.ChatColor;
import net.md_5.bungee.api.CommandSender;

public class ForceCheckSubcommand implements Subcommand {
    private final BuycraftPlugin plugin;

    public ForceCheckSubcommand(final BuycraftPlugin plugin) {
        this.plugin = plugin;
    }

    @Override
    public void execute(CommandSender sender, String[] args) {
        if (args.length != 0) {
            sender.sendMessage(ChatColor.RED + plugin.getI18n().get("no_params"));
            return;
        }

        if (plugin.getApiClient() == null) {
            sender.sendMessage(ChatColor.RED + plugin.getI18n().get("need_secret_key"));
            return;
        }

        if (plugin.getDuePlayerFetcher().inProgress()) {
            sender.sendMessage(ChatColor.RED + plugin.getI18n().get("already_checking_for_purchases"));
            return;
        }

        plugin.getProxy().getScheduler().runAsync(plugin, () -> plugin.getDuePlayerFetcher().run(false));
        sender.sendMessage(ChatColor.GREEN + plugin.getI18n().get("forcecheck_queued"));
    }

    @Override
    public String getDescription() {
        return plugin.getI18n().get("usage_forcecheck");
    }
}


text:
description: How to link Docker Cloud to a hosted cloud services provider or your own hosts
keywords: node, create, understand
redirect_from:
- /docker-cloud/getting-started/use-hosted/
title: Link to your infrastructure
---

To deploy Docker Cloud nodes, you first need to grant Docker Cloud access to your infrastructure.

This could mean granting access to a cloud services provider such as AWS or Azure, or installing the Docker Cloud Agent on your own hosts. Once this is done, you can provision nodes directly from within Docker Cloud using the Web UI, CLI or API.

## Link to a cloud service provider
To link your cloud provider accounts, first go to your [Docker Cloud dashboard](https://cloud.docker.com/).

Then, use the one of the detailed tutorials below to link your account. You should open the detailed linking tutorial in a new tab or window so you can continue the tutorial when you're finished.

  - [Amazon Web Services](../infrastructure/link-aws.md) (uses an Access Key ID + Secret Access Key)
  - [DigitalOcean](../infrastructure/link-do.md) (uses OAuth)
  - [Microsoft Azure](../infrastructure/link-azure.md) (uses OAuth)
  - [IBM SoftLayer](../infrastructure/link-softlayer.md) (uses an API key)
  - [Packet.net](../infrastructure/link-packet.md) (uses an API key)

  You can always come back and link more cloud service providers later.

## Link to your own hosts (Bring Your Own Node - BYON)

If you are not using a cloud services provider and are using your own hosts, you'll need to install the Docker Cloud Agent on those hosts so that Docker Cloud can communicate with them.  Follow the directions at [Bring Your Own Node instructions](../infrastructure/byoh.md). Open these instructions in a new window or tab so you can return to this tutorial once you're done linking your hosts.

## Ready to go?
Once you've linked to your cloud services provider or to your own hosts, [continue the tutorial and deploy your first node](your_first_node.md).


text:
package com.google.cloud.teleport.templates;

import com.google.cloud.teleport.metadata.Template;
import com.google.cloud.teleport.metadata.TemplateCategory;
import com.google.cloud.teleport.templates.DatastoreToText.DatastoreToTextOptions;
import com.google.cloud.teleport.templates.common.DatastoreConverters.DatastoreReadOptions;
import com.google.cloud.teleport.templates.common.DatastoreConverters.ReadJsonEntities;
import com.google.cloud.teleport.templates.common.FirestoreNestedValueProvider;
import com.google.cloud.teleport.templates.common.JavascriptTextTransformer.JavascriptTextTransformerOptions;
import com.google.cloud.teleport.templates.common.JavascriptTextTransformer.TransformTextViaJavascript;
import com.google.cloud.teleport.templates.common.TextConverters.FilesystemWriteOptions;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.options.ValueProvider;

/**
 * Dataflow template which copies Datastore Entities to a Text sink. Text is encoded using JSON
 * encoded entity in the v1/Entity rest format:
 * https://cloud.google.com/datastore/docs/reference/rest/v1/Entity
 */
@Template(
    name = "Datastore_to_GCS_Text",
    category = TemplateCategory.BATCH,
    displayName = "Datastore to Text Files on Cloud Storage [Deprecated]",
    description =
        "Batch pipeline. Reads Datastore entities and writes them to Cloud Storage as text files.",
    optionsClass = DatastoreToTextOptions.class,
    skipOptions = {"firestoreReadNamespace", "firestoreReadGqlQuery", "firestoreReadProjectId"},
    contactInformation = "https://cloud.google.com/support")
@Template(
    name = "Firestore_to_GCS_Text",
    category = TemplateCategory.BATCH,
    displayName = "Firestore (Datastore mode) to Text Files on Cloud Storage",
    description =
        "Batch pipeline. Reads Firestore entities and writes them to Cloud Storage as text files.",
    optionsClass = DatastoreToTextOptions.class,
    skipOptions = {"datastoreReadNamespace", "datastoreReadGqlQuery", "datastoreReadProjectId"},
    contactInformation = "https://cloud.google.com/support")
public class DatastoreToText {

  public static ValueProvider<String> selectProvidedInput(
      ValueProvider<String> datastoreInput, ValueProvider<String> firestoreInput) {
    return new FirestoreNestedValueProvider(datastoreInput, firestoreInput);
  }

  /** Custom PipelineOptions. */
  public interface DatastoreToTextOptions
      extends PipelineOptions,
          DatastoreReadOptions,
          JavascriptTextTransformerOptions,
          FilesystemWriteOptions {}

  /**
   * Runs a pipeline which reads in Entities from Datastore, passes in the JSON encoded Entities to
   * a Javascript UDF, and writes the JSON to TextIO sink.
   *
   * @param args arguments to the pipeline
   */
  public static void main(String[] args) {
    DatastoreToTextOptions options =
        PipelineOptionsFactory.fromArgs(args).withValidation().as(DatastoreToTextOptions.class);

    Pipeline pipeline = Pipeline.create(options);

    pipeline
        .apply(
            ReadJsonEntities.newBuilder()
                .setGqlQuery(
                    selectProvidedInput(
                        options.getDatastoreReadGqlQuery(), options.getFirestoreReadGqlQuery()))
                .setProjectId(
                    selectProvidedInput(
                        options.getDatastoreReadProjectId(), options.getFirestoreReadProjectId()))
                .setNamespace(
                    selectProvidedInput(
                        options.getDatastoreReadNamespace(), options.getFirestoreReadNamespace()))
                .build())
        .apply(
            TransformTextViaJavascript.newBuilder()
                .setFileSystemPath(options.getJavascriptTextTransformGcsPath())
                .setFunctionName(options.getJavascriptTextTransformFunctionName())
                .build())
        .apply(TextIO.write().to(options.getTextWritePrefix()).withSuffix(".json"));

    pipeline.run();
  }
}


text:
"""Annotation Extension for relational expressions.

   https://link.springer.com/protocol/10.1007/978-1-4939-3743-1_17
   http://wiki.geneontology.org/index.php/Annotation_Extension

   Correlated function associations between genes and GOs containing contextual information.

   With contextual information identify gene products that perform a role:
    - only under certain conditions
    - in the presence of specific factors

   Gene products can have different roles:
     - in different cells
     - in different tissues

"""

import sys
from goatools.anno.extensions.extensions import AnnotationExtensions
from goatools.anno.extensions.extension import AnnotationExtension

__copyright__ = "Copyright (C) 2016-present, DV Klopfenstein, H Tang. All rights reserved."
__author__ = "DV Klopfenstein"

def get_extensions(extstr):
    """Return zero or greater Annotation Extensions, given a line of text."""
    # Extension examples:
    #   has_direct_input(UniProtKB:P37840),occurs_in(GO:0005576)
    #   part_of(UBERON:0006618),part_of(UBERON:0002302)
    #   occurs_in(CL:0000988)|occurs_in(CL:0001021)
    if not extstr:
        return None
    exts = []
    for ext_lst in extstr.split('|'):
        grp = []
        for ext in ext_lst.split(','):
            idx = ext.find('(')
            if idx != -1 and ext[-1] == ')':
                grp.append(AnnotationExtension(ext[:idx], ext[idx+1:-1]))
            else:
                # Ignore improperly formatted Extensions
                sys.stdout.write('BAD Extension({E})\n'.format(E=ext))
        exts.append(grp)
    return AnnotationExtensions(exts)

# Copyright (C) 2016-present, DV Klopfenstein, H Tang. All rights reserved."


text:
<?php

declare(strict_types=1);

namespace JsonStreamingParser\Listener;

interface ListenerInterface
{
    public function startDocument(): void;

    public function endDocument(): void;

    public function startObject(): void;

    public function endObject(): void;

    public function startArray(): void;

    public function endArray(): void;

    public function key(string $key): void;

    /**
     * @param mixed $value the value as read from the parser, it may be a string, integer, boolean, etc
     */
    public function value($value);

    public function whitespace(string $whitespace): void;
}


text:
<?php

include ("dbcon.php");

function insert($dbconnect, $table, $insertData) { //Dynamisk insert funksjon - setter inn med hensyn på hvilken tabell og verdi som skal inn i databasen
    $columns = implode(", ",array_keys($insertData));
    $escaped_values = array_map(array($dbconnect, 'real_escape_string'), array_values($insertData));
    $values  = "'" . implode("', '", array_values($escaped_values)) . "'";
    $sql = "INSERT INTO $table ($columns) VALUES ($values);";
    echo "
$sql"."
";
    mysqli_query($dbconnect, $sql) or die(mysqli_error($dbconnect));
    echo "
 Vellykket
";

}
?>


text:
groupBy.js
==========

Group an array of objects according to a common key

##About
Easily group an array of objects according to a common key. No duplication of identical fields, and automatic array creation when fields with the same key clash. It ignores non-object elements and concatenates arrays in fields on merge.

##Example
Let's say you have an array of objects:

```javascript
	var array = [
		{ name : 'steve', email : 'steve@example.com' },
		{ name : 'joan', email : 'joan@example.com' },
		{ name : 'mary', email : 'mary@example.com' },
		{ name : 'steve', email : ['steve2@example.com', 'steve3@example.com'] },
		{ otherkey : 'super' }
	];
```

```array.groupBy('name')``` would return a new array:

```javascript
	[
		{
			name  : 'steve',
			email : [
				'steve@example.com',
				'steve2@example.com',
				'steve3@example.com'
			]
		},
		{ name : 'joan', email : 'joan@example.com' },
		{ name : 'mary', email : 'mary@example.com' }
	];
```

##Warning
- Any object in the array not containing the key **will not be included** in the output array.
- By design, groupBy will merge two objects with the same key but different values into an array.
- If either (or both) field is an array, it will concat the two. This means it **will always produce a flat array**.

###Examples of output
Two fields are strings
```javascript
	var a = { key : 'admin', word : 'hello' }
	var b = { key : 'admin', word : 'world' }
	var c = { key : 'user', word : 'sweet' }

	[a,b,c].groupBy('key') =>
	[
		{ key : 'admin', word : ['hello','world'] },
		{ key : 'user', word : 'sweet' }
	]
```

One field is an array
```javascript
	var a = { key : 'admin', word : [1,2,3] }
	var b = { key : 'admin', word : ['string', 4] }
	[a,b].groupBy('key') => [{key : admin, word : [1,2,3,'string',4]}]
```


text:
module Fix
  module Protocol
    module Messages
      #
      # A FIX heartbeat message
      #
      class Heartbeat < ::Fix::Protocol::Message
        unordered :body do
          # common fields
          field :app_ver_id,      tag: 1128
          field :sender_comp_id,  tag: 49,    required: true
          field :target_comp_id,  tag: 56,    required: true
          field :msg_seq_num,     tag: 34,    required: true, type: :integer
          field :sending_time,    tag: 52,    required: true, type: :timestamp, default: proc { Time.now.utc }

          field :test_req_id, tag: 112
        end
      end
    end
  end
end