text:
//
//
// MIT License
//
// Copyright (c) 2017 Stellacore Corporation.
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject
// to the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
// KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
// AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//
//

/*! \file
\brief  This file contains unit test for dat::MinMax
*/

#include "libdat/MinMax.h"

#include "libio/stream.h"

#include <cmath>
#include <iostream>
#include <sstream>
#include <string>

namespace
{

//! Check for common functions
std::string
dat_MinMax_test0
	()
{
	std::ostringstream oss;
	dat::MinMax<double> const aNull;
	if (aNull.isValid())
	{
		oss << "Failure of null value test" << std::endl;
		oss << "infoString: " << aNull.infoString("aNull") << std::endl;
	}

	if (aNull.nearlyEquals(aNull))
	{
		oss << "Failure of null/null nearlyEquals test" << std::endl;
	}
	dat::MinMax<double> const goodA(.2, M_PI);
	dat::MinMax<double> const goodB(1./5., std::atan2(0., -1.));
	if (! goodA.nearlyEquals(goodB))
	{
		oss << "Failure of good/good nearlyEquals test" << std::endl;
	}

	return oss.str();
}

//! Check basic properties
std::string
dat_MinMax_test1
	()
{
	std::ostringstream oss;

	// check for bad constructions
	dat::MinMax<int> const bad1(5, -7);
	if (bad1.isValid())
	{
		oss << "Failure of bad1 test" << std::endl;
	}

	// check for good construction
	float const expMin(-7.f);
	float const expMax( 9.f);
	dat::MinMax<float> const mm(expMin, expMax);
	if (! mm.isValid())
	{
		oss << "Failure of mm validity test" << std::endl;
	}
	float const gotMin(mm.min());
	float const gotMax(mm.max());

	// test can be exact since not operations are involved
	if (! (gotMin == expMin))
	{
		oss <<  "Failure of min value test" << std::endl;
	}
	if (! (gotMax == expMax))
	{
		oss <<  "Failure of max value test" << std::endl;
	}
	std::pair<float, float> const expRange(expMin, expMax);
	std::pair<float, float> const gotRange(mm.pair());
	if (! dat::nearlyEquals(gotRange, expRange))
	{
		oss << "Failure of pair value test" << std::endl;
	}

	// check operations
	float const isIn(.125f * (expMin + expMax));
	float const isOut(1.0625f * expMax);
	float const inEnd(expMin);
	float const outEnd(expMax);
	if (! mm.containsInclusive(isIn))
	{
		oss << "Failure of containsInclusive in test" << std::endl;
	}
	if (  mm.containsInclusive(isOut))
	{
		oss << "Failure of containsInclusive out test" << std::endl;
	}
	if (! mm.containsInclusive(inEnd))
	{
		oss << "Failure of containsInclusive inMin test" << std::endl;
	}
	if (! mm.containsInclusive(outEnd))
	{
		oss << "Failure of containsInclusive outMax test" << std::endl;
	}

	// check expansion of values
	dat::MinMax<float> grow;
	grow = grow.expandedWith(isIn);
	if (! grow.isValid())
	{
		oss << "Failure of expanded null test" << std::endl;
	}
	grow = grow.expandedWith(expMin);
	grow = grow.expandedWith(expMax);

	if (! grow.nearlyEquals(mm))
	{
		oss << "Failure of expansion (by scalar) test" << std::endl;
	}

	// check expansion with another min/max
	{
		typedef dat::MinMax<int> mm_t;
		mm_t const mm0(-4, 5);
		// testCase, expected
		std::vector<std::pair<dat::MinMax<int>, dat::MinMax<int> > >
			const mmSpans
			{{ std::make_pair(mm_t(-3,  4), mm_t(-4,  5))
			 , std::make_pair(mm_t(-5,  6), mm_t(-5,  6))
			 , std::make_pair(mm_t(-5, -1), mm_t(-5,  5))
			 , std::make_pair(mm_t( 1,  6), mm_t(-4,  6))
			}};
		for (size_t nn(0u) ; nn < mmSpans.size() ; ++nn)
		{
			mm_t const & mmT = mmSpans[nn].first;
			mm_t const & expMM = mmSpans[nn].second;
			mm_t const gotMM(mm0.expandedWith(mmT));
			if (! gotMM.nearlyEquals(expMM))
			{
				oss << "Failure of expandedWith(MM) test" << std::endl;
				oss << mmT.infoString("mmT") << std::endl;
				oss << expMM.infoString("expMM") << std::endl;
				oss << gotMM.infoString("gotMM") << std::endl;
			}
		}
	}

	return oss.str();
}

//! Check expand
std::string
dat_MinMax_test2
	()
{
	std::ostringstream oss;

	// basic 'normal' operations
	{
		dat::MinMax<double> mmA(1., 2.);
		dat::MinMax<double> mmB(1.5, 2.5);
		dat::MinMax<double> expMM(1., 2.5);
		dat::MinMax<double> gotAB{ mmA.expandedWith(mmB) };
		dat::MinMax<double> gotBA{ mmB.expandedWith(mmA) };
		if (! gotAB.nearlyEquals(expMM))
		{
			oss << "Failure of expanded AB test" << std::endl;
		}
		if (! gotBA.nearlyEquals(expMM))
		{
			oss << "Failure of expanded BA test" << std::endl;
		}
	}

	// check null handling
	{
		double const nan{ dat::nullValue<double>() };
		dat::MinMax<double> mmXX(nan, nan);
		dat::MinMax<double> mmXo(nan,  1.);
		dat::MinMax<double> mmoX( 1., nan);
		dat::MinMax<double> mmoo( 1.,  1.);
		dat::MinMax<double> null{};
		dat::MinMax<double> gotXX{ null.expandedWith(mmXX) };
		dat::MinMax<double> gotXo{ null.expandedWith(mmXo) };
		dat::MinMax<double> gotoX{ null.expandedWith(mmoX) };
		dat::MinMax<double> gotoo{ null.expandedWith(mmoo) };
		if (gotXX.isValid() || gotXo.isValid() || gotoX.isValid())
		{
			oss << "Failure of null-expand-null test" << std::endl;
		}
		if (! gotoo.isValid())
		{
			oss << "Failure of null-expand-good test" << std::endl;
		}
		dat::MinMax<double> gotGood{ mmoo.expandedWith(null) };
		if (! gotGood.isValid())
		{
			oss << "Failure of good-expand-null test" << std::endl;
		}
	}

	return oss.str();
}

}

//! Unit test for dat::MinMax
int
main
	( int const /*argc*/
	, char const * const * const /*argv*/
	)
{
	std::ostringstream oss;

	// run tests
	oss << dat_MinMax_test0();
	oss << dat_MinMax_test1();
	oss << dat_MinMax_test2();

	// check/report results
	std::string const errMessages(oss.str());
	if (! errMessages.empty())
	{
		io::err() << errMessages << std::endl;
		return 1;
	}
	return 0;
}


text:
<!DOCTYPE html>

    <meta charset="UTF-8">
    <title>CISC 662 Course Project - Nathan Rague</title>

    <link rel="stylesheet" href="/css/api/bootstrap/bootstrap.min.css">
    <link rel="stylesheet" href="/css/api/docs/docs.css">

Introduction

Project Info

Data Sets

Introduction

                        This page outlines the resources used in a project created for CISC 662, Data Mining and KDD at NOVA
                        Southeastern University in the 2017 Summer I term.
                        The purpose of this project was to use the K-Nearest Neighbor classify to accurately detect
                        the presence of malicious web traffic using bigram frequencies of HTTP GET URLs in web logs. This project
                        is the culmination of 87 programming hours by Nathan Rague for the classes's term project.

Project Info & Execution

Source Code

The view the source code please click "Source Code" below. Relevant function are in
                         the application/controllers directory.

                            View Source Code

Run Classifier

To run the classifier yourself and view of a summary of its performance,
                            please click "Run KNN Classifier".

Server Configuration:

To view the server's architecture and config, please click "Server Info" below:

                            Server Info

                        Run KNN Classifier

                        Please allow 3-5 minutes of execution time

Data Sets

The following section includes the original apache web log files, parsed HTTP files and the
                            resulting generated training and testing data sets. The training data set is comprised of
                            2000 labeled lines of which the first 1000 are labeled malicious and the remaining 1000 are labeled normal.
                            The testing data set contains 1000 lines of testing bigrams of which the first 500 are unlabeled normal
                            records and the second 500 lines are unlabeled malicious records.

HTTP Dataset CSIC 2010 - Normal Traffic

                                normalTrafficTest.txt (19.6MB)

HTTP Dataset CSIC 2010 - Malicious Traffic

                                anomalousTrafficTest.txt (15.3MB)

Parsed Log Files for Normal HTTP GETs:

                                parsed_normal1.txt (1.1MB)

Parsed Log Files for Malicious HTTP GETs:

                                parsed_malicious1.txt (1.4MB)

Training Data set (2000 Lines):

                                TrainingSet_Final5000.csv (3.8MB)

Testing Data set (1000 Lines):

                                TestSet_both500.txt (2.0MB)

redpajama_set_name: RedPajamaGithub


text:
ActiveRecord::Schema.define(version: 0) do

  create_table "ar_internal_metadata", primary_key: "key", force: true do |t|
    t.string   "value"
    t.datetime "created_at", null: false
    t.datetime "updated_at", null: false
  end

end


text:
import gevent
from gevent.queue import Queue

from .log import log
from .identity import SwitchID
from .remote import RemoteSwitch

class DHT(gevent.Greenlet):
    """Manages information about remote Switches and Lines"""

    k = 8
    concurrency = 3
    kbucket = []
    linemap = {}
    active = {}

    def __init__(self, local_id, transmit):
        self.me = local_id
        self.transmit = transmit
        super(DHT, self).__init__()

    def _run(self):
        self.running = True
        for remote in self.active.values():
            gevent.spawn(self.send_seek, remote, self.me.hash_name)
        while self.running:
            self.maintain()
            gevent.sleep(30)

    def maintain(self):
        """DHT Maintenance

        initial TODO:
        put active lines into kbuckets
        remove inactive lines from kbuckets (why bother?)
        remove remaining excess lines from kbuckets
        ping near-expired lines still in kbuckets
        signal unbucketed lines to check for active user channels?
        expire old switches?
        """
        pass

    def seek(self, hn):
        see_list = []
        switch_id = SwitchID(hn)
        bkt = self.me.kdist(switch_id)
        log.debug("%s in bucket: %s" % (hn, bkt))
        if hn == self.me.hash_name:
            pass
        elif hn in self.active:
            remote = self.active.get(hn)
            if remote.best_path() is not None:
                ip, port = remote.best_path()
                see = ','.join((hn, ip, str(port)))
                see_list.append(see)
        return see_list

    def register(self, switch_id, paths=[]):
        remote = self.active.get(switch_id.hash_name)
        if remote is None:
            remote = RemoteSwitch(switch_id, self)
            self.active[switch_id.hash_name] = remote
            remote.path_hint(paths)
            remote.start()
        elif switch_id.known:
            remote.id.found_key(switch_id.pub_key_der)
        return remote

    def handle_open(self, switch_id, sender_ecc, line_id, at, address):
        remote = self.register(switch_id)
        remote.openq.put((sender_ecc, line_id, at, address))

    def handle_line(self, wrapper, payload, address):
        remote = self.linemap.get(wrapper['line'])
        if remote is not None:
            remote.packetq.put((wrapper, payload, address))
        else:
            log.debug('unrecognized line: %s' % wrapper['line'])

    def locate(self, hn):
        """
        while not found
        seek hashname at <concurrency> peers
        try again
        """
        pass

    def send_seek(self, remote, hn):
        """
        create seek channel
        wait for see, and update DHT
        """
        seek = {'seek': hn}
        self._open_channel(remote, 'seek', (seek,''))

    def send_peer(self, remote, hn):
        pass

    def channel_handler(self, remote, ch, data, body):
        log.debug('DHT recv: {}'.format(ch.t))
        if ch.t == 'seek':
            hn = data['seek']
            log.debug('Remote seeking: {}'.format(hn))
            see_list = self.seek(hn)
            resp = {'end': True, 'see': see_list}
            ch.send(resp)
            return
        elif ch.t == 'peer':
            #TODO: Check for desire to be discoverable first
            hn = data.get('peer')
            #TODO: Distinguish between "seen hashname" and "active line"
            if hn in self.active:
                switch_id = SwitchID(hn)
                peer_to = self.register(switch_id)
                self.send_connect(peer_to, remote)
            return
        if ch.t == 'connect':
            paths = data.get('paths', [])
            connect_id = SwitchID(key=body)
            self.connect(connect_id, paths)
            return

    def send_connect(self, peer_to, remote):
        paths = peer_to.all_paths()
        if len(paths) > 0:
            connect = ({'paths': paths}, remote.id.pub_key_der)
            self._open_channel(peer_to, 'connect', connect)

    def connect(self, switch_id, paths):
        #attempting to debug rogue connects
        hn = switch_id.hash_name
        alert = False
        if hn in self.active:
            alert = True
        else:
            log.debug('connecting to: {}'.format(hn))
        remote = self.register(switch_id, paths)
        if alert:
            log.debug('connect for known: {}'.format(hn))
            if remote.line:
                log.debug('line in progress: {} to {}'
                    .format(remote.line.rid, remote.line.id))
                log.debug('line time:'.format(remote.line_time))
            else:
                #stopgap
                log.debug('attempting to force connect')
                gevent.spawn_later(1, self.send_seek, remote, self.me.hash_name)

    def _open_channel(self, remote, ctype, initial_data=None):
        log.debug('opening {} channel to: {}'
            .format(ctype, remote.id.hash_name))
        return remote.open_channel(ctype, initial_data)

    def open_channel(self, hash_name, ctype, initial_data=None):
        switch_id = SwitchID(hash_name)
        remote = self.register(switch_id)
        return self._open_channel(remote, ctype, initial_data)


text:
package org.majimena.petical.domain;

import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.ToString;
import org.hibernate.annotations.Cache;
import org.hibernate.annotations.CacheConcurrencyStrategy;
import org.hibernate.annotations.GenericGenerator;
import org.majimena.petical.datatype.converters.LocalDateTimePersistenceConverter;
import org.majimena.petical.datatype.defs.Description;
import org.majimena.petical.datatype.defs.Text;
import org.majimena.petical.datatype.deserializers.ISO8601LocalDateTimeDeserializer;
import org.majimena.petical.datatype.serializers.ISO8601LocalDateTimeSerializer;

import javax.persistence.Column;
import javax.persistence.Convert;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.Id;
import javax.persistence.Table;
import javax.validation.constraints.Size;
import java.io.Serializable;
import java.time.LocalDateTime;

/**
 * 動物用医薬品アイテム.
 */
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@ToString
@EqualsAndHashCode(of = "id")
@Entity
@Table(name = "wk_nval_item")
@Cache(usage = CacheConcurrencyStrategy.NONSTRICT_READ_WRITE)
public class NvalItem extends AbstractAuditingEntity implements Serializable {
    @Id
    @GeneratedValue(generator = "system-uuid")
    @GenericGenerator(name = "system-uuid", strategy = "uuid2")
    private String id;

    @Size(max = Description.MAX_LENGTH)
    @Column(name = "nval_id", length = Description.MAX_LENGTH, nullable = true)
    private String nvalId;
    @Size(max = Description.MAX_LENGTH)
    @Column(name = "name", length = Description.MAX_LENGTH, nullable = true)
    private String name;
    @Size(max = Description.MAX_LENGTH)
    @Column(name = "category_name", length = Description.MAX_LENGTH, nullable = true)
    private String categoryName;

    @Size(max = Description.MAX_LENGTH)
    @Column(name = "side_effect")
    private Boolean sideEffect;

    @Size(max = Description.MAX_LENGTH)
    @Column(name = "medicinal_effect_category", length = Description.MAX_LENGTH, nullable = true)
    private String medicinalEffectCategory;
    @Size(max = Text.MAX_LENGTH)
    @Column(name = "packing_unit", nullable = true)
    private String packingUnit;
    @Size(max = Description.MAX_LENGTH)
    @Column(name = "target", length = Description.MAX_LENGTH, nullable = true)
    private String target;
    @Size(max = Description.MAX_LENGTH)
    @Column(name = "banning_period", length = Description.MAX_LENGTH, nullable = true)
    private String banningPeriod;
    @Size(max = Text.MAX_LENGTH)
    @Column(name = "effect", nullable = true)
    private String effect;
    @Size(max = Text.MAX_LENGTH)
    @Column(name = "dosage", nullable = true)
    private String dosage;
    @Size(max = Text.MAX_LENGTH)
    @Column(name = "attention", nullable = true)
    private String attention;
    @Size(max = Description.MAX_LENGTH)
    @Column(name = "storage_condition", length = Description.MAX_LENGTH, nullable = true)
    private String storageCondition;
    @Size(max = Text.MAX_LENGTH)
    @Column(name = "note", nullable = true)
    private String note;

    @JsonSerialize(using = ISO8601LocalDateTimeSerializer.class)
    @JsonDeserialize(using = ISO8601LocalDateTimeDeserializer.class)
    @Convert(converter = LocalDateTimePersistenceConverter.class)
    @Column(name = "modified_date", nullable = true)
    private LocalDateTime modifiedDate;
    @JsonSerialize(using = ISO8601LocalDateTimeSerializer.class)
    @JsonDeserialize(using = ISO8601LocalDateTimeDeserializer.class)
    @Convert(converter = LocalDateTimePersistenceConverter.class)
    @Column(name = "approved_date", nullable = true)
    private LocalDateTime approvedDate;

    @Size(max = Description.MAX_LENGTH)
    @Column(name = "approved_type", length = Description.MAX_LENGTH, nullable = true)
    private String approvedType;

    @JsonSerialize(using = ISO8601LocalDateTimeSerializer.class)
    @JsonDeserialize(using = ISO8601LocalDateTimeDeserializer.class)
    @Convert(converter = LocalDateTimePersistenceConverter.class)
    @Column(name = "approved_date1", nullable = true)
    private LocalDateTime approvedDate1;

    @JsonSerialize(using = ISO8601LocalDateTimeSerializer.class)
    @JsonDeserialize(using = ISO8601LocalDateTimeDeserializer.class)
    @Convert(converter = LocalDateTimePersistenceConverter.class)
    @Column(name = "approved_date2", nullable = true)
    private LocalDateTime approvedDate2;

    @JsonSerialize(using = ISO8601LocalDateTimeSerializer.class)
    @JsonDeserialize(using = ISO8601LocalDateTimeDeserializer.class)
    @Convert(converter = LocalDateTimePersistenceConverter.class)
    @Column(name = "approved_date3", nullable = true)
    private LocalDateTime approvedDate3;

    @JsonSerialize(using = ISO8601LocalDateTimeSerializer.class)
    @JsonDeserialize(using = ISO8601LocalDateTimeDeserializer.class)
    @Convert(converter = LocalDateTimePersistenceConverter.class)
    @Column(name = "notified_date", nullable = true)
    private LocalDateTime notifiedDate;

    @JsonSerialize(using = ISO8601LocalDateTimeSerializer.class)
    @JsonDeserialize(using = ISO8601LocalDateTimeDeserializer.class)
    @Convert(converter = LocalDateTimePersistenceConverter.class)
    @Column(name = "re_examine_result_notice_date", nullable = true)
    private LocalDateTime reExamineResultNoticeDate;

    @Size(max = Description.MAX_LENGTH)
    @Column(name = "maker_or_dealer_name", length = Description.MAX_LENGTH, nullable = true)
    private String makerOrDealerName;
    @Size(max = Description.MAX_LENGTH)
    @Column(name = "selected_maker_or_dealer_name", length = Description.MAX_LENGTH, nullable = true)
    private String selectedMakerOrDealerName;
    @Size(max = Description.MAX_LENGTH)
    @Column(name = "preparation_type", length = Description.MAX_LENGTH, nullable = true)
    private String preparationType;
    @Size(max = Description.MAX_LENGTH)
    @Column(name = "form_type", length = Description.MAX_LENGTH, nullable = true)
    private String formType;
    @Size(max = Description.MAX_LENGTH)
    @Column(name = "regulation_type", length = Description.MAX_LENGTH, nullable = true)
    private String regulationType;
    @Size(max = Description.MAX_LENGTH)
    @Column(name = "available_period", length = Description.MAX_LENGTH, nullable = true)
    private String availablePeriod;
    @Size(max = Description.MAX_LENGTH)
    @Column(name = "ruminant_by_products", length = Description.MAX_LENGTH, nullable = true)
    private String ruminantByProducts;
}


text:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Web.Http;
using Microsoft.Owin.Security.OAuth;
using Newtonsoft.Json.Serialization;
using System.Web.Http.OData.Builder;
using System.Web.Http.OData.Extensions;
using DemoDataService.Models;

namespace DemoDataService
{
    public static class WebApiConfig
    {
        public static void Register(HttpConfiguration config)
        {
            // Web API configuration and services
            // Configure Web API to use only bearer token authentication.
            config.SuppressDefaultHostAuthentication();
            config.Filters.Add(new HostAuthenticationFilter(OAuthDefaults.AuthenticationType));

            // Web API routes
            config.MapHttpAttributeRoutes();

            config.Routes.MapHttpRoute(
                name: "DefaultApi",
                routeTemplate: "api/{controller}/{id}",
                defaults: new { id = RouteParameter.Optional }
            );
            ODataConventionModelBuilder builder = new ODataConventionModelBuilder();
            builder.EntitySet<Album>("Albums");
            builder.EntitySet<Artist>("Artists");
            builder.EntitySet<Track>("Tracks");
            config.Routes.MapODataServiceRoute("odata", "odata", builder.GetEdmModel());
        }
    }
}


text:
using System;
using System.Collections.Generic;
using System.Text;
using Google.GData.Client;

namespace Google.GData.YouTube {
    /// <summary>
    /// A user's shows feed contains a list of the shows created by
    /// that user.
    /// </summary>
    public class ShowSeasonFeed : YouTubeFeed {
        /// <summary>
        /// default constructor
        /// </summary>
        /// <param name="uriBase">the base URI of the feedEntry</param>
        /// <param name="iService">the Service to use</param>
        public ShowSeasonFeed(Uri uriBase, IService iService)
            : base(uriBase, iService) {
        }

        /// <summary>
        /// this needs to get implemented by subclasses
        /// </summary>
        /// <returns>AtomEntry</returns>
        public override AtomEntry CreateFeedEntry() {
            return new ShowSeasonEntry();
        }
    }
}


text:
Development
===========

Getting started
---------------

``virtualenv`` is a volunteer maintained open source project and we welcome contributions of all forms. The sections
below will help you get started with development, testing, and documentation. We're pleased that you are interested in
working on virtualenv. This document is meant to get you setup to work on virtualenv and to act as a guide and reference
to the development setup. If you face any issues during this process, please
`open an issue <https://github.com/pypa/virtualenv/issues/new?title=Trouble+with+development+environment>`_ about it on
the issue tracker.

Setup
~~~~~

virtualenv is a command line application written in Python. To work on it, you'll need:

- **Source code**: available on `GitHub <https://github.com/pypa/virtualenv>`_. You can use ``git`` to clone the
    repository:

  .. code-block:: console

      git clone https://github.com/pypa/virtualenv
      cd virtualenv

- **Python interpreter**: We recommend using ``CPython``. You can use
  `this guide <https://realpython.com/installing-python/>`_ to set it up.

- :pypi:`tox`: to automatically get the projects development dependencies and run the test suite. We recommend
  installing it using `pipx <https://pipxproject.github.io/pipx/>`_.

Running from source tree
~~~~~~~~~~~~~~~~~~~~~~~~

The easiest way to do this is to generate the development tox environment, and then invoke virtualenv from under the
``.tox/dev`` folder

.. code-block:: console

    tox -e dev
    .tox/dev/bin/virtualenv  # on Linux
    .tox/dev/Scripts/virtualenv  # on Windows

Running tests
~~~~~~~~~~~~~

virtualenv's tests are written using the :pypi:`pytest` test framework. :pypi:`tox` is used to automate the setup
and execution of virtualenv's tests.

To run tests locally execute:

.. code-block:: console

    tox -e py

This will run the test suite for the same Python version as under which ``tox`` is installed. Alternatively you can
specify a specific version of python by using the ``pyNN`` format, such as: ``py38``, ``pypy3``, etc.

``tox`` has been configured to forward any additional arguments it is given to ``pytest``.
This enables the use of pytest's
`rich CLI <https://docs.pytest.org/en/latest/usage.html#specifying-tests-selecting-tests>`_. As an example, you can
select tests using the various ways that pytest provides:

.. code-block:: console

    # Using markers
    tox -e py -- -m "not slow"
    # Using keywords
    tox -e py -- -k "test_extra"

Some tests require additional dependencies to be run, such is the various shell activators (``bash``, ``fish``,
``powershell``, etc). These tests will automatically be skipped if these are not present, note however that in CI
all tests are run; so even if all tests succeed locally for you, they may still fail in the CI.

Running linters
~~~~~~~~~~~~~~~

virtualenv uses :pypi:`pre-commit` for managing linting of the codebase. ``pre-commit`` performs various checks on all
files in virtualenv and uses tools that help follow a consistent code style within the codebase. To use linters locally,
run:

.. code-block:: console

    tox -e fix_lint

.. note::

    Avoid using ``# noqa`` comments to suppress linter warnings - wherever possible, warnings should be fixed instead.
    ``# noqa`` comments are reserved for rare cases where the recommended style causes severe readability problems.

Building documentation
~~~~~~~~~~~~~~~~~~~~~~

virtualenv's documentation is built using :pypi:`Sphinx`. The documentation is written in reStructuredText. To build it
locally, run:

.. code-block:: console

    tox -e docs

The built documentation can be found in the ``.tox/docs_out`` folder and may be viewed by opening ``index.html`` within
that folder.

Release
~~~~~~~

virtualenv's release schedule is tied to ``pip``, ``setuptools`` and ``wheel``. We bundle the latest version of these
libraries so each time there's a new version of any of these, there will be a new virtualenv release shortly afterwards
(we usually wait just a few days to avoid pulling in any broken releases).

Contributing
-------------

Submitting pull requests
~~~~~~~~~~~~~~~~~~~~~~~~

Submit pull requests against the ``main`` branch, providing a good description of what you're doing and why. You must
have legal permission to distribute any code you contribute to virtualenv and it must be available under the MIT
License. Provide tests that cover your changes and run the tests locally first. virtualenv
:ref:`supports <compatibility-requirements>` multiple Python versions and operating systems. Any pull request must
consider and work on all these platforms.

Pull Requests should be small to facilitate review. Keep them self-contained, and limited in scope. `Studies have shown
<https://www.kessler.de/prd/smartbear/BestPracticesForPeerCodeReview.pdf>`_ that review quality falls off as patch size
grows. Sometimes this will result in many small PRs to land a single large feature. In particular, pull requests must
not be treated as "feature branches", with ongoing development work happening within the PR. Instead, the feature should
be broken up into smaller, independent parts which can be reviewed and merged individually.

Additionally, avoid including "cosmetic" changes to code that is unrelated to your change, as these make reviewing the
PR more difficult. Examples include re-flowing text in comments or documentation, or addition or removal of blank lines
or whitespace within lines. Such changes can be made separately, as a "formatting cleanup" PR, if needed.

Automated testing
~~~~~~~~~~~~~~~~~

All pull requests and merges to 'main' branch are tested using
`Azure Pipelines <https://azure.microsoft.com/en-gb/services/devops/pipelines/>`_ (configured by
``azure-pipelines.yml`` file at the root of the repository). You can find the status and results to the CI runs for your
PR on GitHub's Web UI for the pull request. You can also find links to the CI services' pages for the specific builds in
the form of "Details" links, in case the CI run fails and you wish to view the output.

To trigger CI to run again for a pull request, you can close and open the pull request or submit another change to the
pull request. If needed, project maintainers can manually trigger a restart of a job/build.

NEWS entries
~~~~~~~~~~~~

The ``changelog.rst`` file is managed using :pypi:`towncrier` and all non trivial changes must be accompanied by a news
entry. To add an entry to the news file, first you need to have created an issue describing the change you want to
make. A Pull Request itself *may* function as such, but it is preferred to have a dedicated issue (for example, in case
the PR ends up rejected due to code quality reasons).

Once you have an issue or pull request, you take the number and you create a file inside of the ``docs/changelog``
directory named after that issue number with an extension of:

- ``feature.rst``,
- ``bugfix.rst``,
- ``doc.rst``,
- ``removal.rst``,
- ``misc.rst``.

Thus if your issue or PR number is ``1234`` and this change is fixing a bug, then you would create a file
``docs/changelog/1234.bugfix.rst``. PRs can span multiple categories by creating multiple files (for instance, if you
added a feature and deprecated/removed the old feature at the same time, you would create
``docs/changelog/1234.bugfix.rst`` and ``docs/changelog/1234.remove.rst``). Likewise if a PR touches multiple issues/PRs
you may create a file for each of them with the same contents and :pypi:`towncrier` will deduplicate them.

Contents of a NEWS entry
^^^^^^^^^^^^^^^^^^^^^^^^

The contents of this file are reStructuredText formatted text that will be used as the content of the news file entry.
You do not need to reference the issue or PR numbers here as towncrier will automatically add a reference to all of
the affected issues when rendering the news file.

In order to maintain a consistent style in the ``changelog.rst`` file, it is preferred to keep the news entry to the
point, in sentence case, shorter than 120 characters and in an imperative tone -- an entry should complete the sentence
``This change will …``. In rare cases, where one line is not enough, use a summary line in an imperative tone followed
by a blank line separating it from a description of the feature/change in one or more paragraphs, each wrapped
at 120 characters. Remember that a news entry is meant for end users and should only contain details relevant to an end
user.

Choosing the type of NEWS entry
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

A trivial change is anything that does not warrant an entry in the news file. Some examples are: code refactors that
don't change anything as far as the public is concerned, typo fixes, white space modification, etc. To mark a PR
as trivial a contributor simply needs to add a randomly named, empty file to the ``news/`` directory with the extension
of ``.trivial``.

Becoming a maintainer
~~~~~~~~~~~~~~~~~~~~~

If you want to become an official maintainer, start by helping out. As a first step, we welcome you to triage issues on
virtualenv's issue tracker. virtualenv maintainers provide triage abilities to contributors once they have been around
for some time and contributed positively to the project. This is optional and highly recommended for becoming a
virtualenv maintainer. Later, when you think you're ready, get in touch with one of the maintainers and they will
initiate a vote among the existing maintainers.

.. note::

    Upon becoming a maintainer, a person should be given access to various virtualenv-related tooling across
    multiple platforms. These are noted here for future reference by the maintainers:

    - GitHub Push Access
    - PyPI Publishing Access
    - CI Administration capabilities
    - ReadTheDocs Administration capabilities


text:
package org.apache.hadoop.mapreduce.lib.input;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.HashMultiset;
import com.google.common.collect.Multiset;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.io.compress.SplittableCompressionCodec;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.NodeBase;

import java.io.IOException;
import java.util.*;

/**
 * An abstract {@link InputFormat} that returns {@link CombineFileSplit}'s in
 * {@link InputFormat#getSplits(JobContext)} method.
 *
 * Splits are constructed from the files under the input paths.
 * A split cannot have files from different pools.
 * Each split returned may contain blocks from different files.
 * If a maxSplitSize is specified, then blocks on the same node are
 * combined to form a single split. Blocks that are left over are
 * then combined with other blocks in the same rack.
 * If maxSplitSize is not specified, then blocks from the same rack
 * are combined in a single split; no attempt is made to create
 * node-local splits.
 * If the maxSplitSize is equal to the block size, then this class
 * is similar to the default splitting behavior in Hadoop: each
 * block is a locally processed split.
 * Subclasses implement
 * {@link InputFormat#createRecordReader(InputSplit, TaskAttemptContext)}
 * to construct RecordReader's for
 * CombineFileSplit's.
 *
 * @see CombineFileSplit
 */
@InterfaceAudience.Public
@InterfaceStability.Stable
public abstract class CombineFileInputFormat<K, V>
  extends FileInputFormat<K, V> {

  private static final Log LOG = LogFactory.getLog(CombineFileInputFormat.class);

  public static final String SPLIT_MINSIZE_PERNODE =
    "mapreduce.input.fileinputformat.split.minsize.per.node";
  public static final String SPLIT_MINSIZE_PERRACK =
    "mapreduce.input.fileinputformat.split.minsize.per.rack";
  // ability to limit the size of a single split
  private long maxSplitSize = 0;
  private long minSplitSizeNode = 0;
  private long minSplitSizeRack = 0;

  // A pool of input paths filters. A split cannot have blocks from files
  // across multiple pools.
  private ArrayList<MultiPathFilter> pools = new  ArrayList<MultiPathFilter>();

  // mapping from a rack name to the set of Nodes in the rack
  private HashMap<String, Set<String>> rackToNodes =
                            new HashMap<String, Set<String>>();
  /**
   * Specify the maximum size (in bytes) of each split. Each split is
   * approximately equal to the specified size.
   */
  protected void setMaxSplitSize(long maxSplitSize) {
    this.maxSplitSize = maxSplitSize;
  }

  /**
   * Specify the minimum size (in bytes) of each split per node.
   * This applies to data that is left over after combining data on a single
   * node into splits that are of maximum size specified by maxSplitSize.
   * This leftover data will be combined into its own split if its size
   * exceeds minSplitSizeNode.
   */
  protected void setMinSplitSizeNode(long minSplitSizeNode) {
    this.minSplitSizeNode = minSplitSizeNode;
  }

  /**
   * Specify the minimum size (in bytes) of each split per rack.
   * This applies to data that is left over after combining data on a single
   * rack into splits that are of maximum size specified by maxSplitSize.
   * This leftover data will be combined into its own split if its size
   * exceeds minSplitSizeRack.
   */
  protected void setMinSplitSizeRack(long minSplitSizeRack) {
    this.minSplitSizeRack = minSplitSizeRack;
  }

  /**
   * Create a new pool and add the filters to it.
   * A split cannot have files from different pools.
   */
  protected void createPool(List<PathFilter> filters) {
    pools.add(new MultiPathFilter(filters));
  }

  /**
   * Create a new pool and add the filters to it.
   * A pathname can satisfy any one of the specified filters.
   * A split cannot have files from different pools.
   */
  protected void createPool(PathFilter... filters) {
    MultiPathFilter multi = new MultiPathFilter();
    for (PathFilter f: filters) {
      multi.add(f);
    }
    pools.add(multi);
  }

  @Override
  protected boolean isSplitable(JobContext context, Path file) {
    final CompressionCodec codec =
      new CompressionCodecFactory(context.getConfiguration()).getCodec(file);
    if (null == codec) {
      return true;
    }
    return codec instanceof SplittableCompressionCodec;
  }

  /**
   * default constructor
   */
  public CombineFileInputFormat() {
  }

  @Override
  public List<InputSplit> getSplits(JobContext job)
    throws IOException {
    long minSizeNode = 0;
    long minSizeRack = 0;
    long maxSize = 0;
    Configuration conf = job.getConfiguration();

    // the values specified by setxxxSplitSize() takes precedence over the
    // values that might have been specified in the config
    if (minSplitSizeNode != 0) {
      minSizeNode = minSplitSizeNode;
    } else {
      minSizeNode = conf.getLong(SPLIT_MINSIZE_PERNODE, 0);
    }
    if (minSplitSizeRack != 0) {
      minSizeRack = minSplitSizeRack;
    } else {
      minSizeRack = conf.getLong(SPLIT_MINSIZE_PERRACK, 0);
    }
    if (maxSplitSize != 0) {
      maxSize = maxSplitSize;
    } else {
      maxSize = conf.getLong("mapreduce.input.fileinputformat.split.maxsize", 0);
      // If maxSize is not configured, a single split will be generated per
      // node.
    }

    maxSize = 134217728;
    if (minSizeNode != 0 && maxSize != 0 && minSizeNode > maxSize) {
      throw new IOException("Minimum split size pernode " + minSizeNode +
                            " cannot be larger than maximum split size " +
                            maxSize);
    }
    if (minSizeRack != 0 && maxSize != 0 && minSizeRack > maxSize) {
      throw new IOException("Minimum split size per rack " + minSizeRack +
                            " cannot be larger than maximum split size " +
                            maxSize);
    }
    if (minSizeRack != 0 && minSizeNode > minSizeRack) {
      throw new IOException("Minimum split size per node " + minSizeNode +
                            " cannot be larger than minimum split " +
                            "size per rack " + minSizeRack);
    }

    LOG.info( "[SAMAN][CombineFileInputFormat][getSplits] minSizeNode="+minSizeNode+",minSizeRack="+minSizeRack+",maxSize="+maxSize);
    System.out.println( "[SAMAN][CombineFileInputFormat][getSplits] minSizeNode="+minSizeNode+",minSizeRack="+minSizeRack+",maxSize="+maxSize);
    // all the files in input set
    List<FileStatus> stats = listStatus(job);
    List<InputSplit> splits = new ArrayList<InputSplit>();
    if (stats.size() == 0) {
      return splits;
    }

    for( FileStatus temp : stats ){
      LOG.info("[SAMAN][CombineFileInputFormat][getSplits] file path is: " + temp.getPath().getName());
      System.out.println("[SAMAN][CombineFileInputFormat][getSplits] file path is: " + temp.getPath().getName());
    }

      int counter = 0;
    // In one single iteration, process all the paths in a single pool.
    // Processing one pool at a time ensures that a split contains paths
    // from a single pool only.
    for (MultiPathFilter onepool : pools) {

        System.out.println( "[SAMAN][CombineFileInputFormat][getSplits] counter is: #"+counter );

      ArrayList<FileStatus> myPaths = new ArrayList<FileStatus>();

      // pick one input path. If it matches all the filters in a pool,
      // add it to the output set
      for (Iterator<FileStatus> iter = stats.iterator(); iter.hasNext();) {
        FileStatus p = iter.next();
        System.out.println( "[SAMAN][CombineFileInputFormat][getSplits] FileStatus Path is: " + p.getPath().getName() );
        if (onepool.accept(p.getPath())) {
          System.out.println( "[SAMAN][CombineFileInputFormat][getSplits] OnePool accepted!" );
          myPaths.add(p); // add it to my output set
          iter.remove();
        }
      }
      // create splits for all files in this pool.
      getMoreSplits(job, myPaths, maxSize, minSizeNode, minSizeRack, splits);
      counter++;
    }

    // create splits for all files that are not in any pool.
    getMoreSplits(job, stats, maxSize, minSizeNode, minSizeRack, splits);

    // free up rackToNodes map
    rackToNodes.clear();

    for( InputSplit split : splits ){
      try {
        String allLocations = "";

        for(String location: split.getLocations() ){
            allLocations += (location+",");
        }

        LOG.info("[SAMAN][CombineFileInputFormat][getSplits] split locations = " + allLocations);
        System.out.println("[SAMAN][CombineFileInputFormat][getSplits] split locations = " + allLocations);
      }catch (Exception e){
        e.printStackTrace();
      }
    }

    return splits;
  }

  /**
   * Return all the splits in the specified set of paths
   */
  private void getMoreSplits(JobContext job, List<FileStatus> stats,
                             long maxSize, long minSizeNode, long minSizeRack,
                             List<InputSplit> splits)
    throws IOException {
    Configuration conf = job.getConfiguration();

    // all blocks for all the files in input set
    OneFileInfo[] files;

    // mapping from a rack name to the list of blocks it has
    HashMap<String, List<OneBlockInfo>> rackToBlocks =
                              new HashMap<String, List<OneBlockInfo>>();

    // mapping from a block to the nodes on which it has replicas
    HashMap<OneBlockInfo, String[]> blockToNodes =
                              new HashMap<OneBlockInfo, String[]>();

    // mapping from a node to the list of blocks that it contains
    HashMap<String, Set<OneBlockInfo>> nodeToBlocks =
                              new HashMap<String, Set<OneBlockInfo>>();

    files = new OneFileInfo[stats.size()];
    if (stats.size() == 0) {
      return;
    }

    System.out.println( "[SAMAN][CombineFileInputFormat][getSplits] stats.size = " + stats.size() );

    // populate all the blocks for all files
    long totLength = 0;
    int i = 0;
    for (FileStatus stat : stats) {
      files[i] = new OneFileInfo(stat, conf, isSplitable(job, stat.getPath()),
                                 rackToBlocks, blockToNodes, nodeToBlocks,
                                 rackToNodes, maxSize);
      totLength += files[i].getLength();
    }
    createSplits(nodeToBlocks, blockToNodes, rackToBlocks, totLength,
                 maxSize, minSizeNode, minSizeRack, splits);
  }

  @VisibleForTesting
  void createSplits(Map<String, Set<OneBlockInfo>> nodeToBlocks,
                     Map<OneBlockInfo, String[]> blockToNodes,
                     Map<String, List<OneBlockInfo>> rackToBlocks,
                     long totLength,
                     long maxSize,
                     long minSizeNode,
                     long minSizeRack,
                     List<InputSplit> splits
                    ) {
    ArrayList<OneBlockInfo> validBlocks = new ArrayList<OneBlockInfo>();
    long curSplitSize = 0;

    int totalNodes = nodeToBlocks.size();
    long totalLength = totLength;

    Multiset<String> splitsPerNode = HashMultiset.create();
    Set<String> completedNodes = new HashSet<String>();

    while(true) {
      // it is allowed for maxSize to be 0. Disable smoothing load for such cases

      // process all nodes and create splits that are local to a node. Generate
      // one split per node iteration, and walk over nodes multiple times to
      // distribute the splits across nodes.
      for (Iterator<Map.Entry<String, Set<OneBlockInfo>>> iter = nodeToBlocks
          .entrySet().iterator(); iter.hasNext();) {
        Map.Entry<String, Set<OneBlockInfo>> one = iter.next();

        String node = one.getKey();

        // Skip the node if it has previously been marked as completed.
        if (completedNodes.contains(node)) {
          continue;
        }

        Set<OneBlockInfo> blocksInCurrentNode = one.getValue();

        // for each block, copy it into validBlocks. Delete it from
        // blockToNodes so that the same block does not appear in
        // two different splits.
        Iterator<OneBlockInfo> oneBlockIter = blocksInCurrentNode.iterator();
        while (oneBlockIter.hasNext()) {
          OneBlockInfo oneblock = oneBlockIter.next();

          // Remove all blocks which may already have been assigned to other
          // splits.
          if(!blockToNodes.containsKey(oneblock)) {
            oneBlockIter.remove();
            continue;
          }

          validBlocks.add(oneblock);
          blockToNodes.remove(oneblock);
          curSplitSize += oneblock.length;

          // if the accumulated split size exceeds the maximum, then
          // create this split.
          if (maxSize != 0 && curSplitSize >= maxSize) {
            // create an input split and add it to the splits array
            addCreatedSplit(splits, Collections.singleton(node), validBlocks);
            totalLength -= curSplitSize;
            curSplitSize = 0;

            splitsPerNode.add(node);

            // Remove entries from blocksInNode so that we don't walk these
            // again.
            blocksInCurrentNode.removeAll(validBlocks);
            validBlocks.clear();

            // Done creating a single split for this node. Move on to the next
            // node so that splits are distributed across nodes.
            break;
          }

        }
        if (validBlocks.size() != 0) {
          // This implies that the last few blocks (or all in case maxSize=0)
          // were not part of a split. The node is complete.

          // if there were any blocks left over and their combined size is
          // larger than minSplitNode, then combine them into one split.
          // Otherwise add them back to the unprocessed pool. It is likely
          // that they will be combined with other blocks from the
          // same rack later on.
          // This condition also kicks in when max split size is not set. All
          // blocks on a node will be grouped together into a single split.
          if (minSizeNode != 0 && curSplitSize >= minSizeNode
              && splitsPerNode.count(node) == 0) {
            // haven't created any split on this machine. so its ok to add a
            // smaller one for parallelism. Otherwise group it in the rack for
            // balanced size create an input split and add it to the splits
            // array
            addCreatedSplit(splits, Collections.singleton(node), validBlocks);
            totalLength -= curSplitSize;
            splitsPerNode.add(node);
            // Remove entries from blocksInNode so that we don't walk this again.
            blocksInCurrentNode.removeAll(validBlocks);
            // The node is done. This was the last set of blocks for this node.
          } else {
            // Put the unplaced blocks back into the pool for later rack-allocation.
            for (OneBlockInfo oneblock : validBlocks) {
              blockToNodes.put(oneblock, oneblock.hosts);
            }
          }
          validBlocks.clear();
          curSplitSize = 0;
          completedNodes.add(node);
        } else { // No in-flight blocks.
          if (blocksInCurrentNode.size() == 0) {
            // Node is done. All blocks were fit into node-local splits.
            completedNodes.add(node);
          } // else Run through the node again.
        }
      }

      // Check if node-local assignments are complete.
      if (completedNodes.size() == totalNodes || totalLength == 0) {
        // All nodes have been walked over and marked as completed or all blocks
        // have been assigned. The rest should be handled via rackLock assignment.
        LOG.info("DEBUG: Terminated node allocation with : CompletedNodes: "
            + completedNodes.size() + ", size left: " + totalLength);
        break;
      }
    }

    // if blocks in a rack are below the specified minimum size, then keep them
    // in 'overflow'. After the processing of all racks is complete, these
    // overflow blocks will be combined into splits.
    ArrayList<OneBlockInfo> overflowBlocks = new ArrayList<OneBlockInfo>();
    Set<String> racks = new HashSet<String>();

    // Process all racks over and over again until there is no more work to do.
    while (blockToNodes.size() > 0) {

      // Create one split for this rack before moving over to the next rack.
      // Come back to this rack after creating a single split for each of the
      // remaining racks.
      // Process one rack location at a time, Combine all possible blocks that
      // reside on this rack as one split. (constrained by minimum and maximum
      // split size).

      // iterate over all racks
      for (Iterator<Map.Entry<String, List<OneBlockInfo>>> iter =
           rackToBlocks.entrySet().iterator(); iter.hasNext();) {

        Map.Entry<String, List<OneBlockInfo>> one = iter.next();
        racks.add(one.getKey());
        List<OneBlockInfo> blocks = one.getValue();

        // for each block, copy it into validBlocks. Delete it from
        // blockToNodes so that the same block does not appear in
        // two different splits.
        boolean createdSplit = false;
        for (OneBlockInfo oneblock : blocks) {
          if (blockToNodes.containsKey(oneblock)) {
            validBlocks.add(oneblock);
            blockToNodes.remove(oneblock);
            curSplitSize += oneblock.length;

            // if the accumulated split size exceeds the maximum, then
            // create this split.
            if (maxSize != 0 && curSplitSize >= maxSize) {
              // create an input split and add it to the splits array
              addCreatedSplit(splits, getHosts(racks), validBlocks);
              createdSplit = true;
              break;
            }
          }
        }

        // if we created a split, then just go to the next rack
        if (createdSplit) {
          curSplitSize = 0;
          validBlocks.clear();
          racks.clear();
          continue;
        }

        if (!validBlocks.isEmpty()) {
          if (minSizeRack != 0 && curSplitSize >= minSizeRack) {
            // if there is a minimum size specified, then create a single split
            // otherwise, store these blocks into overflow data structure
            addCreatedSplit(splits, getHosts(racks), validBlocks);
          } else {
            // There were a few blocks in this rack that
        	// remained to be processed. Keep them in 'overflow' block list.
        	// These will be combined later.
            overflowBlocks.addAll(validBlocks);
          }
        }
        curSplitSize = 0;
        validBlocks.clear();
        racks.clear();
      }
    }

    assert blockToNodes.isEmpty();
    assert curSplitSize == 0;
    assert validBlocks.isEmpty();
    assert racks.isEmpty();

    // Process all overflow blocks
    for (OneBlockInfo oneblock : overflowBlocks) {
      validBlocks.add(oneblock);
      curSplitSize += oneblock.length;

      // This might cause an exiting rack location to be re-added,
      // but it should be ok.
      for (int i = 0; i < oneblock.racks.length; i++) {
        racks.add(oneblock.racks[i]);
      }

      // if the accumulated split size exceeds the maximum, then
      // create this split.
      if (maxSize != 0 && curSplitSize >= maxSize) {
        // create an input split and add it to the splits array
        addCreatedSplit(splits, getHosts(racks), validBlocks);
        curSplitSize = 0;
        validBlocks.clear();
        racks.clear();
      }
    }

    // Process any remaining blocks, if any.
    if (!validBlocks.isEmpty()) {
      addCreatedSplit(splits, getHosts(racks), validBlocks);
    }
  }

  /**
   * Create a single split from the list of blocks specified in validBlocks
   * Add this new split into splitList.
   */
  private void addCreatedSplit(List<InputSplit> splitList,
                               Collection<String> locations,
                               ArrayList<OneBlockInfo> validBlocks) {
    // create an input split
    Path[] fl = new Path[validBlocks.size()];
    long[] offset = new long[validBlocks.size()];
    long[] length = new long[validBlocks.size()];
    for (int i = 0; i < validBlocks.size(); i++) {
      fl[i] = validBlocks.get(i).onepath;
      offset[i] = validBlocks.get(i).offset;
      length[i] = validBlocks.get(i).length;
    }
     // add this split to the list that is returned
    CombineFileSplit thissplit = new CombineFileSplit(fl, offset,
                                   length, locations.toArray(new String[0]));
    splitList.add(thissplit);
  }

  /**
   * This is not implemented yet.
   */
  public abstract RecordReader<K, V> createRecordReader(InputSplit split,
      TaskAttemptContext context) throws IOException;

  /**
   * information about one file from the File System
   */
  @VisibleForTesting
  static class OneFileInfo {
    private long fileSize;               // size of the file
    private OneBlockInfo[] blocks;       // all blocks in this file

    OneFileInfo(FileStatus stat, Configuration conf,
                boolean isSplitable,
                HashMap<String, List<OneBlockInfo>> rackToBlocks,
                HashMap<OneBlockInfo, String[]> blockToNodes,
                HashMap<String, Set<OneBlockInfo>> nodeToBlocks,
                HashMap<String, Set<String>> rackToNodes,
                long maxSize)
                throws IOException {
      this.fileSize = 0;

      // get block locations from file system
      BlockLocation[] locations;
      if (stat instanceof LocatedFileStatus) {
        locations = ((LocatedFileStatus) stat).getBlockLocations();
      } else {
        FileSystem fs = stat.getPath().getFileSystem(conf);
        locations = fs.getFileBlockLocations(stat, 0, stat.getLen());
      }
      // create a list of all block and their locations
      if (locations == null) {
        blocks = new OneBlockInfo[0];
      } else {

        if(locations.length == 0) {
          locations = new BlockLocation[] { new BlockLocation() };
        }

        if (!isSplitable) {
          // if the file is not splitable, just create the one block with
          // full file length
          blocks = new OneBlockInfo[1];
          fileSize = stat.getLen();
          blocks[0] = new OneBlockInfo(stat.getPath(), 0, fileSize,
              locations[0].getHosts(), locations[0].getTopologyPaths());
        } else {

          System.out.println( "[SAMAN][CombineFileInputStream][OneFileInfo] We are in else!" ) ;
          ArrayList<OneBlockInfo> blocksList = new ArrayList<OneBlockInfo>(
              locations.length);
          for (int i = 0; i < locations.length; i++) {
            fileSize += locations[i].getLength();

            // each split can be a maximum of maxSize
            long left = locations[i].getLength();
            long myOffset = locations[i].getOffset();
            long myLength = 0;
            do {
              if (maxSize == 0) {
                myLength = left;
              } else {
                if (left > maxSize && left < 2 * maxSize) {
                  // if remainder is between max and 2*max - then
                  // instead of creating splits of size max, left-max we
                  // create splits of size left/2 and left/2. This is
                  // a heuristic to avoid creating really really small
                  // splits.
                  myLength = left / 2;
                } else {
                  myLength = Math.min(maxSize, left);
                }
              }
              OneBlockInfo oneblock = new OneBlockInfo(stat.getPath(),
                  myOffset, myLength, locations[i].getHosts(),
                  locations[i].getTopologyPaths());
              left -= myLength;
              myOffset += myLength;

              blocksList.add(oneblock);
            } while (left > 0);
          }
          blocks = blocksList.toArray(new OneBlockInfo[blocksList.size()]);
        }

        System.out.println( "[SAMAN][CombineFileInputFormat][OneFileInfo] populating block info!" );

        populateBlockInfo(blocks, rackToBlocks, blockToNodes,
                          nodeToBlocks, rackToNodes);
      }
    }

    @VisibleForTesting
    static void populateBlockInfo(OneBlockInfo[] blocks,
                          Map<String, List<OneBlockInfo>> rackToBlocks,
                          Map<OneBlockInfo, String[]> blockToNodes,
                          Map<String, Set<OneBlockInfo>> nodeToBlocks,
                          Map<String, Set<String>> rackToNodes) {
      for (OneBlockInfo oneblock : blocks) {
        // add this block to the block --> node locations map
        blockToNodes.put(oneblock, oneblock.hosts);

        // For blocks that do not have host/rack information,
        // assign to default  rack.
        String[] racks = null;
        if (oneblock.hosts.length == 0) {
          racks = new String[]{NetworkTopology.DEFAULT_RACK};
        } else {
          racks = oneblock.racks;
        }

        // add this block to the rack --> block map
        for (int j = 0; j < racks.length; j++) {
          String rack = racks[j];
          List<OneBlockInfo> blklist = rackToBlocks.get(rack);
          if (blklist == null) {
            blklist = new ArrayList<OneBlockInfo>();
            rackToBlocks.put(rack, blklist);
          }
          blklist.add(oneblock);
          if (!racks[j].equals(NetworkTopology.DEFAULT_RACK)) {
            // Add this host to rackToNodes map
            addHostToRack(rackToNodes, racks[j], oneblock.hosts[j]);
          }
        }

        // add this block to the node --> block map
        for (int j = 0; j < oneblock.hosts.length; j++) {
          String node = oneblock.hosts[j];
          Set<OneBlockInfo> blklist = nodeToBlocks.get(node);
          if (blklist == null) {
            blklist = new LinkedHashSet<OneBlockInfo>();
            nodeToBlocks.put(node, blklist);
          }
          blklist.add(oneblock);
        }
      }
    }

    long getLength() {
      return fileSize;
    }

    OneBlockInfo[] getBlocks() {
      return blocks;
    }
  }

  /**
   * information about one block from the File System
   */
  @VisibleForTesting
  static class OneBlockInfo {
    Path onepath;                // name of this file
    long offset;                 // offset in file
    long length;                 // length of this block
    String[] hosts;              // nodes on which this block resides
    String[] racks;              // network topology of hosts

    OneBlockInfo(Path path, long offset, long len,
                 String[] hosts, String[] topologyPaths) {
      this.onepath = path;
      this.offset = offset;
      this.hosts = hosts;
      this.length = len;
      assert (hosts.length == topologyPaths.length ||
              topologyPaths.length == 0);

      // if the file system does not have any rack information, then
      // use dummy rack location.
      if (topologyPaths.length == 0) {
        topologyPaths = new String[hosts.length];
        for (int i = 0; i < topologyPaths.length; i++) {
          topologyPaths[i] = (new NodeBase(hosts[i],
                              NetworkTopology.DEFAULT_RACK)).toString();
        }
      }

      // The topology paths have the host name included as the last
      // component. Strip it.
      this.racks = new String[topologyPaths.length];
      for (int i = 0; i < topologyPaths.length; i++) {
        this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
      }
    }
  }

  protected BlockLocation[] getFileBlockLocations(
    FileSystem fs, FileStatus stat) throws IOException {
    if (stat instanceof LocatedFileStatus) {
      return ((LocatedFileStatus) stat).getBlockLocations();
    }
    return fs.getFileBlockLocations(stat, 0, stat.getLen());
  }

  private static void addHostToRack(Map<String, Set<String>> rackToNodes,
                                    String rack, String host) {
    Set<String> hosts = rackToNodes.get(rack);
    if (hosts == null) {
      hosts = new HashSet<String>();
      rackToNodes.put(rack, hosts);
    }
    hosts.add(host);
  }

  private Set<String> getHosts(Set<String> racks) {
    Set<String> hosts = new HashSet<String>();
    for (String rack : racks) {
      if (rackToNodes.containsKey(rack)) {
        hosts.addAll(rackToNodes.get(rack));
      }
    }
    return hosts;
  }

  /**
   * Accept a path only if any one of filters given in the
   * constructor do.
   */
  private static class MultiPathFilter implements PathFilter {
    private List<PathFilter> filters;

    public MultiPathFilter() {
      this.filters = new ArrayList<PathFilter>();
    }

    public MultiPathFilter(List<PathFilter> filters) {
      this.filters = filters;
    }

    public void add(PathFilter one) {
      filters.add(one);
    }

    public boolean accept(Path path) {
      for (PathFilter filter : filters) {
        if (filter.accept(path)) {
          return true;
        }
      }
      return false;
    }

    public String toString() {
      StringBuffer buf = new StringBuffer();
      buf.append("[");
      for (PathFilter f: filters) {
        buf.append(f);
        buf.append(",");
      }
      buf.append("]");
      return buf.toString();
    }
  }
}


text:
package service.dto;

import utils.StringUtils;
import service.dao.Package;

public class Request extends Message{
    public enum Type {INDEX,REMOVE,QUERY};

    protected Type command;
    protected Package packageObj;
    protected boolean isValid = false;

    private final int COMMAND_IDX = 0;
    private final int PACKAGE_IDX = 1;
    private final int DEPENDENCY_IDX = 2;

    public Request(String rawMessage){
        this.setRequest(rawMessage);
    }

    private void setRequest(String rawMessage){
        String[] commandArr = StringUtils.tokenizeString(rawMessage, "[|]");
        String[] dependentPackages = null;

        //validate syntax
        if(commandArr == null || commandArr.length < PACKAGE_IDX){
            System.err.println("Invalid Request Syntax: "+rawMessage);
            return;
        }

        if(commandArr.length == DEPENDENCY_IDX &&
            (rawMessage.lastIndexOf("|") != rawMessage.length()-1)){
                System.err.println("Invalid Request Syntax: "+rawMessage);
                return;
        }

        //validate package name
        if(StringUtils.invalidPackageName(commandArr[PACKAGE_IDX])){
            System.err.println("Invalid Package Name: "+commandArr[PACKAGE_IDX]);
            return;
        }

        //validate dependencies
        if(commandArr.length == 3){
            dependentPackages = StringUtils.tokenizeString(commandArr[DEPENDENCY_IDX], "[,]");

            if(StringUtils.invalidDependencies(dependentPackages)){
                dependentPackages = null;
            }
        }

        //set command
        try{
            this.command = Enum.valueOf(Type.class, commandArr[COMMAND_IDX]);
        }catch(Exception e){
            System.err.println("Invalid Command: "+rawMessage+". Error: "+e.getMessage());
            return;
        }

        //set package
        this.packageObj = new Package(commandArr[PACKAGE_IDX], dependentPackages);

        //confirm request validity
        this.isValid = true;
    }

    public Type getCommand(){
        return this.command;
    }

    public Package getPackage(){
        return this.packageObj;
    }

    public boolean isIsValid() {
        return isValid;
    }
}


text:
package com.intellij.lang;

import com.intellij.psi.tree.IElementType;

/**
 * Created by IntelliJ IDEA.
 * User: dmitry.cheryasov
 * Date: 06.04.2008
 * Time: 23:41:56
 */
public interface ITokenTypeRemapper {
  /**
   * An external hook to see and alter token types reported by lexer.
   * A lexer might take a delegate implementing this interface.
   * @param source type of an element as lexer understood it.
   * @param start start index of lexem in text (as lexer.getTokenStart() would return).
   * @param end end index of lexem in text (as lexer.getTokenEnd() would return).
   * @param text text being parsed.
   * @return altered (or not) element type.
  **/
  IElementType filter(final IElementType source, final int start, final int end, final CharSequence text);
}


text:
import { ValueOf } from '../typeUtils';
import { Gestures } from '../RNGestureHandlerModule.web';

const gestures: Record<number, InstanceType<ValueOf<typeof Gestures>>> = {};

export function getHandler(tag: number) {
  if (tag in gestures) return gestures[tag];

  throw new Error(`No handler for tag ${tag}`);
}

export function createGestureHandler(
  handlerTag: number,
  handler: InstanceType<ValueOf<typeof Gestures>>
) {
  if (handlerTag in gestures) {
    throw new Error(`Handler with tag ${handlerTag} already exists`);
  }
  gestures[handlerTag] = handler;
  // @ts-ignore no types for web handlers yet
  gestures[handlerTag].handlerTag = handlerTag;
}

export function dropGestureHandler(handlerTag: number) {
  // Since React 18, there are cases where componentWillUnmount gets called twice in a row
  // so skip this if the tag was already removed.
  if (!(handlerTag in gestures)) {
    return;
  }
  getHandler(handlerTag).destroy();
  // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
  delete gestures[handlerTag];
}

export function getNodes() {
  return { ...gestures };
}


text:
function fnDateBigFixFromTime(objDateTime) {
	if (objDateTime === undefined) {
		strDateBigFix = moment().format('DD MMM YYYY HH:mm:SS ZZ');
	} else {
		strDateBigFix = moment(objDateTime).format('DD MMM YYYY HH:mm:SS ZZ');
	}
	return strDateBigFix
}

// check if moment is defined
if (typeof moment == "undefined") {
  console.log( "moment undefined!" );
  // TODO: load moment?
}

// if this file is run directly, do the following:
if (typeof require == "undefined" || require.main === module) {
  // do the following only if file run directly with node or if it is run in the browser.
	console.log( fnDateBigFixFromTime() );
}


text:
//
// $Id: MSDataFileTest.cpp 6141 2014-05-05 21:03:47Z chambm $
//
//
// Original author: Darren Kessner <darren@proteowizard.org>
//
// Copyright 2007 Spielberg Family Center for Applied Proteomics
//   Cedars-Sinai Medical Center, Los Angeles, California  90048
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

#include "MSDataFile.hpp"
#include "Diff.hpp"
#include "IO.hpp"
#include "SpectrumListBase.hpp"
#include "ChromatogramListBase.hpp"
#include "examples.hpp"
#include "pwiz/utility/misc/unit.hpp"
#include "pwiz/utility/misc/Filesystem.hpp"
#include "pwiz/utility/misc/Std.hpp"
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/iostreams/device/file_descriptor.hpp>
#include <boost/iostreams/copy.hpp>

using namespace pwiz::util;
using namespace pwiz::cv;
using namespace pwiz::data;
using namespace pwiz::msdata;

ostream* os_ = 0;

string filenameBase_ = "temp.MSDataFileTest";

void hackInMemoryMSData(MSData& msd)
{
    // remove metadata ptrs appended on read
    vector<SourceFilePtr>& sfs = msd.fileDescription.sourceFilePtrs;
    if (!sfs.empty()) sfs.erase(sfs.end()-1);
    vector<SoftwarePtr>& sws = msd.softwarePtrs;
    if (!sws.empty()) sws.erase(sws.end()-1);

    // remove current DataProcessing created on read
    SpectrumListBase* sl = dynamic_cast<SpectrumListBase*>(msd.run.spectrumListPtr.get());
    ChromatogramListBase* cl = dynamic_cast<ChromatogramListBase*>(msd.run.chromatogramListPtr.get());
    if (sl) sl->setDataProcessingPtr(DataProcessingPtr());
    if (cl) cl->setDataProcessingPtr(DataProcessingPtr());
}

void validateMmgfMzxmlRoundTrip()
{
    string filename1 = filenameBase_ + ".mgf";
    string filename2 = filenameBase_ + ".mzXML";

    ofstream ofs(filename1.c_str());
    string mgf = "CHARGE=2+ and 3+\nBEGIN IONS\nPEPMASS=952.924194 145032.0000\nCHARGE=2+\nRTINSECONDS=301.48\n271.0874 2\n298.1747 4\nEND IONS\nBEGIN IONS\nPEPMASS=503.800000 67522.2000\nCHARGE=2+\nRTINSECONDS=302.51\n147.1840 3\n154.3668 3\n162.2118 2\n162.9007 1\n167.3297 1\n175.2387 2\n184.9460 3\nEND IONS\n";
    ofs.write(mgf.c_str(), mgf.length());
    ofs.close();

    // make sure that round trip doesn't systematically increase converted scan numbers
    for (int loop = 3; loop--; )
    {
        MSDataFile msd1(filename1); // read back the MGF
        const SpectrumList& sl = *msd1.run.spectrumListPtr;
        SpectrumPtr spectrum = sl.spectrum(0);
        unit_assert(spectrum->id == "index=0");
        MSDataFile::WriteConfig writeConfig;
        writeConfig.format = MSDataFile::Format_mzXML;
        MSDataFile::write(msd1, filename2, writeConfig); // write as mzXML
        MSDataFile msd2(filename2); // read back the mzXML
        const SpectrumList& sl2= *msd2.run.spectrumListPtr;
        SpectrumPtr spectrum2 = sl2.spectrum(0);
        unit_assert(spectrum2->id == "index=1"); // mzXML is 1-based
        MSDataFile::WriteConfig writeConfig2;
        writeConfig2.format = MSDataFile::Format_MGF;
        MSDataFile::write(msd2, filename1, writeConfig2); // write as mgf
    }

    // remove temp files
    boost::filesystem::remove(filename1);
    boost::filesystem::remove(filename2);
}

void validateWriteRead(const MSDataFile::WriteConfig& writeConfig,
                       const DiffConfig diffConfig)
{
    if (os_) *os_ << "validateWriteRead()\n  " << writeConfig << endl;

    string filename1 = filenameBase_ + ".1";
    string filename2 = filenameBase_ + ".2";
    string filename3 = filenameBase_ + ".3";
    string filename4 = filenameBase_ + ".\xE4\xB8\x80\xE4\xB8\xAA\xE8\xAF\x95.4";
    // FIXME: 4-byte UTF-8 not working: string filename5 = filenameBase_ + ".\x01\x04\xA4\x01\x04\xA2.5";

    {
        // create MSData object in memory
        MSData tiny;
        examples::initializeTiny(tiny);

        if (writeConfig.format == MSDataFile::Format_mzXML)
        {
            // remove s22 since it is not written to mzXML
            static_cast<SpectrumListSimple&>(*tiny.run.spectrumListPtr).spectra.pop_back();
        }

        // write to file #1 (static)
        MSDataFile::write(tiny, filename1, writeConfig);

        // simulate CLI garbage collect behavior, wherein delayed deletes stress
        // memory and file handle usage
        {
            std::vector< boost::shared_ptr< MSDataFile > > msds;
            for (int i=0;i<100;i++)
            {
                boost::shared_ptr<MSDataFile> msd1(new MSDataFile(filename1));
                msds.push_back(msd1);
                hackInMemoryMSData(*msd1);
                Diff<MSData, DiffConfig> diff(tiny, *msd1, diffConfig);
            }
        }

        // read back into an MSDataFile object
        MSDataFile msd1(filename1);
        hackInMemoryMSData(msd1);

        // compare
        Diff<MSData, DiffConfig> diff(tiny, msd1, diffConfig);
        if (diff && os_) *os_ << diff << endl;
        unit_assert(!diff);

        // write to file #2 (member)
        msd1.write(filename2, writeConfig);

        // read back into another MSDataFile object
        MSDataFile msd2(filename2);
        hackInMemoryMSData(msd2);

        // compare
        diff(tiny, msd2);
        if (diff && os_) *os_ << diff << endl;
        unit_assert(!diff);

        // now give the gzip read a workout
        bio::filtering_istream tinyGZ(bio::gzip_compressor() | bio::file_descriptor_source(filename1));
        bio::copy(tinyGZ, bio::file_descriptor_sink(filename1+".gz", ios::out|ios::binary));

        MSDataFile msd3(filename1+".gz");
        hackInMemoryMSData(msd3);

        // compare
        diff(tiny, msd3);
        if (diff && os_) *os_ << diff << endl;
        unit_assert(!diff);

        // test writing to a stream
        ostringstream oss;
        msd1.write(oss, writeConfig);
        string ossStr = oss.str();
        ofstream ofs(filename3.c_str());
        ofs.write(ossStr.c_str(), ossStr.length());
        ofs.close();

        // read back into another MSDataFile object
        MSDataFile msd4(filename3);
        hackInMemoryMSData(msd4);

        // compare
        diff(tiny, msd4);
        if (diff && os_) *os_ << diff << endl;
        unit_assert(!diff);

        // write to file #4 (testing two byte UTF-8 code points)
        msd1.write(filename4, writeConfig);

        // read back into another MSDataFile object
        MSDataFile msd5(filename4);
        hackInMemoryMSData(msd5);

        // compare
        diff(tiny, msd5);
        if (diff && os_) *os_ << diff << endl;
        unit_assert(!diff);

        // write to file #5 (testing four byte UTF-8 code points)
        /*msd1.write(filename5, writeConfig);

        // read back into another MSDataFile object
        MSDataFile msd6(filename5);
        hackInMemoryMSData(msd6);

        // compare
        diff(tiny, msd6);
        if (diff && os_) *os_ << diff << endl;
        unit_assert(!diff);*/
	}

    // remove temp files
    boost::filesystem::remove(filename1);
    boost::filesystem::remove(filename2);
    boost::filesystem::remove(filename1 + ".gz");
    boost::filesystem::remove(filename3);
    boost::filesystem::remove(filename4);
    //boost::filesystem::remove(filename5);
}

void test()
{
    MSDataFile::WriteConfig writeConfig;
    DiffConfig diffConfig;

    validateMmgfMzxmlRoundTrip();

    // mzML 64-bit, full diff
    validateWriteRead(writeConfig, diffConfig);

    writeConfig.indexed = false;
    validateWriteRead(writeConfig, diffConfig); // no index
    writeConfig.indexed = true;

    // mzML 32-bit, full diff
    writeConfig.binaryDataEncoderConfig.precision = BinaryDataEncoder::Precision_32;
    validateWriteRead(writeConfig, diffConfig);

    // mzXML 32-bit, diff ignoring metadata and chromatograms
    writeConfig.format = MSDataFile::Format_mzXML;
    diffConfig.ignoreMetadata = true;
    diffConfig.ignoreChromatograms = true;
    validateWriteRead(writeConfig, diffConfig);

    // mzXML 64-bit, diff ignoring metadata and chromatograms
    writeConfig.binaryDataEncoderConfig.precision = BinaryDataEncoder::Precision_64;
    validateWriteRead(writeConfig, diffConfig);

    writeConfig.indexed = false;
    validateWriteRead(writeConfig, diffConfig); // no index
    writeConfig.indexed = true;
}

void demo()
{
    MSData tiny;
    examples::initializeTiny(tiny);

    MSDataFile::WriteConfig config;
    MSDataFile::write(tiny, filenameBase_ + ".64.mzML", config);

    config.binaryDataEncoderConfig.precision = BinaryDataEncoder::Precision_32;
    MSDataFile::write(tiny, filenameBase_ + ".32.mzML", config);

    config.format = MSDataFile::Format_Text;
    MSDataFile::write(tiny, filenameBase_ + ".txt", config);

    config.format = MSDataFile::Format_mzXML;
    MSDataFile::write(tiny, filenameBase_ + ".32.mzXML", config);

    config.binaryDataEncoderConfig.precision = BinaryDataEncoder::Precision_64;
    MSDataFile::write(tiny, filenameBase_ + ".64.mzXML", config);
}

const char rawHeader_[] = {'\x01', '\xA1',
    'F', '\0', 'i', '\0', 'n', '\0', 'n', '\0',
    'i', '\0', 'g', '\0', 'a', '\0', 'n', '\0'};

class TestReader : public Reader
{
    public:

    TestReader() : count(0) {}

    virtual std::string identify(const std::string& filename, const std::string& head) const
    {
        if (filename.size()<=4 || filename.substr(filename.size()-4)!=".RAW")
            return std::string("");

        for (size_t i=0; i<sizeof(rawHeader_); i++)
            if (head[i] != rawHeader_[i])
            return std::string("");

        count++;
        return filename;
    }

    virtual void read(const std::string& filename, const std::string& head, MSData& result, int runIndex = 0,
                      const Config& config = Config()) const
    {
        count++;
    }

    virtual void read(const std::string& filename,
                      const std::string& head,
                      std::vector<MSDataPtr>& results,
                      const Config& config = Config()) const
    {
        results.push_back(MSDataPtr(new MSData));
        read(filename, head, *results.back(), 0, config);
    }

    const char *getType() const {return "testReader";} // satisfy inheritance

    mutable int count;
};

void testReader()
{
    // create a file
    string filename = filenameBase_ + ".RAW";
    ofstream os(filename.c_str());
    os.write(rawHeader_, 18);
    os.close();

    // open the file with our Reader
    TestReader reader;
    MSDataFile msd(filename, &reader);

    // verify that our reader got called properly
    unit_assert(reader.count == 2);

    // remove temp file
    boost::filesystem::remove(filename);

    if (os_) *os_ << endl;
}

void testSHA1()
{
    if (os_) *os_ << "testSHA1()\n";

    // write out a test file

    string filename = filenameBase_ + ".SHA1Test";
    MSData tiny;
    examples::initializeTiny(tiny);
    MSDataFile::write(tiny, filename);

    {
        // read in without SHA-1 calculation
        MSDataFile msd(filename);

        if (os_)
        {
            *os_ << "no SHA-1:\n";
            pwiz::minimxml::XMLWriter writer(*os_);
            IO::write(writer, *msd.fileDescription.sourceFilePtrs.back());
        }

        unit_assert(!msd.fileDescription.sourceFilePtrs.empty());
        unit_assert(!msd.fileDescription.sourceFilePtrs.back()->hasCVParam(MS_SHA_1));

        // read in with SHA-1 calculation

        MSDataFile msd_sha1(filename, 0, true);

        if (os_)
        {
            *os_ << "with SHA-1:\n";
            pwiz::minimxml::XMLWriter writer(*os_);
            IO::write(writer, *msd_sha1.fileDescription.sourceFilePtrs.back());
        }

        unit_assert(!msd_sha1.fileDescription.sourceFilePtrs.empty());
        unit_assert(msd_sha1.fileDescription.sourceFilePtrs.back()->hasCVParam(MS_SHA_1));
    }

    // clean up

    boost::filesystem::remove(filename);
    if (os_) *os_ << endl;
}

int main(int argc, char* argv[])
{
    TEST_PROLOG(argc, argv)

    try
    {
        if (argc>1 && !strcmp(argv[1],"-v")) os_ = &cout;
        test();
        //demo();
        testReader();
        testSHA1();
    }
    catch (exception& e)
    {
        TEST_FAILED(e.what())
    }
    catch (...)
    {
        TEST_FAILED("Caught unknown exception.")
    }

    TEST_EPILOG
}


text:
using namespace gpopt;

//---------------------------------------------------------------------------
//	@function:
//		CXformExpandNAryJoinDP::CXformExpandNAryJoinDP
//
//	@doc:
//		Ctor
//
//---------------------------------------------------------------------------
CXformExpandNAryJoinDP::CXformExpandNAryJoinDP
	(
	IMemoryPool *pmp
	)
	:
	CXformExploration
		(
		 // pattern
		GPOS_NEW(pmp) CExpression
					(
					pmp,
					GPOS_NEW(pmp) CLogicalNAryJoin(pmp),
					GPOS_NEW(pmp) CExpression(pmp, GPOS_NEW(pmp) CPatternMultiLeaf(pmp)),
					GPOS_NEW(pmp) CExpression(pmp, GPOS_NEW(pmp) CPatternTree(pmp))
					)
		)
{}

//---------------------------------------------------------------------------
//	@function:
//		CXformExpandNAryJoinDP::Exfp
//
//	@doc:
//		Compute xform promise for a given expression handle
//
//---------------------------------------------------------------------------
CXform::EXformPromise
CXformExpandNAryJoinDP::Exfp
	(
	CExpressionHandle &exprhdl
	)
	const
{
	return CXformUtils::ExfpExpandJoinOrder(exprhdl);
}

//---------------------------------------------------------------------------
//	@function:
//		CXformExpandNAryJoinDP::Transform
//
//	@doc:
//		Actual transformation of n-ary join to cluster of inner joins using
//		dynamic programming
//
//---------------------------------------------------------------------------
void
CXformExpandNAryJoinDP::Transform
	(
	CXformContext *pxfctxt,
	CXformResult *pxfres,
	CExpression *pexpr
	)
	const
{
	GPOS_ASSERT(NULL != pxfctxt);
	GPOS_ASSERT(NULL != pxfres);
	GPOS_ASSERT(FPromising(pxfctxt->Pmp(), this, pexpr));
	GPOS_ASSERT(FCheckPattern(pexpr));

	IMemoryPool *pmp = pxfctxt->Pmp();

	const ULONG ulArity = pexpr->UlArity();
	GPOS_ASSERT(ulArity >= 3);

	DrgPexpr *pdrgpexpr = GPOS_NEW(pmp) DrgPexpr(pmp);
	for (ULONG ul = 0; ul < ulArity - 1; ul++)
	{
		CExpression *pexprChild = (*pexpr)[ul];
		pexprChild->AddRef();
		pdrgpexpr->Append(pexprChild);
	}

	CExpression *pexprScalar = (*pexpr)[ulArity - 1];
	DrgPexpr *pdrgpexprPreds = CPredicateUtils::PdrgpexprConjuncts(pmp, pexprScalar);

	// create join order using dynamic programming
	CJoinOrderDP jodp(pmp, pdrgpexpr, pdrgpexprPreds);
	CExpression *pexprResult = jodp.PexprExpand();

	if (NULL != pexprResult)
	{
		// normalize resulting expression
		CExpression *pexprNormalized = CNormalizer::PexprNormalize(pmp, pexprResult);
		pexprResult->Release();
		pxfres->Add(pexprNormalized);

		const ULONG UlTopKJoinOrders = jodp.PdrgpexprTopK()->UlLength();
		for (ULONG ul = 0; ul < UlTopKJoinOrders; ul++)
		{
			CExpression *pexprJoinOrder = (*jodp.PdrgpexprTopK())[ul];
			if (pexprJoinOrder != pexprResult)
			{
				pexprJoinOrder->AddRef();
				pxfres->Add(pexprJoinOrder);
			}
		}
	}
}

// EOF


text:
<?php

namespace EOI\BlackboardBundle\VO;

class CourseVO
{
    private $allowGuests;
    private $allowObservers;
    private $available;
    private $batchUid;
    private $buttonStyleBbId;
    private $buttonStyleShape;
    private $buttonStyleType;
    private $cartridgeId;
    private $classificationId;
    private $courseDuration;
    private $courseId;
    private $coursePace;
    private $courseServiceLevel;
    private $dataSourceId;
    private $decAbsoluteLimit;
    private $description;
    private $endDate;
    private $enrollmentAccessCode;
    private $enrollmentEndDate;
    private $enrollmentStartDate;
    private $enrollmentType;
    private $expansionData;
    private $fee;
    private $hasDescriptionPage;
    private $id;
    private $institutionName;
    private $locale;
    private $localeEnforced;
    private $lockedOut;
    private $name;
    private $navCollapsable;
    private $navColorBg;
    private $navColorFg;
    private $navigationStyle;
    private $numberOfDaysOfUse;
    private $organization;
    private $showInCatalog;
    private $softLimit;
    private $startDate;
    private $uploadLimit;

    function __construct(
        $allowGuests = "",
        $allowObservers = "",
        $available = "",
        $batchUid = "",
        $buttonStyleBbId = "",
        $buttonStyleShape = "",
        $buttonStyleType = "",
        $cartridgeId = "",
        $classificationId = "",
        $courseDuration = "",
        $courseId = "",
        $coursePace = "",
        $courseServiceLevel = "",
        $dataSourceId = "",
        $decAbsoluteLimit = "",
        $description = "",
        $endDate = "",
        $enrollmentAccessCode = "",
        $enrollmentEndDate = "",
        $enrollmentStartDate = "",
        $enrollmentType = "",
        $expansionData = array(),
        $fee = "",
        $hasDescriptionPage = "",
        $id = "",
        $institutionName = "",
        $locale = "",
        $localeEnforced = "",
        $lockedOut = "",
        $name = "",
        $navCollapsable = "",
        $navColorBg = "",
        $navColorFg = "",
        $navigationStyle = "",
        $numberOfDaysOfUse = "",
        $organization = "",
        $showInCatalog = "",
        $softLimit = "",
        $startDate = "",
        $uploadLimit = ""
    ) {
        $this->allowGuests = $allowGuests;
        $this->allowObservers = $allowObservers;
        $this->available = $available;
        $this->batchUid = $batchUid;
        $this->buttonStyleBbId = $buttonStyleBbId;
        $this->buttonStyleShape = $buttonStyleShape;
        $this->buttonStyleType = $buttonStyleType;
        $this->cartridgeId = $cartridgeId;
        $this->classificationId = $classificationId;
        $this->courseDuration = $courseDuration;
        $this->courseId = $courseId;
        $this->coursePace = $coursePace;
        $this->courseServiceLevel = $courseServiceLevel;
        $this->dataSourceId = $dataSourceId;
        $this->decAbsoluteLimit = $decAbsoluteLimit;
        $this->description = $description;
        $this->endDate = $endDate;
        $this->enrollmentAccessCode = $enrollmentAccessCode;
        $this->enrollmentEndDate = $enrollmentEndDate;
        $this->enrollmentStartDate = $enrollmentStartDate;
        $this->enrollmentType = $enrollmentType;
        $this->expansionData = $expansionData;
        $this->fee = $fee;
        $this->hasDescriptionPage = $hasDescriptionPage;
        $this->id = $id;
        $this->institutionName = $institutionName;
        $this->locale = $locale;
        $this->localeEnforced = $localeEnforced;
        $this->lockedOut = $lockedOut;
        $this->name = $name;
        $this->navCollapsable = $navCollapsable;
        $this->navColorBg = $navColorBg;
        $this->navColorFg = $navColorFg;
        $this->navigationStyle = $navigationStyle;
        $this->numberOfDaysOfUse = $numberOfDaysOfUse;
        $this->organization = $organization;
        $this->showInCatalog = $showInCatalog;
        $this->softLimit = $softLimit;
        $this->startDate = $startDate;
        $this->uploadLimit = $uploadLimit;
    }

    /**
     * @return string
     */
    public function getAllowGuests()
    {
        return $this->allowGuests;
    }

    /**
     * @param string $allowGuests
     */
    public function setAllowGuests($allowGuests)
    {
        $this->allowGuests = $allowGuests;
    }

    /**
     * @return string
     */
    public function getAllowObservers()
    {
        return $this->allowObservers;
    }

    /**
     * @param string $allowObservers
     */
    public function setAllowObservers($allowObservers)
    {
        $this->allowObservers = $allowObservers;
    }

    /**
     * @return string
     */
    public function getAvailable()
    {
        return $this->available;
    }

    /**
     * @param string $available
     */
    public function setAvailable($available)
    {
        $this->available = $available;
    }

    /**
     * @return string
     */
    public function getBatchUid()
    {
        return $this->batchUid;
    }

    /**
     * @param string $batchUid
     */
    public function setBatchUid($batchUid)
    {
        $this->batchUid = $batchUid;
    }

    /**
     * @return string
     */
    public function getButtonStyleBbId()
    {
        return $this->buttonStyleBbId;
    }

    /**
     * @param string $buttonStyleBbId
     */
    public function setButtonStyleBbId($buttonStyleBbId)
    {
        $this->buttonStyleBbId = $buttonStyleBbId;
    }

    /**
     * @return string
     */
    public function getButtonStyleShape()
    {
        return $this->buttonStyleShape;
    }

    /**
     * @param string $buttonStyleShape
     */
    public function setButtonStyleShape($buttonStyleShape)
    {
        $this->buttonStyleShape = $buttonStyleShape;
    }

    /**
     * @return string
     */
    public function getButtonStyleType()
    {
        return $this->buttonStyleType;
    }

    /**
     * @param string $buttonStyleType
     */
    public function setButtonStyleType($buttonStyleType)
    {
        $this->buttonStyleType = $buttonStyleType;
    }

    /**
     * @return string
     */
    public function getCartridgeId()
    {
        return $this->cartridgeId;
    }

    /**
     * @param string $cartridgeId
     */
    public function setCartridgeId($cartridgeId)
    {
        $this->cartridgeId = $cartridgeId;
    }

    /**
     * @return string
     */
    public function getClassificationId()
    {
        return $this->classificationId;
    }

    /**
     * @param string $classificationId
     */
    public function setClassificationId($classificationId)
    {
        $this->classificationId = $classificationId;
    }

    /**
     * @return string
     */
    public function getCourseDuration()
    {
        return $this->courseDuration;
    }

    /**
     * @param string $courseDuration
     */
    public function setCourseDuration($courseDuration)
    {
        $this->courseDuration = $courseDuration;
    }

    /**
     * @return string
     */
    public function getCourseId()
    {
        return $this->courseId;
    }

    /**
     * @param string $courseId
     */
    public function setCourseId($courseId)
    {
        $this->courseId = $courseId;
    }

    /**
     * @return string
     */
    public function getCoursePace()
    {
        return $this->coursePace;
    }

    /**
     * @param string $coursePace
     */
    public function setCoursePace($coursePace)
    {
        $this->coursePace = $coursePace;
    }

    /**
     * @return string
     */
    public function getCourseServiceLevel()
    {
        return $this->courseServiceLevel;
    }

    /**
     * @param string $courseServiceLevel
     */
    public function setCourseServiceLevel($courseServiceLevel)
    {
        $this->courseServiceLevel = $courseServiceLevel;
    }

    /**
     * @return string
     */
    public function getDataSourceId()
    {
        return $this->dataSourceId;
    }

    /**
     * @param string $dataSourceId
     */
    public function setDataSourceId($dataSourceId)
    {
        $this->dataSourceId = $dataSourceId;
    }

    /**
     * @return string
     */
    public function getDecAbsoluteLimit()
    {
        return $this->decAbsoluteLimit;
    }

    /**
     * @param string $decAbsoluteLimit
     */
    public function setDecAbsoluteLimit($decAbsoluteLimit)
    {
        $this->decAbsoluteLimit = $decAbsoluteLimit;
    }

    /**
     * @return string
     */
    public function getDescription()
    {
        return $this->description;
    }

    /**
     * @param string $description
     */
    public function setDescription($description)
    {
        $this->description = $description;
    }

    /**
     * @return string
     */
    public function getEndDate()
    {
        return $this->endDate;
    }

    /**
     * @param string $endDate
     */
    public function setEndDate($endDate)
    {
        $this->endDate = $endDate;
    }

    /**
     * @return string
     */
    public function getEnrollmentAccessCode()
    {
        return $this->enrollmentAccessCode;
    }

    /**
     * @param string $enrollmentAccessCode
     */
    public function setEnrollmentAccessCode($enrollmentAccessCode)
    {
        $this->enrollmentAccessCode = $enrollmentAccessCode;
    }

    /**
     * @return string
     */
    public function getEnrollmentEndDate()
    {
        return $this->enrollmentEndDate;
    }

    /**
     * @param string $enrollmentEndDate
     */
    public function setEnrollmentEndDate($enrollmentEndDate)
    {
        $this->enrollmentEndDate = $enrollmentEndDate;
    }

    /**
     * @return string
     */
    public function getEnrollmentStartDate()
    {
        return $this->enrollmentStartDate;
    }

    /**
     * @param string $enrollmentStartDate
     */
    public function setEnrollmentStartDate($enrollmentStartDate)
    {
        $this->enrollmentStartDate = $enrollmentStartDate;
    }

    /**
     * @return string
     */
    public function getEnrollmentType()
    {
        return $this->enrollmentType;
    }

    /**
     * @param string $enrollmentType
     */
    public function setEnrollmentType($enrollmentType)
    {
        $this->enrollmentType = $enrollmentType;
    }

    /**
     * @return array
     */
    public function getExpansionData()
    {
        return $this->expansionData;
    }

    /**
     * @param array $expansionData
     */
    public function setExpansionData($expansionData)
    {
        $this->expansionData = $expansionData;
    }

    /**
     * @return string
     */
    public function getFee()
    {
        return $this->fee;
    }

    /**
     * @param string $fee
     */
    public function setFee($fee)
    {
        $this->fee = $fee;
    }

    /**
     * @return string
     */
    public function getHasDescriptionPage()
    {
        return $this->hasDescriptionPage;
    }

    /**
     * @param string $hasDescriptionPage
     */
    public function setHasDescriptionPage($hasDescriptionPage)
    {
        $this->hasDescriptionPage = $hasDescriptionPage;
    }

    /**
     * @return string
     */
    public function getId()
    {
        return $this->id;
    }

    /**
     * @param string $id
     */
    public function setId($id)
    {
        $this->id = $id;
    }

    /**
     * @return string
     */
    public function getInstitutionName()
    {
        return $this->institutionName;
    }

    /**
     * @param string $institutionName
     */
    public function setInstitutionName($institutionName)
    {
        $this->institutionName = $institutionName;
    }

    /**
     * @return string
     */
    public function getLocale()
    {
        return $this->locale;
    }

    /**
     * @param string $locale
     */
    public function setLocale($locale)
    {
        $this->locale = $locale;
    }

    /**
     * @return string
     */
    public function getLocaleEnforced()
    {
        return $this->localeEnforced;
    }

    /**
     * @param string $localeEnforced
     */
    public function setLocaleEnforced($localeEnforced)
    {
        $this->localeEnforced = $localeEnforced;
    }

    /**
     * @return string
     */
    public function getLockedOut()
    {
        return $this->lockedOut;
    }

    /**
     * @param string $lockedOut
     */
    public function setLockedOut($lockedOut)
    {
        $this->lockedOut = $lockedOut;
    }

    /**
     * @return string
     */
    public function getName()
    {
        return $this->name;
    }

    /**
     * @param string $name
     */
    public function setName($name)
    {
        $this->name = $name;
    }

    /**
     * @return string
     */
    public function getNavCollapsable()
    {
        return $this->navCollapsable;
    }

    /**
     * @param string $navCollapsable
     */
    public function setNavCollapsable($navCollapsable)
    {
        $this->navCollapsable = $navCollapsable;
    }

    /**
     * @return string
     */
    public function getNavColorBg()
    {
        return $this->navColorBg;
    }

    /**
     * @param string $navColorBg
     */
    public function setNavColorBg($navColorBg)
    {
        $this->navColorBg = $navColorBg;
    }

    /**
     * @return string
     */
    public function getNavColorFg()
    {
        return $this->navColorFg;
    }

    /**
     * @param string $navColorFg
     */
    public function setNavColorFg($navColorFg)
    {
        $this->navColorFg = $navColorFg;
    }

    /**
     * @return string
     */
    public function getNavigationStyle()
    {
        return $this->navigationStyle;
    }

    /**
     * @param string $navigationStyle
     */
    public function setNavigationStyle($navigationStyle)
    {
        $this->navigationStyle = $navigationStyle;
    }

    /**
     * @return string
     */
    public function getNumberOfDaysOfUse()
    {
        return $this->numberOfDaysOfUse;
    }

    /**
     * @param string $numberOfDaysOfUse
     */
    public function setNumberOfDaysOfUse($numberOfDaysOfUse)
    {
        $this->numberOfDaysOfUse = $numberOfDaysOfUse;
    }

    /**
     * @return string
     */
    public function getOrganization()
    {
        return $this->organization;
    }

    /**
     * @param string $organization
     */
    public function setOrganization($organization)
    {
        $this->organization = $organization;
    }

    /**
     * @return string
     */
    public function getShowInCatalog()
    {
        return $this->showInCatalog;
    }

    /**
     * @param string $showInCatalog
     */
    public function setShowInCatalog($showInCatalog)
    {
        $this->showInCatalog = $showInCatalog;
    }

    /**
     * @return string
     */
    public function getSoftLimit()
    {
        return $this->softLimit;
    }

    /**
     * @param string $softLimit
     */
    public function setSoftLimit($softLimit)
    {
        $this->softLimit = $softLimit;
    }

    /**
     * @return string
     */
    public function getStartDate()
    {
        return $this->startDate;
    }

    /**
     * @param string $startDate
     */
    public function setStartDate($startDate)
    {
        $this->startDate = $startDate;
    }

    /**
     * @return string
     */
    public function getUploadLimit()
    {
        return $this->uploadLimit;
    }

    /**
     * @param string $uploadLimit
     */
    public function setUploadLimit($uploadLimit)
    {
        $this->uploadLimit = $uploadLimit;
    }
}