text:
package org.apache.hadoop.mapreduce.task.reduce;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.ChecksumFileSystem;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.IFile;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapOutputFile;
import org.apache.hadoop.mapred.Merger;
import org.apache.hadoop.mapred.RawKeyValueIterator;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.Task;
import org.apache.hadoop.mapred.IFile.Reader;
import org.apache.hadoop.mapred.IFile.Writer;
import org.apache.hadoop.mapred.Merger.Segment;
import org.apache.hadoop.mapred.Task.CombineOutputCollector;
import org.apache.hadoop.mapred.Task.CombineValuesIterator;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.CryptoUtils;
import org.apache.hadoop.mapreduce.task.reduce.MapOutput.MapOutputComparator;
import org.apache.hadoop.util.Progress;
import org.apache.hadoop.util.ReflectionUtils;

import com.google.common.annotations.VisibleForTesting;

@SuppressWarnings(value={"unchecked"})
@InterfaceAudience.LimitedPrivate({"MapReduce"})
@InterfaceStability.Unstable
public class MergeManagerImpl<K, V> implements MergeManager<K, V> {

  private static final Log LOG = LogFactory.getLog(MergeManagerImpl.class);

  /* Maximum percentage of the in-memory limit that a single shuffle can
   * consume*/
  private static final float DEFAULT_SHUFFLE_MEMORY_LIMIT_PERCENT
    = 0.25f;

  private final TaskAttemptID reduceId;

  private final JobConf jobConf;
  private final FileSystem localFS;
  private final FileSystem rfs;
  private final LocalDirAllocator localDirAllocator;

  protected MapOutputFile mapOutputFile;

  Set<InMemoryMapOutput<K, V>> inMemoryMergedMapOutputs =
    new TreeSet<InMemoryMapOutput<K,V>>(new MapOutputComparator<K, V>());
  private IntermediateMemoryToMemoryMerger memToMemMerger;

  Set<InMemoryMapOutput<K, V>> inMemoryMapOutputs =
    new TreeSet<InMemoryMapOutput<K,V>>(new MapOutputComparator<K, V>());
  private final MergeThread<InMemoryMapOutput<K,V>, K,V> inMemoryMerger;

  Set<CompressAwarePath> onDiskMapOutputs = new TreeSet<CompressAwarePath>();
  private final OnDiskMerger onDiskMerger;

  private final long memoryLimit;
  private long usedMemory;
  private long commitMemory;
  private final long maxSingleShuffleLimit;

  private final int memToMemMergeOutputsThreshold;
  private final long mergeThreshold;

  private final int ioSortFactor;

  private final Reporter reporter;
  private final ExceptionReporter exceptionReporter;

  /**
   * Combiner class to run during in-memory merge, if defined.
   */
  private final Class<? extends Reducer> combinerClass;

  /**
   * Resettable collector used for combine.
   */
  private final CombineOutputCollector<K,V> combineCollector;

  private final Counters.Counter spilledRecordsCounter;

  private final Counters.Counter reduceCombineInputCounter;

  private final Counters.Counter mergedMapOutputsCounter;

  private final CompressionCodec codec;

  private final Progress mergePhase;

  public MergeManagerImpl(TaskAttemptID reduceId, JobConf jobConf,
                      FileSystem localFS,
                      LocalDirAllocator localDirAllocator,
                      Reporter reporter,
                      CompressionCodec codec,
                      Class<? extends Reducer> combinerClass,
                      CombineOutputCollector<K,V> combineCollector,
                      Counters.Counter spilledRecordsCounter,
                      Counters.Counter reduceCombineInputCounter,
                      Counters.Counter mergedMapOutputsCounter,
                      ExceptionReporter exceptionReporter,
                      Progress mergePhase, MapOutputFile mapOutputFile) {
    this.reduceId = reduceId;
    this.jobConf = jobConf;
    this.localDirAllocator = localDirAllocator;
    this.exceptionReporter = exceptionReporter;

    this.reporter = reporter;
    this.codec = codec;
    this.combinerClass = combinerClass;
    this.combineCollector = combineCollector;
    this.reduceCombineInputCounter = reduceCombineInputCounter;
    this.spilledRecordsCounter = spilledRecordsCounter;
    this.mergedMapOutputsCounter = mergedMapOutputsCounter;
    this.mapOutputFile = mapOutputFile;
    this.mapOutputFile.setConf(jobConf);

    this.localFS = localFS;
    this.rfs = ((LocalFileSystem)localFS).getRaw();

    final float maxInMemCopyUse =
      jobConf.getFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT,
          MRJobConfig.DEFAULT_SHUFFLE_INPUT_BUFFER_PERCENT);
    if (maxInMemCopyUse > 1.0 || maxInMemCopyUse < 0.0) {
      throw new IllegalArgumentException("Invalid value for " +
          MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT + ": " +
          maxInMemCopyUse);
    }

    // Allow unit tests to fix Runtime memory
    this.memoryLimit =
      (long)(jobConf.getLong(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES,
          Math.min(Runtime.getRuntime().maxMemory(), Integer.MAX_VALUE))
        * maxInMemCopyUse);

    this.ioSortFactor = jobConf.getInt(MRJobConfig.IO_SORT_FACTOR, 100);

    final float singleShuffleMemoryLimitPercent =
        jobConf.getFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT,
            DEFAULT_SHUFFLE_MEMORY_LIMIT_PERCENT);
    if (singleShuffleMemoryLimitPercent <= 0.0f
        || singleShuffleMemoryLimitPercent > 1.0f) {
      throw new IllegalArgumentException("Invalid value for "
          + MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT + ": "
          + singleShuffleMemoryLimitPercent);
    }

    usedMemory = 0L;
    commitMemory = 0L;
    this.maxSingleShuffleLimit =
      (long)(memoryLimit * singleShuffleMemoryLimitPercent);
    this.memToMemMergeOutputsThreshold =
            jobConf.getInt(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, ioSortFactor);
    this.mergeThreshold = (long)(this.memoryLimit *
                          jobConf.getFloat(
                            MRJobConfig.SHUFFLE_MERGE_PERCENT,
                            MRJobConfig.DEFAULT_SHUFFLE_MERGE_PERCENT));
    LOG.info("MergerManager: memoryLimit=" + memoryLimit + ", " +
             "maxSingleShuffleLimit=" + maxSingleShuffleLimit + ", " +
             "mergeThreshold=" + mergeThreshold + ", " +
             "ioSortFactor=" + ioSortFactor + ", " +
             "memToMemMergeOutputsThreshold=" + memToMemMergeOutputsThreshold);

    if (this.maxSingleShuffleLimit >= this.mergeThreshold) {
      throw new RuntimeException("Invalid configuration: "
          + "maxSingleShuffleLimit should be less than mergeThreshold"
          + "maxSingleShuffleLimit: " + this.maxSingleShuffleLimit
          + "mergeThreshold: " + this.mergeThreshold);
    }

    boolean allowMemToMemMerge =
      jobConf.getBoolean(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, false);
    if (allowMemToMemMerge) {
      this.memToMemMerger =
        new IntermediateMemoryToMemoryMerger(this,
                                             memToMemMergeOutputsThreshold);
      this.memToMemMerger.start();
    } else {
      this.memToMemMerger = null;
    }

    this.inMemoryMerger = createInMemoryMerger();
    this.inMemoryMerger.start();

    this.onDiskMerger = new OnDiskMerger(this);
    this.onDiskMerger.start();

    this.mergePhase = mergePhase;
  }

  protected MergeThread<InMemoryMapOutput<K,V>, K,V> createInMemoryMerger() {
    return new InMemoryMerger(this);
  }

  protected MergeThread<CompressAwarePath,K,V> createOnDiskMerger() {
    return new OnDiskMerger(this);
  }

  TaskAttemptID getReduceId() {
    return reduceId;
  }

  @VisibleForTesting
  ExceptionReporter getExceptionReporter() {
    return exceptionReporter;
  }

  @Override
  public void waitForResource() throws InterruptedException {
    inMemoryMerger.waitForMerge();
  }

  private boolean canShuffleToMemory(long requestedSize) {
    return (requestedSize < maxSingleShuffleLimit);
  }

  @Override
  public synchronized MapOutput<K,V> reserve(TaskAttemptID mapId,
                                             long requestedSize,
                                             int fetcher
                                             ) throws IOException {
    if (!canShuffleToMemory(requestedSize)) {
      LOG.info(mapId + ": Shuffling to disk since " + requestedSize +
               " is greater than maxSingleShuffleLimit (" +
               maxSingleShuffleLimit + ")");
      return new OnDiskMapOutput<K,V>(mapId, reduceId, this, requestedSize,
                                      jobConf, mapOutputFile, fetcher, true);
    }

    // Stall shuffle if we are above the memory limit

    // It is possible that all threads could just be stalling and not make
    // progress at all. This could happen when:
    //
    // requested size is causing the used memory to go above limit &&
    // requested size < singleShuffleLimit &&
    // current used size < mergeThreshold (merge will not get triggered)
    //
    // To avoid this from happening, we allow exactly one thread to go past
    // the memory limit. We check (usedMemory > memoryLimit) and not
    // (usedMemory + requestedSize > memoryLimit). When this thread is done
    // fetching, this will automatically trigger a merge thereby unlocking
    // all the stalled threads

    if (usedMemory > memoryLimit) {
      LOG.debug(mapId + ": Stalling shuffle since usedMemory (" + usedMemory
          + ") is greater than memoryLimit (" + memoryLimit + ")." +
          " CommitMemory is (" + commitMemory + ")");
      return null;
    }

    // Allow the in-memory shuffle to progress
    LOG.debug(mapId + ": Proceeding with shuffle since usedMemory ("
        + usedMemory + ") is lesser than memoryLimit (" + memoryLimit + ")."
        + "CommitMemory is (" + commitMemory + ")");
    return unconditionalReserve(mapId, requestedSize, true);
  }

  /**
   * Unconditional Reserve is used by the Memory-to-Memory thread
   * @return
   */
  private synchronized InMemoryMapOutput<K, V> unconditionalReserve(
      TaskAttemptID mapId, long requestedSize, boolean primaryMapOutput) {
    usedMemory += requestedSize;
    return new InMemoryMapOutput<K,V>(jobConf, mapId, this, (int)requestedSize,
                                      codec, primaryMapOutput);
  }

  synchronized void unreserve(long size) {
    usedMemory -= size;
  }

  public synchronized void closeInMemoryFile(InMemoryMapOutput<K,V> mapOutput) {
    inMemoryMapOutputs.add(mapOutput);
    LOG.info("closeInMemoryFile -> map-output of size: " + mapOutput.getSize()
        + ", inMemoryMapOutputs.size() -> " + inMemoryMapOutputs.size()
        + ", commitMemory -> " + commitMemory + ", usedMemory ->" + usedMemory);

    commitMemory+= mapOutput.getSize();

    // Can hang if mergeThreshold is really low.
    if (commitMemory >= mergeThreshold) {
      LOG.info("Starting inMemoryMerger's merge since commitMemory=" +
          commitMemory + " > mergeThreshold=" + mergeThreshold +
          ". Current usedMemory=" + usedMemory);
      inMemoryMapOutputs.addAll(inMemoryMergedMapOutputs);
      inMemoryMergedMapOutputs.clear();
      inMemoryMerger.startMerge(inMemoryMapOutputs);
      commitMemory = 0L;  // Reset commitMemory.
    }

    if (memToMemMerger != null) {
      if (inMemoryMapOutputs.size() >= memToMemMergeOutputsThreshold) {
        memToMemMerger.startMerge(inMemoryMapOutputs);
      }
    }
  }

  public synchronized void closeInMemoryMergedFile(InMemoryMapOutput<K,V> mapOutput) {
    inMemoryMergedMapOutputs.add(mapOutput);
    LOG.info("closeInMemoryMergedFile -> size: " + mapOutput.getSize() +
             ", inMemoryMergedMapOutputs.size() -> " +
             inMemoryMergedMapOutputs.size());
  }

  public synchronized void closeOnDiskFile(CompressAwarePath file) {
    onDiskMapOutputs.add(file);

    if (onDiskMapOutputs.size() >= (2 * ioSortFactor - 1)) {
      onDiskMerger.startMerge(onDiskMapOutputs);
    }
  }

  @Override
  public RawKeyValueIterator close() throws Throwable {
    // Wait for on-going merges to complete
    if (memToMemMerger != null) {
      memToMemMerger.close();
    }
    inMemoryMerger.close();
    onDiskMerger.close();

    List<InMemoryMapOutput<K, V>> memory =
      new ArrayList<InMemoryMapOutput<K, V>>(inMemoryMergedMapOutputs);
    inMemoryMergedMapOutputs.clear();
    memory.addAll(inMemoryMapOutputs);
    inMemoryMapOutputs.clear();
    List<CompressAwarePath> disk = new ArrayList<CompressAwarePath>(onDiskMapOutputs);
    onDiskMapOutputs.clear();
    return finalMerge(jobConf, rfs, memory, disk);
  }

  private class IntermediateMemoryToMemoryMerger
  extends MergeThread<InMemoryMapOutput<K, V>, K, V> {

    public IntermediateMemoryToMemoryMerger(MergeManagerImpl<K, V> manager,
                                            int mergeFactor) {
      super(manager, mergeFactor, exceptionReporter);
      setName("InMemoryMerger - Thread to do in-memory merge of in-memory " +
      		    "shuffled map-outputs");
      setDaemon(true);
    }

    @Override
    public void merge(List<InMemoryMapOutput<K, V>> inputs) throws IOException {
      if (inputs == null || inputs.size() == 0) {
        return;
      }

      TaskAttemptID dummyMapId = inputs.get(0).getMapId();
      List<Segment<K, V>> inMemorySegments = new ArrayList<Segment<K, V>>();
      long mergeOutputSize =
        createInMemorySegments(inputs, inMemorySegments, 0);
      int noInMemorySegments = inMemorySegments.size();

      InMemoryMapOutput<K, V> mergedMapOutputs =
        unconditionalReserve(dummyMapId, mergeOutputSize, false);

      Writer<K, V> writer =
        new InMemoryWriter<K, V>(mergedMapOutputs.getArrayStream());

      LOG.info("Initiating Memory-to-Memory merge with " + noInMemorySegments +
               " segments of total-size: " + mergeOutputSize);

      RawKeyValueIterator rIter =
        Merger.merge(jobConf, rfs,
                     (Class<K>)jobConf.getMapOutputKeyClass(),
                     (Class<V>)jobConf.getMapOutputValueClass(),
                     inMemorySegments, inMemorySegments.size(),
                     new Path(reduceId.toString()),
                     (RawComparator<K>)jobConf.getOutputKeyComparator(),
                     reporter, null, null, null);
      Merger.writeFile(rIter, writer, reporter, jobConf);
      writer.close();

      LOG.info(reduceId +
               " Memory-to-Memory merge of the " + noInMemorySegments +
               " files in-memory complete.");

      // Note the output of the merge
      closeInMemoryMergedFile(mergedMapOutputs);
    }
  }

  private class InMemoryMerger extends MergeThread<InMemoryMapOutput<K,V>, K,V> {

    public InMemoryMerger(MergeManagerImpl<K, V> manager) {
      super(manager, Integer.MAX_VALUE, exceptionReporter);
      setName
      ("InMemoryMerger - Thread to merge in-memory shuffled map-outputs");
      setDaemon(true);
    }

    @Override
    public void merge(List<InMemoryMapOutput<K,V>> inputs) throws IOException {
      if (inputs == null || inputs.size() == 0) {
        return;
      }

      //name this output file same as the name of the first file that is
      //there in the current list of inmem files (this is guaranteed to
      //be absent on the disk currently. So we don't overwrite a prev.
      //created spill). Also we need to create the output file now since
      //it is not guaranteed that this file will be present after merge
      //is called (we delete empty files as soon as we see them
      //in the merge method)

      //figure out the mapId
      TaskAttemptID mapId = inputs.get(0).getMapId();
      TaskID mapTaskId = mapId.getTaskID();

      List<Segment<K, V>> inMemorySegments = new ArrayList<Segment<K, V>>();
      long mergeOutputSize =
        createInMemorySegments(inputs, inMemorySegments,0);
      int noInMemorySegments = inMemorySegments.size();

      Path outputPath =
        mapOutputFile.getInputFileForWrite(mapTaskId,
                                           mergeOutputSize).suffix(
                                               Task.MERGED_OUTPUT_PREFIX);

      FSDataOutputStream out = CryptoUtils.wrapIfNecessary(jobConf, rfs.create(outputPath));
      Writer<K, V> writer = new Writer<K, V>(jobConf, out,
          (Class<K>) jobConf.getMapOutputKeyClass(),
          (Class<V>) jobConf.getMapOutputValueClass(), codec, null, true);

      RawKeyValueIterator rIter = null;
      CompressAwarePath compressAwarePath;
      try {
        LOG.info("Initiating in-memory merge with " + noInMemorySegments +
                 " segments...");

        rIter = Merger.merge(jobConf, rfs,
                             (Class<K>)jobConf.getMapOutputKeyClass(),
                             (Class<V>)jobConf.getMapOutputValueClass(),
                             inMemorySegments, inMemorySegments.size(),
                             new Path(reduceId.toString()),
                             (RawComparator<K>)jobConf.getOutputKeyComparator(),
                             reporter, spilledRecordsCounter, null, null);

        if (null == combinerClass) {
          Merger.writeFile(rIter, writer, reporter, jobConf);
        } else {
          combineCollector.setWriter(writer);
          combineAndSpill(rIter, reduceCombineInputCounter);
        }
        writer.close();
        compressAwarePath = new CompressAwarePath(outputPath,
            writer.getRawLength(), writer.getCompressedLength());

        LOG.info(reduceId +
            " Merge of the " + noInMemorySegments +
            " files in-memory complete." +
            " Local file is " + outputPath + " of size " +
            localFS.getFileStatus(outputPath).getLen());
      } catch (IOException e) {
        //make sure that we delete the ondisk file that we created
        //earlier when we invoked cloneFileAttributes
        localFS.delete(outputPath, true);
        throw e;
      }

      // Note the output of the merge
      closeOnDiskFile(compressAwarePath);
    }

  }

  private class OnDiskMerger extends MergeThread<CompressAwarePath,K,V> {

    public OnDiskMerger(MergeManagerImpl<K, V> manager) {
      super(manager, ioSortFactor, exceptionReporter);
      setName("OnDiskMerger - Thread to merge on-disk map-outputs");
      setDaemon(true);
    }

    @Override
    public void merge(List<CompressAwarePath> inputs) throws IOException {
      // sanity check
      if (inputs == null || inputs.isEmpty()) {
        LOG.info("No ondisk files to merge...");
        return;
      }

      long approxOutputSize = 0;
      int bytesPerSum =
        jobConf.getInt("io.bytes.per.checksum", 512);

      LOG.info("OnDiskMerger: We have  " + inputs.size() +
               " map outputs on disk. Triggering merge...");

      // 1. Prepare the list of files to be merged.
      for (CompressAwarePath file : inputs) {
        approxOutputSize += localFS.getFileStatus(file).getLen();
      }

      // add the checksum length
      approxOutputSize +=
        ChecksumFileSystem.getChecksumLength(approxOutputSize, bytesPerSum);

      // 2. Start the on-disk merge process
      Path outputPath =
        localDirAllocator.getLocalPathForWrite(inputs.get(0).toString(),
            approxOutputSize, jobConf).suffix(Task.MERGED_OUTPUT_PREFIX);

      FSDataOutputStream out = CryptoUtils.wrapIfNecessary(jobConf, rfs.create(outputPath));
      Writer<K, V> writer = new Writer<K, V>(jobConf, out,
          (Class<K>) jobConf.getMapOutputKeyClass(),
          (Class<V>) jobConf.getMapOutputValueClass(), codec, null, true);

      RawKeyValueIterator iter  = null;
      CompressAwarePath compressAwarePath;
      Path tmpDir = new Path(reduceId.toString());
      try {
        iter = Merger.merge(jobConf, rfs,
                            (Class<K>) jobConf.getMapOutputKeyClass(),
                            (Class<V>) jobConf.getMapOutputValueClass(),
                            codec, inputs.toArray(new Path[inputs.size()]),
                            true, ioSortFactor, tmpDir,
                            (RawComparator<K>) jobConf.getOutputKeyComparator(),
                            reporter, spilledRecordsCounter, null,
                            mergedMapOutputsCounter, null);

        Merger.writeFile(iter, writer, reporter, jobConf);
        writer.close();
        compressAwarePath = new CompressAwarePath(outputPath,
            writer.getRawLength(), writer.getCompressedLength());
      } catch (IOException e) {
        localFS.delete(outputPath, true);
        throw e;
      }

      closeOnDiskFile(compressAwarePath);

      LOG.info(reduceId +
          " Finished merging " + inputs.size() +
          " map output files on disk of total-size " +
          approxOutputSize + "." +
          " Local output file is " + outputPath + " of size " +
          localFS.getFileStatus(outputPath).getLen());
    }
  }

  private void combineAndSpill(
      RawKeyValueIterator kvIter,
      Counters.Counter inCounter) throws IOException {
    JobConf job = jobConf;
    Reducer combiner = ReflectionUtils.newInstance(combinerClass, job);
    Class<K> keyClass = (Class<K>) job.getMapOutputKeyClass();
    Class<V> valClass = (Class<V>) job.getMapOutputValueClass();
    RawComparator<K> comparator =
      (RawComparator<K>)job.getCombinerKeyGroupingComparator();
    try {
      CombineValuesIterator values = new CombineValuesIterator(
          kvIter, comparator, keyClass, valClass, job, Reporter.NULL,
          inCounter);
      while (values.more()) {
        combiner.reduce(values.getKey(), values, combineCollector,
                        Reporter.NULL);
        values.nextKey();
      }
    } finally {
      combiner.close();
    }
  }

  private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs,
                                      List<Segment<K, V>> inMemorySegments,
                                      long leaveBytes
                                      ) throws IOException {
    long totalSize = 0L;
    // We could use fullSize could come from the RamManager, but files can be
    // closed but not yet present in inMemoryMapOutputs
    long fullSize = 0L;
    for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) {
      fullSize += mo.getMemory().length;
    }
    while(fullSize > leaveBytes) {
      InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0);
      byte[] data = mo.getMemory();
      long size = data.length;
      totalSize += size;
      fullSize -= size;
      Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this,
                                                   mo.getMapId(),
                                                   data, 0, (int)size, jobConf);
      inMemorySegments.add(new Segment<K,V>(reader, true,
                                            (mo.isPrimaryMapOutput() ?
                                            mergedMapOutputsCounter : null)));
    }
    return totalSize;
  }

  class RawKVIteratorReader extends IFile.Reader<K,V> {

    private final RawKeyValueIterator kvIter;

    public RawKVIteratorReader(RawKeyValueIterator kvIter, long size)
        throws IOException {
      super(null, null, size, null, spilledRecordsCounter);
      this.kvIter = kvIter;
    }
    public boolean nextRawKey(DataInputBuffer key) throws IOException {
      if (kvIter.next()) {
        final DataInputBuffer kb = kvIter.getKey();
        final int kp = kb.getPosition();
        final int klen = kb.getLength() - kp;
        key.reset(kb.getData(), kp, klen);
        bytesRead += klen;
        return true;
      }
      return false;
    }
    public void nextRawValue(DataInputBuffer value) throws IOException {
      final DataInputBuffer vb = kvIter.getValue();
      final int vp = vb.getPosition();
      final int vlen = vb.getLength() - vp;
      value.reset(vb.getData(), vp, vlen);
      bytesRead += vlen;
    }
    public long getPosition() throws IOException {
      return bytesRead;
    }

    public void close() throws IOException {
      kvIter.close();
    }
  }

  private RawKeyValueIterator finalMerge(JobConf job, FileSystem fs,
                                       List<InMemoryMapOutput<K,V>> inMemoryMapOutputs,
                                       List<CompressAwarePath> onDiskMapOutputs
                                       ) throws IOException {
    LOG.info("finalMerge called with " +
             inMemoryMapOutputs.size() + " in-memory map-outputs and " +
             onDiskMapOutputs.size() + " on-disk map-outputs");

    final float maxRedPer =
      job.getFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 0f);
    if (maxRedPer > 1.0 || maxRedPer < 0.0) {
      throw new IOException(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT +
                            maxRedPer);
    }
    int maxInMemReduce = (int)Math.min(
        Runtime.getRuntime().maxMemory() * maxRedPer, Integer.MAX_VALUE);

    // merge config params
    Class<K> keyClass = (Class<K>)job.getMapOutputKeyClass();
    Class<V> valueClass = (Class<V>)job.getMapOutputValueClass();
    boolean keepInputs = job.getKeepFailedTaskFiles();
    final Path tmpDir = new Path(reduceId.toString());
    final RawComparator<K> comparator =
      (RawComparator<K>)job.getOutputKeyComparator();

    // segments required to vacate memory
    List<Segment<K,V>> memDiskSegments = new ArrayList<Segment<K,V>>();
    long inMemToDiskBytes = 0;
    boolean mergePhaseFinished = false;
    if (inMemoryMapOutputs.size() > 0) {
      TaskID mapId = inMemoryMapOutputs.get(0).getMapId().getTaskID();
      inMemToDiskBytes = createInMemorySegments(inMemoryMapOutputs,
                                                memDiskSegments,
                                                maxInMemReduce);
      final int numMemDiskSegments = memDiskSegments.size();
      if (numMemDiskSegments > 0 &&
            ioSortFactor > onDiskMapOutputs.size()) {

        // If we reach here, it implies that we have less than io.sort.factor
        // disk segments and this will be incremented by 1 (result of the
        // memory segments merge). Since this total would still be
        // <= io.sort.factor, we will not do any more intermediate merges,
        // the merge of all these disk segments would be directly fed to the
        // reduce method

        mergePhaseFinished = true;
        // must spill to disk, but can't retain in-mem for intermediate merge
        final Path outputPath =
          mapOutputFile.getInputFileForWrite(mapId,
                                             inMemToDiskBytes).suffix(
                                                 Task.MERGED_OUTPUT_PREFIX);
        final RawKeyValueIterator rIter = Merger.merge(job, fs,
            keyClass, valueClass, memDiskSegments, numMemDiskSegments,
            tmpDir, comparator, reporter, spilledRecordsCounter, null,
            mergePhase);

        FSDataOutputStream out = CryptoUtils.wrapIfNecessary(job, fs.create(outputPath));
        Writer<K, V> writer = new Writer<K, V>(job, out, keyClass, valueClass,
            codec, null, true);
        try {
          Merger.writeFile(rIter, writer, reporter, job);
          writer.close();
          onDiskMapOutputs.add(new CompressAwarePath(outputPath,
              writer.getRawLength(), writer.getCompressedLength()));
          writer = null;
          // add to list of final disk outputs.
        } catch (IOException e) {
          if (null != outputPath) {
            try {
              fs.delete(outputPath, true);
            } catch (IOException ie) {
              // NOTHING
            }
          }
          throw e;
        } finally {
          if (null != writer) {
            writer.close();
          }
        }
        LOG.info("Merged " + numMemDiskSegments + " segments, " +
                 inMemToDiskBytes + " bytes to disk to satisfy " +
                 "reduce memory limit");
        inMemToDiskBytes = 0;
        memDiskSegments.clear();
      } else if (inMemToDiskBytes != 0) {
        LOG.info("Keeping " + numMemDiskSegments + " segments, " +
                 inMemToDiskBytes + " bytes in memory for " +
                 "intermediate, on-disk merge");
      }
    }

    // segments on disk
    List<Segment<K,V>> diskSegments = new ArrayList<Segment<K,V>>();
    long onDiskBytes = inMemToDiskBytes;
    long rawBytes = inMemToDiskBytes;
    CompressAwarePath[] onDisk = onDiskMapOutputs.toArray(
        new CompressAwarePath[onDiskMapOutputs.size()]);
    for (CompressAwarePath file : onDisk) {
      long fileLength = fs.getFileStatus(file).getLen();
      onDiskBytes += fileLength;
      rawBytes += (file.getRawDataLength() > 0) ? file.getRawDataLength() : fileLength;

      LOG.debug("Disk file: " + file + " Length is " + fileLength);
      diskSegments.add(new Segment<K, V>(job, fs, file, codec, keepInputs,
                                         (file.toString().endsWith(
                                             Task.MERGED_OUTPUT_PREFIX) ?
                                          null : mergedMapOutputsCounter), file.getRawDataLength()
                                        ));
    }
    LOG.info("Merging " + onDisk.length + " files, " +
             onDiskBytes + " bytes from disk");
    Collections.sort(diskSegments, new Comparator<Segment<K,V>>() {
      public int compare(Segment<K, V> o1, Segment<K, V> o2) {
        if (o1.getLength() == o2.getLength()) {
          return 0;
        }
        return o1.getLength() < o2.getLength() ? -1 : 1;
      }
    });

    // build final list of segments from merged backed by disk + in-mem
    List<Segment<K,V>> finalSegments = new ArrayList<Segment<K,V>>();
    long inMemBytes = createInMemorySegments(inMemoryMapOutputs,
                                             finalSegments, 0);
    LOG.info("Merging " + finalSegments.size() + " segments, " +
             inMemBytes + " bytes from memory into reduce");
    if (0 != onDiskBytes) {
      final int numInMemSegments = memDiskSegments.size();
      diskSegments.addAll(0, memDiskSegments);
      memDiskSegments.clear();
      // Pass mergePhase only if there is a going to be intermediate
      // merges. See comment where mergePhaseFinished is being set
      Progress thisPhase = (mergePhaseFinished) ? null : mergePhase;
      RawKeyValueIterator diskMerge = Merger.merge(
          job, fs, keyClass, valueClass, codec, diskSegments,
          ioSortFactor, numInMemSegments, tmpDir, comparator,
          reporter, false, spilledRecordsCounter, null, thisPhase);
      diskSegments.clear();
      if (0 == finalSegments.size()) {
        return diskMerge;
      }
      finalSegments.add(new Segment<K,V>(
            new RawKVIteratorReader(diskMerge, onDiskBytes), true, rawBytes));
    }
    return Merger.merge(job, fs, keyClass, valueClass,
                 finalSegments, finalSegments.size(), tmpDir,
                 comparator, reporter, spilledRecordsCounter, null,
                 null);

  }

  static class CompressAwarePath extends Path {
    private long rawDataLength;
    private long compressedSize;

    public CompressAwarePath(Path path, long rawDataLength, long compressSize) {
      super(path.toUri());
      this.rawDataLength = rawDataLength;
      this.compressedSize = compressSize;
    }

    public long getRawDataLength() {
      return rawDataLength;
    }

    public long getCompressedSize() {
      return compressedSize;
    }

    @Override
    public boolean equals(Object other) {
      return super.equals(other);
    }

    @Override
    public int hashCode() {
      return super.hashCode();
    }

    @Override
    public int compareTo(Object obj) {
      if(obj instanceof CompressAwarePath) {
        CompressAwarePath compPath = (CompressAwarePath) obj;
        if(this.compressedSize < compPath.getCompressedSize()) {
          return -1;
        } else if (this.getCompressedSize() > compPath.getCompressedSize()) {
          return 1;
        }
        // Not returning 0 here so that objects with the same size (but
        // different paths) are still added to the TreeSet.
      }
      return super.compareTo(obj);
    }
  }
}


text:
//  DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
//
//   Copyright 2016-2018 Pascal ECHEMANN.
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

/**
 * The GlassCatErrorCode class defines a set of standard errors
 * references that are used to indicates the reason why a
 * GlassCatError exception is thrown.
 */
export enum GlassCatErrorCode {

  /**
   * Indicates that the exception is due to the attempt of instantiating a
   * singleton, instead of using the getInstance() method.
   * This error code is initialized to 0.
   */
  SINGLETON_ERROR = 0,

  /**
   * Indicates that the exception is due to the loading failure of a
   * configuration file.
   * This error code is initialized to 1.
   */
  CONFIG_LOADING_FAILURE = 1,

  /**
   * Indicates that the exception is due to an attempt of serializing an invalid
   * configuration object.
   * This error code is initialized to 2.
   */
  CONFIG_SERIALIZATION_ERROR = 2,

  /**
   * Indicates that the exception is due to an error thrown while updating a
   * configuration object.
   * This error code is initialized to 3.
   */
  CONFIG_UPDATE_ERROR = 3,

  /**
   * Indicates that the specified EjpConfig object must not be
   * null.
   * This error code is initialized to 4.
   */
  NULL_EJP_CONFIG = 4,

  /**
   * Indicates that the specified EjpConfig object does not define
   * a required property.
   * This error code is initialized to 5.
   */
  EJP_CONFIG_MISSING_PROPERTY = 5,

  /**
   * Indicates that the specified EjpConfig object define an
   * invalid property.
   * This error code is initialized to 6.
   */
  EJP_CONFIG_INVALID_PROPERTY = 6,

  /**
   * Indicates that the specified EjpConfig object define an
   * invalid login configuration.
   * This error code is initialized to 7.
   */
  EJP_CONFIG_INVALID_LOGIN = 7,

  /**
   * Indicates that the specified EjpConfig object define an
   * invalid realm configuration.
   * This error code is initialized to 8.
   */
  EJP_CONFIG_INVALID_REALM = 8,

  /**
   * Indicates that the specified secutity context object is not valid.
   * This error code is initialized to 9.
   */
  INVALID_SECURITY_CONTEXT = 9,

  /**
   * Indicates that the specified encryption key is not valid.
   * This error code is initialized to 10.
   */
  INVALID_ENCRYPTION_KEY = 10,

  /**
   * Indicates that the specified context object is not valid.
   * This error code is initialized to 11.
   */
  INVALID_CONTEXT = 11,

  /**
   * Indicates that an error ocurred while initializing the admin realm.
   * This error code is initialized to 12.
   */
  ADMIN_REALM_INIT_FAILURE = 12,

  /**
   * Indicates that the specified jslet configuration is not valid.
   * This error code is initialized to 13.
   */
  INVALID_JSLET_CONFIG = 13,

  /**
   * Indicates that the specified bootstrap configuration is not valid.
   * This error code is initialized to 14.
   */
  INVALID_BOOTSTRAP_CONFIG = 14
}

redpajama_set_name: RedPajamaGithub


text:
Neuron-Example
==============


text:
"""Bindings for the Barnes Hut TSNE algorithm with fast nearest neighbors

Refs:
References
[1] van der Maaten, L.J.P.; Hinton, G.E. Visualizing High-Dimensional Data
Using t-SNE. Journal of Machine Learning Research 9:2579-2605, 2008.
[2] van der Maaten, L.J.P. t-Distributed Stochastic Neighbor Embedding
http://homepage.tudelft.nl/19j49/t-SNE.html

"""

import numpy as N
import ctypes
from ctypes import c_int, c_float, c_bool, POINTER
import os
from collections import namedtuple

def ord_string(s):
    b = bytearray()
    arr = b.extend(map(ord, s))
    return N.array([x for x in b] + [0]).astype(N.uint8)

TsneConfig = namedtuple(
    'TsneConfig',
    ['result', 'points', 'dims', 'perplexity', 'learning_rate', 'early_exaggeration',
     'magnitude_factor', 'num_neighbors', 'iterations', 'iterations_no_progress',
     'force_magnify_iters', 'perplexity_search_epsilon', 'pre_exaggeration_momentum',
     'post_exaggeration_momentum', 'theta', 'epssq', 'min_gradient_norm', 'initialization_type',
     'preinit_data', 'dump_points', 'dump_file', 'dump_interval', 'use_interactive',
     'viz_server', 'viz_timeout', 'verbosity', 'print_interval', 'gpu_device', 'return_style',
     'num_snapshots', 'distance_metric'])

class TSNE(object):

    def __init__(self,
                 n_components=2,
                 perplexity=50.0,
                 early_exaggeration=12.0,
                 learning_rate=200.0,
                 num_neighbors=32,
                 force_magnify_iters=250,
                 pre_momentum=0.5,
                 post_momentum=0.8,
                 theta=0.5,
                 epssq=0.0025,
                 n_iter=1000,
                 n_iter_without_progress=1000,
                 min_grad_norm=0.0,
                 perplexity_epsilon=1e-3,
                 metric='euclidean',
                 init='random',
                 return_style='once',
                 num_snapshots=5,
                 verbose=0,
                 random_seed=None,
                 use_interactive=False,
                 viz_timeout=10000,
                 viz_server="tcp://localhost:5556",
                 dump_points=False,
                 dump_file="dump.txt",
                 dump_interval=1,
                 print_interval=10,
                 device=0,
                 magnitude_factor=5):
        """Initialization method for barnes hut T-SNE class.
        """

        # Initialize the variables
        self.n_components = int(n_components)
        if self.n_components != 2:
            raise NotImplementedError('The current fit-tsne implementation does not support '
                                      'projection into dimensions other than 2 for now.')
        if metric in ('euclidean', 'L2'):
            self.metric = 1
        elif metric == 'innerproduct':
            self.metric = 0
        # elif metric == 'L1':
        #     self.metric = 3
        # elif metric == 'LInf':
        #     self.metric = 4
        # elif metric == 'canberra':
        #     self.metric = 20
        # elif metric == 'braycurtis':
        #     self.metric = 5
        # elif metric == 'jensenshannon':
        #     self.metric = 6
        else:
            raise NotImplementedError(
                "Only 'euclidean' and 'innerproduct' metrics are currently supported.")
        if init not in ['random']:
            raise NotImplementedError("Non-Random initialization is not currently supported. "
                                      "Please use init='random' for now.")

        self.perplexity = float(perplexity)
        self.learning_rate = float(learning_rate)
        self.early_exaggeration = float(early_exaggeration)
        self.iterations = int(n_iter)
        self.iterations_no_progress = int(n_iter_without_progress)
        self.min_gradient_norm = float(min_grad_norm)
        self.init = init
        self.verbosity = int(verbose)

        # Initialize non-sklearn variables
        self.num_neighbors = int(num_neighbors)
        self.force_magnify_iters = int(force_magnify_iters)
        self.perplexity_search_epsilon = float(perplexity_epsilon)
        self.pre_exaggeration_momentum = float(pre_momentum)
        self.post_exaggeration_momentum = float(post_momentum)
        self.theta = float(theta)
        self.epssq = float(epssq)
        self.gpu_device = int(device)
        self.print_interval = int(print_interval)
        self.magnitude_factor = float(magnitude_factor)

        # Point dumping
        self.dump_file = str(dump_file)
        self.dump_points = bool(dump_points)
        self.dump_interval = int(dump_interval)

        # Viz
        self.use_interactive = bool(use_interactive)
        self.viz_server = str(viz_server)
        self.viz_timeout = int(viz_timeout)

        # Return style
        if return_style not in ['once', 'snapshots']:
            raise ValueError('Invalid return style...')
        elif return_style == 'once':
            self.return_style = 0
        elif return_style == 'snapshots':
            self.return_style = 1
        self.num_snapshots = int(num_snapshots)

        # Build the hooks for the BH T-SNE library
        self._path = os.path.dirname(__file__)
        self._lib = N.ctypeslib.load_library(
            'libtsnecuda', self._path)  # Load the ctypes library

        # Hook the BH T-SNE function
        self._lib.pymodule_tsne.restype = None
        tsne_argtypes = TsneConfig(
            result=N.ctypeslib.ndpointer(
                N.float32, ndim=2, flags='ALIGNED, F_CONTIGUOUS, WRITEABLE'),
            points=N.ctypeslib.ndpointer(
                N.float32, ndim=2, flags='ALIGNED, CONTIGUOUS'),
            dims=POINTER(N.ctypeslib.c_intp),
            perplexity=c_float,
            learning_rate=c_float,
            early_exaggeration=c_float,
            magnitude_factor=c_float,
            num_neighbors=c_int,
            iterations=c_int,
            iterations_no_progress=c_int,
            force_magnify_iters=c_int,
            perplexity_search_epsilon=c_float,
            pre_exaggeration_momentum=c_float,
            post_exaggeration_momentum=c_float,
            theta=c_float,
            epssq=c_float,
            min_gradient_norm=c_float,
            initialization_type=c_int,
            preinit_data=N.ctypeslib.ndpointer(
                N.float32, ndim=2, flags='ALIGNED, F_CONTIGUOUS'),
            dump_points=c_bool,
            dump_file=N.ctypeslib.ndpointer(
                N.uint8, flags='ALIGNED, CONTIGUOUS'),
            dump_interval=c_int,
            use_interactive=c_bool,
            viz_server=N.ctypeslib.ndpointer(
                N.uint8, flags='ALIGNED, CONTIGUOUS'),
            viz_timeout=c_int,
            verbosity=c_int,
            print_interval=c_int,
            gpu_device=c_int,
            return_style=c_int,
            num_snapshots=c_int,
            distance_metric=c_int,
        )

        self._lib.pymodule_tsne.argtypes = list(tsne_argtypes)

    def fit_transform(self, X, y=None):
        """Fit X into an embedded space and return that transformed output.

        Arguments:
            X {array} -- Input array, shape: (n_points, n_dimensions)

        Keyword Arguments:
            y {None} -- The initialization to use for the T-SNE (default: {None})
        """

        # Setup points/embedding requirements
        self.points = N.require(X, N.float32, ['CONTIGUOUS', 'ALIGNED'])
        self.embedding = N.zeros(shape=(X.shape[0], self.n_components))
        self.embedding = N.require(self.embedding, N.float32, [
                                   'F_CONTIGUOUS', 'ALIGNED', 'WRITEABLE'])

        # Handle Initialization
        if y is None:
            self.initialization_type = 1
            self.preinit_data = N.require(N.zeros((1, 1)), N.float32, [
                                          'CONTIGUOUS', 'ALIGNED'])
        else:
            self.initialization_type = 3
            self.preinit_data = N.require(
                y, N.float32, ['F_CONTIGUOUS', 'ALIGNED'])

        # Handle dumping and viz strings
        self.dump_file_ = N.require(ord_string(self.dump_file), N.uint8, [
                                    'CONTIGUOUS', 'ALIGNED'])
        self.viz_server_ = N.require(ord_string(self.viz_server), N.uint8, [
                                     'CONTIGUOUS', 'ALIGNED'])

        tsne_args = self._construct_tsne_args(self.embedding, self.points)

        self._lib.pymodule_tsne(*tsne_args)
        return self.embedding

    def _construct_tsne_args(self, result_array, input_array):
        tsne_args = TsneConfig(
            result=result_array,
            points=input_array,
            dims=input_array.ctypes.shape,
            perplexity=c_float(self.perplexity),
            learning_rate=c_float(self.learning_rate),
            early_exaggeration=c_float(self.early_exaggeration),
            magnitude_factor=c_float(self.magnitude_factor),
            num_neighbors=c_int(self.num_neighbors),
            iterations=c_int(self.iterations),
            iterations_no_progress=c_int(self.iterations_no_progress),
            force_magnify_iters=c_int(self.force_magnify_iters),
            perplexity_search_epsilon=c_float(self.perplexity_search_epsilon),
            pre_exaggeration_momentum=c_float(self.pre_exaggeration_momentum),
            post_exaggeration_momentum=c_float(
                self.post_exaggeration_momentum),
            theta=c_float(self.theta),
            epssq=c_float(self.epssq),
            min_gradient_norm=c_float(self.min_gradient_norm),
            initialization_type=c_int(self.initialization_type),
            preinit_data=self.preinit_data,
            dump_points=c_bool(self.dump_points),
            dump_file=ord_string(self.dump_file),
            dump_interval=c_int(self.dump_interval),
            use_interactive=c_bool(self.use_interactive),
            viz_server=ord_string(self.viz_server),
            viz_timeout=c_int(self.viz_timeout),
            verbosity=c_int(self.verbosity),
            print_interval=c_int(self.print_interval),
            gpu_device=c_int(self.gpu_device),
            return_style=c_int(self.return_style),
            num_snapshots=c_int(self.num_snapshots),
            distance_metric=c_int(self.metric)
        )

        return tsne_args


text:
package chart

import "github.com/wcharczuk/go-chart/v2/drawing"

// ValuesProvider is a type that produces values.
type ValuesProvider interface {
	Len() int
	GetValues(index int) (float64, float64)
}

// BoundedValuesProvider allows series to return a range.
type BoundedValuesProvider interface {
	Len() int
	GetBoundedValues(index int) (x, y1, y2 float64)
}

// FirstValuesProvider is a special type of value provider that can return it's (potentially computed) first value.
type FirstValuesProvider interface {
	GetFirstValues() (x, y float64)
}

// LastValuesProvider is a special type of value provider that can return it's (potentially computed) last value.
type LastValuesProvider interface {
	GetLastValues() (x, y float64)
}

// BoundedLastValuesProvider is a special type of value provider that can return it's (potentially computed) bounded last value.
type BoundedLastValuesProvider interface {
	GetBoundedLastValues() (x, y1, y2 float64)
}

// FullValuesProvider is an interface that combines `ValuesProvider` and `LastValuesProvider`
type FullValuesProvider interface {
	ValuesProvider
	LastValuesProvider
}

// FullBoundedValuesProvider is an interface that combines `BoundedValuesProvider` and `BoundedLastValuesProvider`
type FullBoundedValuesProvider interface {
	BoundedValuesProvider
	BoundedLastValuesProvider
}

// SizeProvider is a provider for integer size.
type SizeProvider func(xrange, yrange Range, index int, x, y float64) float64

// ColorProvider is a general provider for color ranges based on values.
type ColorProvider func(v, vmin, vmax float64) drawing.Color

// DotColorProvider is a provider for dot color.
type DotColorProvider func(xrange, yrange Range, index int, x, y float64) drawing.Color


text:
// This file has been auto-generated by code_generator_v8.py. DO NOT MODIFY!

#include "config.h"
#include "V8WebGLCompressedTextureETC1.h"

#include "bindings/core/v8/ExceptionState.h"
#include "bindings/core/v8/V8DOMConfiguration.h"
#include "bindings/core/v8/V8GCController.h"
#include "bindings/core/v8/V8ObjectConstructor.h"
#include "core/dom/ContextFeatures.h"
#include "core/dom/Document.h"
#include "core/dom/Element.h"
#include "platform/RuntimeEnabledFeatures.h"
#include "platform/TraceEvent.h"
#include "wtf/GetPtr.h"
#include "wtf/RefPtr.h"

namespace blink {

// Suppress warning: global constructors, because struct WrapperTypeInfo is trivial
// and does not depend on another global objects.
#if defined(COMPONENT_BUILD) && defined(WIN32) && COMPILER(CLANG)
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wglobal-constructors"
#endif
const WrapperTypeInfo V8WebGLCompressedTextureETC1::wrapperTypeInfo = { gin::kEmbedderBlink, V8WebGLCompressedTextureETC1::domTemplate, V8WebGLCompressedTextureETC1::refObject, V8WebGLCompressedTextureETC1::derefObject, V8WebGLCompressedTextureETC1::trace, 0, V8WebGLCompressedTextureETC1::visitDOMWrapper, V8WebGLCompressedTextureETC1::preparePrototypeObject, V8WebGLCompressedTextureETC1::installConditionallyEnabledProperties, "WebGLCompressedTextureETC1", 0, WrapperTypeInfo::WrapperTypeObjectPrototype, WrapperTypeInfo::ObjectClassId, WrapperTypeInfo::NotInheritFromEventTarget, WrapperTypeInfo::Dependent, WrapperTypeInfo::WillBeGarbageCollectedObject };
#if defined(COMPONENT_BUILD) && defined(WIN32) && COMPILER(CLANG)
#pragma clang diagnostic pop
#endif

// This static member must be declared by DEFINE_WRAPPERTYPEINFO in WebGLCompressedTextureETC1.h.
// For details, see the comment of DEFINE_WRAPPERTYPEINFO in
// bindings/core/v8/ScriptWrappable.h.
const WrapperTypeInfo& WebGLCompressedTextureETC1::s_wrapperTypeInfo = V8WebGLCompressedTextureETC1::wrapperTypeInfo;

namespace WebGLCompressedTextureETC1V8Internal {

} // namespace WebGLCompressedTextureETC1V8Internal

void V8WebGLCompressedTextureETC1::visitDOMWrapper(v8::Isolate* isolate, ScriptWrappable* scriptWrappable, const v8::Persistent<v8::Object>& wrapper)
{
    WebGLCompressedTextureETC1* impl = scriptWrappable->toImpl<WebGLCompressedTextureETC1>();
    // The canvas() method may return a reference or a pointer.
    if (Node* owner = WTF::getPtr(impl->canvas())) {
        Node* root = V8GCController::opaqueRootForGC(isolate, owner);
        isolate->SetReferenceFromGroup(v8::UniqueId(reinterpret_cast<intptr_t>(root)), wrapper);
        return;
    }
}

static void installV8WebGLCompressedTextureETC1Template(v8::Local<v8::FunctionTemplate> functionTemplate, v8::Isolate* isolate)
{
    functionTemplate->ReadOnlyPrototype();

    v8::Local<v8::Signature> defaultSignature;
    defaultSignature = V8DOMConfiguration::installDOMClassTemplate(isolate, functionTemplate, "WebGLCompressedTextureETC1", v8::Local<v8::FunctionTemplate>(), V8WebGLCompressedTextureETC1::internalFieldCount,
        0, 0,
        0, 0,
        0, 0);
    v8::Local<v8::ObjectTemplate> instanceTemplate = functionTemplate->InstanceTemplate();
    ALLOW_UNUSED_LOCAL(instanceTemplate);
    v8::Local<v8::ObjectTemplate> prototypeTemplate = functionTemplate->PrototypeTemplate();
    ALLOW_UNUSED_LOCAL(prototypeTemplate);
    static const V8DOMConfiguration::ConstantConfiguration V8WebGLCompressedTextureETC1Constants[] = {
        {"COMPRESSED_RGB_ETC1_WEBGL", 0x8D64, 0, 0, V8DOMConfiguration::ConstantTypeUnsignedLong},
    };
    V8DOMConfiguration::installConstants(isolate, functionTemplate, prototypeTemplate, V8WebGLCompressedTextureETC1Constants, WTF_ARRAY_LENGTH(V8WebGLCompressedTextureETC1Constants));

    // Custom toString template
    functionTemplate->Set(v8AtomicString(isolate, "toString"), V8PerIsolateData::from(isolate)->toStringTemplate());
}

v8::Local<v8::FunctionTemplate> V8WebGLCompressedTextureETC1::domTemplate(v8::Isolate* isolate)
{
    return V8DOMConfiguration::domClassTemplate(isolate, const_cast<WrapperTypeInfo*>(&wrapperTypeInfo), installV8WebGLCompressedTextureETC1Template);
}

bool V8WebGLCompressedTextureETC1::hasInstance(v8::Local<v8::Value> v8Value, v8::Isolate* isolate)
{
    return V8PerIsolateData::from(isolate)->hasInstance(&wrapperTypeInfo, v8Value);
}

v8::Local<v8::Object> V8WebGLCompressedTextureETC1::findInstanceInPrototypeChain(v8::Local<v8::Value> v8Value, v8::Isolate* isolate)
{
    return V8PerIsolateData::from(isolate)->findInstanceInPrototypeChain(&wrapperTypeInfo, v8Value);
}

WebGLCompressedTextureETC1* V8WebGLCompressedTextureETC1::toImplWithTypeCheck(v8::Isolate* isolate, v8::Local<v8::Value> value)
{
    return hasInstance(value, isolate) ? toImpl(v8::Local<v8::Object>::Cast(value)) : 0;
}

void V8WebGLCompressedTextureETC1::refObject(ScriptWrappable* scriptWrappable)
{
#if !ENABLE(OILPAN)
    scriptWrappable->toImpl<WebGLCompressedTextureETC1>()->ref();
#endif
}

void V8WebGLCompressedTextureETC1::derefObject(ScriptWrappable* scriptWrappable)
{
#if !ENABLE(OILPAN)
    scriptWrappable->toImpl<WebGLCompressedTextureETC1>()->deref();
#endif
}

} // namespace blink


text:
export     MAIL_USERNAME='wikomigas@gmail.com'
export     MAIL_PASSWORD='wikomigas5321'
export	   FLASKY_ADMIN='Wikomega'


text:
package pack1;

public class MyRunnable {
    private DImpl d;

    public MyRunnable(DImpl d) {
        this.d = d;
    }

    public void run() {
    d.iAmProtected();
  }
}


text:
/* TEMPLATE GENERATED TESTCASE FILE
Filename: CWE126_Buffer_Overread__new_wchar_t_loop_68b.cpp
Label Definition File: CWE126_Buffer_Overread__new.label.xml
Template File: sources-sink-68b.tmpl.cpp
*/
/*
 * @description
 * CWE: 126 Buffer Over-read
 * BadSource:  Use a small buffer
 * GoodSource: Use a large buffer
 * Sink: loop
 *    BadSink : Copy data to string using a loop
 * Flow Variant: 68 Data flow: data passed as a global variable from one function to another in different source files
 *
 * */

#include "std_testcase.h"

#include <wchar.h>

extern wchar_t * CWE126_Buffer_Overread__new_wchar_t_loop_68_badData;
extern wchar_t * CWE126_Buffer_Overread__new_wchar_t_loop_68_goodG2BData;

namespace CWE126_Buffer_Overread__new_wchar_t_loop_68
{

/* all the sinks are the same, we just want to know where the hit originated if a tool flags one */

#ifndef OMITBAD

void badSink()
{
    wchar_t * data = CWE126_Buffer_Overread__new_wchar_t_loop_68_badData;
    {
        size_t i, destLen;
        wchar_t dest[100];
        wmemset(dest, L'C', 100-1);
        dest[100-1] = L'\0'; /* null terminate */
        destLen = wcslen(dest);
        /* POTENTIAL FLAW: using length of the dest where data
         * could be smaller than dest causing buffer overread */
        for (i = 0; i < destLen; i++)
        {
            dest[i] = data[i];
        }
        dest[100-1] = L'\0';
        printWLine(dest);
        delete [] data;
    }
}

#endif /* OMITBAD */

#ifndef OMITGOOD

/* goodG2B uses the GoodSource with the BadSink */
void goodG2BSink()
{
    wchar_t * data = CWE126_Buffer_Overread__new_wchar_t_loop_68_goodG2BData;
    {
        size_t i, destLen;
        wchar_t dest[100];
        wmemset(dest, L'C', 100-1);
        dest[100-1] = L'\0'; /* null terminate */
        destLen = wcslen(dest);
        /* POTENTIAL FLAW: using length of the dest where data
         * could be smaller than dest causing buffer overread */
        for (i = 0; i < destLen; i++)
        {
            dest[i] = data[i];
        }
        dest[100-1] = L'\0';
        printWLine(dest);
        delete [] data;
    }
}

#endif /* OMITGOOD */

} /* close namespace */


text:
package org.apache.kerberos.kdc.server;

import org.apache.kerberos.kdc.server.ApacheKdcServer;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.nio.ByteBuffer;
import java.nio.channels.SocketChannel;

public class KdcTest {

    private String serverHost = "localhost";
    private short serverPort = 8088;

    private ApacheKdcServer kdcServer;

    @Before
    public void setUp() throws Exception {
        kdcServer = new ApacheKdcServer();
        kdcServer.setKdcHost(serverHost);
        kdcServer.setKdcPort(serverPort);
        kdcServer.init();
        kdcServer.start();
    }

    @Test
    public void testKdc() throws IOException, InterruptedException {
        Thread.sleep(10);

        SocketChannel socketChannel = SocketChannel.open();
        socketChannel.configureBlocking(true);
        SocketAddress sa = new InetSocketAddress(serverHost, serverPort);
        socketChannel.connect(sa);

        String BAD_KRB_MESSAGE = "Hello World!";
        ByteBuffer writeBuffer = ByteBuffer.allocate(4 + BAD_KRB_MESSAGE.getBytes().length);
        writeBuffer.putInt(BAD_KRB_MESSAGE.getBytes().length);
        writeBuffer.put(BAD_KRB_MESSAGE.getBytes());
        writeBuffer.flip();

        socketChannel.write(writeBuffer);
    }

    @After
    public void tearDown() throws Exception {
        kdcServer.stop();
    }
}


text:
ACCEPTED

#### According to
Index Fungorum

#### Published in
Mycotaxon 38: 474 (1990)

#### Original name
Coprobia crassistriata J. Moravec

### Remarks
null


text:
require 'json-schema/attribute'

module JSON
  class Schema
    class PatternPropertiesAttribute < Attribute
      def self.validate(current_schema, data, fragments, processor, validator, options = {})
        return unless data.is_a?(Hash)

        current_schema.schema['patternProperties'].each do |property, property_schema|
          regexp = Regexp.new(property)

          # Check each key in the data hash to see if it matches the regex
          data.each do |key, value|
            next unless regexp.match(key)
            schema = JSON::Schema.new(property_schema, current_schema.uri, validator)
            schema.validate(data[key], fragments + [key], processor, options)
          end
        end
      end
    end
  end
end


text:
import 'mocha';
import { expect } from 'chai';
import { buildQuery } from '../..';
import { Account } from '../assets/sobs';

describe('SubQueries Tests', () => {
    it('select (select x from y)', () => {
        let qry = buildQuery(Account, fields => {
            return {
                select: [
                    fields.subQuery('contacts',subFields => {
                        return {
                            select: subFields.select('id', 'name')
                        };
                    })
                ]
            };
        });
        expect(qry).to.equal(`SELECT (SELECT Id, Name FROM Contacts) FROM Account`);
    });

    it('select (select x.y, x.z from y)', () => {
        let qry = buildQuery(Account, fields => {
            return {
                select: [
                    fields.subQuery('contacts', subFields => {
                        return {
                            select: subFields.parent('owner').select('phone', 'name')
                        };
                    })
                ]
            };
        });
        expect(qry).to.equal(`SELECT (SELECT Owner.Phone, Owner.Name FROM Contacts) FROM Account`);
    });

    it('select (select x.y, x.z from y where x)', () => {
        let qry = buildQuery(Account, fields => {
            return {
                select: [
                    fields.subQuery('contacts', subFields => {
                        return {
                            select: subFields.parent('owner').select('phone', 'name'),
                            where: [
                                {field: fields.select('phone'), op: '=', val: '828-555-5555'}
                            ]
                        };
                    })
                ]
            };
        });
        expect(qry).to.equal(`SELECT (SELECT Owner.Phone, Owner.Name FROM Contacts WHERE Phone = '828-555-5555') FROM Account`);
    });
});


text:
using Android.App;
using Android.OS;
using com.refractored;
using Android.Support.V4.View;
using Toolbar = Android.Support.V7.Widget.Toolbar;
using Android.Support.V7.App;
using Android.Widget;
using System.Collections.Generic;
using System;
using Android.Content;

/*
    Using Listview FD
*/
namespace Sample
{
    [Activity(Label = "Ecoteer House - Projects", ParentActivity = typeof(activity_FuzeEcoteer))]
    [MetaData("android.support.PARENT_ACTIVITY", Value = "activity_FuzeEcoteer")]
    public class EH_Projects : AppCompatActivity
    {
        private List<FD> items_EH;
        private ListView ListView_EH;
        protected override void OnCreate(Bundle bundle)
        {
            base.OnCreate(bundle);
            SetContentView(Resource.Layout.EH_Projects);

            var toolbar = FindViewById<Toolbar>(Resource.Id.toolbarEH_Projects);
            SetSupportActionBar(toolbar);
            SupportActionBar.SetDisplayHomeAsUpEnabled(true);
            SupportActionBar.SetHomeButtonEnabled(true);

            ListView_EH = FindViewById<ListView>(Resource.Id.ListView_EH_Projects);
            items_EH = new List<FD>();
            items_EH.Add(new FD()
            {   Image = Resource.Drawable.EH_Projects_0,
                Title = "Ecoteer House : Community & Conservation Project | Perhentian Islands",
                Description = "Volunteer at Ecoteer House in the tropical island paradise of the Perhentian Islands and you can get involved with the community and teach the children at the number one beach location in Malaysia, as named by CNN in 2013. Ranking 13th in the entire world, these beaches are well worth a visit and deserve to be maintained for future generations to enjoy. Through community tourism, environmental awareness and community interaction, the Ecoteer House aims to teach and support the islanders to care for their islands whilst offering volunteers first-hand experience of community life in a traditional Malaysian village surrounded by turquoise seas, white sands and vibrant marine life."
            });

            items_EH.Add(new FD()
            {   Image = Resource.Drawable.EH_Projects_1,
                Title = "Perhentian Dive Project",
                Description = "Through our Dive Research project you will join hands with researchers to help collect data on the seagrass beds, corals reefs and artificial reefs in the Perhentian Islands. You will be be based in the village allowing you to understand more about the local culture."
            });

            items_EH.Add(new FD()
            {
                Image = Resource.Drawable.EH_Projects_2,
                Title = "Charity Challenge – Cycle for Turtles",
                Description = "Cycle along the east coast of Peninsular Malaysia visiting different turtle projects along the way.\nThis adventure will see you waking up each morning to beautiful sunrise's across the South China Sea.The East coast of Pensinsular Malaysia is almost all continuous beach,\nperfect for nesting season turtles.You will cycle from turtle project to turtle project sleeping under the stars watching turtles lay their eggs.This is an experience not to be missed by anyone!"
            });

            items_EH.Add(new FD()
            {
                Image = Resource.Drawable.EH_Projects_3,
                Title = "3D2N Sea turtle Conservation Holiday – Perhentian Islands",
                Description = "Sitting on the beach at night, under a star lit sky, whilst protecting sea turtles. Snorkel in turquoise waters, take photos of sea turtles and other marine creatures. And take some well-deserved relaxing time on one of the most beautiful beaches of Malaysia after doing a beach clean-up. This is a unique opportunity to actually get close to Malaysians endangered sea turtles and join hands with a local research team."
            });

            ListViewAdapter_FD adapter = new ListViewAdapter_FD(this, items_EH);

            ListView_EH.Adapter = adapter;

            var EH_moreInfo = FindViewById<LinearLayout>(Resource.Id.EH_Projects_moreInfo);

            EH_moreInfo.Click += EH_moreInfo_Click;

        }

        private void EH_moreInfo_Click(object sender, EventArgs e)
        {
            var uri = Android.Net.Uri.Parse("http://ecoteerresponsibletravel.com/responsible-travelvolunteerism/");
            var intent = new Intent(Intent.ActionView, uri);
            StartActivity(intent);
        }

    }
}


text:
<?php
namespace TYPO3\Fluid\Tests\Unit\ViewHelpers\Uri;

require_once(__DIR__ . '/../ViewHelperBaseTestcase.php');

/**
 * Testcase for the action uri view helper
 *
 */
class ActionViewHelperTest extends \TYPO3\Fluid\ViewHelpers\ViewHelperBaseTestcase
{
    /**
     * var \TYPO3\Fluid\ViewHelpers\Uri\ActionViewHelper
     */
    protected $viewHelper;

    public function setUp()
    {
        parent::setUp();
        $this->viewHelper = new \TYPO3\Fluid\ViewHelpers\Uri\ActionViewHelper();
        $this->injectDependenciesIntoViewHelper($this->viewHelper);
        $this->viewHelper->initializeArguments();
    }

    /**
     * @test
     */
    public function renderReturnsUriReturnedFromUriBuilder()
    {
        $this->uriBuilder->expects($this->any())->method('uriFor')->will($this->returnValue('some/uri'));

        $this->viewHelper->initialize();
        $actualResult = $this->viewHelper->render('index');

        $this->assertEquals('some/uri', $actualResult);
    }

    /**
     * @test
     */
    public function renderCorrectlyPassesDefaultArgumentsToUriBuilder()
    {
        $this->uriBuilder->expects($this->once())->method('setSection')->with('');
        $this->uriBuilder->expects($this->once())->method('setCreateAbsoluteUri')->with(false);
        $this->uriBuilder->expects($this->once())->method('setArguments')->with(array());
        $this->uriBuilder->expects($this->once())->method('setAddQueryString')->with(false);
        $this->uriBuilder->expects($this->once())->method('setArgumentsToBeExcludedFromQueryString')->with(array());
        $this->uriBuilder->expects($this->once())->method('setFormat')->with('');
        $this->uriBuilder->expects($this->once())->method('uriFor')->with('theActionName', array(), null, null, null);

        $this->viewHelper->initialize();
        $this->viewHelper->render('theActionName');
    }

    /**
     * @test
     */
    public function renderCorrectlyPassesAllArgumentsToUriBuilder()
    {
        $this->uriBuilder->expects($this->once())->method('setSection')->with('someSection');
        $this->uriBuilder->expects($this->once())->method('setCreateAbsoluteUri')->with(true);
        $this->uriBuilder->expects($this->once())->method('setArguments')->with(array('additional' => 'Parameters'));
        $this->uriBuilder->expects($this->once())->method('setAddQueryString')->with(true);
        $this->uriBuilder->expects($this->once())->method('setArgumentsToBeExcludedFromQueryString')->with(array('arguments' => 'toBeExcluded'));
        $this->uriBuilder->expects($this->once())->method('setFormat')->with('someFormat');
        $this->uriBuilder->expects($this->once())->method('uriFor')->with('someAction', array('some' => 'argument'), 'someController', 'somePackage', 'someSubpackage');

        $this->viewHelper->initialize();
        $this->viewHelper->render('someAction', array('some' => 'argument'), 'someController', 'somePackage', 'someSubpackage', 'someSection', 'someFormat', array('additional' => 'Parameters'), true, true, array('arguments' => 'toBeExcluded'));
    }

    /**
     * @test
     */
    public function renderThrowsViewHelperExceptionIfUriBuilderThrowsFlowException()
    {
        $this->uriBuilder->expects($this->any())->method('uriFor')->will($this->throwException(new \TYPO3\Flow\Exception('Mock Exception', 12345)));
        $this->viewHelper->initialize();
        try {
            $this->viewHelper->render('someAction');
        } catch (\TYPO3\Fluid\Core\ViewHelper\Exception $exception) {
        }
        $this->assertEquals(12345, $exception->getPrevious()->getCode());
    }

    /**
     * @test
     * @expectedException \TYPO3\Fluid\Core\ViewHelper\Exception
     */
    public function renderThrowsExceptionIfUseParentRequestIsSetAndTheCurrentRequestHasNoParentRequest()
    {
        $this->viewHelper->initialize();
        $this->viewHelper->render('someAction', array(), null, null, null, '', '', array(), false, false, array(), true);
    }

    /**
     * @test
     */
    public function renderUsesParentRequestIfUseParentRequestIsSet()
    {
        $viewHelper = new \TYPO3\Fluid\ViewHelpers\Uri\ActionViewHelper();

        $parentRequest = $this->getMockBuilder(\TYPO3\Flow\Mvc\ActionRequest::class)->disableOriginalConstructor()->getMock();

        $this->request = $this->getMockBuilder(\TYPO3\Flow\Mvc\ActionRequest::class)->disableOriginalConstructor()->getMock();
        $this->request->expects($this->atLeastOnce())->method('isMainRequest')->will($this->returnValue(false));
        $this->request->expects($this->atLeastOnce())->method('getParentRequest')->will($this->returnValue($parentRequest));

        $this->controllerContext = $this->getMock(\TYPO3\Flow\Mvc\Controller\ControllerContext::class, array(), array(), '', false);
        $this->controllerContext->expects($this->any())->method('getUriBuilder')->will($this->returnValue($this->uriBuilder));
        $this->controllerContext->expects($this->any())->method('getRequest')->will($this->returnValue($this->request));

        $this->uriBuilder->expects($this->atLeastOnce())->method('setRequest')->with($parentRequest);

        $this->renderingContext->setControllerContext($this->controllerContext);
        $this->injectDependenciesIntoViewHelper($viewHelper);

        $viewHelper->render('someAction', array(), null, null, null, '', '', array(), false, false, array(), true);
    }
}


text:
	<title>Andrew Davies's panel show appearances</title>

	<link rel="stylesheet" media="all" href="../style.css" type="text/css"/>

Andrew Davies's panel show appearances

Andrew Davies has appeared in 3 episodes between 1997-2002. Note that these appearances may be for more than one person if multiple people have the same name.

1997

1998

1999

2000

2001

2002

2002-03-08 / Call My Bluff

2002-02-20 / Call My Bluff

1997-05-09 / Call My Bluff


text:
<?xml version="1.0" encoding="UTF-8"?><?xml-stylesheet href="test-to-html.xsl" type="text/xml"?>

<!--

Copyright (c) 2001-2004 World Wide Web Consortium,
(Massachusetts Institute of Technology, Institut National de
Recherche en Informatique et en Automatique, Keio University).  All
Rights Reserved.  This program is distributed under the W3C's Software
Intellectual Property License.  This program is distributed in the
hope that it will be useful, but WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE.

See W3C License http://www.w3.org/Consortium/Legal/ for more details.

-->

<!DOCTYPE test SYSTEM "dom3.dtd">
<test xmlns="http://www.w3.org/2001/DOM-Test-Suite/Level-3" name="noderemovechild07">
<metadata>
<title>noderemovechild07</title>
<creator>IBM</creator>
<description>
Attempts to remove a notation from a Document node.  Since notations are children of
DocumentType, not Document the operation should fail with a NOT_FOUND_ERR.  Attempting
to remove Document from a Notation should also fail either with a NOT_FOUND_ERR
or a NO_MODIFICATION_ALLOWED_ERR.
</description>
<contributor>Neil Delima</contributor>
<date qualifier="created">2002-06-10</date>
<subject resource="http://www.w3.org/TR/2004/REC-DOM-Level-3-Core-20040407/core#ID-1734834066"/>
<subject resource="http://www.w3.org/Bugs/Public/show_bug.cgi?id=418"/>
</metadata>
<var name="doc" type="Document"/>
<var name="docType" type="DocumentType"/>
<var name="notations" type="NamedNodeMap"/>
<var name="notation" type="Notation"/>
<var name="removedChild" type="Node"/>
<load var="doc" href="hc_staff" willBeModified="true"/>
<doctype var="docType" obj="doc"/>
<notations var="notations" obj="docType"/>
<getNamedItem var="notation" obj="notations" name='"notation1"'/>
<assertDOMException id="NOT_FOUND_ERR_noderemovechild07_1">
<NOT_FOUND_ERR>
<removeChild obj="doc" var="removedChild" oldChild="notation" />
</NOT_FOUND_ERR>
</assertDOMException>
<try>
	<removeChild obj="notation" var="removedChild" oldChild="doc" />
	<catch>
		<DOMException code="NOT_FOUND_ERR"/>
		<DOMException code="NO_MODIFICATION_ALLOWED_ERR"/>
	</catch>
</try>
</test>


text:
===========
### Prerequisites
docker

## Issues

### IPv4 Forwarding
```
/etc/sysctl.conf
net.ipv4.ip_forward=1
```
### Docker bridge (no connection to internet)
Restart bridge
<s>
```
sudo iptables -t nat -F
sudo ifconfig docker0 down
sudo brctl delbr docker0
sudo service command docker restart
```
</s>

## Torified

### Implementation

Uses: [privoxy](https://www.privoxy.org/), and [tor](https://www.torproject.org/).  Has normal proxy at port @ 8123 and secure (using Tor) @ 8118

Default: user:password

```
docker compose build --force-rm --no-cache
docker compose up -d
```

#### Debug (in shell)
```
docker compose run --service-ports torified bash
```


text:
<?php
ob_start();
if (class_exists("\ZapiClosure")) {
    echo "internal class ZapiClosure exists\n";
    $refl = new ReflectionClass("ZapiClosure");
    if ($refl->isFinal()) {
        echo "class ZapiClosure is final\n";
    }
}
if (class_exists("\ClosureTestClass")) {
    echo "internal class ClosureTestClass exists\n";
    $obj = new \ClosureTestClass();
    $obj->testClosureCallable();
    $callable = $obj->getNoArgAndReturnCallable();
    if (is_callable($callable)) {
        echo "\$callable is callable\n";
        $ret = $callable();
        echo "the return of callable is " . $ret."\n";
    }
    $hasParamCallable = $obj->getArgAndReturnCallable();
    if (is_callable($hasParamCallable)) {
        echo "\$hasParamCallable is callable\n";
        $ret = $hasParamCallable();
        echo "the return of hasParamCallable is " . $ret."\n";
        $ret = $hasParamCallable(123);
        echo "the return of hasParamCallable is " . $ret."\n";
        $ret = $hasParamCallable(3.14);
        echo "the return of hasParamCallable is " . $ret."\n";
        $ret = $hasParamCallable(true);
        echo "the return of hasParamCallable is " . $ret."\n";
    }
}

$ret = trim(ob_get_clean());
$expect = <<<'EOF'
internal class ZapiClosure exists
class ZapiClosure is final
internal class ClosureTestClass exists
$callable is callable
print_something called
the return of callable is print_some
$hasParamCallable is callable
have_ret_and_have_arg called
the return of hasParamCallable is have_ret_and_have_arg
have_ret_and_have_arg called
the return of hasParamCallable is 123
have_ret_and_have_arg called
the return of hasParamCallable is 3.14
have_ret_and_have_arg called
the return of hasParamCallable is 1
EOF;

if ($ret != $expect) {
    exit(1);
}


text:
"""Jax3d public API."""
# pylint: disable=unused-import

from jax3d import math
from jax3d import utils
from jax3d.math import volume_rendering
from jax3d.utils import testing
from jax3d.utils import tree_utils as tree
from jax3d.utils import typing
from jax3d.utils.file_utils import j3d_dir
from jax3d.utils.file_utils import nf_dir
from jax3d.utils.geo_utils import get_coords_grid
from jax3d.utils.random import RandomState
from jax3d.utils.random import uniform_points_on_sphere
from jax3d.utils.random import uniform_polar_points_on_sphere
from jax3d.utils.shape_utils import shape_dtype_like
from jax3d.utils.shape_utils import tensor_spec_like
from jax3d.utils.shape_utils import types_like
from jax3d.utils.shape_utils import zeros_like
from jax3d.utils.shape_validation import assert_typing
from jax3d.utils.xmap import xmap


text:
package com.grasp.model.entity;

import lombok.Getter;
import lombok.Setter;

import javax.persistence.*;
import java.util.UUID;

@Getter
@Setter
@Entity
@Table(name = "user_subjects", schema = "course")
public class UserSubject {

    @Id
    @Column(name = "id")
    @GeneratedValue(strategy = GenerationType.IDENTITY)
    private Long id;
    @Column(name = "uid")
    private UUID userId;
    @Column(name="subject")
    private String subject;

    public UserSubject() {
    }

    public UserSubject(UUID userId, String subject) {
        this.userId = userId;
        this.subject = subject;
    }

    @Override
    public boolean equals(Object o) {
        if (this == o) {
            return true;
        }
        if (!(o instanceof UserSubject)) {
            return false;
        }

        UserSubject that = (UserSubject) o;

        if (!getId().equals(that.getId())) {
            return false;
        }
        if (!getUserId().equals(that.getUserId())) {
            return false;
        }
        return getSubject().equals(that.getSubject());
    }

    @Override
    public int hashCode() {
        int result = getId().hashCode();
        result = 31 * result + getUserId().hashCode();
        result = 31 * result + getSubject().hashCode();
        return result;
    }
}


text:
import Ember from 'ember';
import RSVP from 'rsvp';

const { isBlank, inject } = Ember;

export default Ember.Route.extend({
  flash: inject.service('flash-notification'),

  model() {
    return this.modelFor('register.event-registration.show');
  },

  afterModel(model) {
    const event = model.event;

    if (!event.get('isHousingEnabled')) {
      this.transitionTo('register.event-registration.show.edit.shirts');
    }
  }
});


text:
//
//  CZKVItem.h
//  CZCache
//  https://github.com/rtyu128/CZCache
//  Created by Anchor on 16/6/13.
//  Copyright © 2016年 Anchor. All rights reserved.
//

#import <Foundation/Foundation.h>
#import "CZCacheDefines.h"

NS_ASSUME_NONNULL_BEGIN

/**
 CZKVItem is a kind of metadata used to store key-value pairs infos.
 */
@interface CZKVItem : NSObject

// The key of key-value pair.
@property (nonatomic, strong) NSString *key;

// The key's associated object value.
@property (nonatomic, strong) NSData *value;

// If needed, the value may be store as a file. This value represent the value's associated filename.
@property (nonatomic, strong, nullable) NSString *filename;

// The size of the value (in bytes).
@property (nonatomic, assign) int size;

// The expireDate of key-value pair. Equal to secends from 1970-01-01 8:00 to deadline(UTC).
@property (nonatomic, assign) long expireDate;

// The extended date of key-value pair(e.g.Description).
@property (nonatomic, strong, nullable) NSData *extendedData;

// Returns a boolean value indicates whether the key-value pair is valid.
- (BOOL)isValid;

// Calculates the remain lifetime of key-valus pair, if the key-value pair is invalid, just return -1.0.
- (NSTimeInterval)remainLife;

@end

NS_ASSUME_NONNULL_END


text:
using Telerik.Charting;
using Telerik.UI.Automation.Peers;
using Windows.UI.Xaml;
using Windows.UI.Xaml.Automation.Peers;
using Windows.UI.Xaml.Controls;
using Windows.UI.Xaml.Media;
using Windows.UI.Xaml.Shapes;

namespace Telerik.UI.Xaml.Controls.Chart
{
    /// <summary>
    /// Allows you highlight the area between two values drawn by the <see cref="CartesianPlotBandAnnotation.Axis"/>
    /// in the <see cref="RadCartesianChart"/>.
    /// </summary>
    public class CartesianPlotBandAnnotation : CartesianStrokedAnnotation
    {
        /// <summary>
        /// Identifies the <see cref="Axis"/> dependency property.
        /// </summary>
        public static readonly DependencyProperty AxisProperty =
            DependencyProperty.Register(nameof(Axis), typeof(CartesianAxis), typeof(CartesianPlotBandAnnotation), new PropertyMetadata(null, OnAxisPropertyChanged));

        /// <summary>
        /// Identifies the <see cref="From"/> dependency property.
        /// </summary>
        public static readonly DependencyProperty FromProperty =
            DependencyProperty.Register(nameof(From), typeof(object), typeof(CartesianPlotBandAnnotation), new PropertyMetadata(null, OnFromPropertyChanged));

        /// <summary>
        /// Identifies the <see cref="To"/> dependency property.
        /// </summary>
        public static readonly DependencyProperty ToProperty =
            DependencyProperty.Register(nameof(To), typeof(object), typeof(CartesianPlotBandAnnotation), new PropertyMetadata(null, OnToPropertyChanged));

        /// <summary>
        /// Identifies the <see cref="Fill"/> dependency property.
        /// </summary>
        public static readonly DependencyProperty FillProperty =
            DependencyProperty.Register(nameof(Fill), typeof(Brush), typeof(CartesianPlotBandAnnotation), new PropertyMetadata(null, OnFillPropertyChanged));

        private Rectangle presenter;
        private CartesianPlotBandAnnotationModel model;

        /// <summary>
        /// Initializes a new instance of the <see cref="CartesianPlotBandAnnotation" /> class.
        /// </summary>
        public CartesianPlotBandAnnotation()
        {
            this.DefaultStyleKey = typeof(CartesianPlotBandAnnotation);

            this.model = new CartesianPlotBandAnnotationModel();
            this.presenter = new Rectangle();
        }

        /// <summary>
        /// Gets or sets the axis.
        /// </summary>
        /// <value>The axis.</value>
        public CartesianAxis Axis
        {
            get
            {
                return (CartesianAxis)this.GetValue(AxisProperty);
            }
            set
            {
                this.SetValue(AxisProperty, value);
            }
        }

        /// <summary>
        /// Gets or sets from.
        /// </summary>
        /// <value>From.</value>
        public object From
        {
            get
            {
                return this.GetValue(FromProperty);
            }
            set
            {
                this.SetValue(FromProperty, value);
            }
        }

        /// <summary>
        /// Gets or sets to.
        /// </summary>
        /// <value>To.</value>
        public object To
        {
            get
            {
                return this.GetValue(ToProperty);
            }
            set
            {
                this.SetValue(ToProperty, value);
            }
        }

        /// <summary>
        /// Gets or sets the Brush that specifies how the shape's interior is painted.
        /// </summary>
        /// <value>The fill.</value>
        public Brush Fill
        {
            get
            {
                return (Brush)this.GetValue(FillProperty);
            }
            set
            {
                this.SetValue(FillProperty, value);
            }
        }

        internal override ChartAnnotationModel Model
        {
            get
            {
                return this.model;
            }
        }

        /// <summary>
        /// Gets the presenter.
        /// </summary>
        /// <value>The presenter.</value>
        protected override Shape Presenter
        {
            get
            {
                return this.presenter;
            }
        }

        /// <summary>
        /// Gets a value indicating whether the stroke goes inwards by the full <see cref="Shape.StrokeThickness"/>.
        /// </summary>
        /// <value>
        /// <c>true</c> if this instance is stroke inset; otherwise, <c>false</c>.
        /// </value>
        protected override bool IsStrokeInset
        {
            get
            {
                return true;
            }
        }

        internal override void SetPresenterVisibility(Visibility annotationVisibility)
        {
            if (this.presenter != null)
            {
                this.presenter.Visibility = annotationVisibility;
            }

            if (this.labelPresenter != null)
            {
                this.labelPresenter.Visibility = annotationVisibility;
            }
        }

        internal override Visibility GetPresenterVisibility()
        {
            if (this.presenter != null)
            {
                return this.presenter.Visibility;
            }
            else
            {
                return this.Visibility;
            }
        }

        internal override void UpdateVisibility()
        {
            if (this.model.firstPlotInfo != null && this.model.secondPlotInfo != null)
            {
                base.UpdateVisibility();
            }
            else
            {
                this.SetPresenterVisibility(Visibility.Collapsed);
            }
        }

        internal override void UpdatePresenters()
        {
            base.UpdatePresenters();

            Canvas.SetLeft(this.presenter, this.model.layoutSlot.X);
            Canvas.SetTop(this.presenter, this.model.layoutSlot.Y);
            this.presenter.Width = this.model.layoutSlot.Width;
            this.presenter.Height = this.model.layoutSlot.Height;
        }

        internal override ChartAnnotationLabelDefinition CreateDefaultLabelDefinition()
        {
            return new ChartAnnotationLabelDefinition()
            {
                Location = ChartAnnotationLabelLocation.Inside,
                HorizontalAlignment = HorizontalAlignment.Left,
                VerticalAlignment = VerticalAlignment.Top,
                HorizontalOffset = 2
            };
        }

        /// <inheritdoc/>
        protected override void UnapplyTemplateCore()
        {
            base.UnapplyTemplateCore();

            if (this.renderSurface != null)
            {
                this.renderSurface.Children.Remove(this.presenter);
            }
        }

        /// <summary>
        /// Initializes the render surface template part.
        /// </summary>
        protected override bool ApplyTemplateCore()
        {
            bool applied = base.ApplyTemplateCore();

            if (applied)
            {
                this.renderSurface.Children.Add(this.presenter);
            }

            return applied;
        }

        protected override AutomationPeer OnCreateAutomationPeer()
        {
            return new CartesianPlotBandAnnotationAutomationPeer(this);
        }

        private static void OnAxisPropertyChanged(DependencyObject sender, DependencyPropertyChangedEventArgs e)
        {
            CartesianPlotBandAnnotation annotation = sender as CartesianPlotBandAnnotation;
            if (annotation.Axis != null)
            {
                annotation.model.Axis = annotation.Axis.model;
            }
            else
            {
                annotation.model.Axis = null;
            }
        }

        private static void OnFromPropertyChanged(DependencyObject sender, DependencyPropertyChangedEventArgs e)
        {
            CartesianPlotBandAnnotation annotation = sender as CartesianPlotBandAnnotation;
            annotation.model.From = annotation.From;
        }

        private static void OnToPropertyChanged(DependencyObject sender, DependencyPropertyChangedEventArgs e)
        {
            CartesianPlotBandAnnotation annotation = sender as CartesianPlotBandAnnotation;
            annotation.model.To = annotation.To;
        }

        private static void OnFillPropertyChanged(DependencyObject sender, DependencyPropertyChangedEventArgs e)
        {
            CartesianPlotBandAnnotation annotation = sender as CartesianPlotBandAnnotation;
            annotation.presenter.Fill = e.NewValue as Brush;
        }
    }
}


text:
@protocol DialogSendOptionDelegate <NSObject>
- (void)selectChangeAddress;
@end

@interface DialogSendOption : DialogCentered
- (instancetype)initWithDelegate:(NSObject <DialogSendOptionDelegate> *)delegate;

@property(weak) NSObject <DialogSendOptionDelegate> *delegate;
@end


text:
using System;
using System.Collections.Generic;
using NUnit.Framework;
using osu.Framework.Audio.Sample;
using osu.Framework.Bindables;
using osu.Framework.Graphics;
using osu.Framework.Graphics.Containers;
using osu.Framework.Graphics.OpenGL.Textures;
using osu.Framework.Graphics.Textures;
using osu.Framework.Testing;
using osu.Game.Audio;
using osu.Game.Rulesets.Objects.Drawables;
using osu.Game.Rulesets.Objects.Legacy;
using osu.Game.Rulesets.Objects.Types;
using osu.Game.Skinning;
using osu.Game.Tests.Visual;
using osuTK.Graphics;

namespace osu.Game.Tests.Gameplay
{
    [HeadlessTest]
    public class TestSceneHitObjectAccentColour : OsuTestScene
    {
        private Container skinContainer;

        [SetUp]
        public void Setup() => Schedule(() => Child = skinContainer = new SkinProvidingContainer(new TestSkin()));

        [Test]
        public void TestChangeComboIndexBeforeLoad()
        {
            TestDrawableHitObject hitObject = null;

            AddStep("set combo and add hitobject", () =>
            {
                hitObject = new TestDrawableHitObject();
                hitObject.HitObject.ComboIndex = 1;

                skinContainer.Add(hitObject);
            });

            AddAssert("combo colour is green", () => hitObject.AccentColour.Value == Color4.Green);
        }

        [Test]
        public void TestChangeComboIndexDuringLoad()
        {
            TestDrawableHitObject hitObject = null;

            AddStep("add hitobject and set combo", () =>
            {
                skinContainer.Add(hitObject = new TestDrawableHitObject());
                hitObject.HitObject.ComboIndex = 1;
            });

            AddAssert("combo colour is green", () => hitObject.AccentColour.Value == Color4.Green);
        }

        [Test]
        public void TestChangeComboIndexAfterLoad()
        {
            TestDrawableHitObject hitObject = null;

            AddStep("add hitobject", () => skinContainer.Add(hitObject = new TestDrawableHitObject()));
            AddAssert("combo colour is red", () => hitObject.AccentColour.Value == Color4.Red);

            AddStep("change combo", () => hitObject.HitObject.ComboIndex = 1);
            AddAssert("combo colour is green", () => hitObject.AccentColour.Value == Color4.Green);
        }

        private class TestDrawableHitObject : DrawableHitObject<TestHitObjectWithCombo>
        {
            public TestDrawableHitObject()
                : base(new TestHitObjectWithCombo())
            {
            }
        }

        private class TestHitObjectWithCombo : ConvertHitObject, IHasComboInformation
        {
            public bool NewCombo { get; set; }
            public int ComboOffset => 0;

            public Bindable<int> IndexInCurrentComboBindable { get; } = new Bindable<int>();

            public int IndexInCurrentCombo
            {
                get => IndexInCurrentComboBindable.Value;
                set => IndexInCurrentComboBindable.Value = value;
            }

            public Bindable<int> ComboIndexBindable { get; } = new Bindable<int>();

            public int ComboIndex
            {
                get => ComboIndexBindable.Value;
                set => ComboIndexBindable.Value = value;
            }

            public Bindable<int> ComboIndexWithOffsetsBindable { get; } = new Bindable<int>();

            public int ComboIndexWithOffsets
            {
                get => ComboIndexWithOffsetsBindable.Value;
                set => ComboIndexWithOffsetsBindable.Value = value;
            }

            public Bindable<bool> LastInComboBindable { get; } = new Bindable<bool>();

            public bool LastInCombo
            {
                get => LastInComboBindable.Value;
                set => LastInComboBindable.Value = value;
            }
        }

        private class TestSkin : ISkin
        {
            public readonly List<Color4> ComboColours = new List<Color4>
            {
                Color4.Red,
                Color4.Green
            };

            public Drawable GetDrawableComponent(ISkinComponent component) => throw new NotImplementedException();

            public Texture GetTexture(string componentName, WrapMode wrapModeS, WrapMode wrapModeT) => throw new NotImplementedException();

            public ISample GetSample(ISampleInfo sampleInfo) => throw new NotImplementedException();

            public ISkin FindProvider(Func<ISkin, bool> lookupFunction) => null;

            public IBindable<TValue> GetConfig<TLookup, TValue>(TLookup lookup)
            {
                switch (lookup)
                {
                    case SkinComboColourLookup comboColour:
                        return SkinUtils.As<TValue>(new Bindable<Color4>(ComboColours[comboColour.ColourIndex % ComboColours.Count]));
                }

                throw new NotImplementedException();
            }
        }
    }
}


text:
<!--
  ~ Copyright 2016 Red Hat, Inc. and/or its affiliates
  ~ and other contributors as indicated by the @author tags.
  ~
  ~ Licensed under the Apache License, Version 2.0 (the "License");
  ~ you may not use this file except in compliance with the License.
  ~ You may obtain a copy of the License at
  ~
  ~ http://www.apache.org/licenses/LICENSE-2.0
  ~
  ~ Unless required by applicable law or agreed to in writing, software
  ~ distributed under the License is distributed on an "AS IS" BASIS,
  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  ~ See the License for the specific language governing permissions and
  ~ limitations under the License.
  -->

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
    <parent>
        <artifactId>keycloak-parent</artifactId>
        <groupId>org.keycloak</groupId>
        <version>2.2.1.Final</version>
    </parent>

    <name>Providers</name>
    <description/>
    <modelVersion>4.0.0</modelVersion>

    <artifactId>keycloak-providers-parent</artifactId>
    <packaging>pom</packaging>

    <!--build>
      <pluginManagement>
          <plugins>
              <plugin>
                  <groupId>org.apache.maven.plugins</groupId>
                  <artifactId>maven-deploy-plugin</artifactId>
                  <configuration>
                      <skip>true</skip>
		  </configuration>
              </plugin>
          </plugins>
      </pluginManagement>
      <plugins>
          <plugin>
              <groupId>org.apache.maven.plugins</groupId>
              <artifactId>maven-war-plugin</artifactId>
	      <configuration>
		   <failOnMissingWeXml>false</failOnMissingWeXml>
              </configuration>
          </plugin>
      </plugins>
    </build-->

    <!--properties>
        <maven.compiler.target>1.8</maven.compiler.target>
        <maven.compiler.source>1.8</maven.compiler.source>
    </properties-->

    <modules>
        <module>registration-mobile-phone-action</module>
    </modules>
</project>