text:
package org.apache.tez.runtime.library.common.sort.impl.dflt;

import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.IntBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.tez.runtime.library.api.IOInterruptedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.util.IndexedSortable;
import org.apache.hadoop.util.Progress;
import org.apache.tez.common.TezUtilsInternal;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.OutputContext;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.common.ConfigUtils;
import org.apache.tez.runtime.library.common.shuffle.ShuffleUtils;
import org.apache.tez.runtime.library.common.sort.impl.ExternalSorter;
import org.apache.tez.runtime.library.common.sort.impl.IFile;
import org.apache.tez.runtime.library.common.sort.impl.TezIndexRecord;
import org.apache.tez.runtime.library.common.sort.impl.TezMerger;
import org.apache.tez.runtime.library.common.sort.impl.TezRawKeyValueIterator;
import org.apache.tez.runtime.library.common.sort.impl.TezSpillRecord;
import org.apache.tez.runtime.library.common.sort.impl.IFile.Writer;
import org.apache.tez.runtime.library.common.sort.impl.TezMerger.Segment;

import com.google.common.base.Preconditions;

@SuppressWarnings({"unchecked", "rawtypes"})
public class DefaultSorter extends ExternalSorter implements IndexedSortable {

  private static final Logger LOG = LoggerFactory.getLogger(DefaultSorter.class);

  // TODO NEWTEZ Progress reporting to Tez framework. (making progress vs %complete)

  /**
   * The size of each record in the index file for the map-outputs.
   */
  public static final int MAP_OUTPUT_INDEX_RECORD_LENGTH = 24;

  private final static int APPROX_HEADER_LENGTH = 150;

  // k/v accounting
  private final IntBuffer kvmeta; // metadata overlay on backing store
  int kvstart;            // marks origin of spill metadata
  int kvend;              // marks end of spill metadata
  int kvindex;            // marks end of fully serialized records

  int equator;            // marks origin of meta/serialization
  int bufstart;           // marks beginning of spill
  int bufend;             // marks beginning of collectable
  int bufmark;            // marks end of record
  int bufindex;           // marks end of collected
  int bufvoid;            // marks the point where we should stop
                          // reading at the end of the buffer

  private final byte[] kvbuffer;        // main output buffer
  private final byte[] b0 = new byte[0];

  protected static final int VALSTART = 0;         // val offset in acct
  protected static final int KEYSTART = 1;         // key offset in acct
  protected static final int PARTITION = 2;        // partition offset in acct
  protected static final int VALLEN = 3;           // length of value
  protected static final int NMETA = 4;            // num meta ints
  protected static final int METASIZE = NMETA * 4; // size in bytes

  // spill accounting
  final int maxRec;
  final int softLimit;
  boolean spillInProgress;
  int bufferRemaining;
  volatile Throwable sortSpillException = null;

  final int minSpillsForCombine;
  final ReentrantLock spillLock = new ReentrantLock();
  final Condition spillDone = spillLock.newCondition();
  final Condition spillReady = spillLock.newCondition();
  final BlockingBuffer bb = new BlockingBuffer();
  volatile boolean spillThreadRunning = false;
  final SpillThread spillThread = new SpillThread();

  final ArrayList<TezSpillRecord> indexCacheList =
    new ArrayList<TezSpillRecord>();
  private final int indexCacheMemoryLimit;
  private int totalIndexCacheMemory;

  private long totalKeys = 0;
  private long sameKey = 0;

  public static final int MAX_IO_SORT_MB = 1800;

  public DefaultSorter(OutputContext outputContext, Configuration conf, int numOutputs,
      long initialMemoryAvailable) throws IOException {
    super(outputContext, conf, numOutputs, initialMemoryAvailable);
    // sanity checks
    final float spillper = this.conf.getFloat(
        TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT,
        TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT_DEFAULT);
    final int sortmb = computeSortBufferSize((int) availableMemoryMb);

    Preconditions.checkArgument(spillper <= (float) 1.0 && spillper > (float) 0.0,
        TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT
            + " should be greater than 0 and less than or equal to 1");

    indexCacheMemoryLimit = this.conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES,
                                       TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES_DEFAULT);

    boolean confPipelinedShuffle = this.conf.getBoolean(TezRuntimeConfiguration
        .TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED, TezRuntimeConfiguration
        .TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED_DEFAULT);

    if (confPipelinedShuffle) {
      LOG.warn(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED  + " does not work "
          + "with DefaultSorter. It is supported only with PipelinedSorter.");
    }

    // buffers and accounting
    int maxMemUsage = sortmb << 20;
    maxMemUsage -= maxMemUsage % METASIZE;
    kvbuffer = new byte[maxMemUsage];
    bufvoid = kvbuffer.length;
    kvmeta = ByteBuffer.wrap(kvbuffer)
       .order(ByteOrder.nativeOrder())
       .asIntBuffer();
    setEquator(0);
    bufstart = bufend = bufindex = equator;
    kvstart = kvend = kvindex;

    maxRec = kvmeta.capacity() / NMETA;
    softLimit = (int)(kvbuffer.length * spillper);
    bufferRemaining = softLimit;
    if (LOG.isInfoEnabled()) {
      LOG.info(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB + ": " + sortmb);
      LOG.info("soft limit at " + softLimit);
      LOG.info("bufstart = " + bufstart + "; bufvoid = " + bufvoid);
      LOG.info("kvstart = " + kvstart + "; length = " + maxRec + "; finalMergeEnabled = " + isFinalMergeEnabled());
    }

    // k/v serialization
    valSerializer.open(bb);
    keySerializer.open(bb);

    spillInProgress = false;
    minSpillsForCombine = this.conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINE_MIN_SPILLS, 3);
    spillThread.setDaemon(true);
    spillThread.setName("SpillThread ["
        + TezUtilsInternal.cleanVertexName(outputContext.getDestinationVertexName() + "]"));
    spillLock.lock();
    try {
      spillThread.start();
      while (!spillThreadRunning) {
        spillDone.await();
      }
    } catch (InterruptedException e) {
      //interrupt spill thread
      spillThread.interrupt();
      Thread.currentThread().interrupt();
      throw new IOException("Spill thread failed to initialize", e);
    } finally {
      spillLock.unlock();
    }
    if (sortSpillException != null) {
      throw new IOException("Spill thread failed to initialize",
          sortSpillException);
    }
  }

  @VisibleForTesting
  static int computeSortBufferSize(int availableMemoryMB) {

    if (availableMemoryMB <= 0) {
      throw new RuntimeException(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB +
          "=" + availableMemoryMB + ". It should be > 0");
    }

    if (availableMemoryMB > MAX_IO_SORT_MB) {
      LOG.warn("Scaling down " + TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB +
          "=" + availableMemoryMB + " to " + MAX_IO_SORT_MB
          + " (max sort buffer size supported forDefaultSorter)");
    }

    // cap sort buffer to MAX_IO_SORT_MB for DefaultSorter.
    // Not using 2047 to avoid any ArrayIndexOutofBounds in collect() phase.
    return Math.min(MAX_IO_SORT_MB, availableMemoryMB);
  }

  @Override
  public void write(Object key, Object value)
      throws IOException {
    collect(
        key, value, partitioner.getPartition(key, value, partitions));
  }

  /**
   * Serialize the key, value to intermediate storage.
   * When this method returns, kvindex must refer to sufficient unused
   * storage to store one METADATA.
   */
  synchronized void collect(Object key, Object value, final int partition
                                   ) throws IOException {

    if (key.getClass() != keyClass) {
      throw new IOException("Type mismatch in key from map: expected "
                            + keyClass.getName() + ", received "
                            + key.getClass().getName());
    }
    if (value.getClass() != valClass) {
      throw new IOException("Type mismatch in value from map: expected "
                            + valClass.getName() + ", received "
                            + value.getClass().getName());
    }
    if (partition < 0 || partition >= partitions) {
      throw new IOException("Illegal partition for " + key + " (" +
          partition + ")" + ", TotalPartitions: " + partitions);
    }
    checkSpillException();
    bufferRemaining -= METASIZE;
    if (bufferRemaining <= 0) {
      // start spill if the thread is not running and the soft limit has been
      // reached
      spillLock.lock();
      try {
        do {
          if (!spillInProgress) {
            final int kvbidx = 4 * kvindex;
            final int kvbend = 4 * kvend;
            // serialized, unspilled bytes always lie between kvindex and
            // bufindex, crossing the equator. Note that any void space
            // created by a reset must be included in "used" bytes
            final int bUsed = distanceTo(kvbidx, bufindex);
            final boolean bufsoftlimit = bUsed >= softLimit;
            if ((kvbend + METASIZE) % kvbuffer.length !=
                equator - (equator % METASIZE)) {
              // spill finished, reclaim space
              resetSpill();
              bufferRemaining = Math.min(
                  distanceTo(bufindex, kvbidx) - 2 * METASIZE,
                  softLimit - bUsed) - METASIZE;
              continue;
            } else if (bufsoftlimit && kvindex != kvend) {
              // spill records, if any collected; check latter, as it may
              // be possible for metadata alignment to hit spill pcnt
              startSpill();
              final int avgRec = (int)
                (mapOutputByteCounter.getValue() /
                mapOutputRecordCounter.getValue());
              // leave at least half the split buffer for serialization data
              // ensure that kvindex >= bufindex
              final int distkvi = distanceTo(bufindex, kvbidx);
              /**
               * Reason for capping sort buffer to MAX_IO_SORT_MB
               * E.g
               * kvbuffer.length = 2146435072 (2047 MB)
               * Corner case: bufIndex=2026133899, kvbidx=523629312.
               * distkvi = mod - i + j = 2146435072 - 2026133899 + 523629312 = 643930485
               * newPos = (2026133899 + (max(.., min(643930485/2, 271128624))) (This would
               * overflow)
               */
              final int newPos = (bufindex +
                Math.max(2 * METASIZE - 1,
                        Math.min(distkvi / 2,
                                 distkvi / (METASIZE + avgRec) * METASIZE)))
                % kvbuffer.length;
              setEquator(newPos);
              bufmark = bufindex = newPos;
              final int serBound = 4 * kvend;
              // bytes remaining before the lock must be held and limits
              // checked is the minimum of three arcs: the metadata space, the
              // serialization space, and the soft limit
              bufferRemaining = Math.min(
                  // metadata max
                  distanceTo(bufend, newPos),
                  Math.min(
                    // serialization max
                    distanceTo(newPos, serBound),
                    // soft limit
                    softLimit)) - 2 * METASIZE;
            }
          }
        } while (false);
      } finally {
        spillLock.unlock();
      }
    }

    try {
      // serialize key bytes into buffer
      int keystart = bufindex;
      keySerializer.serialize(key);
      if (bufindex < keystart) {
        // wrapped the key; must make contiguous
        bb.shiftBufferedKey();
        keystart = 0;
      }
      // serialize value bytes into buffer
      final int valstart = bufindex;
      valSerializer.serialize(value);
      // It's possible for records to have zero length, i.e. the serializer
      // will perform no writes. To ensure that the boundary conditions are
      // checked and that the kvindex invariant is maintained, perform a
      // zero-length write into the buffer. The logic monitoring this could be
      // moved into collect, but this is cleaner and inexpensive. For now, it
      // is acceptable.
      bb.write(b0, 0, 0);

      // the record must be marked after the preceding write, as the metadata
      // for this record are not yet written
      int valend = bb.markRecord();

      mapOutputRecordCounter.increment(1);
      mapOutputByteCounter.increment(
          distanceTo(keystart, valend, bufvoid));

      // write accounting info
      kvmeta.put(kvindex + PARTITION, partition);
      kvmeta.put(kvindex + KEYSTART, keystart);
      kvmeta.put(kvindex + VALSTART, valstart);
      kvmeta.put(kvindex + VALLEN, distanceTo(valstart, valend));
      // advance kvindex
      kvindex = (int)(((long)kvindex - NMETA + kvmeta.capacity()) % kvmeta.capacity());
      totalKeys++;
    } catch (MapBufferTooSmallException e) {
      LOG.info("Record too large for in-memory buffer: " + e.getMessage());
      spillSingleRecord(key, value, partition);
      mapOutputRecordCounter.increment(1);
      return;
    }
  }

  /**
   * Set the point from which meta and serialization data expand. The meta
   * indices are aligned with the buffer, so metadata never spans the ends of
   * the circular buffer.
   */
  private void setEquator(int pos) {
    equator = pos;
    // set index prior to first entry, aligned at meta boundary
    final int aligned = pos - (pos % METASIZE);
    // Cast one of the operands to long to avoid integer overflow
    kvindex = (int) (((long) aligned - METASIZE + kvbuffer.length) % kvbuffer.length) / 4;
    if (LOG.isInfoEnabled()) {
      LOG.info("(EQUATOR) " + pos + " kvi " + kvindex +
          "(" + (kvindex * 4) + ")");
    }
  }

  /**
   * The spill is complete, so set the buffer and meta indices to be equal to
   * the new equator to free space for continuing collection. Note that when
   * kvindex == kvend == kvstart, the buffer is empty.
   */
  private void resetSpill() {
    final int e = equator;
    bufstart = bufend = e;
    final int aligned = e - (e % METASIZE);
    // set start/end to point to first meta record
    // Cast one of the operands to long to avoid integer overflow
    kvstart = kvend = (int) (((long) aligned - METASIZE + kvbuffer.length) % kvbuffer.length) / 4;
    if (LOG.isInfoEnabled()) {
      LOG.info("(RESET) equator " + e + " kv " + kvstart + "(" +
        (kvstart * 4) + ")" + " kvi " + kvindex + "(" + (kvindex * 4) + ")");
    }
  }

  /**
   * Compute the distance in bytes between two indices in the serialization
   * buffer.
   * @see #distanceTo(int,int,int)
   */
  final int distanceTo(final int i, final int j) {
    return distanceTo(i, j, kvbuffer.length);
  }

  /**
   * Compute the distance between two indices in the circular buffer given the
   * max distance.
   */
  int distanceTo(final int i, final int j, final int mod) {
    return i <= j
      ? j - i
      : mod - i + j;
  }

  /**
   * For the given meta position, return the offset into the int-sized
   * kvmeta buffer.
   */
  int offsetFor(int metapos) {
    return (metapos % maxRec) * NMETA;

  }

  /**
   * Compare logical range, st i, j MOD offset capacity.
   * Compare by partition, then by key.
   * @see IndexedSortable#compare
   */
  public int compare(final int mi, final int mj) {
    final int kvi = offsetFor(mi);
    final int kvj = offsetFor(mj);
    final int kvip = kvmeta.get(kvi + PARTITION);
    final int kvjp = kvmeta.get(kvj + PARTITION);
    // sort by partition
    if (kvip != kvjp) {
      return kvip - kvjp;
    }
    // sort by key
    int result = comparator.compare(kvbuffer,
        kvmeta.get(kvi + KEYSTART),
        kvmeta.get(kvi + VALSTART) - kvmeta.get(kvi + KEYSTART),
        kvbuffer,
        kvmeta.get(kvj + KEYSTART),
        kvmeta.get(kvj + VALSTART) - kvmeta.get(kvj + KEYSTART));
    if (result == 0) {
      sameKey++;
    }
    return result;
  }

  final byte META_BUFFER_TMP[] = new byte[METASIZE];
  /**
   * Swap metadata for items i,j
   * @see IndexedSortable#swap
   */
  public void swap(final int mi, final int mj) {
    int iOff = (mi % maxRec) * METASIZE;
    int jOff = (mj % maxRec) * METASIZE;
    System.arraycopy(kvbuffer, iOff, META_BUFFER_TMP, 0, METASIZE);
    System.arraycopy(kvbuffer, jOff, kvbuffer, iOff, METASIZE);
    System.arraycopy(META_BUFFER_TMP, 0, kvbuffer, jOff, METASIZE);
  }

  /**
   * Inner class managing the spill of serialized records to disk.
   */
  protected class BlockingBuffer extends DataOutputStream {

    public BlockingBuffer() {
      super(new Buffer());
    }

    /**
     * Mark end of record. Note that this is required if the buffer is to
     * cut the spill in the proper place.
     */
    public int markRecord() {
      bufmark = bufindex;
      return bufindex;
    }

    /**
     * Set position from last mark to end of writable buffer, then rewrite
     * the data between last mark and kvindex.
     * This handles a special case where the key wraps around the buffer.
     * If the key is to be passed to a RawComparator, then it must be
     * contiguous in the buffer. This recopies the data in the buffer back
     * into itself, but starting at the beginning of the buffer. Note that
     * this method should <b>only</b> be called immediately after detecting
     * this condition. To call it at any other time is undefined and would
     * likely result in data loss or corruption.
     * @see #markRecord()
     */
    protected void shiftBufferedKey() throws IOException {
      // spillLock unnecessary; both kvend and kvindex are current
      int headbytelen = bufvoid - bufmark;
      bufvoid = bufmark;
      final int kvbidx = 4 * kvindex;
      final int kvbend = 4 * kvend;
      final int avail =
        Math.min(distanceTo(0, kvbidx), distanceTo(0, kvbend));
      if (bufindex + headbytelen < avail) {
        System.arraycopy(kvbuffer, 0, kvbuffer, headbytelen, bufindex);
        System.arraycopy(kvbuffer, bufvoid, kvbuffer, 0, headbytelen);
        bufindex += headbytelen;
        bufferRemaining -= kvbuffer.length - bufvoid;
      } else {
        byte[] keytmp = new byte[bufindex];
        System.arraycopy(kvbuffer, 0, keytmp, 0, bufindex);
        bufindex = 0;
        out.write(kvbuffer, bufmark, headbytelen);
        out.write(keytmp);
      }
    }
  }

  public class Buffer extends OutputStream {
    private final byte[] scratch = new byte[1];

    @Override
    public void write(int v)
        throws IOException {
      scratch[0] = (byte)v;
      write(scratch, 0, 1);
    }

    /**
     * Attempt to write a sequence of bytes to the collection buffer.
     * This method will block if the spill thread is running and it
     * cannot write.
     * @throws MapBufferTooSmallException if record is too large to
     *    deserialize into the collection buffer.
     */
    @Override
    public void write(byte b[], int off, int len)
        throws IOException {
      // must always verify the invariant that at least METASIZE bytes are
      // available beyond kvindex, even when len == 0
      bufferRemaining -= len;
      if (bufferRemaining <= 0) {
        // writing these bytes could exhaust available buffer space or fill
        // the buffer to soft limit. check if spill or blocking are necessary
        boolean blockwrite = false;
        spillLock.lock();
        try {
          do {
            checkSpillException();

            final int kvbidx = 4 * kvindex;
            final int kvbend = 4 * kvend;
            // ser distance to key index
            final int distkvi = distanceTo(bufindex, kvbidx);
            // ser distance to spill end index
            final int distkve = distanceTo(bufindex, kvbend);

            // if kvindex is closer than kvend, then a spill is neither in
            // progress nor complete and reset since the lock was held. The
            // write should block only if there is insufficient space to
            // complete the current write, write the metadata for this record,
            // and write the metadata for the next record. If kvend is closer,
            // then the write should block if there is too little space for
            // either the metadata or the current write. Note that collect
            // ensures its metadata requirement with a zero-length write
            blockwrite = distkvi <= distkve
              ? distkvi <= len + 2 * METASIZE
              : distkve <= len || distanceTo(bufend, kvbidx) < 2 * METASIZE;

            if (!spillInProgress) {
              if (blockwrite) {
                if ((kvbend + METASIZE) % kvbuffer.length !=
                    equator - (equator % METASIZE)) {
                  // spill finished, reclaim space
                  // need to use meta exclusively; zero-len rec & 100% spill
                  // pcnt would fail
                  resetSpill(); // resetSpill doesn't move bufindex, kvindex
                  bufferRemaining = Math.min(
                      distkvi - 2 * METASIZE,
                      softLimit - distanceTo(kvbidx, bufindex)) - len;
                  continue;
                }
                // we have records we can spill; only spill if blocked
                if (kvindex != kvend) {
                  startSpill();
                  // Blocked on this write, waiting for the spill just
                  // initiated to finish. Instead of repositioning the marker
                  // and copying the partial record, we set the record start
                  // to be the new equator
                  setEquator(bufmark);
                } else {
                  // We have no buffered records, and this record is too large
                  // to write into kvbuffer. We must spill it directly from
                  // collect
                  final int size = distanceTo(bufstart, bufindex) + len;
                  setEquator(0);
                  bufstart = bufend = bufindex = equator;
                  kvstart = kvend = kvindex;
                  bufvoid = kvbuffer.length;
                  throw new MapBufferTooSmallException(size + " bytes");
                }
              }
            }

            if (blockwrite) {
              // wait for spill
              try {
                while (spillInProgress) {
                  spillDone.await();
                }
              } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                  throw new IOInterruptedException(
                      "Buffer interrupted while waiting for the writer", e);
              }
            }
          } while (blockwrite);
        } finally {
          spillLock.unlock();
        }
      }
      // here, we know that we have sufficient space to write
      // int overflow possible with (bufindex + len)
      long futureBufIndex = (long) bufindex + len;
      if (futureBufIndex > bufvoid) {
        final int gaplen = bufvoid - bufindex;
        System.arraycopy(b, off, kvbuffer, bufindex, gaplen);
        len -= gaplen;
        off += gaplen;
        bufindex = 0;
      }
      System.arraycopy(b, off, kvbuffer, bufindex, len);
      bufindex += len;
    }
  }

  void interruptSpillThread() throws IOException {
    assert !spillLock.isHeldByCurrentThread();
    // shut down spill thread and wait for it to exit. Since the preceding
    // ensures that it is finished with its work (and sortAndSpill did not
    // throw), we elect to use an interrupt instead of setting a flag.
    // Spilling simultaneously from this thread while the spill thread
    // finishes its work might be both a useful way to extend this and also
    // sufficient motivation for the latter approach.
    try {
      spillThread.interrupt();
      spillThread.join();
    } catch (InterruptedException e) {
      LOG.info("Spill thread interrupted");
      //Reset status
      Thread.currentThread().interrupt();
      throw new IOInterruptedException("Spill failed", e);
    }
  }

  @Override
  public void flush() throws IOException {
    LOG.info("Starting flush of map output");
    if (Thread.currentThread().isInterrupted()) {
      /**
       * Possible that the thread got interrupted when flush was happening or when the flush was
       * never invoked. As a part of cleanup activity in TezTaskRunner, it would invoke close()
       * on all I/O. At that time, this is safe to cleanup
       */
      if (cleanup) {
        cleanup();
      }
      try {
        interruptSpillThread();
      } catch(IOException e) {
        //safe to ignore
      }
      return;
    }

    spillLock.lock();
    try {
      while (spillInProgress) {
        spillDone.await();
      }
      checkSpillException();

      final int kvbend = 4 * kvend;
      if ((kvbend + METASIZE) % kvbuffer.length !=
          equator - (equator % METASIZE)) {
        // spill finished
        resetSpill();
      }
      if (kvindex != kvend) {
        kvend = (kvindex + NMETA) % kvmeta.capacity();
        bufend = bufmark;
        if (LOG.isInfoEnabled()) {
          LOG.info("Sorting & Spilling map output");
          LOG.info("bufstart = " + bufstart + "; bufend = " + bufmark +
                   "; bufvoid = " + bufvoid);
          LOG.info("kvstart = " + kvstart + "(" + (kvstart * 4) +
                   "); kvend = " + kvend + "(" + (kvend * 4) +
                   "); length = " + (distanceTo(kvend, kvstart,
                         kvmeta.capacity()) + 1) + "/" + maxRec);
        }
        sortAndSpill();
      }
    } catch (InterruptedException e) {
      //Reset status
      Thread.currentThread().interrupt();
      interruptSpillThread();
      throw new IOException("Interrupted while waiting for the writer", e);
    } finally {
      spillLock.unlock();
    }

    interruptSpillThread();
    // release sort buffer before the mergecl
    //FIXME
    //kvbuffer = null;

    try {
      mergeParts();
    } catch (InterruptedException e) {
      cleanup();
      Thread.currentThread().interrupt();
    }
    if (isFinalMergeEnabled()) {
      fileOutputByteCounter.increment(rfs.getFileStatus(finalOutputFile).getLen());
    }
  }

  protected class SpillThread extends Thread {

    @Override
    public void run() {
      spillLock.lock();
      try {
        spillThreadRunning = true;
        while (true) {
          spillDone.signal();
          while (!spillInProgress) {
            spillReady.await();
          }
          try {
            spillLock.unlock();
            sortAndSpill();
          } catch (Throwable t) {
            LOG.warn("Got an exception in sortAndSpill", t);
            sortSpillException = t;
          } finally {
            spillLock.lock();
            if (bufend < bufstart) {
              bufvoid = kvbuffer.length;
            }
            kvstart = kvend;
            bufstart = bufend;
            spillInProgress = false;
          }
        }
      } catch (InterruptedException e) {
        LOG.info("Spill thread interrupted");
        Thread.currentThread().interrupt();
      } finally {
        spillLock.unlock();
        spillThreadRunning = false;
      }
    }
  }

  private void checkSpillException() throws IOException {
    final Throwable lspillException = sortSpillException;
    if (lspillException != null) {
      if (lspillException instanceof Error) {
        final String logMsg = "Task " + outputContext.getUniqueIdentifier()
            + " failed : " + ExceptionUtils.getStackTrace(lspillException);
        outputContext.fatalError(lspillException, logMsg);
      }
      if (lspillException instanceof InterruptedException) {
        throw new IOInterruptedException("Spill failed", lspillException);
      } else {
        throw new IOException("Spill failed", lspillException);
      }
    }
  }

  private void startSpill() {
    assert !spillInProgress;
    kvend = (kvindex + NMETA) % kvmeta.capacity();
    bufend = bufmark;
    spillInProgress = true;
    if (LOG.isInfoEnabled()) {
      LOG.info("Spilling map output");
      LOG.info("bufstart = " + bufstart + "; bufend = " + bufmark +
               "; bufvoid = " + bufvoid);
      LOG.info("kvstart = " + kvstart + "(" + (kvstart * 4) +
               "); kvend = " + kvend + "(" + (kvend * 4) +
               "); length = " + (distanceTo(kvend, kvstart,
                     kvmeta.capacity()) + 1) + "/" + maxRec);
    }
    spillReady.signal();
  }

  int getMetaStart() {
    return kvend / NMETA;
  }

  int getMetaEnd() {
    return 1 + // kvend is a valid record
        (kvstart >= kvend
        ? kvstart
        : kvmeta.capacity() + kvstart) / NMETA;
  }

  private boolean isRLENeeded() {
    return (sameKey > (0.1 * totalKeys)) || (sameKey < 0);
  }

  protected void sortAndSpill()
      throws IOException, InterruptedException {
    final int mstart = getMetaStart();
    final int mend = getMetaEnd();
    sorter.sort(this, mstart, mend, nullProgressable);
    spill(mstart, mend);
  }

  private void adjustSpillCounters(long rawLen, long compLength) {
    if (!isFinalMergeEnabled()) {
      outputBytesWithOverheadCounter.increment(rawLen);
    } else {
      if (numSpills > 0) {
        additionalSpillBytesWritten.increment(compLength);
        // Reset the value will be set during the final merge.
        outputBytesWithOverheadCounter.setValue(0);
      } else {
        // Set this up for the first write only. Subsequent ones will be handled in the final merge.
        outputBytesWithOverheadCounter.increment(rawLen);
      }
    }
  }

  protected void spill(int mstart, int mend)
      throws IOException, InterruptedException {

    //approximate the length of the output file to be the length of the
    //buffer + header lengths for the partitions
    final long size = (bufend >= bufstart
        ? bufend - bufstart
        : (bufvoid - bufend) + bufstart) +
                partitions * APPROX_HEADER_LENGTH;
    FSDataOutputStream out = null;
    try {
      // create spill file
      final TezSpillRecord spillRec = new TezSpillRecord(partitions);
      final Path filename =
          mapOutputFile.getSpillFileForWrite(numSpills, size);
      spillFilePaths.put(numSpills, filename);
      out = rfs.create(filename);

      int spindex = mstart;
      final InMemValBytes value = createInMemValBytes();
      boolean rle = isRLENeeded();
      for (int i = 0; i < partitions; ++i) {
        IFile.Writer writer = null;
        try {
          long segmentStart = out.getPos();
          writer = new Writer(conf, out, keyClass, valClass, codec,
                                    spilledRecordsCounter, null, rle);
          if (combiner == null) {
            // spill directly
            DataInputBuffer key = new DataInputBuffer();
            while (spindex < mend &&
                kvmeta.get(offsetFor(spindex) + PARTITION) == i) {
              final int kvoff = offsetFor(spindex);
              int keystart = kvmeta.get(kvoff + KEYSTART);
              int valstart = kvmeta.get(kvoff + VALSTART);
              key.reset(kvbuffer, keystart, valstart - keystart);
              getVBytesForOffset(kvoff, value);
              writer.append(key, value);
              ++spindex;
            }
          } else {
            int spstart = spindex;
            while (spindex < mend &&
                kvmeta.get(offsetFor(spindex)
                          + PARTITION) == i) {
              ++spindex;
            }
            // Note: we would like to avoid the combiner if we've fewer
            // than some threshold of records for a partition
            if (spstart != spindex) {
              TezRawKeyValueIterator kvIter =
                new MRResultIterator(spstart, spindex);
              if (LOG.isDebugEnabled()) {
                LOG.debug("Running combine processor");
              }
              runCombineProcessor(kvIter, writer);
            }
          }

          // close the writer
          writer.close();
          adjustSpillCounters(writer.getRawLength(), writer.getCompressedLength());
          // record offsets
          final TezIndexRecord rec =
              new TezIndexRecord(
                  segmentStart,
                  writer.getRawLength(),
                  writer.getCompressedLength());
          spillRec.putIndex(rec, i);
          if (!isFinalMergeEnabled() && reportPartitionStats()) {
            partitionStats[i] += writer.getCompressedLength();
          }

          writer = null;
        } finally {
          if (null != writer) writer.close();
        }
      }

      if (totalIndexCacheMemory >= indexCacheMemoryLimit) {
        // create spill index file
        Path indexFilename =
            mapOutputFile.getSpillIndexFileForWrite(numSpills, partitions
                * MAP_OUTPUT_INDEX_RECORD_LENGTH);
        spillFileIndexPaths.put(numSpills, indexFilename);
        spillRec.writeToFile(indexFilename, conf);
      } else {
        indexCacheList.add(spillRec);
        totalIndexCacheMemory +=
          spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH;
      }
      LOG.info("Finished spill " + numSpills);
      ++numSpills;
      if (!isFinalMergeEnabled()) {
        numShuffleChunks.setValue(numSpills);
      } else if (numSpills > 1) {
        //Increment only when there was atleast one previous spill
        numAdditionalSpills.increment(1);
      }
    } finally {
      if (out != null) out.close();
    }
  }

  /**
   * Handles the degenerate case where serialization fails to fit in
   * the in-memory buffer, so we must spill the record from collect
   * directly to a spill file. Consider this "losing".
   */
  private void spillSingleRecord(final Object key, final Object value,
                                 int partition) throws IOException {
    long size = kvbuffer.length + partitions * APPROX_HEADER_LENGTH;
    FSDataOutputStream out = null;
    try {
      // create spill file
      final TezSpillRecord spillRec = new TezSpillRecord(partitions);
      final Path filename =
          mapOutputFile.getSpillFileForWrite(numSpills, size);
      spillFilePaths.put(numSpills, filename);
      out = rfs.create(filename);

      // we don't run the combiner for a single record
      for (int i = 0; i < partitions; ++i) {
        IFile.Writer writer = null;
        try {
          long segmentStart = out.getPos();
          // Create a new codec, don't care!
          writer = new IFile.Writer(conf, out, keyClass, valClass, codec,
                                          spilledRecordsCounter, null);

          if (i == partition) {
            final long recordStart = out.getPos();
            writer.append(key, value);
            // Note that our map byte count will not be accurate with
            // compression
            mapOutputByteCounter.increment(out.getPos() - recordStart);
          }
          writer.close();

          adjustSpillCounters(writer.getRawLength(), writer.getCompressedLength());

          // record offsets
          TezIndexRecord rec =
              new TezIndexRecord(
                  segmentStart,
                  writer.getRawLength(),
                  writer.getCompressedLength());
          spillRec.putIndex(rec, i);

          writer = null;
        } catch (IOException e) {
          if (null != writer) writer.close();
          throw e;
        }
      }
      if (totalIndexCacheMemory >= indexCacheMemoryLimit) {
        // create spill index file
        Path indexFilename =
            mapOutputFile.getSpillIndexFileForWrite(numSpills, partitions
                * MAP_OUTPUT_INDEX_RECORD_LENGTH);
        spillFileIndexPaths.put(numSpills, indexFilename);
         spillRec.writeToFile(indexFilename, conf);
      } else {
        indexCacheList.add(spillRec);
        totalIndexCacheMemory +=
          spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH;
      }
      ++numSpills;
      if (!isFinalMergeEnabled()) {
        numShuffleChunks.setValue(numSpills);
      } else if (numSpills > 1) {
        //Increment only when there is atleast one previous spill
        numAdditionalSpills.increment(1);
      }
    } finally {
      if (out != null) out.close();
    }
  }

  protected int getInMemVBytesLength(int kvoff) {
    // get the keystart for the next serialized value to be the end
    // of this value. If this is the last value in the buffer, use bufend
    final int vallen = kvmeta.get(kvoff + VALLEN);
    assert vallen >= 0;
    return vallen;
  }

  /**
   * Given an offset, populate vbytes with the associated set of
   * deserialized value bytes. Should only be called during a spill.
   */
  int getVBytesForOffset(int kvoff, InMemValBytes vbytes) {
    int vallen = getInMemVBytesLength(kvoff);
    vbytes.reset(kvbuffer, kvmeta.get(kvoff + VALSTART), vallen);
    return vallen;
  }

  /**
   * Inner class wrapping valuebytes, used for appendRaw.
   */
  static class InMemValBytes extends DataInputBuffer {
    private byte[] buffer;
    private int start;
    private int length;
    private final int bufvoid;

    public InMemValBytes(int bufvoid) {
      this.bufvoid = bufvoid;
    }

    public void reset(byte[] buffer, int start, int length) {
      this.buffer = buffer;
      this.start = start;
      this.length = length;

      if (start + length > bufvoid) {
        this.buffer = new byte[this.length];
        final int taillen = bufvoid - start;
        System.arraycopy(buffer, start, this.buffer, 0, taillen);
        System.arraycopy(buffer, 0, this.buffer, taillen, length-taillen);
        this.start = 0;
      }

      super.reset(this.buffer, this.start, this.length);
    }
  }

  InMemValBytes createInMemValBytes() {
    return new InMemValBytes(bufvoid);
  }

  protected class MRResultIterator implements TezRawKeyValueIterator {
    private final DataInputBuffer keybuf = new DataInputBuffer();
    private final InMemValBytes vbytes = createInMemValBytes();
    private final int end;
    private int current;
    public MRResultIterator(int start, int end) {
      this.end = end;
      current = start - 1;
    }
    public boolean next() throws IOException {
      return ++current < end;
    }
    public DataInputBuffer getKey() throws IOException {
      final int kvoff = offsetFor(current);
      keybuf.reset(kvbuffer, kvmeta.get(kvoff + KEYSTART),
          kvmeta.get(kvoff + VALSTART) - kvmeta.get(kvoff + KEYSTART));
      return keybuf;
    }
    public DataInputBuffer getValue() throws IOException {
      getVBytesForOffset(offsetFor(current), vbytes);
      return vbytes;
    }
    public Progress getProgress() {
      return null;
    }

    @Override
    public boolean isSameKey() throws IOException {
      return false;
    }

    public void close() { }
  }

  private void maybeSendEventForSpill(List<Event> events, boolean isLastEvent,
      TezSpillRecord spillRecord, int index, boolean sendEvent) throws IOException {
    if (isFinalMergeEnabled()) {
      return;
    }
    Preconditions.checkArgument(spillRecord != null, "Spill record can not be null");

    String pathComponent = (outputContext.getUniqueIdentifier() + "_" + index);
    ShuffleUtils.generateEventOnSpill(events, isFinalMergeEnabled(), isLastEvent,
        outputContext, index, spillRecord, partitions, sendEmptyPartitionDetails, pathComponent,
        partitionStats);

    LOG.info("Adding spill event for spill (final update=" + isLastEvent + "), spillId=" + index);

    if (sendEvent) {
      outputContext.sendEvents(events);
    }
  }

  private void maybeAddEventsForSpills() throws IOException {
    if (isFinalMergeEnabled()) {
      return;
    }
    List<Event> events = Lists.newLinkedList();
    for(int i=0; i<numSpills; i++) {

      TezSpillRecord spillRecord = indexCacheList.get(i);
      if (spillRecord == null) {
        //File was already written and location is stored in spillFileIndexPaths
        spillRecord = new TezSpillRecord(spillFileIndexPaths.get(i), conf);
      } else {
        //Double check if this file has to be written
        if (spillFileIndexPaths.get(i) == null) {
          Path indexPath = mapOutputFile.getSpillIndexFileForWrite(i, partitions *
              MAP_OUTPUT_INDEX_RECORD_LENGTH);
          spillRecord.writeToFile(indexPath, conf);
        }
      }

      maybeSendEventForSpill(events, (i == numSpills - 1), spillRecord, i, false);
      fileOutputByteCounter.increment(rfs.getFileStatus(spillFilePaths.get(i)).getLen());
    }
    outputContext.sendEvents(events);
  }

  private void mergeParts() throws IOException, InterruptedException {
    // get the approximate size of the final output/index files
    long finalOutFileSize = 0;
    long finalIndexFileSize = 0;
    final Path[] filename = new Path[numSpills];
    final String taskIdentifier = outputContext.getUniqueIdentifier();

    for(int i = 0; i < numSpills; i++) {
      filename[i] = spillFilePaths.get(i);
      finalOutFileSize += rfs.getFileStatus(filename[i]).getLen();
    }
    if (numSpills == 1) { //the spill is the final output
      TezSpillRecord spillRecord = null;
      if (isFinalMergeEnabled()) {
        finalOutputFile = mapOutputFile.getOutputFileForWriteInVolume(filename[0]);
        finalIndexFile = mapOutputFile.getOutputIndexFileForWriteInVolume(filename[0]);
        sameVolRename(filename[0], finalOutputFile);
        if (indexCacheList.size() == 0) {
          sameVolRename(spillFileIndexPaths.get(0), finalIndexFile);
          spillRecord = new TezSpillRecord(finalIndexFile, conf);
        } else {
          spillRecord = indexCacheList.get(0);
          spillRecord.writeToFile(finalIndexFile, conf);
        }
      } else {
        List<Event> events = Lists.newLinkedList();
        //Since there is only one spill, spill record would be present in cache.
        spillRecord = indexCacheList.get(0);
        Path indexPath = mapOutputFile.getSpillIndexFileForWrite(numSpills-1, partitions *
            MAP_OUTPUT_INDEX_RECORD_LENGTH);
        spillRecord.writeToFile(indexPath, conf);
        maybeSendEventForSpill(events, true, spillRecord, 0, true);
        fileOutputByteCounter.increment(rfs.getFileStatus(spillFilePaths.get(0)).getLen());
        //No need to populate finalIndexFile, finalOutputFile etc when finalMerge is disabled
      }
      if (spillRecord != null && reportPartitionStats()) {
        for(int i=0; i < spillRecord.size(); i++) {
          partitionStats[i] += spillRecord.getIndex(i).getPartLength();
        }
      }
      numShuffleChunks.setValue(numSpills);
      return;
    }

    // read in paged indices
    for (int i = indexCacheList.size(); i < numSpills; ++i) {
      Path indexFileName = spillFileIndexPaths.get(i);
      indexCacheList.add(new TezSpillRecord(indexFileName, conf));
    }

    //Check if it is needed to do final merge. Or else, exit early.
    if (numSpills > 0 && !isFinalMergeEnabled()) {
      maybeAddEventsForSpills();
      //No need to do final merge.
      return;
    }

    //make correction in the length to include the sequence file header
    //lengths for each partition
    finalOutFileSize += partitions * APPROX_HEADER_LENGTH;
    finalIndexFileSize = partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH;

    if (isFinalMergeEnabled()) {
      finalOutputFile = mapOutputFile.getOutputFileForWrite(finalOutFileSize);
      finalIndexFile = mapOutputFile.getOutputIndexFileForWrite(finalIndexFileSize);
    } else if (numSpills == 0) {
      //e.g attempt_1424502260528_0119_1_07_000058_0_10012_0/file.out when final merge is
      // disabled
      finalOutputFile = mapOutputFile.getSpillFileForWrite(numSpills, finalOutFileSize);
      finalIndexFile = mapOutputFile.getSpillIndexFileForWrite(numSpills, finalIndexFileSize);
    }

    //The output stream for the final single output file
    FSDataOutputStream finalOut = rfs.create(finalOutputFile, true, 4096);

    if (numSpills == 0) {
      // TODO Change event generation to say there is no data rather than generating a dummy file
      //create dummy files

      TezSpillRecord sr = new TezSpillRecord(partitions);
      try {
        for (int i = 0; i < partitions; i++) {
          long segmentStart = finalOut.getPos();
          Writer writer =
            new Writer(conf, finalOut, keyClass, valClass, codec, null, null);
          writer.close();

          TezIndexRecord rec =
              new TezIndexRecord(
                  segmentStart,
                  writer.getRawLength(),
                  writer.getCompressedLength());
          // Covers the case of multiple spills.
          outputBytesWithOverheadCounter.increment(writer.getRawLength());
          sr.putIndex(rec, i);
        }
        sr.writeToFile(finalIndexFile, conf);
      } finally {
        finalOut.close();
      }
      ++numSpills;
      if (!isFinalMergeEnabled()) {
        List<Event> events = Lists.newLinkedList();
        maybeSendEventForSpill(events, true, sr, 0, true);
        fileOutputByteCounter.increment(rfs.getFileStatus(finalOutputFile).getLen());
      }
      numShuffleChunks.setValue(numSpills);
      return;
    }
    else {
      final TezSpillRecord spillRec = new TezSpillRecord(partitions);
      for (int parts = 0; parts < partitions; parts++) {
        //create the segments to be merged
        List<Segment> segmentList =
          new ArrayList<Segment>(numSpills);
        for(int i = 0; i < numSpills; i++) {
          TezIndexRecord indexRecord = indexCacheList.get(i).getIndex(parts);

          Segment s =
            new Segment(rfs, filename[i], indexRecord.getStartOffset(),
                             indexRecord.getPartLength(), codec, ifileReadAhead,
                             ifileReadAheadLength, ifileBufferSize, true);
          segmentList.add(i, s);

          if (LOG.isDebugEnabled()) {
            LOG.debug("TaskIdentifier=" + taskIdentifier + " Partition=" + parts +
                "Spill =" + i + "(" + indexRecord.getStartOffset() + "," +
                indexRecord.getRawLength() + ", " +
                indexRecord.getPartLength() + ")");
          }
        }

        int mergeFactor =
            this.conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR,
                TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR_DEFAULT);
        // sort the segments only if there are intermediate merges
        boolean sortSegments = segmentList.size() > mergeFactor;
        //merge
        TezRawKeyValueIterator kvIter = TezMerger.merge(conf, rfs,
                       keyClass, valClass, codec,
                       segmentList, mergeFactor,
                       new Path(taskIdentifier),
                       (RawComparator)ConfigUtils.getIntermediateOutputKeyComparator(conf),
                       nullProgressable, sortSegments, true,
                       null, spilledRecordsCounter, additionalSpillBytesRead,
                       null); // Not using any Progress in TezMerger. Should just work.

        //write merged output to disk
        long segmentStart = finalOut.getPos();
        Writer writer =
            new Writer(conf, finalOut, keyClass, valClass, codec,
                spilledRecordsCounter, null);
        if (combiner == null || numSpills < minSpillsForCombine) {
          TezMerger.writeFile(kvIter, writer,
              nullProgressable, TezRuntimeConfiguration.TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS_DEFAULT);
        } else {
          runCombineProcessor(kvIter, writer);
        }
        writer.close();
        outputBytesWithOverheadCounter.increment(writer.getRawLength());

        // record offsets
        final TezIndexRecord rec =
            new TezIndexRecord(
                segmentStart,
                writer.getRawLength(),
                writer.getCompressedLength());
        spillRec.putIndex(rec, parts);
        if (reportPartitionStats()) {
          partitionStats[parts] += writer.getCompressedLength();
        }
      }
      numShuffleChunks.setValue(1); //final merge has happened
      spillRec.writeToFile(finalIndexFile, conf);
      finalOut.close();
      for(int i = 0; i < numSpills; i++) {
        rfs.delete(filename[i],true);
      }
    }
  }
}


text:
6502 Macro Assembler in a single c++ file using the struse single file text parsing library. Supports most syntaxes. x65 was recently named Asm6502 but was renamed because Asm6502 is too generic, x65 has no particular meaning.

![x65](bin/x65.png)

In order to minimize the documentation and make this page shorter I've moved the [old documentation](../../wiki/Previous-first-page) here.

The [up to date documentation is here](x65.txt).

x65 can assemble 6502, 65C02 and 65816 source and build executables for c64, Apple II or just raw binary.

Noteworthy features:

* Code with sections, object files and linking or single file fixed
  address, or mix it up with fixed address sections in object files.
* Assembler listing with cycle counting for code review.
* Export multiple binaries with a single link operation.
* C style scoping within '{' and '}' with local and pool labels
  respecting scopes.
* Conditional assembly with if/ifdef/else etc.
* Recursible macro support
* Assembler directives representing a variety of features.
* Local labels can be defined in a number of ways, such as leading
  period (.label) or leading at-sign (@label) or terminating
  dollar sign (label$), and leading colon for merlin (:label).
* String Symbols system allows building user expressions and macros
  during assembly.
* Reassignment of symbols and labels by default.
* No indentation required for instructions, meaning that labels can't
  be mnemonics, macros or directives (merlin requires indentation).
* Supporting the syntax of other 6502 assemblers (Merlin syntax
  requires command line argument, -endm adds support for sources
  using macro/endmacro and repeat/endrepeat combos rather
  than scoeps).
* Apple II GS executable output (relocatable executable).

## Features

* **Code**
* **Linking**
* **Comments**
* **Labels**
* **String Symbols**
* **Directives**
* **Macros**
* **Expressions**
* **List File with Cycle Count**

## Prerequisite

x65.cpp requires struse.h which is a single file text parsing library that can be retrieved from https://github.com/Sakrac/struse.

### References

* [6502 opcodes](http://www.6502.org/tutorials/6502opcodes.html)
* [6502 cheat sheet](https://drive.google.com/file/d/0B4lBG-q6aGsROEtKWXRXVnlTNUE/view)
* [6502 opcode grid](http://www.llx.com/~nparker/a2/opcodes.html)
* [Codebase64 CPU section](http://codebase64.org/doku.php?id=base:6502_6510_coding)
* [6502 illegal opcodes](http://www.oxyron.de/html/opcodes02.html)
* [65816 opcodes](http://wiki.superfamicom.org/snes/show/65816+Reference#fn:14)

### Download Binaries

Please note that releases have moved the Github [releases](https://github.com/Sakrac/x65/releases)

### x65

x65 is the assembler

### x65macro.i

x65macro.i is a 6502 include file that defines a number of standard macros that can assign values, move values, copy values and loop constructs, see x65.txt for details.

### dump_x65

dump_x65 is a tool to inspect the contents of .x65 object files generated by x65 to track down linking issues

### x65dsasm

x65dsasm is a tool to disassemble assembled binary code for review, it will perform a basic analysis and assign labels where appropriate and treats unreferenced bytes as data rather than code. It can also export assemblable code from a binary.

### Acknowledgments

This project would not be completed without the direct or indirect support of great people, some which I can currently remember:

* Marc dePeo, helping me uncover the strange and unique world of Merlin's assembler syntax (and working together with me on True Crime NY gameplay code and more)
* Che Lalic, explaining the murky bits of 65816 (and a Ninja on SNES NBA Hangtime and other projects)
* John Brooks, sharing the Rastan Apple II source code so I could test 65816 and figure out a number of issues with my initial linker, and encouraging the implementation of Apple II GS OS executable file format / OMF export (and helping out with Playstation All-Stars)
* [Brutal Deluxe](http://www.brutaldeluxe.fr) for releasing the excellent OMF Analyzer tool and the source, which was a significant help generating Apple II GS OS executables.
* The C64 demo scene for sharing a great deal of 6502 programming resources and overall inspiration.
* Jordan Mechner, sharing the Prince of Persia Apple II source code so I could test out a significant part of the assembler and the Merlin syntax mode
* Bill Budge, sharing the Pinball Construction Set Apple II source code, although at the point I tried it, all of it just assembled without any assembler issues at all.

### Development Status

Looking for help testing various features of the assembler, I have a large number of tests that pass without fail but there are so many ways for assemblers to break.
Primarily tested with personal archive of sources written for Kick assmebler, DASM, TASM, XASM, etc. and passing most of Apple II Prince of Persia and Pinball Construction set.

**TODO**
* irp (indefinite repeat)

**FIXED**
* Source Debug output file including linkable object files, C64Debugger format
* Adding MERGE directive, Label Pools rewrite, TEXT data can be indexed from a string symbol
* Label Pools were destroyed after each scope so they did not work in include files which defeated their purpose. Label pools are now persistent through scopes.
* Labels reserved from label pools now distinguish between global and local. Use [.!@$] as a prefix to reserve a local label from a label pool (previously always local)
* Merlin macro parameters are not required on the MAC line, scope braces ('{', '}') can be used in the first column in Merlin.
* First line of a Merlin macro was sometimes ignored, two sequential subtractions were ignored in expressions.
* Pushing source contexts (macro, rept, include etc.) will always increment the scope depth.
* Fixed REPT / LUP to not destroy local symbols in the scope it was used in while also destroying local symbols within the repeating block correctly
* Switched over to inttypes.h from built-in types since the word unsigned was used a little too much in the code
* Removed the disassembler and put it into its own project [x65dsasm](http://github.com/sakrac/x65dsasm)
* LUP/REPT directives clean up local symbols each iteration to avoid crossing over an iteration with branches to local labels.
* Fixed Merlin MAC directive which is a little different from normal assembler macros
* Labels can start with numbers and values will only be interpreted as decimal numbers if terminated by a character that is not an alphabetic character or underscore
* INCSYM failed with local labels, this is now properly handled. (fixed again..)
* INCBIN and IMPORT BINARY always failed (force 0 bytes length)
* Using more than 16 bytes of Pool labels was flawed as byte 15 and 16 would be the same address.
* Fixed STRUCT directive (failed if contained line was empty).
* Adding x65macro.i
* Vice symbols will generate breakpoints whenever label 'debugbreak' is encountered
* Evaluating '==' was broken
* Macros can have dots in their names
* Handling double negative in expressions (--35 == 35)
* Macros works within conditionals (if/else/endif, etc)
* String symbols broke late evaluation resulting in garbage references, this has been fixed
* Added string symbols
* Resolved the DirectPage_Stack section vs. Zeropage section for Apple II GS/OS executables.
* OMF export for Apple II GS/OS executables

[(older fixes)](../../wiki/fixes)

Revisions:
* 10 - String Symbols
* 9 - Apple II GS OS executable
* 8 - Fish food / Linking tested and passed with external project (Apple II gs Rastan)
* 7 - 65816 Support
* 6 - 65C02 support
* 5 - Merlin syntax
* 4 - Object files, relative sections and linking
* 3 - 6502 full support
* 2 - Moved file out of struse samples
* 1 - Built a sample of a simple assembler within struse

redpajama_set_name: RedPajamaGithub


text:
/*
#######################################################################
#		SYMBIOT
#
#		Real-time Network Threat Modeling
#		(C) 2002-2008 Symbiot, Inc.	---	ALL RIGHTS RESERVED
#
#		Plugin to report network activity in realtime
#
#		http://www.symbiot.com
#
#######################################################################
#		Author: Borrowed Time, Inc.
#		e-mail: libsymbiot@bti.net
#
#		Created:					11 Nov 2003
#		Last Modified:				10 Jan 2004
#
#######################################################################
*/

//---------------------------------------------------------------------
// Includes
//---------------------------------------------------------------------
#include "packet-objs.h"

#include "plugin-utils.h"

#include <arpa/inet.h>

//---------------------------------------------------------------------
// Definitions
//---------------------------------------------------------------------

#if !defined(ETHERTYPE_IP)
	#define	ETHERTYPE_IP	0x0800
#endif

#if !defined(ETHERTYPE_ARP)
	#define	ETHERTYPE_ARP	0x0806
#endif

#if HAVE_DECL_AF_INET6
	#if !defined(ETHERTYPE_IPV6)
		#define	ETHERTYPE_IPV6	0x86dd
	#endif
#endif

//*********************************************************************
// Class TPacketEthernet
//*********************************************************************

//---------------------------------------------------------------------
// Constructor
//---------------------------------------------------------------------
TPacketEthernet::TPacketEthernet (const TCapturedPacket* packetPtr)
	:	Inherited(packetPtr)
{
	fHeaderPtr = reinterpret_cast<const struct ether_header*>(fRawPacketPtr->Data());
	fPayloadPtr = reinterpret_cast<const u_char*>(fHeaderPtr) + EthernetHeaderSize();
}

//---------------------------------------------------------------------
// Constructor
//---------------------------------------------------------------------
TPacketEthernet::TPacketEthernet (const TPacketEthernet& obj)
	:	Inherited(obj),
		fHeaderPtr(obj.fHeaderPtr),
		fPayloadPtr(obj.fPayloadPtr)
{
}

//---------------------------------------------------------------------
// Destructor
//---------------------------------------------------------------------
TPacketEthernet::~TPacketEthernet ()
{
}

//---------------------------------------------------------------------
// TPacketEthernet::SourceMACAddressAsString
//---------------------------------------------------------------------
string TPacketEthernet::SourceMACAddressAsString () const
{
	return _DecodeMACAddress(EthernetHeaderPtr()->ether_shost);
}

//---------------------------------------------------------------------
// TPacketEthernet::DestinationMACAddressAsString
//---------------------------------------------------------------------
string TPacketEthernet::DestinationMACAddressAsString () const
{
	return _DecodeMACAddress(EthernetHeaderPtr()->ether_dhost);
}

//---------------------------------------------------------------------
// TPacketEthernet::_DecodeMACAddress (static protected)
//---------------------------------------------------------------------
string TPacketEthernet::_DecodeMACAddress (const u_char* encodedMACAddress)
{
	string			macAddress;
	const char		kEncodingTable[] = {'0','1','2','3','4','5',
										'6','7','8','9','A','B',
										'C','D','E','F'
									   };

	for (unsigned long x = 0; x < ETHER_ADDR_LEN; x++)
	{
		unsigned char		charValue = encodedMACAddress[x];
		unsigned char		msb = charValue / 16;
		unsigned char		lsb = charValue % 16;

		macAddress += kEncodingTable[msb];
		macAddress += kEncodingTable[lsb];
		if (x + 1 < ETHER_ADDR_LEN)
			macAddress += ':';
	}

	return macAddress;
}

//*********************************************************************
// Class TPacketIPv4
//*********************************************************************

//---------------------------------------------------------------------
// Constructor
//---------------------------------------------------------------------
TPacketIPv4::TPacketIPv4 (const TCapturedPacket* packetPtr)
	:	Inherited(packetPtr)
{
	fHeaderPtr = reinterpret_cast<const struct ip*>(Inherited::PayloadPtr());
	fPayloadPtr = reinterpret_cast<const u_char*>(fHeaderPtr) + IPHeaderSize();
}

//---------------------------------------------------------------------
// Constructor
//---------------------------------------------------------------------
TPacketIPv4::TPacketIPv4 (const TPacketIPv4& obj)
	:	Inherited(obj),
		fHeaderPtr(obj.fHeaderPtr),
		fPayloadPtr(obj.fPayloadPtr)
{
}

//---------------------------------------------------------------------
// Destructor
//---------------------------------------------------------------------
TPacketIPv4::~TPacketIPv4 ()
{
}

//---------------------------------------------------------------------
// TPacketIPv4::SourceIPAddressAsString
//---------------------------------------------------------------------
string TPacketIPv4::SourceIPAddressAsString () const
{
	return string(inet_ntoa(IPHeaderPtr()->ip_src));
}

//---------------------------------------------------------------------
// TPacketIPv4::DestinationIPAddressAsString
//---------------------------------------------------------------------
string TPacketIPv4::DestinationIPAddressAsString () const
{
	return string(inet_ntoa(IPHeaderPtr()->ip_dst));
}

//*********************************************************************
// Class TPacketTCPv4
//*********************************************************************

//---------------------------------------------------------------------
// Constructor
//---------------------------------------------------------------------
TPacketTCPv4::TPacketTCPv4 (const TCapturedPacket* packetPtr)
	:	Inherited(packetPtr)
{
	fHeaderPtr = reinterpret_cast<const struct tcphdr*>(Inherited::PayloadPtr());
	fPayloadPtr = reinterpret_cast<const u_char*>(fHeaderPtr) + TCPHeaderSize();
}

//---------------------------------------------------------------------
// Constructor
//---------------------------------------------------------------------
TPacketTCPv4::TPacketTCPv4 (const TPacketTCPv4& obj)
	:	Inherited(obj),
		fHeaderPtr(obj.fHeaderPtr),
		fPayloadPtr(obj.fPayloadPtr)
{
}

//---------------------------------------------------------------------
// Destructor
//---------------------------------------------------------------------
TPacketTCPv4::~TPacketTCPv4 ()
{
}

//*********************************************************************
// Class TPacketUDPv4
//*********************************************************************

//---------------------------------------------------------------------
// Constructor
//---------------------------------------------------------------------
TPacketUDPv4::TPacketUDPv4 (const TCapturedPacket* packetPtr)
	:	Inherited(packetPtr)
{
	fHeaderPtr = reinterpret_cast<const struct udphdr*>(Inherited::PayloadPtr());
	fPayloadPtr = reinterpret_cast<const u_char*>(fPayloadPtr) + UDPHeaderSize();
}

//---------------------------------------------------------------------
// Constructor
//---------------------------------------------------------------------
TPacketUDPv4::TPacketUDPv4 (const TPacketUDPv4& obj)
	:	Inherited(obj),
		fHeaderPtr(obj.fHeaderPtr),
		fPayloadPtr(obj.fPayloadPtr)
{
}

//---------------------------------------------------------------------
// Destructor
//---------------------------------------------------------------------
TPacketUDPv4::~TPacketUDPv4 ()
{
}

//*********************************************************************
// Class TPacketICMPv4
//*********************************************************************

//---------------------------------------------------------------------
// Constructor
//---------------------------------------------------------------------
TPacketICMPv4::TPacketICMPv4 (const TCapturedPacket* packetPtr)
	:	Inherited(packetPtr)
{
	fHeaderPtr = reinterpret_cast<const struct icmp*>(Inherited::PayloadPtr());
	fPayloadPtr = reinterpret_cast<const u_char*>(fHeaderPtr) + ICMPHeaderSize();
}

//---------------------------------------------------------------------
// Constructor
//---------------------------------------------------------------------
TPacketICMPv4::TPacketICMPv4 (const TPacketICMPv4& obj)
	:	Inherited(obj),
		fHeaderPtr(obj.fHeaderPtr),
		fPayloadPtr(obj.fPayloadPtr)
{
}

//---------------------------------------------------------------------
// Destructor
//---------------------------------------------------------------------
TPacketICMPv4::~TPacketICMPv4 ()
{
}

//*********************************************************************
// Class TPacketARP
//*********************************************************************

//---------------------------------------------------------------------
// Constructor
//---------------------------------------------------------------------
TPacketARP::TPacketARP (const TCapturedPacket* packetPtr)
	:	Inherited(packetPtr)
{
	fHeaderPtr = reinterpret_cast<const struct arphdr*>(Inherited::PayloadPtr());
	fPayloadPtr = reinterpret_cast<const u_char*>(fHeaderPtr) + ARPHeaderSize();
}

//---------------------------------------------------------------------
// Constructor
//---------------------------------------------------------------------
TPacketARP::TPacketARP (const TPacketARP& obj)
	:	Inherited(obj),
		fHeaderPtr(obj.fHeaderPtr),
		fPayloadPtr(obj.fPayloadPtr)
{
}

//---------------------------------------------------------------------
// Destructor
//---------------------------------------------------------------------
TPacketARP::~TPacketARP ()
{
}

#if HAVE_DECL_AF_INET6
	//*********************************************************************
	// Class TPacketIPv6
	//*********************************************************************

	//---------------------------------------------------------------------
	// Constructor
	//---------------------------------------------------------------------
	TPacketIPv6::TPacketIPv6 (const TCapturedPacket* packetPtr)
		:	Inherited(packetPtr)
	{
		fHeaderPtr = reinterpret_cast<const struct ip6_hdr*>(Inherited::PayloadPtr());
		fPayloadPtr = reinterpret_cast<const u_char*>(fHeaderPtr) + IPv6HeaderSize();
	}

	//---------------------------------------------------------------------
	// Constructor
	//---------------------------------------------------------------------
	TPacketIPv6::TPacketIPv6 (const TPacketIPv6& obj)
		:	Inherited(obj),
			fHeaderPtr(obj.fHeaderPtr),
			fPayloadPtr(obj.fPayloadPtr)
	{
	}

	//---------------------------------------------------------------------
	// Destructor
	//---------------------------------------------------------------------
	TPacketIPv6::~TPacketIPv6 ()
	{
	}

	//---------------------------------------------------------------------
	// TPacketIPv6::SourceIPAddressAsString
	//---------------------------------------------------------------------
	string TPacketIPv6::SourceIPAddressAsString () const
	{
		return _IPAddressAsString(IPv6HeaderPtr()->ip6_src);
	}

	//---------------------------------------------------------------------
	// TPacketIPv6::DestinationIPAddressAsString
	//---------------------------------------------------------------------
	string TPacketIPv6::DestinationIPAddressAsString () const
	{
		return _IPAddressAsString(IPv6HeaderPtr()->ip6_dst);
	}

	//---------------------------------------------------------------------
	// TPacketIPv6::_IPAddressAsString (static private)
	//---------------------------------------------------------------------
	string TPacketIPv6::_IPAddressAsString (const struct in6_addr& addr)
	{
		string					addrStr;
		struct sockaddr_in6		socketInfo;
		int						niFlags = NI_NUMERICHOST;

		#ifdef NI_WITHSCOPEID
			niFlags |= NI_WITHSCOPEID;
		#endif

		addrStr.resize(NI_MAXHOST);
		memset(&socketInfo,0,sizeof(socketInfo));
		socketInfo.sin6_family = AF_INET6;
		#if defined(HAVE_STRUCT_SOCKADDR_IN6_SIN6_LEN)
			socketInfo.sin6_len = sizeof(socketInfo);
		#endif
		socketInfo.sin6_addr = addr;

		if (IN6_IS_ADDR_LINKLOCAL(&addr) && *(reinterpret_cast<u_int16_t*>(&socketInfo.sin6_addr.s6_addr[2])) != 0)
		{
			socketInfo.sin6_scope_id = ntohs(*(reinterpret_cast<u_int16_t*>(&socketInfo.sin6_addr.s6_addr[2])));
			socketInfo.sin6_addr.s6_addr[2] = socketInfo.sin6_addr.s6_addr[3] = 0;
		}

		if (getnameinfo(reinterpret_cast<struct sockaddr*>(&socketInfo),sizeof(socketInfo),const_cast<char*>(addrStr.data()),addrStr.capacity()-1,NULL,0,niFlags) == 0)
			addrStr.resize(strlen(addrStr.c_str()));
		else
			addrStr = "";

		return addrStr;
	}

	//*********************************************************************
	// Class TPacketTCPv6
	//*********************************************************************

	//---------------------------------------------------------------------
	// Constructor
	//---------------------------------------------------------------------
	TPacketTCPv6::TPacketTCPv6 (const TCapturedPacket* packetPtr)
		:	Inherited(packetPtr)
	{
		fHeaderPtr = reinterpret_cast<const struct tcphdr*>(Inherited::PayloadPtr());
		fPayloadPtr = reinterpret_cast<const u_char*>(fHeaderPtr) + TCPHeaderSize();
	}

	//---------------------------------------------------------------------
	// Constructor
	//---------------------------------------------------------------------
	TPacketTCPv6::TPacketTCPv6 (const TPacketTCPv6& obj)
		:	Inherited(obj),
			fHeaderPtr(obj.fHeaderPtr),
			fPayloadPtr(obj.fPayloadPtr)
	{
	}

	//---------------------------------------------------------------------
	// Destructor
	//---------------------------------------------------------------------
	TPacketTCPv6::~TPacketTCPv6 ()
	{
	}

	//*********************************************************************
	// Class TPacketUDPv6
	//*********************************************************************

	//---------------------------------------------------------------------
	// Constructor
	//---------------------------------------------------------------------
	TPacketUDPv6::TPacketUDPv6 (const TCapturedPacket* packetPtr)
		:	Inherited(packetPtr)
	{
		fHeaderPtr = reinterpret_cast<const struct udphdr*>(Inherited::PayloadPtr());
		fPayloadPtr = reinterpret_cast<const u_char*>(fPayloadPtr) + UDPHeaderSize();
	}

	//---------------------------------------------------------------------
	// Constructor
	//---------------------------------------------------------------------
	TPacketUDPv6::TPacketUDPv6 (const TPacketUDPv6& obj)
		:	Inherited(obj),
			fHeaderPtr(obj.fHeaderPtr),
			fPayloadPtr(obj.fPayloadPtr)
	{
	}

	//---------------------------------------------------------------------
	// Destructor
	//---------------------------------------------------------------------
	TPacketUDPv6::~TPacketUDPv6 ()
	{
	}

	//*********************************************************************
	// Class TPacketICMPv6
	//*********************************************************************

	//---------------------------------------------------------------------
	// Constructor
	//---------------------------------------------------------------------
	TPacketICMPv6::TPacketICMPv6 (const TCapturedPacket* packetPtr)
		:	Inherited(packetPtr)
	{
		fHeaderPtr = reinterpret_cast<const struct icmp6_hdr*>(Inherited::PayloadPtr());
		fPayloadPtr = reinterpret_cast<const u_char*>(fHeaderPtr) + ICMPv6HeaderSize();
	}

	//---------------------------------------------------------------------
	// Constructor
	//---------------------------------------------------------------------
	TPacketICMPv6::TPacketICMPv6 (const TPacketICMPv6& obj)
		:	Inherited(obj),
			fHeaderPtr(obj.fHeaderPtr),
			fPayloadPtr(obj.fPayloadPtr)
	{
	}

	//---------------------------------------------------------------------
	// Destructor
	//---------------------------------------------------------------------
	TPacketICMPv6::~TPacketICMPv6 ()
	{
	}
#endif

//*********************************************************************
// Global Functions
//*********************************************************************

//---------------------------------------------------------------------
// CreatePacketObj
//---------------------------------------------------------------------
TPacket* CreatePacketObj (const TCapturedPacket& rawPacket)
{
	TPacket*					packetObjPtr = NULL;
	const struct ether_header*	headerPtr = reinterpret_cast<const struct ether_header*>(rawPacket.Data());

	switch (ntohs(headerPtr->ether_type))
	{
		case ETHERTYPE_IP:
			{
				const struct ip*	ipPtr = reinterpret_cast<const struct ip*>(rawPacket.Data() + ETHER_HDR_LEN);

				if ((ntohs(ipPtr->ip_off) & 0x1fff) == 0)
				{
					switch (ipPtr->ip_p)
					{
						case IPPROTO_TCP:
							packetObjPtr = new TPacketTCPv4(&rawPacket);
							break;

						case IPPROTO_UDP:
							packetObjPtr = new TPacketUDPv4(&rawPacket);
							break;

						case IPPROTO_ICMP:
							packetObjPtr = new TPacketICMPv4(&rawPacket);
							break;
					}
				}
				else
				{
					// It's a fragment, and not the first one
					packetObjPtr = new TPacketIPv4(&rawPacket);
				}

				// When all else fails...
				if (!packetObjPtr)
					packetObjPtr = new TPacketEthernet(&rawPacket);
			}
			break;

	#if HAVE_DECL_AF_INET6
		case ETHERTYPE_IPV6:
			{
				const struct ip6_hdr*	ipPtr = reinterpret_cast<const struct ip6_hdr*>(rawPacket.Data() + ETHER_HDR_LEN);

				switch (ipPtr->ip6_nxt)
				{
					case IPPROTO_TCP:
						packetObjPtr = new TPacketTCPv6(&rawPacket);
						break;

					case IPPROTO_UDP:
						packetObjPtr = new TPacketUDPv6(&rawPacket);
						break;

					case IPPROTO_ICMPV6:
						packetObjPtr = new TPacketICMPv6(&rawPacket);
						break;

					default:
						packetObjPtr = new TPacketIPv6(&rawPacket);
						break;
				}
			}
			break;
	#endif

		case ETHERTYPE_ARP:
			packetObjPtr = new TPacketARP(&rawPacket);
			break;

		default:
			break;
	}

	return packetObjPtr;
}


text:
var Playlist =
  {
    init: function(){
      var instance =
      {
        tracks: [],
        onTrackNumber: 0,
        humanTrackNumber: function(){return(this.onTrackNumber +1)},
        onTrack: function(number){ return(parseInt(number) == this.onTrackNumber) },
        totalTracks: function(){ return(this.tracks.length-1); },
        nextTrack: function(){ this.gotoTrack(this.onTrackNumber + 1); },
        previousTrack: function(){ this.gotoTrack(this.onTrackNumber - 1); },
        gotoTrack: function(number){
          this.currentTrack().setPlayhead(0);
          number = parseInt(number);
          if(number < 0){
            this.onTrackNumber = 0;
          }else if(number > this.totalTracks()){
            this.onTrackNumber = this.totalTracks();
          }else{
            this.onTrackNumber = number;
          }
        },
        currentTrack: function(){ return(this.tracks[this.onTrackNumber]) },
        onLastTrack: function(){ return(this.onTrackNumber == this.totalTracks()) }
      };

      return(instance)
    },

    initWithHTML: function(element){
      var instance = Playlist.init();

      $(element).find(".track").each(function(i){
        instance.tracks.push(Track.initWithHTML(this));
      });

      return(instance);
    },

    initWithJSON: function(info){
      var instance = Playlist.init();

      info = eval("("+info+")");
      for(i in info){
        instance.tracks.push(Track.initWithObjectLiteral(info[i]));
      }

      return(instance);
    }

  }


text:
package io.gatekeeper.configuration.annotation;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

@Target({ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
public @interface Discriminator {
    public String field();

    public DiscriminatorMapping[] map();
}


text:
    <?php echo F::t('Categories') ?>

<?php
$categories = Category::model()->findByPk(3);
$descendants = $categories->descendants()->findAll();
$level = 2;
$k=1;

foreach ($descendants as $category) {
    if ($category->level == $level)
        echo CHtml::closeTag('li') . "\n";
    else if ($category->level > $level)
        echo CHtml::openTag('ul' , array('class'=>'level0')) . "\n";
    else {
        echo CHtml::closeTag('li') . "\n";

        for ($i = $level - $category->level; $i; $i--) {
            echo CHtml::closeTag('ul') . "\n";
            echo CHtml::closeTag('li') . "\n";
        }
    }

    echo CHtml::openTag('li', array('class'=>'level0 nav-'.$k.' level-top'));
    echo CHtml::encode($category->name).$category->getLabel();
    $level = $category->level;

}

for ($i = $level; $i; $i--) {
    echo CHtml::closeTag('li') . "\n";
    echo CHtml::closeTag('ul') . "\n";

}
?>

        <?php
        echo '
';
        $category = Category::model()->findByPk(3);
        $parent = $category->children()->findAll();
        foreach ($parent as $k => $p) {
            ?>
            <li class="level0 nav-<?php echo $k+1 ?> level-top">
                <a href="<?php echo Yii::app()->createUrl('/item/index', array('id' => $p->id)) ?>" class="level-top">
                    <?php echo $p->name ?>
                    <?php if($p->label=='1') {
                        echo 'New';
                    }elseif($p->label=='2'){
                        echo 'Hot!';
                    }
?>

                <?php if($p->getChildCount()>0) { ?>

                <?php } ?>

        <?php } ?>


text:
int main()                                //nagłówek funkcji
{                                         //początek treści funkcji
  using namespace std;                    //uwidocznienie definicji
  cout << "Zabaw się językiem C++.";      //komunikat
  cout << endl;                           //zaczynamy nowy wiersz
  cout << "Nie pożałujesz!" << endl;      //kolejny komunikat
  return 0;                               //zakończenie działania funkcji main()
}                                         //koniec treści funkcji


text:
// Copyright (c) 2007-2017 ppy Pty Ltd <contact@ppy.sh>.
// Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE

using osu.Framework.Allocation;
using osu.Framework.Audio;
using osu.Framework.Graphics;
using osu.Game.Graphics;
using osu.Game.Graphics.UserInterface;

namespace osu.Game.Screens.Play
{
    public class SkipButton : TwoLayerButton
    {
        private readonly double skipDestination;

        public SkipButton()
        {
            Text = @"Skip";
            Icon = FontAwesome.fa_osu_right_o;
            Anchor = Anchor.BottomRight;
            Origin = Anchor.BottomRight;
        }

        [BackgroundDependencyLoader]
        private void load(AudioManager audio, OsuColour colours)
        {
            ActivationSound = audio.Sample.Get(@"Menu/menuhit");
            Colour = colours.Yellow;
            HoverColour = colours.YellowDark;
        }
    }
}


text:
package com.kixeye.chassis.transport.http;

import java.net.InetSocketAddress;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;

import com.codahale.metrics.health.HealthCheckRegistry;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Function;
import com.google.common.collect.Collections2;
import com.kixeye.chassis.transport.serde.JacksonMessageSerDe;
import com.kixeye.chassis.transport.shared.HealthServlet;
import org.apache.commons.lang.StringUtils;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.servlets.GzipFilter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Condition;
import org.springframework.context.annotation.ConditionContext;
import org.springframework.context.annotation.Conditional;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
import org.springframework.core.annotation.Order;
import org.springframework.core.type.AnnotatedTypeMetadata;
import org.springframework.web.context.ConfigurableWebApplicationContext;
import org.springframework.web.context.support.AnnotationConfigWebApplicationContext;
import org.springframework.web.servlet.DispatcherServlet;

import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.jetty9.InstrumentedQueuedThreadPool;
import com.kixeye.chassis.transport.SpringMvcConfiguration;
import com.kixeye.chassis.transport.http.HttpTransportConfiguration.HttpEnabledCondition;
import com.kixeye.chassis.transport.shared.JettyConnectorRegistry;
import com.kixeye.chassis.transport.swagger.SwaggerRegistry;
import com.kixeye.chassis.transport.util.SpringContextWrapper;
import javax.annotation.Nullable;

/**
 * Configures the Http transport.
 *
 * @author ebahtijaragic
 */
@Configuration
@Conditional(HttpEnabledCondition.class)
@ComponentScan(basePackageClasses=HttpTransportConfiguration.class)
public class HttpTransportConfiguration {
    public static final String HTTP_TRANSPORT_CHILD_CONTEXT_BEAN_NAME = "transportWebMvcContext";

    @Autowired(required = false)
    private MetricRegistry metricRegistry;

    @Value("${http.metrics.threadpool.enabled}")
    private boolean monitorThreadpool;

    @Value("${http.metrics.handler.enabled}")
    private boolean monitorHandler;

    @Autowired(required = false)
    private HealthCheckRegistry healthCheckRegistry;

    @Bean(initMethod="start", destroyMethod="stop")
	@Order(0)
	public Server httpServer(
			@Value("${http.enabled:false}") boolean httpEnabled,
            @Value("${http.hostname:}") String httpHostname,
            @Value("${http.port:-1}") int httpPort,

			@Value("${https.enabled:false}") boolean httpsEnabled,
            @Value("${https.hostname:}") String httpsHostname,
            @Value("${https.port:-1}") int httpsPort,
            @Value("${https.selfSigned:false}") boolean selfSigned,
            @Value("${https.mutualSsl:false}") boolean mutualSsl,

			@Value("${https.keyStorePath:}") String keyStorePath,
			@Value("${https.keyStoreData:}") String keyStoreData,
			@Value("${https.keyStorePassword:}") String keyStorePassword,
			@Value("${https.keyManagerPassword:}") String keyManagerPassword,

			@Value("${https.trustStorePath:}") String trustStorePath,
			@Value("${https.trustStoreData:}") String trustStoreData,
			@Value("${https.trustStorePassword:}") String trustStorePassword,

			@Value("${https.excludedCipherSuites:}") String[] excludedCipherSuites,

			ConfigurableWebApplicationContext webApplicationContext) throws Exception {

        // set up servlets
        ServletContextHandler context = servletContextHandler();

		// create a new child application context
		AnnotationConfigWebApplicationContext childApplicationContext = (AnnotationConfigWebApplicationContext)transportWebMvcContext(webApplicationContext, context).getContext();

		// register swagger
		childApplicationContext.getBean(SwaggerRegistry.class).registerSwagger(context, getObjectMappers(webApplicationContext));

		// configure the spring mvc dispatcher
		DispatcherServlet dispatcher = new DispatcherServlet(childApplicationContext);

		// enable gzip
		context.addFilter(GzipFilter.class, "/*", null);

        // map application servlets
		context.addServlet(new ServletHolder(dispatcher), "/");
        if (healthCheckRegistry != null) {
            context.addServlet(new ServletHolder(new HealthServlet(healthCheckRegistry)), "/healthcheck");
        }

        // create the server
    	Server server;
    	if (metricRegistry == null || !monitorThreadpool){
        	server = new Server();

            server.setHandler(context);
        } else {
        	server = new Server(new InstrumentedQueuedThreadPool(metricRegistry));

        	InstrumentedHandler instrumented = new InstrumentedHandler(metricRegistry);
            instrumented.setHandler(context);

            server.setHandler(instrumented);
        }

    	// set up connectors
    	if (httpEnabled) {
        	InetSocketAddress address = StringUtils.isBlank(httpHostname) ? new InetSocketAddress(httpPort) : new InetSocketAddress(httpHostname, httpPort);

        	JettyConnectorRegistry.registerHttpConnector(server, address);
    	}

    	if (httpsEnabled) {
        	InetSocketAddress address = StringUtils.isBlank(httpsHostname) ? new InetSocketAddress(httpsPort) : new InetSocketAddress(httpsHostname, httpsPort);

        	JettyConnectorRegistry.registerHttpsConnector(server, address, selfSigned, mutualSsl, keyStorePath, keyStoreData, keyStorePassword, keyManagerPassword,
        			trustStorePath, trustStoreData, trustStorePassword, excludedCipherSuites);
    	}

		return server;
	}

    private Collection<ObjectMapper> getObjectMappers(ConfigurableWebApplicationContext webApplicationContext) {
        Map<String, JacksonMessageSerDe> serdes = webApplicationContext.getBeansOfType(JacksonMessageSerDe.class);
        if(serdes.isEmpty()){
            return Collections.emptyList();
        }
        return Collections2.transform(serdes.values(), new Function<JacksonMessageSerDe, ObjectMapper>() {
            @Nullable
            @Override
            public ObjectMapper apply(@Nullable JacksonMessageSerDe serDe) {
                return serDe.getObjectMapper();
            }
        });
    }

    @Bean
    public ServletContextHandler servletContextHandler(){
        ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS | ServletContextHandler.NO_SECURITY);
        context.setErrorHandler(null);
        context.setWelcomeFiles(new String[] { "/" });

        return context;
    }

    @Bean(name=HTTP_TRANSPORT_CHILD_CONTEXT_BEAN_NAME)
    public SpringContextWrapper transportWebMvcContext(ConfigurableWebApplicationContext parentContext, ServletContextHandler servletContextHandler){
        AnnotationConfigWebApplicationContext transportWebMvcContext = new AnnotationConfigWebApplicationContext();
        transportWebMvcContext.setDisplayName("httpTransport-webMvcContext");
        transportWebMvcContext.setServletContext(servletContextHandler.getServletContext());
        transportWebMvcContext.setParent(parentContext);
        transportWebMvcContext.setEnvironment(parentContext.getEnvironment());
        transportWebMvcContext.register(SpringMvcConfiguration.class);
        transportWebMvcContext.register(PropertySourcesPlaceholderConfigurer.class);
        transportWebMvcContext.refresh();

        return new SpringContextWrapper(transportWebMvcContext);
    }

    /**
	 * A condition to check whether HTTP is enabled.
	 *
	 * @author ebahtijaragic
	 */
	public static class HttpEnabledCondition implements Condition {
		public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
			return StringUtils.equalsIgnoreCase(context.getEnvironment().resolvePlaceholders("${http.enabled}"), "true") || StringUtils.equalsIgnoreCase(context.getEnvironment().resolvePlaceholders("${https.enabled}"), "true");
		}
	}
}


text:
using System.Collections.Generic;
using System.Linq;
using AllReady.Models;
using MediatR;

namespace AllReady.Features.Event
{
    public class EventsByGeographyQueryHandler : IRequestHandler<EventsByGeographyQuery, List<Models.Event>>
    {
        private readonly IAllReadyDataAccess dataAccess;

        public EventsByGeographyQueryHandler(IAllReadyDataAccess dataAccess)
        {
            this.dataAccess = dataAccess;
        }

        public List<Models.Event> Handle(EventsByGeographyQuery message)
        {
            return dataAccess.EventsByGeography(message.Latitude, message.Longitude, message.Miles).ToList();
        }
    }
}


text:
import { create_audio_model } from 'sound/common/utils';

export default({audio_context})=> {

		const frequency_range = {
			min: 0,
			max: 15
		};
		const gain_range = {
			min: 0,
			max: 2500
		};

		const osc = audio_context.createOscillator();
		const gain = audio_context.createGain();

		const frequency = create_audio_model({
			param: osc.frequency,
			range: frequency_range
		})

		const amplitude = create_audio_model({
			param: gain.gain,
			range: gain_range
		});

		const type = create_audio_model({
			param: {
				set value(value){
					osc.type = value;
				},
				get value(){
					return osc.type;
				}
			}
		});

		return {
			connect({param}){
				osc.connect(gain);
				gain.connect(param);
				osc.start();
			},
			get frequency(){
				return frequency;
			},
			get amplitude(){
				return amplitude;
			},
			get type(){
				return type;
			}
		};
}