text:
package com.dremio.exec.store.hive.exec;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.ListIterator;
import java.util.function.Function;
import java.util.stream.Collectors;

import org.apache.arrow.vector.ValueVector;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.io.orc.OrcFile;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
import org.apache.hadoop.hive.ql.io.orc.OrcSplit;
import org.apache.hadoop.hive.ql.io.orc.Reader;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.orc.OrcConf;
import org.apache.orc.OrcProto;
import org.apache.orc.impl.DataReaderProperties;

import com.dremio.common.exceptions.UserException;
import com.dremio.common.expression.SchemaPath;
import com.dremio.exec.ExecConstants;
import com.dremio.exec.store.ScanFilter;
import com.dremio.exec.store.SplitAndPartitionInfo;
import com.dremio.exec.store.hive.HiveSettings;
import com.dremio.exec.store.hive.HiveUtilities;
import com.dremio.exec.store.hive.exec.HiveORCCopiers.ORCCopier;
import com.dremio.exec.store.hive.exec.apache.HadoopFileSystemWrapper;
import com.dremio.hive.proto.HiveReaderProto.HiveTableXattr;
import com.dremio.sabot.exec.context.OperatorContext;
import com.dremio.sabot.op.scan.ScanOperator;
import com.dremio.sabot.op.scan.ScanOperator.Metric;

/**
 * Use vectorized reader provided by the Hive to read ORC files. We copy one column completely at a time,
 * instead of one row at time.
 */
public class HiveORCVectorizedReader extends HiveAbstractReader {

  /**
   * For transactional orc files, the row data is stored in the struct vector at position 5
   */
  static final int TRANS_ROW_COLUMN_INDEX = 5;
  private org.apache.hadoop.hive.ql.io.orc.RecordReader hiveOrcReader;
  private ORCCopier[] copiers;
  private DremioORCRecordUtils.DefaultDataReader dataReader;

  /**
   * Hive vectorized ORC reader reads into this batch. It is a heap based structure and reused until the reader exhaust
   * records. Most of the heap structures remain constant size on heap, but the variable width structures may get
   * reallocated if there is not enough space.
   */
  private VectorizedRowBatch hiveBatch;
  // non-zero value indicates partially read batch in previous iteration.
  private int offset;

  @Override
  protected void logDebugMessages() {
    super.logDebugMessages();
    if (dataReader != null) {
      logger.debug(dataReader.getClass().getClassLoader().toString());
      dataReader.logMessage();
    }
    if (hiveOrcReader != null) {
      logger.debug(hiveOrcReader.getClass().getClassLoader().toString());
    }
  }

  public HiveORCVectorizedReader(final HiveTableXattr tableAttr, final SplitAndPartitionInfo split,
                                 final List<SchemaPath> projectedColumns, final OperatorContext context, final JobConf jobConf,
                                 final AbstractSerDe tableSerDe, final StructObjectInspector tableOI, final AbstractSerDe partitionSerDe,
                                 final StructObjectInspector partitionOI, final ScanFilter filter, final Collection<List<String>> referencedTables,
                                 final UserGroupInformation readerUgi) {
    super(tableAttr, split, projectedColumns, context, jobConf, tableSerDe, tableOI, partitionSerDe, partitionOI, filter,
      referencedTables, readerUgi);
  }

  private int[] getOrdinalIdsOfSelectedColumns(List<OrcProto.Type> types, List<Integer> selectedColumns, boolean isOriginal) {
    int rootColumn = isOriginal ? 0 : TRANS_ROW_COLUMN_INDEX + 1;
    int[] ids = new int[types.size()];
    OrcProto.Type root = types.get(rootColumn);

    // iterating over only direct children
    for (int i = 0; i < root.getSubtypesCount(); ++i) {
      if (selectedColumns.contains(i)) {
        // find the position of this column in the types list
        ids[i] = root.getSubtypes(i);
      }
    }

    return ids;
  }

  class SearchResult {
    public int index;
    public ObjectInspector oI;
  }

  /*
    PreOrder tree traversal
    position.index will contain preorder tree traversal index starting at root=0
  */
  private static boolean searchAllFields(final ObjectInspector rootOI,
                                         final String name,
                                         final int[] childCounts,
                                         SearchResult position
  ) {
    Category category = rootOI.getCategory();
    if (category == Category.STRUCT) {
      position.index++; // first child is immediately next to parent
      StructObjectInspector sOi = (StructObjectInspector) rootOI;
      for (StructField sf : sOi.getAllStructFieldRefs()) {
        // We depend on the fact that caller takes care of calling current method
        // once for each segment in the selected column path. So, we should always get
        // searched field as immediate child
        if (sf.getFieldName().equalsIgnoreCase(name)) {
          position.oI = sf.getFieldObjectInspector();
          return true;
        } else {
          if (position.index >= childCounts.length) {
            return false;
          }
          position.index += childCounts[position.index];
        }
      }
    } else if (category == Category.MAP) {
      position.index++; // first child is immediately next to parent
      if (name.equalsIgnoreCase(HiveUtilities.MAP_KEY_FIELD_NAME)) {
        ObjectInspector kOi = ((MapObjectInspector) rootOI).getMapKeyObjectInspector();
        position.oI = kOi;
        return true;
      }
      if (position.index >= childCounts.length) {
        return false;
      }
      position.index += childCounts[position.index];
      if (name.equalsIgnoreCase(HiveUtilities.MAP_VALUE_FIELD_NAME)) {
        ObjectInspector vOi = ((MapObjectInspector) rootOI).getMapValueObjectInspector();
        position.oI = vOi;
        return true;
      }
    }
    return false;
  }

  private void includeNodeAndItsChildren(SearchResult position, boolean[] include) {
    // Function starts from a list element. It includes the list and its children.
    ObjectInspector lOI = position.oI;
    // include root node
    include[position.index] = true;
    switch (lOI.getCategory()) {
      case PRIMITIVE:
        // no children. so return.
        break;
      case LIST:
        // move to child
        position.index++;
        if (position.index >= include.length) {
          return;
        }
        lOI = ((ListObjectInspector) lOI).getListElementObjectInspector();
        position.oI = lOI;

        // include child
        includeNodeAndItsChildren(position, include);
        break;
      case MAP:
        position.index++; // first child is immediately next to parent
        if (position.index >= include.length) {
          // input schema has more columns than what reader can read
          return;
        }
        ObjectInspector kOi = ((MapObjectInspector) lOI).getMapKeyObjectInspector();
        position.oI = kOi;
        includeNodeAndItsChildren(position, include);

        position.index++;
        if (position.index >= include.length) {
          // input schema has more columns than what reader can read
          return;
        }
        ObjectInspector vOi = ((MapObjectInspector) lOI).getMapValueObjectInspector();
        position.oI = vOi;
        includeNodeAndItsChildren(position, include);
        break;
      case STRUCT:
        StructObjectInspector sOi = (StructObjectInspector) lOI;
        for (StructField sf : sOi.getAllStructFieldRefs()) {
          // move to next child
          position.index++;
          if (position.index >= include.length) {
            return;
          }
          position.oI = sf.getFieldObjectInspector();

          // include next child
          includeNodeAndItsChildren(position, include);
        }
        break;
      case UNION:
        for (ObjectInspector fOi : ((UnionObjectInspector) lOI).getObjectInspectors()) {
          position.index++;
          if (position.index >= include.length) {
            return;
          }
          position.oI = fOi;
          includeNodeAndItsChildren(position, include);
        }
        break;
    }
  }

  // Takes SchemaPath and sets included bits of only fields in selected schema path
  // and all children of last segment
  // Example: if table schema is  <col1: int, col2:struct<f1:int, f2:string>, col3: string>
  // then childCounts will be [6, 1, 3, 1, 1, 1]
  // calling this method for co2.f2 will set include[2] for struct and include[4] for field f2
  private void getIncludedColumnsFromTableSchema(ObjectInspector rootOI, int rootColumn, SchemaPath selectedField, int[] childCounts, boolean[] include) {
    SearchResult searchResult = new SearchResult();
    searchResult.index = rootColumn;
    searchResult.oI = null;

    List<String> nameSegments = selectedField.getNameSegments();
    ListIterator<String> listIterator = nameSegments.listIterator();
    while (listIterator.hasNext()) {
      String name = listIterator.next();
      boolean found = searchAllFields(rootOI, name, childCounts, searchResult);
      if (found) {
        if (listIterator.hasNext() && searchResult.oI.getCategory() == Category.LIST) {
          includeNodeAndItsChildren(searchResult, include);
          return;
        }
        rootColumn = searchResult.index;
        rootOI = searchResult.oI;
        if (rootColumn < include.length) {
          if (listIterator.hasNext()) {
            include[rootColumn] = true;
          } else {
            int childCount = childCounts[rootColumn];
            for (int child = 0; child < childCount; ++child) {
              include[rootColumn + child] = true;
            }
          }
        }
      } else {
        break;
      }
    }
  }

  private void getIncludedColumnsFromTableSchema(ObjectInspector oi, int rootColumn, int[] childCounts, boolean[] include) {
    if (rootColumn < include.length) {
      include[rootColumn] = true;
    }
    Collection<SchemaPath> selectedColumns = getColumns();
    for (SchemaPath selectedField : selectedColumns) {
      getIncludedColumnsFromTableSchema(oi, rootColumn, selectedField, childCounts, include);
    }
  }

  /*
    For each root, populate total number of nodes in the tree starting from it
   */
  private int getChildCountsFromTableSchema(ObjectInspector rootOI, int position, int[] counts) {
    if (position >= counts.length) {
      return 0;
    }
    Category category = rootOI.getCategory();
    switch (category) {
      case PRIMITIVE:
        counts[position] = 1;
        return counts[position];
      case LIST:
        // total count is children count and 1 extra for itself
        counts[position] = getChildCountsFromTableSchema(((ListObjectInspector) rootOI).getListElementObjectInspector(),
          position + 1, counts) + 1;
        return counts[position];
      case STRUCT: {
        // total count is children count and 1 extra for itself
        int totalCount = 1;
        int childPosition = position + 1;
        StructObjectInspector sOi = (StructObjectInspector) rootOI;
        for (StructField sf : sOi.getAllStructFieldRefs()) {
          int childCount = getChildCountsFromTableSchema(sf.getFieldObjectInspector(),
            childPosition, counts);
          childPosition += childCount;
          totalCount += childCount;
        }
        counts[position] = totalCount;
        return counts[position];
      }
      case MAP: {
        // total count is children count and 1 extra for itself
        int totalCount = 1;
        int childPosition = position + 1;
        ObjectInspector kOi = ((MapObjectInspector) rootOI).getMapKeyObjectInspector();
        int childCount = getChildCountsFromTableSchema(kOi, childPosition, counts);
        childPosition += childCount;
        totalCount += childCount;
        ObjectInspector vOi = ((MapObjectInspector) rootOI).getMapValueObjectInspector();
        childCount = getChildCountsFromTableSchema(vOi, childPosition, counts);
        totalCount += childCount;
        counts[position] = totalCount;
        return counts[position];
      }
      case UNION: {
        // total count is children count and 1 extra for itself
        int totalCount = 1;
        int childPosition = position + 1;
        for (ObjectInspector fOi : ((UnionObjectInspector) rootOI).getObjectInspectors()) {
          int childCount = getChildCountsFromTableSchema(fOi, childPosition, counts);
          childPosition += childCount;
          totalCount += childCount;
        }
        counts[position] = totalCount;
        return counts[position];
      }
      default:
        throw UserException.unsupportedError()
          .message("Vectorized ORC reader is not supported for datatype: %s", category)
          .build(logger);
    }
  }

  @Override
  protected void internalInit(InputSplit inputSplit, JobConf jobConf, ValueVector[] vectors) throws IOException {
    final OrcSplit fSplit = (OrcSplit) inputSplit;
    final Path path = fSplit.getPath();

    final OrcFile.ReaderOptions opts = OrcFile.readerOptions(jobConf);

    // TODO: DX-16001 make enabling async configurable.
    final FileSystem fs = new HadoopFileSystemWrapper(jobConf, path.getFileSystem(jobConf), this.context.getStats());
    opts.filesystem(fs);
    final Reader hiveReader = OrcFile.createReader(path, opts);

    final List<OrcProto.Type> types = hiveReader.getTypes();

    final Reader.Options options = new Reader.Options();
    long offset = fSplit.getStart();
    long length = fSplit.getLength();
    options.schema(fSplit.isOriginal() ? hiveReader.getSchema() : hiveReader.getSchema().getChildren().get(TRANS_ROW_COLUMN_INDEX));
    options.range(offset, length);
    boolean[] include = new boolean[types.size()];
    int[] childCounts = new int[types.size()];

    getChildCountsFromTableSchema(finalOI, fSplit.isOriginal() ? 0 : TRANS_ROW_COLUMN_INDEX + 1, childCounts);
    getIncludedColumnsFromTableSchema(finalOI, fSplit.isOriginal() ? 0 : TRANS_ROW_COLUMN_INDEX + 1, childCounts, include);
    include[0] = true; // always include root. reader always includes, but setting it explicitly here.

    final Boolean zeroCopy = OrcConf.USE_ZEROCOPY.getBoolean(jobConf);
    final boolean useDirectMemory = new HiveSettings(context.getOptions()).useDirectMemoryForOrcReaders();
    options.include(fSplit.isOriginal() ? include : Arrays.copyOfRange(include, TRANS_ROW_COLUMN_INDEX + 1, include.length));
    dataReader = DremioORCRecordUtils.createDefaultDataReader(context.getAllocator(), DataReaderProperties.builder()
      .withBufferSize(hiveReader.getCompressionSize())
      .withCompression(hiveReader.getCompressionKind())
      .withFileSystem(fs)
      .withPath(path)
      .withTypeCount(types.size())
      .withZeroCopy(zeroCopy)
      .build(), useDirectMemory, context.getOptions().getOption(ExecConstants.SCAN_COMPUTE_LOCALITY));
    options.dataReader(dataReader);

    String[] selectedColNames = getColumns().stream().map(x -> x.getAsUnescapedPath().toLowerCase()).toArray(String[]::new);

    // there is an extra level of nesting in the transactional tables
    if (!fSplit.isOriginal()) {
      selectedColNames = ArrayUtils.addAll(new String[]{"row"}, selectedColNames);
    }

    if (filter != null) {
      final HiveProxyingOrcScanFilter orcScanFilter = (HiveProxyingOrcScanFilter) filter;
      final SearchArgument sarg = HiveUtilities.decodeSearchArgumentFromBase64(orcScanFilter.getProxiedOrcScanFilter().getKryoBase64EncodedFilter());
      options.searchArgument(sarg, OrcInputFormat.getSargColumnNames(selectedColNames, types, options.getInclude(), fSplit.isOriginal()));
    }

    hiveOrcReader = hiveReader.rowsOptions(options);
    StructObjectInspector orcFileRootOI = (StructObjectInspector) hiveReader.getObjectInspector();
    if (!fSplit.isOriginal()) {
      orcFileRootOI = (StructObjectInspector) orcFileRootOI.getAllStructFieldRefs().get(TRANS_ROW_COLUMN_INDEX).getFieldObjectInspector();
    }
    hiveBatch = createVectorizedRowBatch(orcFileRootOI, fSplit.isOriginal());

    final List<Integer> projectedColOrdinals = ColumnProjectionUtils.getReadColumnIDs(jobConf);
    final int[] ordinalIdsFromOrcFile = getOrdinalIdsOfSelectedColumns(types, projectedColOrdinals, fSplit.isOriginal());
    HiveORCCopiers.HiveColumnVectorData columnVectorData = new HiveORCCopiers.HiveColumnVectorData(include, childCounts);

    copiers = HiveORCCopiers.createCopiers(columnVectorData,
      projectedColOrdinals, ordinalIdsFromOrcFile,
      vectors, hiveBatch, fSplit.isOriginal(), this.operatorContextOptions);

    // Store the number of vectorized columns for stats/to find whether vectorized ORC reader is used or not
    context.getStats().setLongStat(Metric.NUM_VECTORIZED_COLUMNS, vectors.length);
  }

  @Override
  protected int populateData() {
    try {
      final int numRowsPerBatch = (int) this.numRowsPerBatch;

      int outputIdx = 0;

      // Consume the left over records from previous iteration
      if (offset > 0 && offset < hiveBatch.size) {
        int toRead = Math.min(hiveBatch.size - offset, numRowsPerBatch - outputIdx);
        copy(offset, toRead, outputIdx);
        outputIdx += toRead;
        offset += toRead;
      }

      while (outputIdx < numRowsPerBatch && hiveOrcReader.nextBatch(hiveBatch)) {
        offset = 0;
        int toRead = Math.min(hiveBatch.size, numRowsPerBatch - outputIdx);
        copy(offset, toRead, outputIdx);
        outputIdx += toRead;
        offset = toRead;
      }

      return outputIdx;
    } catch (Throwable t) {
      throw createExceptionWithContext("Failed to read data from ORC file", t);
    }
  }

  private void copy(final int inputIdx, final int count, final int outputIdx) {
    for (ORCCopier copier : copiers) {
      copier.copy(inputIdx, count, outputIdx);
    }
  }

  private boolean isSupportedType(Category category) {
    return (category == Category.PRIMITIVE ||
      category == Category.LIST ||
      category == Category.STRUCT ||
      category == Category.MAP ||
      category == Category.UNION);
  }

  private List<ColumnVector> getVectors(StructObjectInspector rowOI) {
    return rowOI.getAllStructFieldRefs()
      .stream()
      .map((Function<StructField, ColumnVector>) structField -> {
        Category category = structField.getFieldObjectInspector().getCategory();
        if (!isSupportedType(category)) {
          throw UserException.unsupportedError()
            .message("Vectorized ORC reader is not supported for datatype: %s", category)
            .build(logger);
        }
        return getColumnVector(structField.getFieldObjectInspector());
      })
      .collect(Collectors.toList());

  }

  private ColumnVector getColumnVector(ObjectInspector oi) {
    Category category = oi.getCategory();
    switch (category) {

      case PRIMITIVE:
        return getPrimitiveColumnVector((PrimitiveObjectInspector) oi);
      case LIST:
        return getListColumnVector((ListObjectInspector) oi);
      case STRUCT:
        return getStructColumnVector((StructObjectInspector) oi);
      case MAP:
        return getMapColumnVector((MapObjectInspector) oi);
      case UNION:
        return getUnionColumnVector((UnionObjectInspector) oi);
      default:
        throw UserException.unsupportedError()
          .message("Vectorized ORC reader is not supported for datatype: %s", category)
          .build(logger);
    }
  }

  private ColumnVector getUnionColumnVector(UnionObjectInspector uoi) {
    ArrayList<ColumnVector> vectors = new ArrayList<>();
    List<? extends ObjectInspector> members = uoi.getObjectInspectors();
    for (ObjectInspector unionField : members) {
      vectors.add(getColumnVector(unionField));
    }
    ColumnVector[] columnVectors = vectors.toArray(new ColumnVector[0]);
    return new UnionColumnVector(VectorizedRowBatch.DEFAULT_SIZE, columnVectors);
  }

  private ColumnVector getMapColumnVector(MapObjectInspector moi) {
    ColumnVector keys = getColumnVector(moi.getMapKeyObjectInspector());
    ColumnVector values = getColumnVector(moi.getMapValueObjectInspector());
    return new MapColumnVector(VectorizedRowBatch.DEFAULT_SIZE, keys, values);
  }

  private ColumnVector getStructColumnVector(StructObjectInspector soi) {
    ArrayList<ColumnVector> vectors = new ArrayList<>();
    List<? extends StructField> members = soi.getAllStructFieldRefs();
    for (StructField structField : members) {
      vectors.add(getColumnVector(structField.getFieldObjectInspector()));
    }
    ColumnVector[] columnVectors = vectors.toArray(new ColumnVector[0]);
    return new StructColumnVector(VectorizedRowBatch.DEFAULT_SIZE, columnVectors);
  }

  private ColumnVector getListColumnVector(ListObjectInspector loi) {
    ColumnVector lecv = getColumnVector(loi.getListElementObjectInspector());
    return new ListColumnVector(VectorizedRowBatch.DEFAULT_SIZE, lecv);
  }

  private ColumnVector getPrimitiveColumnVector(PrimitiveObjectInspector poi) {
    switch (poi.getPrimitiveCategory()) {
      case BOOLEAN:
      case BYTE:
      case SHORT:
      case INT:
      case LONG:
      case DATE:
        return new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
      case TIMESTAMP:
        return new TimestampColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
      case FLOAT:
      case DOUBLE:
        return new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
      case BINARY:
      case STRING:
      case CHAR:
      case VARCHAR:
        return new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
      case DECIMAL:
        DecimalTypeInfo tInfo = (DecimalTypeInfo) poi.getTypeInfo();
        return new DecimalColumnVector(VectorizedRowBatch.DEFAULT_SIZE,
          tInfo.precision(), tInfo.scale()
        );
      default:
        throw UserException.unsupportedError()
          .message("Vectorized ORC reader is not supported for datatype: %s", poi.getPrimitiveCategory())
          .build(logger);
    }
  }

  /**
   * Helper method that creates {@link VectorizedRowBatch}. For each selected column an input vector is created in the
   * batch. For unselected columns the vector entry is going to be null. The order of input vectors in batch should
   * match the order the columns in ORC file.
   *
   * @param rowOI Used to find the ordinal of the selected column.
   * @return
   */
  private VectorizedRowBatch createVectorizedRowBatch(StructObjectInspector rowOI, boolean isOriginal) {
    final List<? extends StructField> fieldRefs = rowOI.getAllStructFieldRefs();
    final List<ColumnVector> vectors = getVectors(rowOI);

    final VectorizedRowBatch result = new VectorizedRowBatch(fieldRefs.size());

    ColumnVector[] vectorArray = vectors.toArray(new ColumnVector[0]);

    if (!isOriginal) {
      vectorArray = createTransactionalVectors(vectorArray);
    }

    result.cols = vectorArray;
    result.numCols = fieldRefs.size();
    result.reset();
    return result;
  }

  private ColumnVector[] createTransactionalVectors(ColumnVector[] dataVectors) {
    ColumnVector[] transVectors = new ColumnVector[6];

    transVectors[0] = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    transVectors[1] = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    transVectors[2] = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    transVectors[3] = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    transVectors[4] = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);

    transVectors[5] = new StructColumnVector(dataVectors.length, dataVectors);

    return transVectors;
  }

  @Override
  public void close() throws IOException {
    if (hiveOrcReader != null) {
      hiveOrcReader.close();
      hiveOrcReader = null;
    }

    if (dataReader != null) {
      if (dataReader.isRemoteRead()) {
        context.getStats().addLongStat(ScanOperator.Metric.NUM_REMOTE_READERS, 1);
      } else {
        context.getStats().addLongStat(ScanOperator.Metric.NUM_REMOTE_READERS, 0);
      }
      dataReader = null;
    }

    super.close();
  }

}


text:
module.exports = function(grunt) {
  grunt.registerTask('serve',
    'Compile app then start a local web server and API proxy',
    ['connect::keepalive', 'watch']
  );
};

redpajama_set_name: RedPajamaGithub


text:
BASE=../../../../../
BKSCRIPT=$BASE/bookkeeper-server/bin/bookkeeper
HWSCRIPT=$BASE/hedwig-server/bin/hedwig
ZKCLIENT=org.apache.zookeeper.ZooKeeperMain

check_bk_down() {
    NUM_UP=100
    for i in 0 1 2 3 4 5 6 7 8 9; do
	NUM_UP=`sh $BKSCRIPT $ZKCLIENT ls /ledgers/available 2> /dev/null | awk 'BEGIN{SERVERS=0} /^\[/ { gsub(/[,\[\]]/, ""); SERVERS=NF} END{ print SERVERS }'`
	if [ $NUM_UP == 0 ]; then
	    break;
	fi
	sleep 1
    done

    if [ $NUM_UP != 0 ]; then
	echo "Warning: Couldn't stop all bookies"
	exit 1;
    fi
}

check_bk_up() {
    NUM_BOOKIES=$1
    NUM_UP=0
    for i in 0 1 2 3 4 5 6 7 8 9; do
	NUM_UP=`sh $BKSCRIPT $ZKCLIENT ls /ledgers/available 2> /dev/null | awk 'BEGIN{SERVERS=0} /^\[/ { gsub(/[,\[\]]/, ""); SERVERS=NF} END{ print SERVERS }'`
	if [ $NUM_UP == $NUM_BOOKIES ]; then
	    break;
	fi
	sleep 1
    done

    if [ $NUM_UP != $NUM_BOOKIES ]; then
	echo "Couldn't start bookkeeper"
	exit 1;
    fi
}

check_hw_down() {
    REGION=$1
    NUM_UP=100
    for i in 0 1 2 3 4 5 6 7 8 9; do
	NUM_UP=`sh $BKSCRIPT $ZKCLIENT ls /hedwig/$REGION/hosts 2> /dev/null | awk 'BEGIN{SERVERS=0} /^\[/ { gsub(/[,\[\]]/, ""); SERVERS=NF} END{ print SERVERS }'`
	if [ $NUM_UP == 0 ]; then
	    break;
	fi
	sleep 1
    done

    if [ $NUM_UP != 0 ]; then
	echo "Warning: Couldn't stop all hedwig servers"
	exit 1;
    fi
}

check_hw_up() {
    REGION=$1
    NUM_SERVERS=$2
    NUM_UP=0
    for i in 0 1 2 3 4 5 6 7 8 9; do
	NUM_UP=`sh $BKSCRIPT $ZKCLIENT ls /hedwig/$REGION/hosts 2> /dev/null | awk 'BEGIN{SERVERS=0} /^\[/ { gsub(/[,\[\]]/, ""); SERVERS=NF} END{ print SERVERS }'`
	if [ $NUM_UP == $NUM_SERVERS ]; then
	    break;
	fi
	sleep 1
    done

    if [ $NUM_UP != $NUM_SERVERS ]; then
	echo "Couldn't start hedwig"
	exit 1;
    fi
}

start_hw_server () {
    REGION=$1
    COUNT=$2
    PORT=$((4080+$COUNT))

    export HEDWIG_LOG_CONF=/tmp/hw-log4j-$COUNT.properties
    cat > $HEDWIG_LOG_CONF <<EOF
log4j.rootLogger=INFO, ROLLINGFILE
#
# Add ROLLINGFILE to rootLogger to get log file output
#    Log DEBUG level and above messages to a log file
log4j.appender.ROLLINGFILE=org.apache.log4j.DailyRollingFileAppender
log4j.appender.ROLLINGFILE.Threshold=DEBUG
log4j.appender.ROLLINGFILE.File=/tmp/hedwig-server-$COUNT.log
log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout
log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n
# Max log file size of 10MB
log4j.appender.ROLLINGFILE.MaxFileSize=10MB
# uncomment the next line to limit number of backup files
#log4j.appender.ROLLINGFILE.MaxBackupIndex=10
log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout
log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n
EOF

    export HEDWIG_SERVER_CONF=/tmp/hw-server-$COUNT.conf
    cat > $HEDWIG_SERVER_CONF <<EOF
zk_host=localhost:2181
# The number of milliseconds of each tick in ZooKeeper.
zk_timeout=2000
# The port at which the clients will connect.
server_port=$PORT
# The SSL port at which the clients will connect (only if SSL is enabled).
ssl_server_port=9876
# Flag indicating if the server should also operate in SSL mode.
ssl_enabled=false
region=$REGION
EOF
    sh $HWSCRIPT server 2>&1 > hwoutput.$COUNT.log &
    echo $! > hwprocess.$COUNT.pid
}

start_cluster() {
    if [ -e bkprocess.pid ] || [ `ls hwprocess.*.pid 2> /dev/null | wc -l` != 0 ]; then
	stop_cluster;
    fi

    sh $BKSCRIPT localbookie 3 2>&1 > bkoutput.log &
    echo $! > bkprocess.pid
    check_bk_up 3

    for i in 1 2 3; do
	start_hw_server CppUnitTest $i
    done
    check_hw_up CppUnitTest 3
}

stop_cluster() {
    for i in hwprocess.*.pid; do
	if [ ! -e $i ]; then
	    continue;
	fi
	kill `cat $i`;
	rm $i;
    done
    check_hw_down

    if [ ! -e bkprocess.pid ]; then
	return;
    fi

    kill `cat bkprocess.pid`
    rm bkprocess.pid

    check_bk_down
}


text:
..
      Copyright 2011-2013 OpenStack Foundation
      All Rights Reserved.

      Licensed under the Apache License, Version 2.0 (the "License"); you may
      not use this file except in compliance with the License. You may obtain
      a copy of the License at

          http://www.apache.org/licenses/LICENSE-2.0

      Unless required by applicable law or agreed to in writing, software
      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
      License for the specific language governing permissions and limitations
      under the License.

.. _notifications:

Notifications
=============

Notifications can be generated for several events in the image lifecycle.
These can be used for auditing, troubleshooting, etc.

Notification Drivers
--------------------

* log

  This driver uses the standard Python logging infrastructure with
  the notifications ending up in file specified by the log_file
  configuration directive.

* messaging

  This strategy sends notifications to a message queue configured
  using oslo.messaging configuration options.

* noop

  This strategy produces no notifications. It is the default strategy.

Notification Types
------------------

* ``image.create``

  Emitted when an image record is created in Glance.  Image record creation is
  independent of image data upload.

* ``image.prepare``

  Emitted when Glance begins uploading image data to its store.

* ``image.upload``

  Emitted when Glance has completed the upload of image data to its store.

* ``image.activate``

  Emitted when an image goes to `active` status.  This occurs when Glance
  knows where the image data is located.

* ``image.send``

  Emitted upon completion of an image being sent to a consumer.

* ``image.update``

  Emitted when an image record is updated in Glance.

* ``image.delete``

  Emitted when an image deleted from Glance.

* ``task.run``

  Emitted when a task is picked up by the executor to be run.

* ``task.processing``

  Emitted when a task is sent over to the executor to begin processing.

* ``task.success``

  Emitted when a task is successfully completed.

* ``task.failure``

  Emitted when a task fails.

Content
-------

Every message contains a handful of attributes.

* message_id

  UUID identifying the message.

* publisher_id

  The hostname of the glance instance that generated the message.

* event_type

  Event that generated the message.

* priority

  One of WARN, INFO or ERROR.

* timestamp

  UTC timestamp of when event was generated.

* payload

  Data specific to the event type.

Payload
-------

* image.send

  The payload for INFO, WARN, and ERROR events contain the following:

  image_id
    ID of the image (UUID)
  owner_id
    Tenant or User ID that owns this image (string)
  receiver_tenant_id
    Tenant ID of the account receiving the image (string)
  receiver_user_id
    User ID of the account receiving the image (string)
  destination_ip
    The receiver's IP address to which the image was sent (string)
  bytes_sent
    The number of bytes actually sent

* image.create

  For INFO events, it is the image metadata.
  WARN and ERROR events contain a text message in the payload.

* image.prepare

  For INFO events, it is the image metadata.
  WARN and ERROR events contain a text message in the payload.

* image.upload

  For INFO events, it is the image metadata.
  WARN and ERROR events contain a text message in the payload.

* image.activate

  For INFO events, it is the image metadata.
  WARN and ERROR events contain a text message in the payload.

* image.update

  For INFO events, it is the image metadata.
  WARN and ERROR events contain a text message in the payload.

* image.delete

  For INFO events, it is the image id.
  WARN and ERROR events contain a text message in the payload.

* task.run

  The payload for INFO, WARN, and ERROR events contain the following:

  task_id
    ID of the task (UUID)
  owner
    Tenant or User ID that created this task (string)
  task_type
    Type of the task. Example, task_type is "import". (string)
  status,
    status of the task. Status can be "pending", "processing",
    "success" or "failure". (string)
  task_input
    Input provided by the user when attempting to create a task. (dict)
  result
    Resulting output from a successful task. (dict)
  message
    Message shown in the task if it fails. None if task succeeds. (string)
  expires_at
    UTC time at which the task would not be visible to the user. (string)
  created_at
    UTC time at which the task was created. (string)
  updated_at
    UTC time at which the task was latest updated. (string)

  The exceptions are:-
    For INFO events, it is the task dict with result and message as None.
    WARN and ERROR events contain a text message in the payload.

* task.processing

  For INFO events, it is the task dict with result and message as None.
  WARN and ERROR events contain a text message in the payload.

* task.success

  For INFO events, it is the task dict with message as None and result is a
  dict.
  WARN and ERROR events contain a text message in the payload.

* task.failure

  For INFO events, it is the task dict with result as None and message is
  text.
  WARN and ERROR events contain a text message in the payload.


text:
// stdafx.h : include file for standard system include files,
// or project specific include files that are used frequently, but
// are changed infrequently
//
#pragma once
#undef AFX_DATA
#define AFX_DATA AFX_EXT_DATA
//#pragma once
#define _CRT_SECURE_NO_DEPRECATE

// TODO: reference additional headers your program requires here
/* Win headers */
/*#if defined(W32) || defined(W64)
	#include <Windows.h>
#endif*/
#include <stdio.h>
#include <stdlib.h>

/* Domino headers */
#include <global.h>
#include <event.h>
#include <kfm.h>
#include <nsfnote.h>
#include <nsferr.h>
#include <nsfdb.h>
#include <nsfdata.h>
#include <nsfnote.h>
#include <osmisc.h>
#include <misc.h>
#include <ostime.h>
#include <extmgr.h>
#include <OSfile.h>
#include <osmem.h>
#include <miscerr.h>
#include <oserr.h>
#include <ft.h>
#include <nif.h>
#include <niferr.h>
#include <osenv.h>
#include <textlist.h>
#include <google/protobuf/stubs/port.h>
//#include <pool.h>

/* Definitions */
#define DefLogFile "\\extmgr_test.log"
#define RC_STATUS_STOPPED 0
#define RC_STATUS_STARTING 1
#define RC_STATUS_ERROR 2
#define RC_STATUS_RUNNING 3
//
#define HM_STATUS_STOPPED 0
#define HM_STATUS_STARTING 1
#define HM_STATUS_ERROR 2
#define HM_STATUS_RUNNING 3
//
#define NM_STATUS_STOPPED 0
#define NM_STATUS_STARTING 1
#define NM_STATUS_ERROR 2
#define NM_STATUS_RUNNING 3
//
#define RH_ERR_NOERROR 0x0000
#define RH_ERR_RC_CONNECTION_UNDEFINED 0x0001
#define RH_ERR_RC_ERROR 0x0002
#define RH_ERR_RC_STOPPED 0x0003
//
#define RH_ERR_NM_NOERROR 0x0000
#define RH_ERR_NM_ERROR 0x0001
//
#define LOG_FOLDER "\\IBM_TECHNICAL_SUPPORT"
#define LOG_FILENAME "\\redhooks_debug.log"
//
#define INI_PARAM_DBLIST "RED_DB_LIST"
#define INI_RED_URL "RED_URL"
#define INI_RED_PORT "RED_PORT"
//
#define RC_SERVER_STATE_OK 0x000000C8	//200
#define RC_SERVER_STATE_BAD_REQUEST 0x00000190	// 400
#define RC_SERVER_STATE_UNAUTHORIZED 0x00000191	// 401
#define RC_SERVER_STATE_FORBIDDEN 0x00000193	// 403
#define RC_SERVER_STATE_NOT_FOUND 0x00000194	// 404
#define RC_SERVER_STATE_METHOD_NOT_ALLOWED 0x00000195	// 405
#define RC_SERVER_STATE_NOT_ACCEPTABLE 0x00000196	// 406
#define RC_SERVER_STATE_INTERNAL_SERVER_ERROR 0x000001F4	// 500
#define RC_SERVER_STATE_NOT_IMPLEMENTED 0x000001F5	// 501
//
#define R_TYPE_TEXT 1280
#define R_TYPE_NUMBER 768
//
#define UNID_LENGTH 32

// variables/functions definitions
// dllmain.cpp
extern CRITICAL_SECTION gCritSection;
//extern DLL_EXPORT void DLLLogLine(char * line, FILE * stream);
extern BOOL WINAPI DllMain(HINSTANCE hModule, DWORD  ul_reason_for_call, LPVOID lpReserved);
// RED_Interceptors.cpp
extern BOOL gHooksRegistered;
extern char DataDir[];
extern char LogFile[];
extern FILE * gLogStream;
extern WORD gRecursionID;
extern EMHANDLER gNoteOpenHandlerProc;
extern EMHANDLER gNoteOpenUNIDHandlerProc;
extern EMHANDLER gNoteOpenExtHandlerProc;
extern EMHANDLER gDbGetNoteInfoByUnidProc;
extern EMHANDLER gDbReadObjectProc;
extern EMHANDLER gDbGetObjectSizeProc;
extern EMHANDLER gDbGetNoteInfoProc;
extern EMHANDLER gNoteGetInfoProc;
extern EMHANDLER gEMProc;
extern HEMREGISTRATION	  hHandler;
extern HEMREGISTRATION	  hHandlerUNID;
extern HEMREGISTRATION	  hHandlerExt;
extern HEMREGISTRATION hHandlerDbGetNoteInfoByUnid;
extern HEMREGISTRATION hHandlerDbReadObject;
extern HEMREGISTRATION hHandlerDbGetObjectSize;
extern HEMREGISTRATION hHandlerDbGetNoteInfo;
extern HEMREGISTRATION hHandlerNoteGetInfo;
extern HEMREGISTRATION hEM;
extern HCOLLECTION * myColl;
extern unsigned long myCount;
extern void CleanUp(void);
extern STATUS LNPUBLIC MainEntryPoint(void);
#undef AFX_DATA
#define AFX_DATA


text:
package WSDL;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.GeneratedValue;
import javax.persistence.Id;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import javax.persistence.Transient;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;

import org.hibernate.Session;
import org.hibernate.annotations.ForeignKey;
import org.hibernate.annotations.OnDelete;
import org.hibernate.annotations.OnDeleteAction;
import org.hibernate.search.annotations.Analyze;
import org.hibernate.search.annotations.Field;
import org.hibernate.search.annotations.Index;
import org.hibernate.search.annotations.IndexedEmbedded;

import org.hibernate.search.annotations.Store;

import org.hibernate.search.annotations.Analyze;
import org.hibernate.search.annotations.Field;
import org.hibernate.search.annotations.Index;
import org.hibernate.search.annotations.Indexed;
import org.hibernate.search.annotations.IndexedEmbedded;
import org.hibernate.search.annotations.Store;

import WADL.Link;

@XmlRootElement
@Entity
@Table(name="inputMessage")
@Indexed
public class InputMessageModel
{
	//properties

	@Transient
	private List<Link> linkList = new ArrayList<Link>();
	//place holder for all resource model properties

	@Id
	@GeneratedValue
	@Column(name = "inputMessageId")
	private int inputMessageId;

	@Column(name = "name")
	@Field(index=Index.YES, analyze=Analyze.YES, store=Store.NO)
	private String name;

	@OneToMany(fetch = FetchType.EAGER, mappedBy="oInputMessage",orphanRemoval=true)
    @OnDelete(action=OnDeleteAction.CASCADE)
	@IndexedEmbedded
	private Set<InputParameterModel> setOfInputParameter = new HashSet<InputParameterModel>();

	@ManyToOne(fetch = FetchType.EAGER)
	@JoinColumn(name="SOAPOperationId")
    @ForeignKey(name = "fk_SOAPOperation_InputMessage")
	private SOAPOperationModel oSOAPOperation;

	//operations
	//place holder for marshalModel operation

    public InputMessageModel marshalModel(List<Link> linkList)
    {
        this.linkList = linkList;
        return this;
    }
	//place holder for setLinkList operation

    public void setLinkList(List<Link> linkList)
    {
        this.linkList = linkList;
    }
	//place holder for getLinkList operation

    public List<Link> getLinkList()
    {
        return linkList;
    }

	//place holder for all setters and getters of properties

	public int getInputMessageId()
	{
		return inputMessageId;
	}

	public void setInputMessageId(int inputMessageId)
	{
		this.inputMessageId = inputMessageId;
	}

	public String getName()
	{
		return name;
	}

	public void setName(String name)
	{
		this.name = name;
	}

    public void deleteAllCollections(Session hibernateSession)
    {

        Iterator<InputParameterModel> inputParameterIterator = setOfInputParameter.iterator();
        while(inputParameterIterator.hasNext())
        {
            inputParameterIterator.next().deleteAllCollections(hibernateSession);
        }
    }

    @XmlTransient
    public Set<InputParameterModel> getSetOfInputParameter()
    {
        return this.setOfInputParameter;
    }

    public void setSetOfInputParameter( Set<InputParameterModel> setOfInputParameter)
    {
        this.setOfInputParameter = setOfInputParameter;
    }

    @XmlTransient
	public SOAPOperationModel getSOAPOperation()
	{
		return oSOAPOperation;
	}

	public void setSOAPOperation( SOAPOperationModel oSOAPOperation)
	{
		this.oSOAPOperation = oSOAPOperation;
	}

}


text:
require 'spec_helper'

describe DataMapper::Dynamodb::Primitive::Object do
  let(:object) { Object.new }
  let(:primitive) { DataMapper::Dynamodb::Primitive.new(object, 'Object') }

  context '#initializer' do
    it 'should wrap to Object wrapper' do
      primitive.send(:dynamodb_primitive).should be_an_instance_of DataMapper::Dynamodb::Primitive::Object
    end
  end

  context '#to_dynamodb' do
    it 'should save value as string' do
      primitive.to_dynamodb.should eq "BAhvOgtPYmplY3QA\n"
    end
  end

  context '#from_dynamodb' do
    it 'should load value as Object' do
      primitive.value = primitive.to_dynamodb
      primitive.from_dynamodb.should be_an_instance_of Object
    end
  end

  context '#type_in_dynamodb' do
    it 'should return string type for DynamoDB' do
      primitive.type_in_dynamodb.should eq 'b'
    end
  end
end


text:
/**
 * webpack.common
 */

var webpack = require('webpack');
var HtmlWebpackPlugin = require('html-webpack-plugin');
var ExtractTextPlugin = require('extract-text-webpack-plugin');
var helpers = require('./helpers');

module.exports = {
    entry: {
        'polyfills': './src/polyfills.ts',
        'vendor': './src/vendor.ts',
        'app': './src/main.ts'
    },

    resolve: {
        extensions: ['.ts', '.js', '.json'],
        modules: [helpers.root('src'), helpers.root('node_modules')],
    },

    module: {
        rules: [
            {
                test: /\.ts$/,
                use: ['awesome-typescript-loader', 'angular2-template-loader']
            },
            {
                test: /\.html$/,
                use: 'html-loader'
            },
            {
                test: /\.(png|jpe?g|gif|svg|woff|woff2|ttf|eot|ico)$/,
                use: 'file?name=assets/[name].[hash].[ext]'
            },
            {
                test: /\.css$/,
                exclude: helpers.root('src'),
                use: ExtractTextPlugin.extract({fallback: 'style-loader', use: 'css-loader?sourceMap'})
            },
            {
                test: /\.css$/,
                include: helpers.root('src'),
                use: 'to-string-loader!css-loader'
            },

            /*
             * SCSS compile
             *
             * */
            {
                test: /\.scss$/,
                include: helpers.root('src', 'app'),
                use: ['to-string-loader', 'css-loader', 'postcss-loader', 'resolve-url-loader', 'sass-loader']
            },
        ]
    },

    plugins: [
        // Workaround for angular/angular#11580
        new webpack.ContextReplacementPlugin(
            /angular(\\|\/)core(\\|\/)@angular/,
            helpers.root('src'), // location of your src
            {
                // your Angular Async Route paths relative to this root directory
            }
        ),

        new webpack.optimize.CommonsChunkPlugin({
            name: ['app', 'vendor', 'polyfills']
        }),

        new HtmlWebpackPlugin({
            template: 'src/index.html'
        })
    ]
};


text:
package jdo.invoice.model.payment;

import java.util.Date;
import java.util.List;
import java.util.UUID;

import javax.persistence.ElementCollection;
import javax.persistence.Embedded;
import javax.persistence.Entity;
import javax.persistence.Inheritance;
import javax.persistence.InheritanceType;
import javax.persistence.Lob;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;

import jdo.fields.Money;
import jdo.model.BasePersistentModel;

@Entity
@Inheritance(strategy = InheritanceType.SINGLE_TABLE)
public class Payment extends BasePersistentModel {

	/**
	 *
	 */
	private static final long serialVersionUID = 1L;

	@Embedded
	private Money amount;

	@Lob
	private String comment;

	@Temporal(TemporalType.DATE)
	private Date effectiveDate;

	@ManyToOne
	private PaymentMethodType paidVia;

	private String paymentReferenceNumber;

	@OneToMany
	private List<PaymentApplication> usedToPay;

	/**
	 * UUID of PaymentBudgetAllocation.
	 *
	 */
	@ElementCollection
	private List<UUID> bookedAgainst;

	/**
	 * UUID of PaymentAccountingTransaction.
	 *
	 */
	private UUID postedVia;

	public List<UUID> getBookedAgainst() {
		return bookedAgainst;
	}

	public void setBookedAgainst(List<UUID> bookedAgainst) {
		this.bookedAgainst = bookedAgainst;
	}

	public UUID getPostedVia() {
		return postedVia;
	}

	public void setPostedVia(UUID postedVia) {
		this.postedVia = postedVia;
	}

	public Money getAmount() {
		return amount;
	}

	public String getComment() {
		return comment;
	}

	public Date getEffectiveDate() {
		return effectiveDate;
	}

	public PaymentMethodType getPaidVia() {
		return paidVia;
	}

	public String getPaymentReferenceNumber() {
		return paymentReferenceNumber;
	}

	public List<PaymentApplication> getUsedToPay() {
		return usedToPay;
	}

	public void setAmount(Money amount) {
		this.amount = amount;
	}

	public void setComment(String comment) {
		this.comment = comment;
	}

	public void setEffectiveDate(Date effectiveDate) {
		this.effectiveDate = effectiveDate;
	}

	public void setPaidVia(PaymentMethodType paidVia) {
		this.paidVia = paidVia;
	}

	public void setPaymentReferenceNumber(String paymentReferenceNumber) {
		this.paymentReferenceNumber = paymentReferenceNumber;
	}

	public void setUsedToPay(List<PaymentApplication> usedToPay) {
		this.usedToPay = usedToPay;
	}
}


text:
import csv
import logging
import psycopg2 as postgres
from psycopg2.extras import RealDictCursor
import geocoder

weed_cols = ['case_no', 'case_type', 'status', 'opened_date', 'addr_no', 'addr_dir', 'addr_street_name', 'addr_street_type', 'application_name', 'desc', 'assigned_to_staff', 'business_name', 'active_task']

currentService = 4

logging.basicConfig(filename='graffiti.log',level=logging.DEBUG)

con = postgres.connect('dbname=civic user=postgres password=calvin')
cur = con.cursor(cursor_factory=RealDictCursor)

with open("data/graffiti.csv", "r") as weed_file:
    weed_reader = csv.DictReader(weed_file, weed_cols)

    for entry in weed_reader:
        address = "{} {} {} {} Indianapolis, IN, United States".format(entry['addr_no'], entry['addr_dir'], entry['addr_street_name'], entry['addr_street_type'])
        logging.info("Adding violation for address {}.".format(address))

        try:
            if currentService == 0:
                code = geocoder.google(address)
            elif currentService == 1:
                code = geocoder.bing(address)
            elif currentService == 2:
                code = geocoder.osm(address)
            elif currentService == 3:
                code = geocoder.here(address)
            elif currentService == 4:
                code = geocoder.tomtom(address)
            elif currentService == 5:
                code = geocoder.mapquest(address)

            lat, lng = code.latlng

        except ValueError:  # Request failed
            logging.error("Geocoding failed for address {}.".format(address))
            currentService += 1
            currentService %= 6
            continue

        cur.execute("INSERT INTO violations (lattitude, longitude, street_address, date, violation_type_id) VALUES (%s, %s, %s, %s, 4)", (lat, lng, address, entry['opened_date']))
        con.commit()
        logging.info("Adding violation for address {}.".format(address))


text:
package location

import (
	"encoding/csv"
	"errors"
	"io"
	"strconv"
	"strings"
	"time"
)

var (
	ErrWrongNumberOfCols = errors.New("Number of fields and columns differs")
	ErrNoTimestamp       = errors.New("No time found")
	ErrNoLatLng          = errors.New("No latitude or longitude found")
)

var colMappings = map[string]string{
	"latitude": "latitude",
	"lat":      "latitude",

	"longitude": "longitude",
	"long":      "longitude",
	"lon":       "longitude",
	"lng":       "longitude",

	"accuracy": "accuracy",
	"acc":      "accuracy",

	"time":               "time",
	"timestamp":          "time",
	"location timestamp": "time",
}

func parseRow(fields, row []string) (*Location, error) {
	if len(fields) != len(row) {
		return nil, ErrWrongNumberOfCols
	}

	lat, lng := false, false
	var err error
	loc := &Location{}
	for i, k := range fields {
		v := row[i]
		switch k {
		case "latitude":
			lat = true
			loc.Latitude, err = strconv.ParseFloat(v, 64)
		case "longitude":
			lng = true
			loc.Longitude, err = strconv.ParseFloat(v, 64)
		case "accuracy":
			loc.Accuracy, err = strconv.ParseFloat(v, 64)
		case "time":
			loc.Time, err = time.Parse(time.RFC3339, v)
		}
		if err != nil {
			return nil, err
		}
	}

	if !lat || !lng {
		return nil, ErrNoLatLng
	}
	if loc.Time.IsZero() {
		return nil, ErrNoTimestamp
	}
	return loc, err
}

func ReadCSV(r io.Reader) ([]*Location, error) {
	cr := csv.NewReader(r)
	cols, err := cr.Read()
	if err != nil {
		return nil, err
	}

	for i, col := range cols {
		col = strings.ToLower(col)
		cols[i] = colMappings[col]
	}

	locs := make([]*Location, 0)
	for {
		row, err := cr.Read()
		if err != nil {
			if err == io.EOF {
				break
			} else {
				return locs, err
			}
		}

		loc, err := parseRow(cols, row)
		if err != nil {
			return nil, err
		}
		locs = append(locs, loc)
	}
	return locs, nil
}


text:
void main(void) {
	// We're back at IO: since a pin can be used a number of different
	// pursposes. We need to configure the pin to be used for pin IO:

	EVERY_GPIO_SET_FUNCTION(LED_PORT, LED_PIN, PIO, IOCON_IO_ADMODE_DIGITAL);

	// (strictly speaking, this last step wasn't necessary for the LED
	// because port 0, pin 7 is set up for pin IO per default, but some
	// other pins aren't, so it's a good habit to explicitly configure the
	// pin function. Else, in the furture you will use a pin that is not
	// set to IO per default an spend hours wondering why nothing is
	// working...)

	// Finally, pins configured for pin IO can either be OUTPUT (we write
	// a value to them) or INPUT we read the value physically present at
	// the connected pin. We need to set the pin configuration to output
	// mode, because we want to turn the LED on and off:

	every_gpio_set_dir(LED_PORT, LED_PIN, OUTPUT);

	// We defined LED_PORT and LED_PIN to identify the LED in the code
	// above. OUTPUT is a constant of the type every_gpio_direction and
	// defined in everykey/gpio.h which was included indirectly by
	// including: everykey/everykey.h ...

	// Next we need to configure the timer and interrupt. All Cortex M3
	// processors include a "systick" timer, which, when activated
	// triggers the "systick" interrupt to run every 10ms.

	SYSCON_StartSystick_10ms();
}

// The LPC1343 (and many other processors) require an interrupt table
// that keeps track of where the functions that run when an interrupt is
// triggered are located. This table is placed at the very beginning of
// the firmware and it's defined in the file: everykey/startup.c.
//
// If you have a look at the interrupt table, you'll see the first entry
// refers to the top of the stack (ignore that for now) and the second
// entry refers to `bootstrap`. Once the processor is ready to run, an
// event is generated that triggers this bootstrap interrupt routine to
// run. `bootstrap` does a little bit of initialization and then calls
// your `main` function. So if you ever wondering "who calls main", now
// you know.
//
// The interrupt table also contains a `systick` routine which gets
// called for each systick event. This routine is preconfigured to do
// nothing, so we'll just write a new version to blink:

void systick(void) {

	// Like stated above, this interrupt routine is run once every 10ms.
	// This is much to fast to blink the LED. We want it to blink every
	// second for a tenth of a second, so we'll need to count:

	static uint32_t counter = 0;

	// the `static` modifier causes this variable to be reused every time
	// the function is called and maintain it's value. Else, we would get
	// a fresh new counter set to 0 every time the routine is called.

	// We'll increment the counter by 2 every time it's called:
	counter += 2;

	// This is just a trick to remember whether to turn the LED on or off:

	if (counter == 20) {
		// `counter` reaches 20 after systick has executed 10 times, so after
		// a tenth of a second (10 * 10ms (10/1000s) = 1/10s). Once a tenth
		// of a second has elapsed, we turn the LED off.
		every_gpio_write(LED_PORT, LED_PIN, false);

		// here's the trick: we don't reset the counter to 0, but to 1.
		// Since we're always incrementing by 2, we'll now that if the
		// counter has an odd value, we are waiting to turning the LED on.
		// Likewise, once the LED is turned on, we reset the counter to 0
		// and will now that we're waiting to turn it back off because the
		// counter value is even.

		counter = 1;
	} else if (counter == 181) {
		// the counter reaches 181 after 90/100th of a second, because we
		// increment by 2 every 1/100th of a second and start counting at 1.
		// This has the effect that the LED will blink once a second for
		// 1/10s...

		every_gpio_write(LED_PORT, LED_PIN, true);

		// reset counter to 0 in order to remeber to turn it back off!
		counter = 0;
	}
}


text:
class CreateDigestRules < ActiveRecord::Migration
  def change
    create_table :digest_rules, force: true do |t|
      t.references :user
      t.string :name
      t.integer :position
      t.boolean :active
      t.string :recurrent
      t.string :project_selector
      t.text :project_ids
      t.text :event_ids
      t.timestamps
    end
  end
end


text:
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices.WindowsRuntime;
using Windows.Foundation;
using Windows.Foundation.Collections;
using Windows.UI.Xaml;
using Windows.UI.Xaml.Controls;
using Windows.UI.Xaml.Controls.Primitives;
using Windows.UI.Xaml.Data;
using Windows.UI.Xaml.Input;
using Windows.UI.Xaml.Media;
using Windows.UI.Xaml.Navigation;

namespace ManaBob.Windows
{
    public sealed partial class MainPage
    {
        public MainPage()
        {
            this.InitializeComponent();

            //LoadApplication(new ManaBob.App());
        }

        private void button_Click(object sender, Windows.UI.Xaml.RoutedEventArgs e)
        {
            // Delete the file if it exists.

            //String filepath = @"C:\test.txt";
            //StreamWriter str = new StreamWriter();

            //char[] ch = text.ToCharArray();
            //for (int i = 0; i < ch.Length; i++)
            //{
            //   str.AsStreamForWrite(ch[i]);
            //}
            //str.Dispose();

            String text = textBox.Text;
            StorageFolder current = ApplicationData.Current.LocalFolder;
            StorageFile file = await current.CreateFileAsync(text, CreationCollisionOption.ReplaceExisting);
            IRandomAccessStream writeStream = await file.OpenAsync(FileAccessMode.ReadWrite);
            IOutputStream outputStream = writeStream.GetOutputStreamAt(0);
            DataWriter dataWriter = new DataWriter(outputStream);
            dataWriter.WriteString(" ");
            await dataWriter.StoreAsync();
            await outputStream.FlushAsync();

            UnicodeEncoding uniencoding = new UnicodeEncoding();
            string filename = @"c:\Users\exampleuser\Documents\userinputlog.txt";

            using (FileStream SourceStream = File.Open(filename, FileMode.OpenOrCreate))
            {
                SourceStream.Seek(0, SeekOrigin.End);
                await SourceStream.WriteAsync(result, 0, result.Length);
            }

            string path = "C:\\ListColorRGB.txt";
            System.Environment.
            StreamReader sr = new StreamReader();
        }
    }
}


text:
namespace Yabe
{
    partial class ProgressDialog
    {
        /// <summary>
        /// Required designer variable.
        /// </summary>
        private System.ComponentModel.IContainer components = null;

        /// <summary>
        /// Clean up any resources being used.
        /// </summary>
        /// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
        protected override void Dispose(bool disposing)
        {
            if (disposing && (components != null))
            {
                components.Dispose();
            }
            base.Dispose(disposing);
        }

        #region Windows Form Designer generated code

        /// <summary>
        /// Required method for Designer support - do not modify
        /// the contents of this method with the code editor.
        /// </summary>
        private void InitializeComponent()
        {
            this.m_Label = new System.Windows.Forms.Label();
            this.m_progessbar = new System.Windows.Forms.ProgressBar();
            this.m_CancelButton = new System.Windows.Forms.Button();
            this.SuspendLayout();
            //
            // m_Label
            //
            this.m_Label.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)
            | System.Windows.Forms.AnchorStyles.Right)));
            this.m_Label.Location = new System.Drawing.Point(12, 9);
            this.m_Label.Name = "m_Label";
            this.m_Label.Size = new System.Drawing.Size(300, 17);
            this.m_Label.TabIndex = 0;
            this.m_Label.Text = "Progress";
            this.m_Label.TextAlign = System.Drawing.ContentAlignment.MiddleCenter;
            //
            // m_progessbar
            //
            this.m_progessbar.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)
            | System.Windows.Forms.AnchorStyles.Right)));
            this.m_progessbar.Location = new System.Drawing.Point(12, 29);
            this.m_progessbar.Name = "m_progessbar";
            this.m_progessbar.Size = new System.Drawing.Size(300, 23);
            this.m_progessbar.TabIndex = 1;
            //
            // m_CancelButton
            //
            this.m_CancelButton.Anchor = System.Windows.Forms.AnchorStyles.Bottom;
            this.m_CancelButton.DialogResult = System.Windows.Forms.DialogResult.Cancel;
            this.m_CancelButton.Location = new System.Drawing.Point(125, 60);
            this.m_CancelButton.Name = "m_CancelButton";
            this.m_CancelButton.Size = new System.Drawing.Size(75, 23);
            this.m_CancelButton.TabIndex = 3;
            this.m_CancelButton.Text = "Cancel";
            this.m_CancelButton.UseVisualStyleBackColor = true;
            this.m_CancelButton.Click += new System.EventHandler(this.m_CancelButton_Click);
            //
            // ProgressDialog
            //
            this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
            this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
            this.CancelButton = this.m_CancelButton;
            this.ClientSize = new System.Drawing.Size(324, 95);
            this.Controls.Add(this.m_CancelButton);
            this.Controls.Add(this.m_progessbar);
            this.Controls.Add(this.m_Label);
            this.FormBorderStyle = System.Windows.Forms.FormBorderStyle.FixedSingle;
            this.MaximizeBox = false;
            this.MinimizeBox = false;
            this.Name = "ProgressDialog";
            this.ShowIcon = false;
            this.ShowInTaskbar = false;
            this.StartPosition = System.Windows.Forms.FormStartPosition.CenterParent;
            this.Text = "ProgressDialog";
            this.FormClosing += new System.Windows.Forms.FormClosingEventHandler(this.ProgressDialog_FormClosing);
            this.ResumeLayout(false);

        }

        #endregion

        private System.Windows.Forms.Label m_Label;
        private System.Windows.Forms.ProgressBar m_progessbar;
        private System.Windows.Forms.Button m_CancelButton;
    }
}


text:
'use strict';

/**
 * Error message body that will indicate why the operation failed.
 *
 */
class WorkbookError {
  /**
   * Create a WorkbookError.
   * @member {string} [code] Service-defined error code. This code serves as a
   * sub-status for the HTTP error code specified in the response.
   * @member {string} [message] Human-readable representation of the error.
   * @member {array} [details] The list of invalid fields send in request, in
   * case of validation error.
   */
  constructor() {
  }

  /**
   * Defines the metadata of WorkbookError
   *
   * @returns {object} metadata of WorkbookError
   *
   */
  mapper() {
    return {
      required: false,
      serializedName: 'WorkbookError',
      type: {
        name: 'Composite',
        className: 'WorkbookError',
        modelProperties: {
          code: {
            required: false,
            serializedName: 'code',
            type: {
              name: 'String'
            }
          },
          message: {
            required: false,
            serializedName: 'message',
            type: {
              name: 'String'
            }
          },
          details: {
            required: false,
            serializedName: 'details',
            type: {
              name: 'Sequence',
              element: {
                  required: false,
                  serializedName: 'ErrorFieldContractElementType',
                  type: {
                    name: 'Composite',
                    className: 'ErrorFieldContract'
                  }
              }
            }
          }
        }
      }
    };
  }
}

module.exports = WorkbookError;


text:
<?php

namespace DvsaCommonTest\TestUtils;

class MethodNotInvokedException extends \Exception
{
    public function __construct($methodName)
    {
        parent::__construct("Method '" . $methodName. "' has not been invoked'");
    }
}


text:
Spree::Core::Engine.config.to_prepare do
  if Spree.user_class
    Spree.user_class.class_eval do

      include Spree::UserApiAuthentication
      include Spree::UserReporting

      has_many :role_users, foreign_key: "user_id", class_name: "Spree::RoleUser"
      has_many :spree_roles, through: :role_users, source: :role

      has_many :user_stock_locations, foreign_key: "user_id", class_name: "Spree::UserStockLocation"
      has_many :stock_locations, through: :user_stock_locations

      has_many :spree_orders, foreign_key: "user_id", class_name: "Spree::Order"

      has_many :store_credits, -> { includes(:credit_type) }, foreign_key: "user_id", class_name: "Spree::StoreCredit"
      has_many :store_credit_events, through: :store_credits

      belongs_to :ship_address, class_name: 'Spree::Address'
      belongs_to :bill_address, class_name: 'Spree::Address'

      # has_spree_role? simply needs to return true or false whether a user has a role or not.
      def has_spree_role?(role_in_question)
        spree_roles.any? { |role| role.name == role_in_question.to_s }
      end

      # @return [Spree::Order] the most-recently-created incomplete order
      # since the customer's last complete order.
      def last_incomplete_spree_order(store: nil, only_frontend_viewable: true)
        self_orders = self.orders
        self_orders = self_orders.where(frontend_viewable: true) if only_frontend_viewable
        self_orders = self_orders.where(store: store) if store
        last_order = self_orders.order(:created_at).last
        last_order unless last_order.try!(:completed?)
      end

      def total_available_store_credit
        store_credits.reload.to_a.sum{ |credit| credit.amount_remaining }
      end
    end
  end
end


text:
title: >-
  **Shobha Rao** - *Girls Burn Brighter* <br> **Mira T. Lee** - *Everything Here
  Is Beautiful*
published: true
start_at: 2018-03-21 19:00:00
image_path:
book_category:
program: '-writers-and-readers'
ticketed: false
related_books:
  - girls-burn-brighter
  - everything-here-is-beautiful
---

In Shobha Rao's *Girls Burn Brighter*, Poornima and Savitha have three strikes against them: they are poor, they are ambitious, and they are girls. Tragic circumstances bring the girls together, and a friendship blossoms. Suddenly their Indian village doesn't feel quite so claustrophobic - Poornima even begins to imagine a life beyond arranged marriage. But when a devastating act of cruelty drives Savitha away, Poornima leaves behind everything she has ever known to find her friend.

In Mira T. Lee's *Everything Here is Beautiful*, two Chinese-American sisters–Miranda, the older, responsible one, always her younger sister's protector; Lucia, the headstrong, unpredictable one, whose impulses are huge and, often, life changing. When Lucia starts hearing voices, it is Miranda who must find a way to reach her sister. Lucia impetuously plows ahead, but the bitter constant is that she is, in fact, mentally ill. Lucia lives life on a grand scale, until, inevitably, she crashes to earth.

Miranda leaves her own self-contained life in Switzerland to rescue her sister again–but only Lucia can decide whether she wants to be saved. The bonds of sisterly devotion stretch across oceans–but what does it take to break them?


text:
__author__ = 'thorsteinn'
import urllib2
from bs4 import BeautifulSoup
import re
import sys

def mc_soup(imoNumber):
    '''
Soup module that grabs information from MarineConnection
    '''

    opener = urllib2.build_opener()
    opener.addheaders = [('User-agent', 'Mozilla/5.0')]

    # define the proper web address
    try:
        url = 'http://maritime-connector.com/ship/{0:07d}/'.format(imoNumber)
    except ValueError, me:
        print "ValueError: The number {imo_id} could not be formatted.\n{message}".format(
            imo_id=imoNumber,
            message=me.message
        )
        return {'name_s': 'error'}
    try:
        page = opener.open(url)
    except urllib2.HTTPError, me:
        print "calling {imo_id} gave the error code: {code} : {message}. Returning code 'blank'".format(
            imo_id=imoNumber,
            code=me.code,
            message=me.message
        )
        return {'name_s': 'error'}
    except urllib2.URLError, me:
        print "Failed to reach server. Reason: {reason}".format(
            reason=me.reason
        )
        return sys.exit(status=123)

    soup = BeautifulSoup(page.read())

    # find if a valid IMO number has been found
    title = soup.text.strip().split()
    if str(imoNumber) not in title:
        print 'ship not found'
        return {'imo_id': imoNumber, 'name_s': None}

    # passed title find the rows in the data table
    t = soup.find_all('table', attrs={'class': 'ship-data-table'})
    tr = t[0].find_all('tr')

    # break the rows into a dictionary
    retDict = {}
    for aRow in tr:
        head = aRow.find('th').text
        val = aRow.find('td').text

        # former names is a list of the form "SOMESHIP until 2010 Apr..."
        if 'Former names' == head:
            val = re.findall(r'\w+ until \d+ \w\w\w', val)

        # mass names are given as "2939 tons" form
        elif 'tons' in val.split():
            val = val.split()[0]
        retDict[head] = val

    return retDict