text:
package worker

import (
	"log"
	"time"

	"github.com/gaocegege/the-big-brother-is-watching-you/common"
	"github.com/gaocegege/the-big-brother-is-watching-you/source"
	"github.com/gaocegege/the-big-brother-is-watching-you/storage"
)

const (
	fetchTime = 240
)

// Worker fetch all the records and put them to db
type Worker struct {
	sm *source.Manager
	vm *storage.VendorCollectionManager
	rm *storage.RecordCollectionManager
}

// NewWorker returns a new Worker object
func NewWorker(sm *source.Manager, vm *storage.VendorCollectionManager, rm *storage.RecordCollectionManager) *Worker {
	return &Worker{
		sm: sm,
		vm: vm,
		rm: rm,
	}
}

// Work now
func (w *Worker) Work() {
	for _, source := range w.sm.GetSources() {
		var lastTime time.Time

		vendor, err := w.vm.FindVendorByHost(source.GetHostName())

		// if the vendor doesn't exist, alloc a new document
		if err != nil {
			log.Print(err)

			// trick: add a negative duration
			lastTime = time.Now().Add(-time.Duration(fetchTime) * time.Hour)

			vendorDoc := &common.Vendor{
				Host:     source.GetHostName(),
				LastTime: time.Now(),
			}
			vendorID, err := w.vm.NewVendorDocument(vendorDoc)
			if err != nil {
				log.Fatal(err)
			}

			vendor, err = w.vm.FindVendorByID(vendorID)
			if err != nil {
				log.Fatal(err)
			}
		}

		// else set the last time
		lastTime = vendor.LastTime
		vendor.LastTime = time.Now()
		w.vm.UpdateVendorDocument(vendor.VendorID, *vendor)

		// check out the last date, and fetch origin
		records, err := source.FetchFromOrigin(vendor.VendorID, lastTime)
		if err != nil {
			log.Fatal(err)
		}

		// store
		for _, record := range records {
			w.rm.NewRecordDocument(&record)
			w.vm.AddNewRecord(vendor.VendorID, record.RecordID)
		}

		log.Printf("New Records from %s has been fetched", source.GetHostName())
	}
}


text:
import { info } from "./loggers"
import { round } from "lodash"

export default key => {
  const start = process.hrtime()

  return {
    start: () => {
      info(`Loading: ${key}`)
      return start
    },

    end: () => {
      const end = process.hrtime(start)
      const interval = `${end[0]}s ${round(end[1] / 1000000, 3)}ms`
      info(`Elapsed: ${interval} - ${key}`)
      return interval
    },
  }
}

redpajama_set_name: RedPajamaGithub


text:
import mygeneinfo
import data_model as dm
import re

class Term2gene_mapper():
    def __init__(self):
        # a map to hold the term to gene mapping to speed the query.
        self.term_to_gene_map = {}
        self.excluded_id_regex = re.compile('[(),\'\"\s]+')

    def reset(self):
        self.term_to_gene_map = {}

    def get_identifier_without_prefix(self, string):
        # remove the prefix, if there is one

        elements = string.split(':')
        length = len(elements)
        if length is 2:
            return str(elements[1])
        elif length > 2:
            return None
        else:
            return string

    def exclude_id(self, id):
        #if len(id) > 10:
        #    return None
        #if id == "3',5'-cyclic GMP":
        #   print "test!"
        result = self.excluded_id_regex.search(id)
        return result

    def add_network_nodes(self, node_table):
        query_string = ""
        for node_id, node in node_table.items():
            if node.get('name') is not None:
                name = list(node.get('name'))[0]
            else:
                name = list(node.get('represents'))[0]

            if self.exclude_id(name) is None:
                query_string = query_string + " " + name
            if node.get('alias'):
                for alias in node.get('alias'):
                    id = self.get_identifier_without_prefix(alias)
                    if id:
                        if self.exclude_id(id) is None:
                            query_string = query_string + " " + id
                        else:
                            print "skipping " + id

        gene_list = mygeneinfo.query_batch(query_string)
        for hit in gene_list:
            term = hit.get('query')
            symbol = hit.get('symbol')
            id = hit.get('entrezgene')
            if symbol and id:
                gene = dm.Gene(hit.get('symbol'), hit.get('entrezgene'))
                self.term_to_gene_map[term] = gene
                # else:
                # print "no symbol and id for " + term
        return gene_list

    # standardize a term to a gene symbol
    # input: search term as a single string
    # return: Single string if found a gene symbol for the input term, otherwise return None
    def gene_from_term(self, term):
        gene = self.term_to_gene_map.get(term)
        if gene:
            return gene
        else:
            # look for term on external service
            gene = mygeneinfo.query_term_to_gene(term)
            if gene:
                self.term_to_gene_map[term] = gene
            else:
                print term + " doesn't map to any gene"
            return gene

    def get_gene_from_identifier(self, identifier):
        key = self.get_identifier_without_prefix(identifier)
        gene = self.term_to_gene_map.get(key)
        if gene:
            return gene
        else:
            # print key + " doesn't map to any gene"
            return None

    # a variation of the function above. Returns the Gene object rather than a gene symbol
    # returns None if term can't be mapped to a gene.
    def  gene_symbol_and_id_from_term(self, term):
        gene = self.term_to_gene_map.get(term)
        if gene:
            return gene
        else:
            # look for term on external service
            gene = mygeneinfo.query_standard_to_gene_quick(term)
            if gene:
                self.term_to_gene_map[term] = gene
                return gene
            else:
                print term + " doesn't map to any gene"
                return None

    # returns a set of gene symbols found in the given function term
    def genes_from_function_term(self, function_term):
        # if it is a function term, process all genes mentioned
        result = set([])
        for parameter in function_term['args']:
            if type(parameter) == 'str':
                gene = self.gene_from_term(parameter)
                if gene:
                    result.add(gene)
            else:
                result.union(self.genes_from_function_term(parameter))
        return result

    # input: a collection of genes, we assume there is no dupicates in it.
    # returns: a nested hash table contains the mapping of search terms and mapped genes
    # returned data structure: {'matched': { gene_symbol: [search terms]}, 'unmatched':[search_term]}
    def standarize_terms(self, terms):
        result = {'matched': {}, 'unmatched': []}
        for term in terms:
            symbol = self.gene_from_term(term)
            if symbol:
                matched_genes = result['matched']
                if symbol in matched_genes:
                    matched_genes[symbol].append(term)
                else:
                    matched_genes[symbol] = [term]
            else:
                result['unmatched'].append(term)

        return result


text:
#pragma once

#include <react/renderer/attributedstring/AttributedString.h>
#include <react/renderer/attributedstring/ParagraphAttributes.h>
#include <react/renderer/core/LayoutConstraints.h>
#include <react/utils/FloatComparison.h>
#include <react/utils/SimpleThreadSafeCache.h>

namespace facebook {
namespace react {

struct LineMeasurement {
  std::string text;
  Rect frame;
  Float descender;
  Float capHeight;
  Float ascender;
  Float xHeight;
};

using LinesMeasurements = std::vector<LineMeasurement>;

/*
 * Describes a result of text measuring.
 */
class TextMeasurement final {
 public:
  class Attachment final {
   public:
    Rect frame;
    bool isClipped;
  };

  using Attachments = std::vector<Attachment>;

  Size size;
  Attachments attachments;
};

// The Key type that is used for Text Measure Cache.
// The equivalence and hashing operations of this are defined to respect the
// nature of text measuring.
class TextMeasureCacheKey final {
 public:
  AttributedString attributedString{};
  ParagraphAttributes paragraphAttributes{};
  LayoutConstraints layoutConstraints{};
};

/*
 * Maximum size of the Cache.
 * The number was empirically chosen based on approximation of an average amount
 * of meaningful measures per surface.
 */
constexpr auto kSimpleThreadSafeCacheSizeCap = size_t{256};

/*
 * Thread-safe, evicting hash table designed to store text measurement
 * information.
 */
using TextMeasureCache = SimpleThreadSafeCache<
    TextMeasureCacheKey,
    TextMeasurement,
    kSimpleThreadSafeCacheSizeCap>;

inline bool areTextAttributesEquivalentLayoutWise(
    TextAttributes const &lhs,
    TextAttributes const &rhs) {
  // Here we check all attributes that affect layout metrics and don't check any
  // attributes that affect only a decorative aspect of displayed text (like
  // colors).
  return std::tie(
             lhs.fontFamily,
             lhs.fontWeight,
             lhs.fontStyle,
             lhs.fontVariant,
             lhs.allowFontScaling,
             lhs.alignment) ==
      std::tie(
             rhs.fontFamily,
             rhs.fontWeight,
             rhs.fontStyle,
             rhs.fontVariant,
             rhs.allowFontScaling,
             rhs.alignment) &&
      floatEquality(lhs.fontSize, rhs.fontSize) &&
      floatEquality(lhs.fontSizeMultiplier, rhs.fontSizeMultiplier) &&
      floatEquality(lhs.letterSpacing, rhs.letterSpacing) &&
      floatEquality(lhs.lineHeight, rhs.lineHeight);
}

inline size_t textAttributesHashLayoutWise(
    TextAttributes const &textAttributes) {
  // Taking into account the same props as
  // `areTextAttributesEquivalentLayoutWise` mentions.
  return folly::hash::hash_combine(
      0,
      textAttributes.fontFamily,
      textAttributes.fontSize,
      textAttributes.fontSizeMultiplier,
      textAttributes.fontWeight,
      textAttributes.fontStyle,
      textAttributes.fontVariant,
      textAttributes.allowFontScaling,
      textAttributes.letterSpacing,
      textAttributes.lineHeight,
      textAttributes.alignment);
}

inline bool areAttributedStringFragmentsEquivalentLayoutWise(
    AttributedString::Fragment const &lhs,
    AttributedString::Fragment const &rhs) {
  return lhs.string == rhs.string &&
      areTextAttributesEquivalentLayoutWise(
             lhs.textAttributes, rhs.textAttributes) &&
      // LayoutMetrics of an attachment fragment affects the size of a measured
      // attributed string.
      (!lhs.isAttachment() ||
       (lhs.parentShadowView.layoutMetrics ==
        rhs.parentShadowView.layoutMetrics));
}

inline size_t textAttributesHashLayoutWise(
    AttributedString::Fragment const &fragment) {
  // Here we are not taking `isAttachment` and `layoutMetrics` into account
  // because they are logically interdependent and this can break an invariant
  // between hash and equivalence functions (and cause cache misses).
  return folly::hash::hash_combine(
      0,
      fragment.string,
      textAttributesHashLayoutWise(fragment.textAttributes));
}

inline bool areAttributedStringsEquivalentLayoutWise(
    AttributedString const &lhs,
    AttributedString const &rhs) {
  auto &lhsFragment = lhs.getFragments();
  auto &rhsFragment = rhs.getFragments();

  if (lhsFragment.size() != rhsFragment.size()) {
    return false;
  }

  auto size = lhsFragment.size();
  for (auto i = size_t{0}; i < size; i++) {
    if (!areAttributedStringFragmentsEquivalentLayoutWise(
            lhsFragment.at(i), rhsFragment.at(i))) {
      return false;
    }
  }

  return true;
}

inline size_t textAttributedStringHashLayoutWise(
    AttributedString const &attributedString) {
  auto seed = size_t{0};

  for (auto const &fragment : attributedString.getFragments()) {
    seed =
        folly::hash::hash_combine(seed, textAttributesHashLayoutWise(fragment));
  }

  return seed;
}

inline bool operator==(
    TextMeasureCacheKey const &lhs,
    TextMeasureCacheKey const &rhs) {
  return areAttributedStringsEquivalentLayoutWise(
             lhs.attributedString, rhs.attributedString) &&
      lhs.paragraphAttributes == rhs.paragraphAttributes &&
      lhs.layoutConstraints.maximumSize.width ==
      rhs.layoutConstraints.maximumSize.width;
}

inline bool operator!=(
    TextMeasureCacheKey const &lhs,
    TextMeasureCacheKey const &rhs) {
  return !(lhs == rhs);
}

} // namespace react
} // namespace facebook

namespace std {

template <>
struct hash<facebook::react::TextMeasureCacheKey> {
  size_t operator()(facebook::react::TextMeasureCacheKey const &key) const {
    return folly::hash::hash_combine(
        0,
        textAttributedStringHashLayoutWise(key.attributedString),
        key.paragraphAttributes,
        key.layoutConstraints.maximumSize.width);
  }
};

} // namespace std


text:
package org.apache.carbondata.sdk.file;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.annotations.InterfaceStability;
import org.apache.carbondata.common.constants.LoggerAction;
import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.datatype.MapType;
import org.apache.carbondata.core.metadata.datatype.StructField;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.TableSchema;
import org.apache.carbondata.core.metadata.schema.table.TableSchemaBuilder;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.loading.model.CarbonLoadModelBuilder;
import org.apache.carbondata.processing.util.CarbonLoaderUtil;

import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;

/**
 * Builder for {@link CarbonWriter}
 */
@InterfaceAudience.User
@InterfaceStability.Unstable
public class CarbonWriterBuilder {
  private Schema schema;
  private String path;
  //initialize with empty array , as no columns should be selected for sorting in NO_SORT
  private String[] sortColumns = new String[0];
  private int blockletSize;
  private int blockSize;
  private long timestamp;
  private Map<String, String> options;
  private String taskNo;
  private int localDictionaryThreshold;
  private boolean isLocalDictionaryEnabled = Boolean.parseBoolean(
          CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE_DEFAULT);
  private short numOfThreads;
  private Configuration hadoopConf;
  private String writtenByApp;
  private String[] invertedIndexColumns;
  private enum WRITER_TYPE {
    CSV, AVRO, JSON, ROWFORMAT
  }

  private WRITER_TYPE writerType;

  /**
   * Sets the output path of the writer builder
   *
   * @param path is the absolute path where output files are written
   *             This method must be called when building CarbonWriterBuilder
   * @return updated CarbonWriterBuilder
   */
  public CarbonWriterBuilder outputPath(String path) {
    Objects.requireNonNull(path, "path should not be null");
    this.path = path;
    return this;
  }

  /**
   * sets the list of columns that needs to be in sorted order
   *
   * @param sortColumns is a string array of columns that needs to be sorted.
   *                    If it is null or by default all dimensions are selected for sorting
   *                    If it is empty array, no columns are sorted
   * @return updated CarbonWriterBuilder
   */
  public CarbonWriterBuilder sortBy(String[] sortColumns) {
    if (sortColumns != null) {
      for (int i = 0; i < sortColumns.length; i++) {
        sortColumns[i] = sortColumns[i].toLowerCase().trim();
      }
    }
    this.sortColumns = sortColumns;
    return this;
  }

  /**
   * sets the list of columns for which inverted index needs to generated
   * @param invertedIndexColumns is a string array of columns for which inverted index needs to
   * generated.
   * If it is null or an empty array, inverted index will be generated for none of the columns
   * @return updated CarbonWriterBuilder
   */
  public CarbonWriterBuilder invertedIndexFor(String[] invertedIndexColumns) {
    if (invertedIndexColumns != null) {
      for (int i = 0; i < invertedIndexColumns.length; i++) {
        invertedIndexColumns[i] = invertedIndexColumns[i].toLowerCase().trim();
      }
    }
    this.invertedIndexColumns = invertedIndexColumns;
    return this;
  }

  /**
   * sets the taskNo for the writer. SDKs concurrently running
   * will set taskNo in order to avoid conflicts in file's name during write.
   *
   * @param taskNo is the TaskNo user wants to specify.
   *               by default it is system time in nano seconds.
   * @return updated CarbonWriterBuilder
   */
  public CarbonWriterBuilder taskNo(long taskNo) {
    this.taskNo = String.valueOf(taskNo);
    return this;
  }

  /**
   * to set the timestamp in the carbondata and carbonindex index files
   *
   * @param timestamp is a timestamp to be used in the carbondata and carbonindex index files.
   *                  By default set to zero.
   * @return updated CarbonWriterBuilder
   */
  public CarbonWriterBuilder uniqueIdentifier(long timestamp) {
    Objects.requireNonNull(timestamp, "Unique Identifier should not be null");
    this.timestamp = timestamp;
    return this;
  }

  /**
   * To support the load options for sdk writer
   * @param options key,value pair of load options.
   * supported keys values are
   * a. bad_records_logger_enable -- true (write into separate logs), false
   * b. bad_records_action -- FAIL, FORCE, IGNORE, REDIRECT
   * c. bad_record_path -- path
   * d. dateformat -- same as JAVA SimpleDateFormat
   * e. timestampformat -- same as JAVA SimpleDateFormat
   * f. complex_delimiter_level_1 -- value to Split the complexTypeData
   * g. complex_delimiter_level_2 -- value to Split the nested complexTypeData
   * h. quotechar
   * i. escapechar
   *
   * Default values are as follows.
   *
   * a. bad_records_logger_enable -- "false"
   * b. bad_records_action -- "FAIL"
   * c. bad_record_path -- ""
   * d. dateformat -- "" , uses from carbon.properties file
   * e. timestampformat -- "", uses from carbon.properties file
   * f. complex_delimiter_level_1 -- "\001"
   * g. complex_delimiter_level_2 -- "\002"
   * h. quotechar -- "\""
   * i. escapechar -- "\\"
   *
   * @return updated CarbonWriterBuilder
   */
  public CarbonWriterBuilder withLoadOptions(Map<String, String> options) {
    Objects.requireNonNull(options, "Load options should not be null");
    //validate the options.
    for (String option : options.keySet()) {
      if (!option.equalsIgnoreCase("bad_records_logger_enable") &&
          !option.equalsIgnoreCase("bad_records_action") &&
          !option.equalsIgnoreCase("bad_record_path") &&
          !option.equalsIgnoreCase("dateformat") &&
          !option.equalsIgnoreCase("timestampformat") &&
          !option.equalsIgnoreCase("complex_delimiter_level_1") &&
          !option.equalsIgnoreCase("complex_delimiter_level_2") &&
          !option.equalsIgnoreCase("complex_delimiter_level_3") &&
          !option.equalsIgnoreCase("quotechar") &&
          !option.equalsIgnoreCase("escapechar")) {
        throw new IllegalArgumentException("Unsupported option:" + option
            + ". Refer method header or documentation");
      }
    }

    for (Map.Entry<String, String> entry : options.entrySet()) {
      if (entry.getKey().equalsIgnoreCase("bad_records_action")) {
        try {
          LoggerAction.valueOf(entry.getValue().toUpperCase());
        } catch (Exception e) {
          throw new IllegalArgumentException(
              "option BAD_RECORDS_ACTION can have only either " +
                  "FORCE or IGNORE or REDIRECT or FAIL. It shouldn't be " + entry.getValue());
        }
      } else if (entry.getKey().equalsIgnoreCase("bad_records_logger_enable")) {
        boolean isValid;
        isValid = CarbonUtil.validateBoolean(entry.getValue());
        if (!isValid) {
          throw new IllegalArgumentException("Invalid value "
              + entry.getValue() + " for key " + entry.getKey());
        }
      } else if (entry.getKey().equalsIgnoreCase("quotechar")) {
        String quoteChar = entry.getValue();
        if (quoteChar.length() > 1) {
          throw new IllegalArgumentException("QUOTECHAR cannot be more than one character.");
        }
      } else if (entry.getKey().equalsIgnoreCase("escapechar")) {
        String escapeChar = entry.getValue();
        if (escapeChar.length() > 1 && !CarbonLoaderUtil.isValidEscapeSequence(escapeChar)) {
          throw new IllegalArgumentException("ESCAPECHAR cannot be more than one character.");
        }
      }
    }

    if (this.options == null) {
      // convert it to treeMap as keys need to be case insensitive
      this.options = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
    }
    this.options.putAll(options);
    return this;
  }

  /**
   * To support the load options for sdk writer
   *
   * @param key   the key of load option
   * @param value the value of load option
   * @return updated CarbonWriterBuilder object
   */
  public CarbonWriterBuilder withLoadOption(String key, String value) {
    Objects.requireNonNull(key, "key of load properties should not be null");
    Objects.requireNonNull(key, "value of load properties should not be null");
    Map map = new HashMap();
    map.put(key, value);
    withLoadOptions(map);
    return this;
  }

  /**
   * To support the table properties for sdk writer
   *
   * @param options key,value pair of create table properties.
   * supported keys values are
   * a. table_blocksize -- [1-2048] values in MB. Default value is 1024
   * b. table_blocklet_size -- values in MB. Default value is 64 MB
   * c. local_dictionary_threshold -- positive value, default is 10000
   * d. local_dictionary_enable -- true / false. Default is false
   * e. sort_columns -- comma separated column. "c1,c2". Default all dimensions are sorted.
   *                    If empty string "" is passed. No columns are sorted
   * j. sort_scope -- "local_sort", "no_sort", "batch_sort". default value is "local_sort"
   * k. long_string_columns -- comma separated string columns which are more than 32k length.
   *                           default value is null.
   * l. inverted_index -- comma separated string columns for which inverted index needs to be
   *                      generated
   *
   * @return updated CarbonWriterBuilder
   */
  public CarbonWriterBuilder withTableProperties(Map<String, String> options) {
    Objects.requireNonNull(options, "Table properties should not be null");
    Set<String> supportedOptions = new HashSet<>(Arrays
        .asList("table_blocksize", "table_blocklet_size", "local_dictionary_threshold",
            "local_dictionary_enable", "sort_columns", "sort_scope", "long_string_columns",
            "inverted_index", "primary_key_columns"));

    // TODO handle this

//    for (String key : options.keySet()) {
//      if (!supportedOptions.contains(key.toLowerCase())) {
//        throw new IllegalArgumentException(
//            "Unsupported options. " + key + " Refer method header or documentation");
//      }
//    }

    for (Map.Entry<String, String> entry : options.entrySet()) {
      if (entry.getKey().equalsIgnoreCase("table_blocksize")) {
        this.withBlockSize(Integer.parseInt(entry.getValue()));
      } else if (entry.getKey().equalsIgnoreCase("table_blocklet_size")) {
        this.withBlockletSize(Integer.parseInt(entry.getValue()));
      } else if (entry.getKey().equalsIgnoreCase("local_dictionary_threshold")) {
        this.localDictionaryThreshold(Integer.parseInt(entry.getValue()));
      } else if (entry.getKey().equalsIgnoreCase("local_dictionary_enable")) {
        this.enableLocalDictionary((entry.getValue().equalsIgnoreCase("true")));
      } else if (entry.getKey().equalsIgnoreCase("sort_columns")) {
        //sort columns
        String[] sortColumns;
        if (entry.getValue().trim().isEmpty()) {
          sortColumns = new String[0];
        } else {
          sortColumns = entry.getValue().split(",");
        }
        this.sortBy(sortColumns);
      } else if (entry.getKey().equalsIgnoreCase("sort_scope")) {
        this.withSortScope(entry);
      } else if (entry.getKey().equalsIgnoreCase("long_string_columns")) {
        updateToLoadOptions(entry);
      } else if (entry.getKey().equalsIgnoreCase("inverted_index")) {
        //inverted index columns
        String[] invertedIndexColumns;
        if (entry.getValue().trim().isEmpty()) {
          invertedIndexColumns = new String[0];
        } else {
          invertedIndexColumns = entry.getValue().split(",");
        }
        this.invertedIndexFor(invertedIndexColumns);
      } else {
        if (this.options == null) {
          // convert it to treeMap as keys need to be case insensitive
          this.options = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
        }
        this.options.put(entry.getKey(), entry.getValue());
      }
    }
    return this;
  }

  /**
   * To support the table properties for sdk writer
   *
   * @param key   property key
   * @param value property value
   * @return CarbonWriterBuilder object
   */
  public CarbonWriterBuilder withTableProperty(String key, String value) {
    Objects.requireNonNull(key, "key of table properties should not be null");
    Objects.requireNonNull(key, "value of table properties  should not be null");
    Map map = new HashMap();
    map.put(key, value);
    withTableProperties(map);
    return this;
  }

  /**
   * To make sdk writer thread safe.
   *
   * @param numOfThreads should number of threads in which writer is called in multi-thread scenario
   *                     default sdk writer is not thread safe.
   *                     can use one writer instance in one thread only.
   * @return updated CarbonWriterBuilder
   */
  public CarbonWriterBuilder withThreadSafe(short numOfThreads) {
    if (numOfThreads < 1) {
      throw new IllegalArgumentException("number of threads cannot be lesser than 1. "
          + "suggest to keep two times the number of cores available");
    }
    this.numOfThreads = numOfThreads;
    return this;
  }

  /**
   * To support hadoop configuration
   *
   * @param conf hadoop configuration support, can set s3a AK,SK,end point and other conf with this
   * @return updated CarbonWriterBuilder
   */
  public CarbonWriterBuilder withHadoopConf(Configuration conf) {
    if (conf != null) {
      this.hadoopConf = conf;
    }
    return this;
  }

  /**
   * Updates the hadoop configuration with the given key value
   *
   * @param key   key word
   * @param value value
   * @return this object
   */
  public CarbonWriterBuilder withHadoopConf(String key, String value) {
    if (this.hadoopConf == null) {
      this.hadoopConf = new Configuration(true);
    }
    this.hadoopConf.set(key, value);
    return this;
  }

  /**
   * To set the carbondata file size in MB between 1MB-2048MB
   *
   * @param blockSize is size in MB between 1MB to 2048 MB
   *                  default value is 1024 MB
   * @return updated CarbonWriterBuilder
   */
  public CarbonWriterBuilder withBlockSize(int blockSize) {
    if (blockSize <= 0 || blockSize > 2048) {
      throw new IllegalArgumentException("blockSize should be between 1 MB to 2048 MB");
    }
    this.blockSize = blockSize;
    return this;
  }

  /**
   * @param localDictionaryThreshold is localDictionaryThreshold, default is 10000
   * @return updated CarbonWriterBuilder
   */
  public CarbonWriterBuilder localDictionaryThreshold(int localDictionaryThreshold) {
    if (localDictionaryThreshold <= 0) {
      throw new IllegalArgumentException(
          "Local Dictionary Threshold should be greater than 0");
    }
    this.localDictionaryThreshold = localDictionaryThreshold;
    return this;
  }

  /**
   * @param appName appName which is writing the carbondata files
   * @return
   */
  public CarbonWriterBuilder writtenBy(String appName) {
    this.writtenByApp = appName;
    return this;
  }

  /**
   * @param enableLocalDictionary enable local dictionary, default is false
   * @return updated CarbonWriterBuilder
   */
  public CarbonWriterBuilder enableLocalDictionary(boolean enableLocalDictionary) {
    this.isLocalDictionaryEnabled = enableLocalDictionary;
    return this;
  }

  /**
   * To set the blocklet size of CarbonData file
   *
   * @param blockletSize is blocklet size in MB
   *                     default value is 64 MB
   * @return updated CarbonWriterBuilder
   */
  public CarbonWriterBuilder withBlockletSize(int blockletSize) {
    if (blockletSize <= 0) {
      throw new IllegalArgumentException("blockletSize should be greater than zero");
    }
    this.blockletSize = blockletSize;
    return this;
  }

  /**
   * to build a {@link CarbonWriter}, which accepts row in CSV format
   *
   * @param schema carbon Schema object {org.apache.carbondata.sdk.file.Schema}
   * @return CarbonWriterBuilder
   */
  public CarbonWriterBuilder withCsvInput(Schema schema) {
    Objects.requireNonNull(schema, "schema should not be null");
    this.schema = schema;
    this.writerType = WRITER_TYPE.CSV;
    return this;
  }

  /**
   * to build a {@link CarbonWriter}, which accepts row in CSV format
   *
   * @param schema carbon Schema object {org.apache.carbondata.sdk.file.Schema}
   * @return CarbonWriterBuilder
   */
  public CarbonWriterBuilder withRowFormat(Schema schema) {
    Objects.requireNonNull(schema, "schema should not be null");
    this.schema = schema;
    this.writerType = WRITER_TYPE.ROWFORMAT;
    return this;
  }

  /**
   * to build a {@link CarbonWriter}, which accepts row in CSV format
   *
   * @param jsonSchema json Schema string
   * @return CarbonWriterBuilder
   */
  public CarbonWriterBuilder withCsvInput(String jsonSchema) {
    Objects.requireNonNull(jsonSchema, "schema should not be null");
    this.schema = Schema.parseJson(jsonSchema);
    this.writerType = WRITER_TYPE.CSV;
    return this;
  }

  /**
   * to build a {@link CarbonWriter}, which accepts Avro object
   *
   * @param avroSchema avro Schema object {org.apache.avro.Schema}
   * @return CarbonWriterBuilder
   */
  public CarbonWriterBuilder withAvroInput(org.apache.avro.Schema avroSchema) {
    Objects.requireNonNull(avroSchema, "Avro schema should not be null");
    this.schema = AvroCarbonWriter.getCarbonSchemaFromAvroSchema(avroSchema);
    this.writerType = WRITER_TYPE.AVRO;
    return this;
  }

  /**
   * to build a {@link CarbonWriter}, which accepts Json object
   *
   * @param carbonSchema carbon Schema object
   * @return CarbonWriterBuilder
   */
  public CarbonWriterBuilder withJsonInput(Schema carbonSchema) {
    Objects.requireNonNull(carbonSchema, "schema should not be null");
    this.schema = carbonSchema;
    this.writerType = WRITER_TYPE.JSON;
    return this;
  }

  /**
   * Build a {@link CarbonWriter}
   * This writer is not thread safe,
   * use withThreadSafe() configuration in multi thread environment
   *
   * @return CarbonWriter {AvroCarbonWriter/CSVCarbonWriter/JsonCarbonWriter based on Input Type }
   * @throws IOException
   * @throws InvalidLoadOptionException
   */
  public CarbonWriter build() throws IOException, InvalidLoadOptionException {
    Objects.requireNonNull(path, "path should not be null");
    if (this.writerType == null) {
      throw new IOException(
          "'writerType' must be set, use withCsvInput() or withAvroInput() or withJsonInput()  "
              + "API based on input");
    }
    if (this.writtenByApp == null || this.writtenByApp.isEmpty()) {
      throw new RuntimeException(
          "'writtenBy' must be set when writing carbon files, use writtenBy() API to "
              + "set it, it can be the name of the application which is using the SDK");
    }
    CarbonLoadModel loadModel = buildLoadModel(schema);
    loadModel.setSdkWriterCores(numOfThreads);
    CarbonProperties.getInstance()
        .addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME, writtenByApp);
    if (hadoopConf == null) {
      hadoopConf = FileFactory.getConfiguration();
    }
    if (this.writerType == WRITER_TYPE.AVRO) {
      // AVRO records are pushed to Carbon as Object not as Strings. This was done in order to
      // handle multi level complex type support. As there are no conversion converter step is
      // removed from the load. LoadWithoutConverter flag is going to point to the Loader Builder
      // which will skip Conversion Step.
      loadModel.setLoadWithoutConverterStep(true);
      return new AvroCarbonWriter(loadModel, hadoopConf);
    } else if (this.writerType == WRITER_TYPE.JSON) {
      loadModel.setJsonFileLoad(true);
      return new JsonCarbonWriter(loadModel, hadoopConf);
    } else if (this.writerType == WRITER_TYPE.ROWFORMAT) {
      return new RowFormatCarbonWriter(loadModel, hadoopConf);
    } else {
      // CSV
      return new CSVCarbonWriter(loadModel, hadoopConf);
    }
  }

  private void setCsvHeader(CarbonLoadModel model) {
    Field[] fields = schema.getFields();
    StringBuilder builder = new StringBuilder();
    String[] columns = new String[fields.length];
    int i = 0;
    for (Field field : fields) {
      if (null != field) {
        builder.append(field.getFieldName());
        builder.append(",");
        columns[i++] = field.getFieldName();
      }
    }
    String header = builder.toString();
    model.setCsvHeader(header.substring(0, header.length() - 1));
    model.setCsvHeaderColumns(columns);
  }

  public CarbonLoadModel buildLoadModel(Schema carbonSchema)
      throws IOException, InvalidLoadOptionException {
    timestamp = System.nanoTime();
    // validate long_string_column
    Set<String> longStringColumns = new HashSet<>();
    if (options != null && options.get(CarbonCommonConstants.LONG_STRING_COLUMNS) != null) {
      String[] specifiedLongStrings =
          options.get(CarbonCommonConstants.LONG_STRING_COLUMNS).toLowerCase().split(",");
      for (String str : specifiedLongStrings) {
        longStringColumns.add(str.trim());
      }
      validateLongStringColumns(carbonSchema, longStringColumns);
    }
    // for the longstring field, change the datatype from string to varchar
    this.schema = updateSchemaFields(carbonSchema, longStringColumns);
    if (sortColumns != null && sortColumns.length != 0) {
      if (options == null || options.get("sort_scope") == null) {
        // If sort_columns are specified and sort_scope is not specified,
        // change sort scope to local_sort as now by default sort scope is no_sort.
        if (CarbonProperties.getInstance().getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE)
            == null) {
          if (options == null) {
            options = new HashMap<>();
          }
          options.put("sort_scope", "local_sort");
        }
      }
    }
    // build CarbonTable using schema
    CarbonTable table = buildCarbonTable();
    if (writerType == WRITER_TYPE.ROWFORMAT) {
      table.getTableInfo().getFactTable().getTableProperties().put("streaming", "true");
    }
    // build LoadModel
    return buildLoadModel(table, timestamp, taskNo, options);
  }

  private void validateLongStringColumns(Schema carbonSchema, Set<String> longStringColumns) {
    // long string columns must be string or varchar type
    for (Field field :carbonSchema.getFields()) {
      if (longStringColumns.contains(field.getFieldName().toLowerCase()) && (
          (field.getDataType() != DataTypes.STRING) && field.getDataType() != DataTypes.VARCHAR)) {
        throw new RuntimeException(
            "long string column : " + field.getFieldName() + "is not supported for data type: "
                + field.getDataType());
      }
    }
    // long string columns must not be present in sort columns
    if (sortColumns != null) {
      for (String col : sortColumns) {
        // already will be in lower case
        if (longStringColumns.contains(col)) {
          throw new RuntimeException(
              "long string column : " + col + "must not be present in sort columns");
        }
      }
    }
  }

  /**
   * Build a {@link CarbonTable}
   */
  private CarbonTable buildCarbonTable() {
    TableSchemaBuilder tableSchemaBuilder = TableSchema.builder();
    if (blockSize > 0) {
      tableSchemaBuilder = tableSchemaBuilder.blockSize(blockSize);
    }

    if (blockletSize > 0) {
      tableSchemaBuilder = tableSchemaBuilder.blockletSize(blockletSize);
    }
    tableSchemaBuilder.enableLocalDictionary(isLocalDictionaryEnabled);
    tableSchemaBuilder.localDictionaryThreshold(localDictionaryThreshold);
    List<String> sortColumnsList = new ArrayList<>();
    if (sortColumns == null) {
      // If sort columns are not specified, default set all dimensions to sort column.
      // When dimensions are default set to sort column,
      // Inverted index will be supported by default for sort columns.
      //Null check for field to handle hole in field[] ex.
      //  user passed size 4 but supplied only 2 fileds
      for (Field field : schema.getFields()) {
        if (null != field) {
          if (field.getDataType() == DataTypes.STRING ||
              field.getDataType() == DataTypes.DATE  ||
              field.getDataType() == DataTypes.TIMESTAMP) {
            sortColumnsList.add(field.getFieldName());
          }
        }
      }
      sortColumns = new String[sortColumnsList.size()];
      sortColumns = sortColumnsList.toArray(sortColumns);
    } else {
      sortColumnsList = Arrays.asList(sortColumns);
    }
    List<ColumnSchema> sortColumnsSchemaList = new ArrayList<>(sortColumnsList.size());
    List<String> invertedIdxColumnsList = new ArrayList<>();
    if (null != invertedIndexColumns) {
      invertedIdxColumnsList = Arrays.asList(invertedIndexColumns);
    }
    Field[] fields = schema.getFields();
    buildTableSchema(fields, tableSchemaBuilder, sortColumnsList, sortColumnsSchemaList,
        invertedIdxColumnsList);

    tableSchemaBuilder.setSortColumns(sortColumnsSchemaList);
    String tableName;
    String dbName;
    dbName = "";
    tableName = "_tempTable_" + String.valueOf(timestamp);
    TableSchema schema = tableSchemaBuilder.build();
    schema.setTableName(tableName);
    CarbonTable table =
        CarbonTable.builder().tableName(schema.getTableName()).databaseName(dbName).tablePath(path)
            .tableSchema(schema).isTransactionalTable(false).build();
    return table;
  }

  private void buildTableSchema(Field[] fields, TableSchemaBuilder tableSchemaBuilder,
      List<String> sortColumnsList, List<ColumnSchema> sortColumnsSchemaList,
      List<String> invertedIdxColumnsList) {
    Set<String> uniqueFields = new HashSet<>();
    // a counter which will be used in case of complex array type. This valIndex will be assigned
    // to child of complex array type in the order val1, val2 so that each array type child is
    // differentiated to any level
    AtomicInteger valIndex = new AtomicInteger(0);
    validateColumns(fields, sortColumnsList,
        " specified in sort columns does not exist in schema");

    // Check if any of the columns specified in inverted index are missing from schema.
    validateColumns(fields, invertedIdxColumnsList,
        " specified in inverted index columns does not exist in schema");
    String primaryKeyCols = options.get(CarbonCommonConstants.PRIMARY_KEY_COLUMNS);
    List<String> primaryKeyList = new ArrayList<>();
    if (primaryKeyCols != null) {
      String[] split = primaryKeyCols.split(",");
      for (String s : split) {
        if (StringUtils.isNotEmpty(s.trim())) {
          primaryKeyList.add(s.toLowerCase().trim());
        }
      }
    }
    LinkedHashSet<String> sortSet = new LinkedHashSet<>(primaryKeyList);
    sortSet.addAll(sortColumnsList);
    sortColumnsList = new ArrayList<>(sortSet);
    ColumnSchema[] sortColumnsSchemaArray = new ColumnSchema[sortSet.size()];
    // Check if any of the columns specified in inverted index are missing from schema.
    validateColumns(fields, primaryKeyList,
        " specified in primary key columns does not exist in schema");
    for (Field field : fields) {
      if (null != field) {
        if (!uniqueFields.add(field.getFieldName())) {
          throw new RuntimeException(
              "Duplicate column " + field.getFieldName() + " found in table schema");
        }
        int isInvertedIdxColumn = invertedIdxColumnsList.indexOf(field.getFieldName());
        int isSortColumn = getIsSortColumn(sortColumnsList, field);
        int isPrimaryColumn = getIsSortColumn(primaryKeyList, field);
        if (field.getChildren() != null && field.getChildren().size() > 0) {
          if (field.getDataType().getName().equalsIgnoreCase("ARRAY")) {
            // Loop through the inner columns and for a StructData
            DataType complexType =
                DataTypes.createArrayType(field.getChildren().get(0).getDataType());
            tableSchemaBuilder
                .addColumn(new StructField(field.getFieldName(), complexType), valIndex, false,
                    isInvertedIdxColumn > -1);
          } else if (field.getDataType().getName().equalsIgnoreCase("STRUCT")) {
            // Loop through the inner columns and for a StructData
            List<StructField> structFieldsArray =
                new ArrayList<StructField>(field.getChildren().size());
            for (StructField childFld : field.getChildren()) {
              structFieldsArray
                  .add(new StructField(childFld.getFieldName(), childFld.getDataType()));
            }
            DataType complexType = DataTypes.createStructType(structFieldsArray);
            tableSchemaBuilder
                .addColumn(new StructField(field.getFieldName(), complexType), valIndex, false,
                    isInvertedIdxColumn > -1);
          } else if (field.getDataType().getName().equalsIgnoreCase("MAP")) {
            // Loop through the inner columns for MapType
            DataType mapType = DataTypes.createMapType(((MapType) field.getDataType()).getKeyType(),
                field.getChildren().get(0).getDataType());
            tableSchemaBuilder
                .addColumn(new StructField(field.getFieldName(), mapType), valIndex, false,
                    isInvertedIdxColumn > -1);
          }
        } else {
          ColumnSchema columnSchema = tableSchemaBuilder
              .addColumn(new StructField(field.getFieldName(), field.getDataType()), valIndex,
                  isSortColumn > -1, isInvertedIdxColumn > -1);
          if (isSortColumn > -1) {
            columnSchema.setSortColumn(true);
            sortColumnsSchemaArray[isSortColumn] = columnSchema;
          }
          if (isPrimaryColumn > -1) {
            columnSchema.setPrimaryKeyColumn(true);
          }
        }
      }
    }
    sortColumnsSchemaList.addAll(Arrays.asList(sortColumnsSchemaArray));
  }

  private int getIsSortColumn(List<String> sortColumnsList, Field field) {
    int isSortColumn = sortColumnsList.indexOf(field.getFieldName());
    if (isSortColumn > -1) {
      // unsupported types for ("array", "struct", "double", "float", "decimal")
      if (field.getDataType() == DataTypes.DOUBLE || field.getDataType() == DataTypes.FLOAT
          || DataTypes.isDecimal(field.getDataType()) || field.getDataType().isComplexType()
          || field.getDataType() == DataTypes.VARCHAR) {
        String errorMsg =
            "sort columns not supported for array, struct, map, double, float, decimal,"
                + "varchar";
        throw new RuntimeException(errorMsg);
      }
    }
    return isSortColumn;
  }

  private void validateColumns(Field[] fields, List<String> sortColumnsList, String s) {
    for (String sortColumn : sortColumnsList) {
      boolean exists = false;
      for (Field field : fields) {
        if (field.getFieldName().equalsIgnoreCase(sortColumn)) {
          exists = true;
          break;
        }
      }
      if (!exists) {
        throw new RuntimeException("column: " + sortColumn + s);
      }
    }
  }

  /**
   * Build a {@link CarbonLoadModel}
   */
  private CarbonLoadModel buildLoadModel(CarbonTable table, long timestamp, String taskNo,
      Map<String, String> options) throws InvalidLoadOptionException, IOException {
    if (options == null) {
      options = new HashMap<>();
    }
    CarbonLoadModelBuilder builder = new CarbonLoadModelBuilder(table);
    CarbonLoadModel build = builder.build(options, timestamp, taskNo);
    setCsvHeader(build);
    return build;
  }

  /* loop through all the parent column and
  a) change fields name to lower case.
  this is to match with sort column case.
  b) change string fields to varchar type */
  private Schema updateSchemaFields(Schema schema, Set<String> longStringColumns) {
    if (schema == null) {
      return null;
    }
    Field[] fields =  schema.getFields();
    for (int i = 0; i < fields.length; i++) {
      if (fields[i] != null) {
        if (longStringColumns != null) {
          /* Also update the string type to varchar */
          if (longStringColumns.contains(fields[i].getFieldName())) {
            fields[i].updateDataTypeToVarchar();
          }
        }
      }
    }
    return new Schema(fields);
  }

  private void updateToLoadOptions(Map.Entry<String, String> entry) {
    if (this.options == null) {
      // convert it to treeMap as keys need to be case insensitive
      this.options = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
    }
    // update it to load options
    this.options.put(entry.getKey(), entry.getValue());
  }

  private void withSortScope(Map.Entry<String, String> entry) {
    String sortScope = entry.getValue();
    if (sortScope != null) {
      if ((!CarbonUtil.isValidSortOption(sortScope))) {
        throw new IllegalArgumentException("Invalid Sort Scope Option: " + sortScope);
      } else if (sortScope.equalsIgnoreCase("global_sort")) {
        throw new IllegalArgumentException("global sort is not supported");
      }
    }
    // update it to load options
    updateToLoadOptions(entry);
  }
}


text:
from __future__ import absolute_import, division, print_function, unicode_literals

import sys

import traceback
from pytest import raises, yield_fixture
from algorithm import Algorithm, FunctionNotFound, PYTHON_2
from filesystem_tree import FilesystemTree

# fixtures
# ========

@yield_fixture
def fs():
    fs = FilesystemTree()
    yield fs
    fs.remove()

@yield_fixture
def module_scrubber():
    before = set(sys.modules.keys())
    yield
    after = set(sys.modules.keys())
    for name in after - before:
        del sys.modules[name]

@yield_fixture
def sys_path(fs, module_scrubber):
    sys.path.insert(0, fs.root)
    yield fs
    sys.path = sys.path[1:]

FOO_PY = ('foo.py', '''\
def bar(): return {'val': 1}
def baz(): return {'val': 2}
def buz(): return {'val': 3}
''')

# tests
# =====

def test_Algorithm_can_be_instantiated():
    def foo(): pass
    foo_algorithm = Algorithm(foo)
    assert foo_algorithm.functions == [foo]

def test_Algorithm_can_be_instantiated_with_from_dotted_name(sys_path):
    sys_path.mk(('foo/__init__.py', ''), ('foo/bar.py', 'def baz(): pass'))
    foo_algorithm = Algorithm.from_dotted_name('foo.bar')
    from foo.bar import baz
    assert foo_algorithm.functions == [baz]

def test_Algorithm_cant_be_instantiated_with_a_string():
    actual = raises(TypeError, Algorithm, 'foo.bar').value
    u = 'u' if sys.version_info < (3,) else ''
    assert str(actual) == "Not a function: {0}'foo.bar'".format(u)

def test_Algorithm_includes_imported_functions_and_the_order_is_screwy(sys_path):
    sys_path.mk( ('um.py', 'def um(): pass')
               , ('foo/__init__.py', '')
               , ('foo/bar.py', '''
def baz(): pass
from um import um
def blah(): pass
'''))
    foo_algorithm = Algorithm.from_dotted_name('foo.bar')
    import foo.bar, um
    assert foo_algorithm.functions == [um.um, foo.bar.baz, foo.bar.blah]

def test_Algorithm_ignores_functions_starting_with_underscore(sys_path):
    sys_path.mk( ('um.py', 'def um(): pass')
               , ('foo/__init__.py', '')
               , ('foo/bar.py', '''
def baz(): pass
from um import um as _um
def blah(): pass
'''))
    foo_algorithm = Algorithm.from_dotted_name('foo.bar')
    import foo.bar
    assert foo_algorithm.functions == [foo.bar.baz, foo.bar.blah]

def test_can_run_through_algorithm(sys_path):
    sys_path.mk(FOO_PY)
    foo_algorithm = Algorithm.from_dotted_name('foo')
    state = foo_algorithm.run(val=None)
    assert state == {'val': 3, 'exception': None, 'state': state, 'algorithm': foo_algorithm}

def test_can_stop_algorithm_after_a_certain_point(sys_path):
    sys_path.mk(FOO_PY)
    foo_algorithm = Algorithm.from_dotted_name('foo')
    state = foo_algorithm.run(val=None, _return_after='baz')
    assert state == {'val': 2, 'exception': None, 'state': state, 'algorithm': foo_algorithm}

def test_error_raised_if_we_try_to_return_after_an_unknown_function(sys_path):
    sys_path.mk(FOO_PY)
    foo_algorithm = Algorithm.from_dotted_name('foo')
    raises(FunctionNotFound, foo_algorithm.run, val=None, _return_after='blaaaaaah')

def test_inserted_algorithm_steps_run(sys_path):
    sys_path.mk(FOO_PY)
    foo_algorithm = Algorithm.from_dotted_name('foo')

    def biz(): return {'val': 4}

    foo_algorithm.insert_after('buz', biz)
    state = foo_algorithm.run(val=None)

    assert state == {'val': 4, 'exception': None, 'state': state, 'algorithm':foo_algorithm}

# Exception Handling
# ==================

EXCEPT = ('foo.py', '''

    def bar():
        raise heck

    def baz():
        return {'val': 42}

    def clear(exception):
        return {'val': 666, 'exception': None}

''')

def test_exception_fast_forwards(sys_path):
    sys_path.mk(EXCEPT)
    foo_algorithm = Algorithm.from_dotted_name('foo')
    state = foo_algorithm.run()
    assert state == {'val': 666, 'exception': None, 'state': state, 'algorithm': foo_algorithm}

def test_exception_raises_if_uncleared(sys_path):
    sys_path.mk(EXCEPT)
    foo_algorithm = Algorithm.from_dotted_name('foo')
    foo_algorithm.remove('clear')
    raises(NameError, foo_algorithm.run)

def test_traceback_for_uncleared_exception_reaches_back_to_original_raise(sys_path):
    sys_path.mk(EXCEPT)
    foo_algorithm = Algorithm.from_dotted_name('foo')
    foo_algorithm.remove('clear')
    try:
        foo_algorithm.run()
    except:
        tb = traceback.format_exc()

    # We get an extra frame under Python 3, but what we don't want is not
    # enough frames.

    assert len(tb.splitlines()) == (8 if PYTHON_2 else 10)

def test_function_can_have_default_value_for_exception_to_be_always_called(sys_path):
    sys_path.mk(EXCEPT)
    foo_algorithm = Algorithm.from_dotted_name('foo')

    # Add a both-handling function.
    def both(exception=None):
        return {'exception': None, 'um': 'yeah'}
    foo_algorithm.insert_before('clear', both)

    # Exception case.
    assert foo_algorithm.run()['um'] == 'yeah'

    # Non-exception case.
    foo_algorithm.remove('bar')
    assert foo_algorithm.run()['um'] == 'yeah'

def test_exception_raises_immediately_if_told_to_via_constructor(sys_path):
    sys_path.mk(EXCEPT)
    foo_algorithm = Algorithm.from_dotted_name('foo', raise_immediately=True)
    foo_algorithm.remove('clear')
    raises(NameError, foo_algorithm.run)

def test_exception_raises_immediately_if_told_to_via_run_call(sys_path):
    sys_path.mk(EXCEPT)
    foo_algorithm = Algorithm.from_dotted_name('foo')
    foo_algorithm.remove('clear')
    raises(NameError, foo_algorithm.run, _raise_immediately=True)

def test_per_call_trumps_constructor_for_raise_immediately(sys_path):
    sys_path.mk(EXCEPT)
    foo_algorithm = Algorithm.from_dotted_name('foo', raise_immediately=True)
    foo_algorithm.remove('clear')
    raises(NameError, foo_algorithm.run, _raise_immediately=False)


text:
/**
 * Internal helper functions common to parsing byte arrays of any type
 */

/**
 * Reads a string of 8-bit characters from an array of bytes and advances
 * the position by length bytes.  A null terminator will end the string
 * but will not affect advancement of the position.  Trailing and leading
 * spaces are preserved (not trimmed)
 * @param byteArray the byteArray to read from
 * @param position the position in the byte array to read from
 * @param length the maximum number of bytes to parse
 * @returns {string} the parsed string
 * @throws error if buffer overread would occur
 * @access private
 */
export function readFixedString (byteArray, position, length) {
  if (length < 0) {
    throw 'dicomParser.readFixedString - length cannot be less than 0';
  }

  if (position + length > byteArray.length) {
    throw 'dicomParser.readFixedString: attempt to read past end of buffer';
  }

  var result = '';
  var byte;

  for (var i = 0; i < length; i++) {
    byte = byteArray[position + i];
    if (byte === 0) {
      position += length;

      return result;
    }
    result += String.fromCharCode(byte);
  }

  return result;
}


text:
package odesafio;

public class NumeroMuitoAltoException extends RuntimeException {

    public String getMensagem() {
    return "Esse número é muito alto";
    }

}


text:
'use strict';

var glslify = require('glslify');
var Config = require('./../config');

var landscapeFunc = function () {
    var self = this;

    var geometry = new THREE.PlaneGeometry(200, 200, 30, 30);

    var vertexShader = glslify('./shaders/landscape.vert');
    var fragmentShader = glslify('./shaders/landscape.frag');

    this.customUniforms =
    {
        mixAmount: {type: "f", value: Config.mixAmount},
        landscapeAnimMix: {type: "f", value: 1},
        aspectRatio: {type: "f", value: Config.aspectRatio},
        landscape: {type: "t", value: Config.rtTexture},
        landscapeOld: {type: "t", value: Config.rtTextureOld},
        myLightPos: {
            type: "v3",
            value: new THREE.Vector3(0.5, 0.2, 1.0)
        }
    };

    var material = new THREE.ShaderMaterial(
        {
            uniforms: self.customUniforms,
            vertexShader: vertexShader,
            fragmentShader: fragmentShader,
            side: THREE.DoubleSide
            // transparent: true,
            // lights: true
        });

    var mesh = new THREE.Mesh(geometry, material);

    this.changeLandscape = function () {
        self.customUniforms.landscapeAnimMix.value = 0;
    };
    this.update = function () {
        self.customUniforms.mixAmount.value = Config.mixAmount;
        self.customUniforms.aspectRatio.value = Config.aspectRatio;
        self.customUniforms.landscapeAnimMix.value = (Config.time / 1000 - Config.changeLandscapeStartTime) / Config.changeLandscapeLength < 1 ?
        (Config.time / 1000 - Config.changeLandscapeStartTime) / Config.changeLandscapeLength : 1;
    };

    return {mesh: mesh, update: self.update, changeLandscape: self.changeLandscape}
};

module.exports = landscapeFunc;


text:
require 'spec_helper'

describe Gitlab::LegacyGithubImport::ReleaseFormatter do
  let!(:project) { create(:project, namespace: create(:namespace, path: 'octocat')) }
  let(:octocat) { double(id: 123456, login: 'octocat') }
  let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') }
  let(:published_at) { DateTime.strptime('2011-01-26T20:00:00Z') }

  let(:base_data) do
    {
      tag_name: 'v1.0.0',
      name: 'First release',
      draft: false,
      created_at: created_at,
      published_at: published_at,
      body: 'Release v1.0.0'
    }
  end

  subject(:release) { described_class.new(project, raw_data) }

  describe '#attributes' do
    let(:raw_data) { double(base_data) }

    it 'returns formatted attributes' do
      expected = {
        project: project,
        tag: 'v1.0.0',
        name: 'First release',
        description: 'Release v1.0.0',
        created_at: created_at,
        released_at: published_at,
        updated_at: created_at
      }

      expect(release.attributes).to eq(expected)
    end

    context 'with a nil published_at date' do
      let(:published_at) { nil }

      it 'inserts a timestamp for released_at' do
        expect(release.attributes[:released_at]).to be_a(Time)
      end
    end
  end

  describe '#valid' do
    context 'when release is not a draft' do
      let(:raw_data) { double(base_data) }

      it 'returns true' do
        expect(release.valid?).to eq true
      end
    end

    context 'when release is draft' do
      let(:raw_data) { double(base_data.merge(draft: true)) }

      it 'returns false' do
        expect(release.valid?).to eq false
      end
    end
  end
end


text:
@interface LianViewController : UIViewController

@end


text:
' Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

Imports System.Collections.Generic
Imports System.Collections.Immutable
Imports System.Runtime.InteropServices
Imports System.Text
Imports System.Threading
Imports Microsoft.CodeAnalysis.Text
Imports Microsoft.CodeAnalysis.VisualBasic.Symbols
Imports Microsoft.CodeAnalysis.VisualBasic.Syntax
Imports TypeKind = Microsoft.CodeAnalysis.TypeKind

Namespace Microsoft.CodeAnalysis.VisualBasic.Symbols
    Friend Class ErrorTypeSymbol
        Inherits NamedTypeSymbol
        Implements IErrorTypeSymbol

        Friend Shared ReadOnly UnknownResultType As ErrorTypeSymbol = New ErrorTypeSymbol()

        ''' <summary>
        ''' Returns information about the reason that this type is in error.
        ''' </summary>
        Friend Overridable ReadOnly Property ErrorInfo As DiagnosticInfo
            Get
                Return Nothing
            End Get
        End Property

        Friend Overrides Function GetUseSiteErrorInfo() As DiagnosticInfo
            If Me.IsDefinition Then
                Return Me.ErrorInfo
            End If

            ' Base class handles constructed types.
            Return MyBase.GetUseSiteErrorInfo()
        End Function

        Friend Overrides Function GetUnificationUseSiteDiagnosticRecursive(owner As Symbol, ByRef checkedTypes As HashSet(Of TypeSymbol)) As DiagnosticInfo
            Return Nothing
        End Function

        Friend Overrides Function MakeDeclaredBase(basesBeingResolved As ConsList(Of Symbol), diagnostics As DiagnosticBag) As NamedTypeSymbol
            Return Nothing
        End Function

        Friend Overrides Function MakeDeclaredInterfaces(basesBeingResolved As ConsList(Of Symbol), diagnostics As DiagnosticBag) As ImmutableArray(Of NamedTypeSymbol)
            Return ImmutableArray(Of NamedTypeSymbol).Empty
        End Function

        Friend Overrides Function MakeAcyclicBaseType(diagnostics As DiagnosticBag) As NamedTypeSymbol
            Return Nothing
        End Function

        Friend Overrides Function MakeAcyclicInterfaces(diagnostics As DiagnosticBag) As ImmutableArray(Of NamedTypeSymbol)
            Return ImmutableArray(Of NamedTypeSymbol).Empty
        End Function

        Public Overrides ReadOnly Property IsReferenceType As Boolean
            Get
                ' TODO: Consider returning False.
                Return True
            End Get
        End Property

        Public Overrides ReadOnly Property IsValueType As Boolean
            Get
                Return False
            End Get
        End Property

        Public Overrides ReadOnly Property MemberNames As IEnumerable(Of String)
            Get
                Return SpecializedCollections.EmptyEnumerable(Of String)()
            End Get
        End Property

        Public Overrides Function GetMembers() As ImmutableArray(Of Symbol)
            Return ImmutableArray(Of Symbol).Empty
        End Function

        Public Overrides Function GetMembers(name As String) As ImmutableArray(Of Symbol)
            Return ImmutableArray(Of Symbol).Empty
        End Function

        Public Overrides Function GetTypeMembers() As ImmutableArray(Of NamedTypeSymbol)
            Return ImmutableArray(Of NamedTypeSymbol).Empty
        End Function

        Public Overrides Function GetTypeMembers(name As String) As ImmutableArray(Of NamedTypeSymbol)
            Return ImmutableArray(Of NamedTypeSymbol).Empty
        End Function

        Public Overrides Function GetTypeMembers(name As String, arity As Integer) As ImmutableArray(Of NamedTypeSymbol)
            Return ImmutableArray(Of NamedTypeSymbol).Empty
        End Function

        Friend Overrides Function GetFieldsToEmit() As IEnumerable(Of FieldSymbol)
            Return SpecializedCollections.EmptyEnumerable(Of FieldSymbol)()
        End Function

        Public NotOverridable Overrides ReadOnly Property Kind As SymbolKind
            Get
                Return SymbolKind.ErrorType
            End Get
        End Property

        Public NotOverridable Overrides ReadOnly Property TypeKind As TypeKind
            Get
                Return TypeKind.Error
            End Get
        End Property

        Friend NotOverridable Overrides ReadOnly Property IsInterface As Boolean
            Get
                Return False
            End Get
        End Property

        Public Overrides ReadOnly Property ContainingSymbol As Symbol
            Get
                Return Nothing
            End Get
        End Property

        Public Overrides ReadOnly Property Locations As ImmutableArray(Of Location)
            Get
                Return ImmutableArray(Of Location).Empty
            End Get
        End Property

        Public Overrides ReadOnly Property DeclaringSyntaxReferences As ImmutableArray(Of SyntaxReference)
            Get
                Return ImmutableArray(Of SyntaxReference).Empty
            End Get
        End Property

        Public Overrides ReadOnly Property Arity As Integer
            Get
                Return 0
            End Get
        End Property

        Public Overrides ReadOnly Property Name As String
            Get
                Return String.Empty
            End Get
        End Property

        Friend Overrides ReadOnly Property MangleName As Boolean
            Get
                Debug.Assert(Arity = 0)
                Return False
            End Get
        End Property

        Friend NotOverridable Overrides ReadOnly Property HasSpecialName As Boolean
            Get
                Return False
            End Get
        End Property

        Friend NotOverridable Overrides ReadOnly Property IsSerializable As Boolean
            Get
                Return False
            End Get
        End Property

        Friend Overrides ReadOnly Property Layout As TypeLayout
            Get
                Return Nothing
            End Get
        End Property

        Friend Overrides ReadOnly Property MarshallingCharSet As CharSet
            Get
                Return DefaultMarshallingCharSet
            End Get
        End Property

        Friend Overrides ReadOnly Property TypeArgumentsNoUseSiteDiagnostics As ImmutableArray(Of TypeSymbol)
            Get
                Return ImmutableArray(Of TypeSymbol).Empty
            End Get
        End Property

        Friend Overrides ReadOnly Property HasTypeArgumentsCustomModifiers As Boolean
            Get
                Return False
            End Get
        End Property

        Friend Overrides ReadOnly Property TypeArgumentsCustomModifiers As ImmutableArray(Of ImmutableArray(Of CustomModifier))
            Get
                Return ImmutableArray(Of ImmutableArray(Of CustomModifier)).Empty
            End Get
        End Property

        Public Overrides ReadOnly Property TypeParameters As ImmutableArray(Of TypeParameterSymbol)
            Get
                Return ImmutableArray(Of TypeParameterSymbol).Empty
            End Get
        End Property

        Public Overrides ReadOnly Property ConstructedFrom As NamedTypeSymbol
            Get
                Return Me
            End Get
        End Property

        Friend Overrides Function Accept(Of TArgument, TResult)(visitor As VisualBasicSymbolVisitor(Of TArgument, TResult), arg As TArgument) As TResult
            Return visitor.VisitErrorType(Me, arg)
        End Function

        ' Only compiler creates an error symbol.
        Friend Sub New()
        End Sub

        Public NotOverridable Overrides ReadOnly Property DeclaredAccessibility As Accessibility
            Get
                Return Accessibility.Public
            End Get
        End Property

        Public NotOverridable Overrides ReadOnly Property IsMustInherit As Boolean
            Get
                Return False
            End Get
        End Property

        Public NotOverridable Overrides ReadOnly Property IsNotInheritable As Boolean
            Get
                Return False
            End Get
        End Property

        Public NotOverridable Overrides ReadOnly Property MightContainExtensionMethods As Boolean
            Get
                Return False
            End Get
        End Property

        Friend Overrides ReadOnly Property HasEmbeddedAttribute As Boolean
            Get
                Return False
            End Get
        End Property

        Friend Overrides ReadOnly Property IsExtensibleInterfaceNoUseSiteDiagnostics As Boolean
            Get
                Return False
            End Get
        End Property

        Friend Overrides ReadOnly Property IsWindowsRuntimeImport As Boolean
            Get
                Return False
            End Get
        End Property

        Friend Overrides ReadOnly Property ShouldAddWinRTMembers As Boolean
            Get
                Return False
            End Get
        End Property

        Friend Overrides ReadOnly Property IsComImport As Boolean
            Get
                Return False
            End Get
        End Property

        Friend Overrides ReadOnly Property CoClassType As TypeSymbol
            Get
                Return Nothing
            End Get
        End Property

        Friend Overrides Function GetAppliedConditionalSymbols() As ImmutableArray(Of String)
            Return ImmutableArray(Of String).Empty
        End Function

        Friend Overrides Function GetAttributeUsageInfo() As AttributeUsageInfo
            Throw ExceptionUtilities.Unreachable
        End Function

        Friend Overrides ReadOnly Property HasDeclarativeSecurity As Boolean
            Get
                Return False
            End Get
        End Property

        Friend Overrides Function GetSecurityInformation() As IEnumerable(Of Microsoft.Cci.SecurityAttribute)
            Throw ExceptionUtilities.Unreachable
        End Function

        ''' <summary>
        ''' Substitute the given type substitution within this type, returning a new type. If the
        ''' substitution had no effect, return Me.
        ''' !!! Only code implementing construction of generic types is allowed to call this method !!!
        ''' !!! All other code should use Construct methods.                                        !!!
        ''' </summary>
        Friend Overrides Function InternalSubstituteTypeParameters(substitution As TypeSubstitution) As TypeWithModifiers
            Return New TypeWithModifiers(Me)
        End Function

        Friend Overrides ReadOnly Property TypeSubstitution As TypeSubstitution
            Get
                Return Nothing
            End Get
        End Property

        Friend Overrides ReadOnly Property CanConstruct As Boolean
            Get
                Return False
            End Get
        End Property

        Public Overrides Function Construct(typeArguments As ImmutableArray(Of TypeSymbol)) As NamedTypeSymbol
            ' An error symbol has no type parameters, so construction isn't allowed.
            ' TDOD: Remove temporary workaround from constructor of ArrayTypeSymbol.
            Throw New InvalidOperationException()
        End Function

        Friend Overrides ReadOnly Property DefaultPropertyName As String
            Get
                Return Nothing
            End Get
        End Property

        ''' <summary>
        ''' If we believe we know which symbol the user intended, then we should retain that information
        ''' in the corresponding error symbol - it can be useful for deciding how to handle the error.
        ''' </summary>
        Friend ReadOnly Property NonErrorGuessType As NamedTypeSymbol
            Get
                Dim candidates = Me.CandidateSymbols
                If candidates.Length = 1 Then  ' Only return a guess if its unambiguous.
                    Return TryCast(candidates(0), NamedTypeSymbol)
                Else
                    Return Nothing
                End If
            End Get
        End Property

        ''' <summary>
        ''' Return why the candidate symbols were bad.
        ''' </summary>
        Friend Overridable ReadOnly Property ResultKind As LookupResultKind
            Get
                Return LookupResultKind.Empty
            End Get
        End Property

        Friend NotOverridable Overrides ReadOnly Property ObsoleteAttributeData As ObsoleteAttributeData
            Get
                Return Nothing
            End Get
        End Property

        ''' <summary>
        ''' When constructing this ErrorTypeSymbol, there may have been symbols that seemed to
        ''' be what the user intended, but were unsuitable. For example, a type might have been
        ''' inaccessible, or ambiguous. This property returns the possible symbols that the user
        ''' might have intended. It will return no symbols if no possible symbols were found.
        ''' See the CandidateReason property to understand why the symbols were unsuitable.
        ''' </summary>
        Public Overridable ReadOnly Property CandidateSymbols As ImmutableArray(Of Symbol)
            Get
                Return ImmutableArray(Of Symbol).Empty
            End Get
        End Property

        Public ReadOnly Property CandidateReason As CandidateReason Implements IErrorTypeSymbol.CandidateReason
            Get
                If CandidateSymbols.IsEmpty Then
                    Return Microsoft.CodeAnalysis.CandidateReason.None
                Else
                    Debug.Assert(ResultKind <> LookupResultKind.Good, "Shouldn't have good result kind on error symbol")
                    Return ResultKind.ToCandidateReason()
                End If
            End Get
        End Property

        Public Overrides Function Equals(obj As Object) As Boolean
            Return Me Is obj
        End Function

        Public Overrides Function GetHashCode() As Integer
            Return System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(Me)
        End Function

        ''' <summary>
        ''' Force all declaration errors to be generated.
        ''' </summary>
        Friend Overrides Sub GenerateDeclarationErrors(cancellationToken As CancellationToken)
            Throw ExceptionUtilities.Unreachable
        End Sub

#Region "IErrorTypeSymbol members"

        Public ReadOnly Property IErrorTypeSymbol_CandidateSymbols As ImmutableArray(Of ISymbol) Implements IErrorTypeSymbol.CandidateSymbols
            Get
                Return StaticCast(Of ISymbol).From(Me.CandidateSymbols)
            End Get
        End Property

#End Region

    End Class
End Namespace


text:
<?php
namespace Czim\PxlCms\Generator\Writer\Repository\Steps;

/**
 * Should always be executed last, since only then do we know
 * what to include for the imports.
 */
class StubReplaceImportsAndTraits extends AbstractProcessStep
{

    protected function process()
    {
        $this->stubPregReplace('# *{{USETRAITS}}\n?#i', "\n")
             ->stubPregReplace('#{{USEIMPORTS}}\n?#i', $this->getImportsReplace());
    }

    /**
     * Returns the replacement for the use use-imports placeholder
     *
     * @return string
     */
    protected function getImportsReplace()
    {

        // build up import lines
        $importLines = [
            config('pxlcms.generator.repositories.extend_class')
        ];

        // model classname
        $importLines[] = $this->context->makeFqnForModelName( studly_case($this->data->name) );;

        // set them in the right order
        if (config('pxlcms.generator.aesthetics.sort_imports_by_string_length')) {

            // sort from shortest to longest
            usort($importLines, function ($a, $b) {
                return strlen($a) - strlen($b);
            });

        } else {
            sort($importLines);
        }

        // build the actual replacement string
        $replace = "\n";

        foreach ($importLines as $line) {
            $replace .= "use " . $line . ";\n";
        }

        $replace .= "\n";

        return $replace;
    }

}


text:
using System;
using System.Collections.Generic;
using System.Web.Http.Routing;
using Semver;

namespace jnonce.WebApi.VersionedRouting
{
    /// <summary>
    /// Limit constrained route to particular Api versions
    /// </summary>
    [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method, Inherited = false, AllowMultiple = true)]
    public sealed class ApiVersionAttribute : Attribute, IHttpRouteConstraintProvider
    {
        /// <summary>
        /// Initializes a new instance of the <see cref="ApiVersionAttribute"/>
        /// </summary>
        /// <param name="version">The version this api supports.</param>
        public ApiVersionAttribute(string version)
        {
            this.MinVersion = SemVersion.Parse(version);
            this.MaxVersion = SemVersion.Parse(version);
        }

        /// <summary>
        /// Initializes a new instance of the <see cref="ApiVersionAttribute" />
        /// </summary>
        /// <param name="minVersion">The minimum version.</param>
        /// <param name="maxVersion">The maximum version.</param>
        public ApiVersionAttribute(string minVersion, string maxVersion)
        {
            this.MinVersion = SemVersion.Parse(minVersion);
            this.MaxVersion = SemVersion.Parse(maxVersion);
        }

        /// <summary>
        /// Gets the minimum version for the Api.
        /// </summary>
         public SemVersion MinVersion { get; private set; }

        /// <summary>
        /// Gets the maximum version for the Api.
        /// </summary>
         public SemVersion MaxVersion { get; private set; }

         /// <summary>
         /// Gets the constraints.
         /// </summary>
         /// <returns></returns>
        public IEnumerable<KeyValuePair<string, IHttpRouteConstraint>> GetConstraints()
        {
            var c = new ApiVersionRouteConstraint(
                givenVersion => (givenVersion >= MinVersion) && (givenVersion <= MaxVersion));

            yield return new KeyValuePair<string, IHttpRouteConstraint>(ApiVersionRouteConstraint.ConstraintKey, c);
        }
    }
}


text:
namespace open_vcdiff {
namespace {

// Provides an address_stream_ buffer and functions to manually encode
// values into the buffer, and to manually decode and verify test results
// from the buffer.
//
class VCDiffAddressCacheTest : public testing::Test {
 public:
  typedef std::string string;

  VCDiffAddressCacheTest() : decode_position_(NULL),
                             decode_position_end_(NULL),
                             verify_encode_position_(NULL),
                             last_encode_size_(0),
                             last_decode_position_(NULL) { }

  virtual ~VCDiffAddressCacheTest() { }

  virtual void SetUp() {
    EXPECT_TRUE(cache_.Init());
  }

  // Benchmarks for timing encode/decode operations
  void BM_Setup(int test_size);
  void BM_CacheEncode(int iterations, int test_size);
  void BM_CacheDecode(int iterations, int test_size);

 protected:
  virtual void TestBody() { }  // to allow instantiation of this class

  void BeginDecode() {
    decode_position_ = address_stream_.data();
    EXPECT_TRUE(decode_position_ != NULL);
    last_decode_position_ = decode_position_;
    decode_position_end_ = decode_position_ + address_stream_.size();
  }

  void ExpectEncodedSizeInBytes(int n) {
    EXPECT_EQ(last_encode_size_ + n, address_stream_.size());
    last_encode_size_ = address_stream_.size();
  }

  void ExpectDecodedSizeInBytes(int n) {
    EXPECT_EQ(last_decode_position_ + n, decode_position_);
    last_decode_position_ = decode_position_;
  }

  void ManualEncodeVarint(VCDAddress value) {
    VarintBE<VCDAddress>::AppendToString(value, &address_stream_);
  }

  void ManualEncodeByte(unsigned char byte) {
    address_stream_.push_back(byte);
  }

  void ExpectEncodedVarint(VCDAddress expected_value, int expected_size) {
    if (!verify_encode_position_) {
      verify_encode_position_ = address_stream_.data();
    }
    EXPECT_EQ(expected_size, VarintBE<VCDAddress>::Length(expected_value));
    VCDAddress output_val = VarintBE<VCDAddress>::Parse(
        address_stream_.data() + address_stream_.size(),
        &verify_encode_position_);
    EXPECT_EQ(expected_value, output_val);
  }

  void ExpectEncodedByte(unsigned char expected_value) {
    if (!verify_encode_position_) {
      verify_encode_position_ = address_stream_.data();
    }
    EXPECT_EQ(expected_value, *verify_encode_position_);
    ++verify_encode_position_;
  }

  void TestEncode(VCDAddress address,
                  VCDAddress here_address,
                  unsigned char mode,
                  int size) {
    VCDAddress encoded_addr = 0;
    EXPECT_EQ(mode, cache_.EncodeAddress(address, here_address, &encoded_addr));
    if (cache_.WriteAddressAsVarintForMode(mode)) {
      ManualEncodeVarint(encoded_addr);
    } else {
      EXPECT_GT(256, encoded_addr);
      ManualEncodeByte(static_cast<unsigned char>(encoded_addr));
    }
    ExpectEncodedSizeInBytes(size);
  }

  VCDiffAddressCache cache_;
  string address_stream_;
  const char* decode_position_;
  const char* decode_position_end_;
  string large_address_stream_;
  std::vector<unsigned char> mode_stream_;
  std::vector<VCDAddress> verify_stream_;

 private:
  const char* verify_encode_position_;
  string::size_type last_encode_size_;
  const char*       last_decode_position_;
};

#ifdef GTEST_HAS_DEATH_TEST
// This synonym is needed for the tests that use ASSERT_DEATH
typedef VCDiffAddressCacheTest VCDiffAddressCacheDeathTest;
#endif  // GTEST_HAS_DEATH_TEST

// Having either or both cache size == 0 is acceptable
TEST_F(VCDiffAddressCacheTest, ZeroCacheSizes) {
  VCDiffAddressCache zero_cache(0, 0);
  EXPECT_TRUE(zero_cache.Init());
}

// VCD_MAX_MODES is the maximum number of modes, including SAME and HERE modes.
// So neither the SAME cache nor the HERE cache can be larger than
// (VCD_MAX_MODES - 2).
TEST_F(VCDiffAddressCacheTest, NearCacheSizeIsTooBig) {
  VCDiffAddressCache cache(VCD_MAX_MODES - 1, 0);
  EXPECT_FALSE(cache.Init());
}

TEST_F(VCDiffAddressCacheTest, SameCacheSizeIsTooBig) {
  VCDiffAddressCache cache(0, VCD_MAX_MODES - 1);
  EXPECT_FALSE(cache.Init());
}

TEST_F(VCDiffAddressCacheTest, CombinedSizesAreTooBig) {
  VCDiffAddressCache cache1((VCD_MAX_MODES / 2), (VCD_MAX_MODES / 2) - 1);
  EXPECT_FALSE(cache1.Init());
  VCDiffAddressCache cache2(VCD_MAX_MODES - 1, VCD_MAX_MODES - 1);
  EXPECT_FALSE(cache2.Init());
}

TEST_F(VCDiffAddressCacheTest, MaxLegalNearCacheSize) {
  VCDiffAddressCache cache(VCD_MAX_MODES - 2, 0);
  EXPECT_TRUE(cache.Init());
}

TEST_F(VCDiffAddressCacheTest, MaxLegalSameCacheSize) {
  VCDiffAddressCache cache(0, VCD_MAX_MODES - 2);
  EXPECT_TRUE(cache.Init());
}

TEST_F(VCDiffAddressCacheTest, MaxLegalCombinedSizes) {
  VCDiffAddressCache cache((VCD_MAX_MODES / 2) - 1, (VCD_MAX_MODES / 2) - 1);
  EXPECT_TRUE(cache.Init());
}

TEST_F(VCDiffAddressCacheTest, DestroyWithoutInitialization) {
  VCDiffAddressCache no_init_cache(4, 3);
  // Should be destroyed without crashing
}

TEST_F(VCDiffAddressCacheTest, DestroyDefaultWithoutInitialization) {
  VCDiffAddressCache no_init_cache;
  // Should be destroyed without crashing
}

TEST_F(VCDiffAddressCacheTest, CacheContentsInitiallyZero) {
  VCDAddress test_address = 0;
  // Check that caches are initially set to zero
  for (test_address = 0; test_address < 4; ++test_address) {
    EXPECT_EQ(0, cache_.NearAddress(test_address));
  }
  for (test_address = 0; test_address < 256 * 3; ++test_address) {
    EXPECT_EQ(0, cache_.SameAddress(test_address));
  }
}

// Inserts values 1, 2, ... , 10 into the cache and tests its entire
// contents for consistency.
//
TEST_F(VCDiffAddressCacheTest, InsertFirstTen) {
  VCDAddress test_address = 0;
  for (test_address = 1; test_address <= 10; ++test_address) {
    cache_.UpdateCache(test_address);
  }
  EXPECT_EQ(9, cache_.NearAddress(0));   // slot 0: 1 => 5 => 9
  EXPECT_EQ(10, cache_.NearAddress(1));  // slot 1: 2 => 6 => 10
  EXPECT_EQ(7, cache_.NearAddress(2));   // slot 2: 3 => 7
  EXPECT_EQ(8, cache_.NearAddress(3));   // slot 3: 4 => 8
  EXPECT_EQ(0, cache_.SameAddress(0));
  for (test_address = 1; test_address <= 10; ++test_address) {
    EXPECT_EQ(test_address, cache_.SameAddress(test_address));
  }
  for (test_address = 11; test_address < 256 * 3; ++test_address) {
    EXPECT_EQ(0, cache_.SameAddress(test_address));
  }
}

TEST_F(VCDiffAddressCacheTest, InsertIntMax) {
  cache_.UpdateCache(INT_MAX);
  EXPECT_EQ(INT_MAX, cache_.NearAddress(0));
  EXPECT_EQ(INT_MAX, cache_.SameAddress(INT_MAX % (256 * 3)));
  EXPECT_EQ(0, cache_.SameAddress((INT_MAX - 256) % (256 * 3)));
  EXPECT_EQ(0, cache_.SameAddress((INT_MAX - 512) % (256 * 3)));
}

// Exercises all four addressing mode types by encoding five values
// with EncodeAddress.
// Checks to see that the proper mode was selected in each case,
// and that the encoding is correct.
//
TEST_F(VCDiffAddressCacheTest, EncodeAddressModes) {
  TestEncode(0x0000FFFF, 0x10000000,  VCD_SELF_MODE, 3);
  TestEncode(0x10000000, 0x10000010,  VCD_HERE_MODE, 1);
  TestEncode(0x10000004, 0x10000020,  cache_.FirstNearMode() + 0x01, 1);
  TestEncode(0x0FFFFFFE, 0x10000030,  VCD_HERE_MODE, 1);
  TestEncode(0x10000004, 0x10000040,  cache_.FirstSameMode() + 0x01, 1);
  ExpectEncodedVarint(0xFFFF, 3);  // SELF mode: addr 0x0000FFFF
  ExpectEncodedVarint(0x10, 1);    // HERE mode: here - 0x10 = 0x10000000
  ExpectEncodedVarint(0x04, 1);    // NEAR cache #1:
                                   // last addr + 0x4 = 0x10000004
  ExpectEncodedVarint(0x32, 1);    // HERE mode: here - 0x32 = 0x0FFFFFFE
  ExpectEncodedByte(0x04);         // SAME cache #1: 0x10000004 hits
}

// Exercises all four addressing mode types by manually encoding six values
// and calling DecodeAddress on each one.
//
TEST_F(VCDiffAddressCacheTest, DecodeAddressModes) {
  ManualEncodeVarint(0xCAFE);
  ManualEncodeVarint(0xCAFE);
  ManualEncodeVarint(0x1000);
  ManualEncodeByte(0xFE);   // SAME mode uses a byte, not a Varint
  ManualEncodeVarint(0xFE);
  ManualEncodeVarint(0x1000);
  BeginDecode();
  EXPECT_EQ(0xCAFE,
            cache_.DecodeAddress(0x10000,
                                 VCD_SELF_MODE,
                                 &decode_position_,
                                 decode_position_end_));
  ExpectDecodedSizeInBytes(VarintBE<VCDAddress>::Length(0xCAFE));
  EXPECT_EQ(0x20000 - 0xCAFE,
            cache_.DecodeAddress(0x20000,
                                 VCD_HERE_MODE,
                                 &decode_position_,
                                 decode_position_end_));
  ExpectDecodedSizeInBytes(VarintBE<VCDAddress>::Length(0xCAFE));
  EXPECT_EQ(0xDAFE,
            cache_.DecodeAddress(0x30000,
                                 cache_.FirstNearMode(),
                                 &decode_position_,
                                 decode_position_end_));
  ExpectDecodedSizeInBytes(VarintBE<VCDAddress>::Length(0x1000));
  EXPECT_EQ(0xCAFE,
            cache_.DecodeAddress(0x40000,
                                 cache_.FirstSameMode() + (0xCA % 3),
                                 &decode_position_,
                                 decode_position_end_));
  ExpectDecodedSizeInBytes(sizeof(unsigned char));  // a byte, not a Varint
  EXPECT_EQ(0xFE,
            cache_.DecodeAddress(0x50000,
                                 VCD_SELF_MODE,
                                 &decode_position_,
                                 decode_position_end_));
  ExpectDecodedSizeInBytes(VarintBE<VCDAddress>::Length(0xFE));
  // NEAR mode #0 has been overwritten by fifth computed addr (wrap around)
  EXPECT_EQ(0x10FE,
            cache_.DecodeAddress(0x60000,
                                 cache_.FirstNearMode(),
                                 &decode_position_,
                                 decode_position_end_));
  ExpectDecodedSizeInBytes(VarintBE<VCDAddress>::Length(0x1000));
}

// Test with both cache sizes == 0.  The encoder should not choose
// a SAME or NEAR mode under these conditions.
TEST_F(VCDiffAddressCacheTest, EncodeAddressZeroCacheSizes) {
  VCDAddress encoded_addr = 0;
  VCDiffAddressCache zero_cache(0, 0);
  EXPECT_TRUE(zero_cache.Init());
  EXPECT_EQ(VCD_SELF_MODE,
            zero_cache.EncodeAddress(0x0000FFFF, 0x10000000, &encoded_addr));
  EXPECT_EQ(0xFFFF, encoded_addr);
  EXPECT_EQ(VCD_HERE_MODE,
            zero_cache.EncodeAddress(0x10000000, 0x10000010, &encoded_addr));
  EXPECT_EQ(0x10, encoded_addr);
  EXPECT_EQ(VCD_HERE_MODE,
            zero_cache.EncodeAddress(0x10000004, 0x10000020, &encoded_addr));
  EXPECT_EQ(0x1C, encoded_addr);
  EXPECT_EQ(VCD_HERE_MODE,
            zero_cache.EncodeAddress(0x0FFFFFFE, 0x10000030, &encoded_addr));
  EXPECT_EQ(0x32, encoded_addr);
  EXPECT_EQ(VCD_HERE_MODE,
            zero_cache.EncodeAddress(0x10000004, 0x10000040, &encoded_addr));
  EXPECT_EQ(0x3C, encoded_addr);
}

TEST_F(VCDiffAddressCacheTest, DecodeAddressZeroCacheSizes) {
  VCDiffAddressCache zero_cache(0, 0);
  EXPECT_TRUE(zero_cache.Init());
  ManualEncodeVarint(0xCAFE);
  ManualEncodeVarint(0xCAFE);
  ManualEncodeVarint(0xDAFE);
  BeginDecode();
  EXPECT_EQ(0xCAFE, zero_cache.DecodeAddress(0x10000,
                                             VCD_SELF_MODE,
                                             &decode_position_,
                                             decode_position_end_));
  ExpectDecodedSizeInBytes(VarintBE<VCDAddress>::Length(0xCAFE));
  EXPECT_EQ(0x20000 - 0xCAFE, zero_cache.DecodeAddress(0x20000,
                                                       VCD_HERE_MODE,
                                                       &decode_position_,
                                                       decode_position_end_));
  ExpectDecodedSizeInBytes(VarintBE<VCDAddress>::Length(0xCAFE));
  EXPECT_EQ(0xDAFE, zero_cache.DecodeAddress(0x30000,
                                             VCD_SELF_MODE,
                                             &decode_position_,
                                             decode_position_end_));
  ExpectDecodedSizeInBytes(VarintBE<VCDAddress>::Length(0xDAFE));
}

#ifdef GTEST_HAS_DEATH_TEST
TEST_F(VCDiffAddressCacheDeathTest, EncodeNegativeAddress) {
  VCDAddress dummy_encoded_address = 0;
  EXPECT_DEBUG_DEATH(cache_.EncodeAddress(-1, -1, &dummy_encoded_address),
                     "negative");
}

TEST_F(VCDiffAddressCacheDeathTest, EncodeAddressPastHereAddress) {
  VCDAddress dummy_encoded_address = 0;
  EXPECT_DEBUG_DEATH(cache_.EncodeAddress(0x100, 0x100, &dummy_encoded_address),
                     "address.*<.*here_address");
  EXPECT_DEBUG_DEATH(cache_.EncodeAddress(0x200, 0x100, &dummy_encoded_address),
                     "address.*<.*here_address");
}

TEST_F(VCDiffAddressCacheDeathTest, DecodeInvalidMode) {
  ManualEncodeVarint(0xCAFE);
  BeginDecode();
  EXPECT_DEBUG_DEATH(EXPECT_EQ(RESULT_ERROR,
                               cache_.DecodeAddress(0x10000000,
                                                    cache_.LastMode() + 1,
                                                    &decode_position_,
                                                    decode_position_end_)),
                     "mode");
  EXPECT_DEBUG_DEATH(EXPECT_EQ(RESULT_ERROR,
                               cache_.DecodeAddress(0x10000000,
                                                    0xFF,
                                                    &decode_position_,
                                                    decode_position_end_)),
                     "mode");
  ExpectDecodedSizeInBytes(0);  // Should not modify decode_position_
}

TEST_F(VCDiffAddressCacheDeathTest, DecodeZeroOrNegativeHereAddress) {
  ManualEncodeVarint(0xCAFE);
  ManualEncodeVarint(0xCAFE);
  BeginDecode();
  // Using a Debug build, the check will fail; using a Release build,
  // the check will not occur, and the SELF mode does not depend on
  // the value of here_address, so DecodeAddress() will succeed.
  EXPECT_DEBUG_DEATH(cache_.DecodeAddress(-1,
                                          VCD_SELF_MODE,
                                          &decode_position_,
                                          decode_position_end_),
                     "negative");
  // A zero value for here_address should not kill the decoder,
  // but instead should return an error value.  A delta file may contain
  // a window that has no source segment and that (erroneously)
  // uses a COPY instruction as its first instruction.  This should
  // cause an error to be reported, not a debug check failure.
  EXPECT_EQ(RESULT_ERROR, cache_.DecodeAddress(0,
                                         VCD_SELF_MODE,
                                         &decode_position_,
                                         decode_position_end_));
}
#endif  // GTEST_HAS_DEATH_TEST

TEST_F(VCDiffAddressCacheTest, DecodeAddressPastHereAddress) {
  ManualEncodeVarint(0xCAFE);
  BeginDecode();
  EXPECT_EQ(RESULT_ERROR, cache_.DecodeAddress(0x1000,
                                         VCD_SELF_MODE,
                                         &decode_position_,
                                         decode_position_end_));
  ExpectDecodedSizeInBytes(0);  // Should not modify decode_position_
}

TEST_F(VCDiffAddressCacheTest, HereModeAddressTooLarge) {
  ManualEncodeVarint(0x10001);  // here_address + 1
  BeginDecode();
  EXPECT_EQ(RESULT_ERROR, cache_.DecodeAddress(0x10000,
                                         VCD_HERE_MODE,
                                         &decode_position_,
                                         decode_position_end_));
  ExpectDecodedSizeInBytes(0);  // Should not modify decode_position_
}

TEST_F(VCDiffAddressCacheTest, NearModeAddressOverflow) {
  ManualEncodeVarint(0xCAFE);
  ManualEncodeVarint(0x7FFFFFFF);
  BeginDecode();
  EXPECT_EQ(0xCAFE, cache_.DecodeAddress(0x10000,
                                         VCD_SELF_MODE,
                                         &decode_position_,
                                         decode_position_end_));
  ExpectDecodedSizeInBytes(VarintBE<VCDAddress>::Length(0xCAFE));
  // Now decode a NEAR mode address of base address 0xCAFE
  // (the first decoded address) + offset 0x7FFFFFFF.  This will cause
  // an integer overflow and should signal an error.
  EXPECT_EQ(RESULT_ERROR, cache_.DecodeAddress(0x10000000,
                                         cache_.FirstNearMode(),
                                         &decode_position_,
                                         decode_position_end_));
  ExpectDecodedSizeInBytes(0);  // Should not modify decode_position_
}

// A Varint should contain at most 9 bytes that have their continuation bit
// (the uppermost, or 7 bit) set.  A longer string of bytes that all have
// bit 7 set is not a valid Varint.  Try to parse such a string as a Varint
// and confirm that it does not run off the end of the input buffer and
// it returns an error value (RESULT_ERROR).
//
TEST_F(VCDiffAddressCacheTest, DecodeInvalidVarint) {
  address_stream_.clear();
  // Write 512 0xFE bytes
  address_stream_.append(512, static_cast<char>(0xFE));
  BeginDecode();
  EXPECT_EQ(RESULT_ERROR, cache_.DecodeAddress(0x10000000,
                                         VCD_SELF_MODE,
                                         &decode_position_,
                                         decode_position_end_));
  ExpectDecodedSizeInBytes(0);  // Should not modify decode_position_
}

// If only part of a Varint appears in the data to be decoded,
// then DecodeAddress should return RESULT_END_OF_DATA,
// which means that the Varint *may* be valid if there is more
// data expected to be returned.
//
TEST_F(VCDiffAddressCacheTest, DecodePartialVarint) {
  address_stream_.clear();
  ManualEncodeByte(0xFE);
  ManualEncodeByte(0xFE);
  ManualEncodeByte(0xFE);
  BeginDecode();
  EXPECT_EQ(RESULT_END_OF_DATA,
            cache_.DecodeAddress(0x10000000,
                                 VCD_SELF_MODE,
                                 &decode_position_,
                                 decode_position_end_));
  ExpectDecodedSizeInBytes(0);  // Should not modify decode_position_
  // Now add the missing last byte (supposedly read from a stream of data)
  // and verify that the Varint is now valid.
  ManualEncodeByte(0x01);  // End the Varint with an additional byte
  BeginDecode();  // Reset read position to start of data
  EXPECT_EQ(0xFDFBF01,
            cache_.DecodeAddress(0x10000000,
                                 VCD_SELF_MODE,
                                 &decode_position_,
                                 decode_position_end_));
  ExpectDecodedSizeInBytes(4);  // ManualEncodeByte was called for 4 byte values
}

#ifdef GTEST_HAS_DEATH_TEST
TEST_F(VCDiffAddressCacheDeathTest, DecodeBadMode) {
  ManualEncodeVarint(0xCAFE);
  BeginDecode();
  EXPECT_DEBUG_DEATH(EXPECT_EQ(RESULT_ERROR,
                               cache_.DecodeAddress(0x10000,
                                                    cache_.LastMode() + 1,
                                                    &decode_position_,
                                                    decode_position_end_)),
                     "maximum");
  ExpectDecodedSizeInBytes(0);
}
#endif  // GTEST_HAS_DEATH_TEST

TEST_F(VCDiffAddressCacheTest, DecodeInvalidHereAddress) {
  ManualEncodeVarint(0x10001);  // offset larger than here_address
  BeginDecode();
  EXPECT_EQ(RESULT_ERROR, cache_.DecodeAddress(0x10000,
                                         VCD_HERE_MODE,
                                         &decode_position_,
                                         decode_position_end_));
  ExpectDecodedSizeInBytes(0);
}

TEST_F(VCDiffAddressCacheTest, DecodeInvalidNearAddress) {
  ManualEncodeVarint(0xCAFE);
  ManualEncodeVarint(INT_MAX);  // offset will cause integer overflow
  BeginDecode();
  EXPECT_EQ(0xCAFE,
            cache_.DecodeAddress(0x10000,
                                 VCD_SELF_MODE,
                                 &decode_position_,
                                 decode_position_end_));
  ExpectDecodedSizeInBytes(VarintBE<VCDAddress>::Length(0xCAFE));
  EXPECT_EQ(RESULT_ERROR, cache_.DecodeAddress(0x10000,
                                         cache_.FirstNearMode(),
                                         &decode_position_,
                                         decode_position_end_));
  ExpectDecodedSizeInBytes(0);
}

void VCDiffAddressCacheTest::BM_Setup(int test_size) {
  mode_stream_.resize(test_size);
  verify_stream_.resize(test_size);
  VCDAddress here_address = 1;
  srand(1);
  for (int i = 0; i < test_size; ++i) {
    verify_stream_[i] = PortableRandomInRange(here_address - 1);
    here_address += 4;
  }
  BM_CacheEncode(1, test_size);  // populate large_address_stream_, mode_stream_
}

void VCDiffAddressCacheTest::BM_CacheEncode(int iterations, int test_size) {
  VCDAddress here_address = 1;
  VCDAddress encoded_addr = 0;
  for (int test_iteration = 0; test_iteration < iterations; ++test_iteration) {
    cache_.Init();
    large_address_stream_.clear();
    here_address = 1;
    for (int i = 0; i < test_size; ++i) {
      const unsigned char mode = cache_.EncodeAddress(verify_stream_[i],
                                                      here_address,
                                                      &encoded_addr);
      if (cache_.WriteAddressAsVarintForMode(mode)) {
        VarintBE<VCDAddress>::AppendToString(encoded_addr,
                                             &large_address_stream_);
      } else {
        EXPECT_GT(256, encoded_addr);
        large_address_stream_.push_back(
            static_cast<unsigned char>(encoded_addr));
      }
      mode_stream_[i] = mode;
      here_address += 4;
    }
  }
}

void VCDiffAddressCacheTest::BM_CacheDecode(int iterations, int test_size) {
  VCDAddress here_address = 1;
  for (int test_iteration = 0; test_iteration < iterations; ++test_iteration) {
    cache_.Init();
    const char* large_decode_pointer = large_address_stream_.data();
    const char* const end_of_encoded_data =
        large_decode_pointer + large_address_stream_.size();
    here_address = 1;
    for (int i = 0; i < test_size; ++i) {
      EXPECT_EQ(verify_stream_[i],
                cache_.DecodeAddress(here_address,
                                     mode_stream_[i],
                                     &large_decode_pointer,
                                     end_of_encoded_data));
      here_address += 4;
    }
    EXPECT_EQ(end_of_encoded_data, large_decode_pointer);
  }
}

TEST_F(VCDiffAddressCacheTest, PerformanceTest) {
  const int test_size = 20 * 1024;  // 20K random encode/decode operations
  const int num_iterations = 40;  // run test 40 times and take average
  BM_Setup(test_size);
  {
    CycleTimer encode_timer;
    encode_timer.Start();
    BM_CacheEncode(num_iterations, test_size);
    encode_timer.Stop();
    double encode_time_in_ms =
        static_cast<double>(encode_timer.GetInUsec()) / 1000;
    std::cout << "Time to encode: "
              << (encode_time_in_ms / num_iterations) << " ms" << std::endl;
  }
  {
    CycleTimer decode_timer;
    decode_timer.Start();
    BM_CacheDecode(num_iterations, test_size);
    decode_timer.Stop();
    double decode_time_in_ms =
        static_cast<double>(decode_timer.GetInUsec()) / 1000;
    std::cout << "Time to decode: "
              << (decode_time_in_ms / num_iterations) << " ms" << std::endl;
  }
}

}  // unnamed namespace
}  // namespace open_vcdiff