text:
package org.apache.hadoop.hive.ql.optimizer;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Stack;

import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FilterOperator;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorFactory;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.RowSchema;
import org.apache.hadoop.hive.ql.exec.SelectOperator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.spark.SparkUtilities;
import org.apache.hadoop.hive.ql.io.AcidUtils.Operation;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.SemanticNodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.spark.CombineEquivalentWorkResolver;
import org.apache.hadoop.hive.ql.optimizer.spark.SparkPartitionPruningSinkDesc;
import org.apache.hadoop.hive.ql.parse.GenTezUtils;
import org.apache.hadoop.hive.ql.parse.GenTezUtils.DynamicListContext;
import org.apache.hadoop.hive.ql.parse.GenTezUtils.DynamicPartitionPrunerContext;
import org.apache.hadoop.hive.ql.parse.OptimizeTezProcContext;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
import org.apache.hadoop.hive.ql.parse.RuntimeValuesInfo;
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.SemiJoinBranchInfo;
import org.apache.hadoop.hive.ql.parse.SemiJoinHint;
import org.apache.hadoop.hive.ql.parse.spark.OptimizeSparkProcContext;
import org.apache.hadoop.hive.ql.parse.spark.SparkPartitionPruningSinkOperator;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc;
import org.apache.hadoop.hive.ql.plan.DynamicValue;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.ExprNodeDynamicValueDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.FilterDesc;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
import org.apache.hadoop.hive.ql.plan.SelectDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFBloomFilter.GenericUDAFBloomFilterEvaluator;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
import org.apache.hadoop.hive.ql.util.NullOrdering;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Preconditions;

/**
 * This optimization looks for expressions of the kind "x IN (RS[n])". If such
 * an expression made it to a table scan operator and x is a partition column we
 * can use an existing join to dynamically prune partitions. This class sets up
 * the infrastructure for that.
 */
public class DynamicPartitionPruningOptimization implements SemanticNodeProcessor {

  static final private Logger LOG = LoggerFactory.getLogger(DynamicPartitionPruningOptimization.class
      .getName());

  @Override
  public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs)
      throws SemanticException {
    ParseContext parseContext;
    if (procCtx instanceof OptimizeTezProcContext) {
      parseContext = ((OptimizeTezProcContext) procCtx).parseContext;
    } else if (procCtx instanceof OptimizeSparkProcContext) {
      parseContext = ((OptimizeSparkProcContext) procCtx).getParseContext();
    } else {
      throw new IllegalArgumentException("expected parseContext to be either " +
          "OptimizeTezProcContext or OptimizeSparkProcContext, but found " +
          procCtx.getClass().getName());
    }

    FilterOperator filter = (FilterOperator) nd;
    FilterDesc desc = filter.getConf();

    if (!parseContext.getConf().getBoolVar(ConfVars.TEZ_DYNAMIC_PARTITION_PRUNING) &&
        !parseContext.getConf().isSparkDPPAny()) {
      // nothing to do when the optimization is off
      return null;
    }

    TableScanOperator ts = null;

    if (filter.getParentOperators().size() == 1
        && filter.getParentOperators().get(0) instanceof TableScanOperator) {
      ts = (TableScanOperator) filter.getParentOperators().get(0);
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("Parent: " + filter.getParentOperators().get(0));
      LOG.debug("Filter: " + desc.getPredicateString());
      LOG.debug("TableScan: " + ts);
    }

    DynamicPartitionPrunerContext removerContext = new DynamicPartitionPrunerContext();

    // collect the dynamic pruning conditions
    removerContext.dynLists.clear();
    GenTezUtils.collectDynamicPruningConditions(desc.getPredicate(), removerContext);

    if (ts == null) {
      // Replace the synthetic predicate with true and bail out
      for (DynamicListContext ctx : removerContext) {
        ExprNodeDesc constNode =
                new ExprNodeConstantDesc(ctx.parent.getTypeInfo(), true);
        replaceExprNode(ctx, desc, constNode);
      }
      return false;
    }

    boolean semiJoin = parseContext.getConf().getBoolVar(ConfVars.TEZ_DYNAMIC_SEMIJOIN_REDUCTION);
    if (HiveConf.getVar(parseContext.getConf(), HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
      //TODO HIVE-16862: Implement a similar feature like "hive.tez.dynamic.semijoin.reduction" in hive on spark
      semiJoin = false;
    }

    List<ExprNodeDesc> newBetweenNodes = new ArrayList<>();
    List<ExprNodeDesc> newBloomFilterNodes = new ArrayList<>();
    for (DynamicListContext ctx : removerContext) {
      String column = ExprNodeDescUtils.extractColName(ctx.parent);
      boolean semiJoinAttempted = false;

      ExprNodeDesc constNode =
          new ExprNodeConstantDesc(ctx.parent.getTypeInfo(), true);
      if (column != null) {
        // Need unique IDs to refer to each min/max key value in the DynamicValueRegistry
        String keyBaseAlias = "";

        Table table = ts.getConf().getTableMetadata();

        boolean nonEquiJoin = isNonEquiJoin(ctx.parent);
        if (table != null && table.isPartitionKey(column) && !nonEquiJoin) {
          String columnType = table.getPartColByName(column).getType();
          String alias = ts.getConf().getAlias();
          PrunedPartitionList plist = parseContext.getPrunedPartitions(alias, ts);
          if (LOG.isDebugEnabled()) {
            LOG.debug("alias: " + alias);
            LOG.debug("pruned partition list: ");
            if (plist != null) {
              for (Partition p : plist.getPartitions()) {
                LOG.debug(p.getCompleteName());
              }
            }
          }
          // If partKey is a constant, we can check whether the partitions
          // have been already filtered
          if (plist == null || plist.getPartitions().size() != 0) {
            LOG.info("Dynamic partitioning: " + table.getCompleteName() + "." + column);
            generateEventOperatorPlan(ctx, parseContext, ts, column, columnType);
          } else {
            // all partitions have been statically removed
            LOG.debug("No partition pruning necessary.");
          }
        } else if (table.isNonNative() &&
          table.getStorageHandler().addDynamicSplitPruningEdge(desc.getPredicate())){
          generateEventOperatorPlan(ctx, parseContext, ts, column,
            table.getCols().stream().filter(e -> e.getName().equals(column)).
          map(e -> e.getType()).findFirst().get());
        } else { // semijoin
          LOG.debug("Column " + column + " is not a partition column");
          if (semiJoin && !disableSemiJoinOptDueToExternalTable(parseContext.getConf(), ts, ctx)
                  && ts.getConf().getFilterExpr() != null && !nonEquiJoin) {
            LOG.debug("Initiate semijoin reduction for " + column + " ("
                + ts.getConf().getFilterExpr().getExprString());

            StringBuilder internalColNameBuilder = new StringBuilder();
            StringBuilder colNameBuilder = new StringBuilder();

            // Apply best effort to fetch the correct table alias. If not
            // found, fallback to old logic.
            StringBuilder tabAliasBuilder = new StringBuilder();
            if (getColumnInfo(ctx, internalColNameBuilder, colNameBuilder, tabAliasBuilder)) {
              String colName = colNameBuilder.toString();
              String tableAlias;
              if (tabAliasBuilder.length() > 0) {
                tableAlias = tabAliasBuilder.toString();
              } else {
                //falling back
                Operator<?> op = ctx.generator;

                while (!(op == null || op instanceof TableScanOperator)) {
                  op = op.getParentOperators().get(0);
                }
                tableAlias = (op == null ? "" : ((TableScanOperator) op).
                        getConf().getAlias());
              }

              // Use the tableAlias to generate keyBaseAlias
              keyBaseAlias = ctx.generator.getOperatorId() + "_" + tableAlias
                      + "_" + colName;
              Map<String, List<SemiJoinHint>> hints = parseContext.getSemiJoinHints();
              if (hints != null) {
                // Create semijoin optimizations ONLY for hinted columns
                semiJoinAttempted = processSemiJoinHints(
                        parseContext, ctx, hints, tableAlias,
                        internalColNameBuilder.toString(), colName, ts,
                        keyBaseAlias);
              } else {
                // fallback to regular logic
                semiJoinAttempted = generateSemiJoinOperatorPlan(
                        ctx, parseContext, ts, keyBaseAlias,
                        internalColNameBuilder.toString(), colName, null);
              }
            }
          }
        }

        // If semijoin is attempted then replace the condition with a min-max filter
        // and bloom filter else,
        // we always remove the condition by replacing it with "true"
        if (semiJoinAttempted) {
          List<ExprNodeDesc> betweenArgs = new ArrayList<ExprNodeDesc>();
          betweenArgs.add(new ExprNodeConstantDesc(Boolean.FALSE)); // Do not invert between result
          // add column expression here
          betweenArgs.add(ctx.parent.getChildren().get(0));
          betweenArgs.add(new ExprNodeDynamicValueDesc(new DynamicValue(keyBaseAlias + "_min", ctx.desc.getTypeInfo())));
          betweenArgs.add(new ExprNodeDynamicValueDesc(new DynamicValue(keyBaseAlias + "_max", ctx.desc.getTypeInfo())));
          ExprNodeDesc betweenNode = ExprNodeGenericFuncDesc.newInstance(
                  FunctionRegistry.getFunctionInfo("between").getGenericUDF(), betweenArgs);
          // add column expression for bloom filter
          List<ExprNodeDesc> bloomFilterArgs = new ArrayList<ExprNodeDesc>();
          bloomFilterArgs.add(ctx.parent.getChildren().get(0));
          bloomFilterArgs.add(new ExprNodeDynamicValueDesc(
                  new DynamicValue(keyBaseAlias + "_bloom_filter",
                          TypeInfoFactory.binaryTypeInfo)));
          ExprNodeDesc bloomFilterNode = ExprNodeGenericFuncDesc.newInstance(
                  FunctionRegistry.getFunctionInfo("in_bloom_filter").
                          getGenericUDF(), bloomFilterArgs);
          newBetweenNodes.add(betweenNode);
          newBloomFilterNodes.add(bloomFilterNode);
        }
      }
      replaceExprNode(ctx, desc, constNode);
    }

    if (!newBetweenNodes.isEmpty()) {
      // We need to add the new nodes: first the between nodes, then the bloom filters
      if (FunctionRegistry.isOpAnd(desc.getPredicate())) { // AND
        desc.getPredicate().getChildren().addAll(newBetweenNodes);
        desc.getPredicate().getChildren().addAll(newBloomFilterNodes);
      } else {
        List<ExprNodeDesc> andArgs = new ArrayList<>();
        andArgs.add(desc.getPredicate());
        andArgs.addAll(newBetweenNodes);
        andArgs.addAll(newBloomFilterNodes);
        ExprNodeGenericFuncDesc andExpr = ExprNodeGenericFuncDesc.newInstance(
            FunctionRegistry.getFunctionInfo("and").getGenericUDF(), andArgs);
        // Also pass in filter as tableScan filterExpr
        ts.getConf().setFilterExpr(andExpr);
        desc.setPredicate(andExpr);
      }
    }

    // if we pushed the predicate into the table scan we need to remove the
    // synthetic conditions there.
    cleanTableScanFilters(ts);

    return false;
  }

  private boolean disableSemiJoinOptDueToExternalTable(HiveConf conf, TableScanOperator ts, DynamicListContext ctx) {
    boolean disableSemiJoin = false;
    if (conf.getBoolVar(HiveConf.ConfVars.HIVE_DISABLE_UNSAFE_EXTERNALTABLE_OPERATIONS)) {
      // We already have the TableScan for one side of the join. Check this now.
      if (MetaStoreUtils.isExternalTable(ts.getConf().getTableMetadata().getTTable())) {
        LOG.debug("Disabling semijoin optimzation on {} since it is an external table.",
            ts.getConf().getTableMetadata().getFullyQualifiedName());
        disableSemiJoin = true;
      } else {
        // Check the other side of the join, using the DynamicListContext
        ExprNodeDesc exprNodeDesc = ctx.getKeyCol();
        ExprNodeColumnDesc colExpr = ExprNodeDescUtils.getColumnExpr(exprNodeDesc);

        if (colExpr != null) {
          // fetch table alias
          ExprNodeDescUtils.ColumnOrigin columnOrigin =
                  ExprNodeDescUtils.findColumnOrigin(exprNodeDesc, ctx.generator);
          if (columnOrigin != null && columnOrigin.op instanceof TableScanOperator) {
            // Join key origin has been traced to a table column. Check if the table is external.
            TableScanOperator joinKeyTs = (TableScanOperator) columnOrigin.op;
            if (MetaStoreUtils.isExternalTable(joinKeyTs.getConf().getTableMetadata().getTTable())) {
              LOG.debug("Join key {} is from {} which is an external table. Disabling semijoin optimization.",
                  columnOrigin.col,
                  joinKeyTs.getConf().getTableMetadata().getFullyQualifiedName());
              disableSemiJoin = true;
            }
          }
        }
      }
    }
    return disableSemiJoin;
  }

  // Given a key, find the corresponding column name.
  private boolean getColumnInfo(DynamicListContext ctx, StringBuilder internalColName,
                                StringBuilder colName, StringBuilder tabAlias) {
    ExprNodeDesc exprNodeDesc = ctx.getKeyCol();
    ExprNodeColumnDesc colExpr = ExprNodeDescUtils.getColumnExpr(exprNodeDesc);

    if (colExpr == null) {
      return false;
    }
    internalColName.append(colExpr.getColumn());

    // fetch table ablias
    ExprNodeDescUtils.ColumnOrigin columnOrigin =
            ExprNodeDescUtils.findColumnOrigin(exprNodeDesc, ctx.generator);

    if (columnOrigin != null) {
      // get both tableAlias and column name from columnOrigin
      assert columnOrigin.op instanceof TableScanOperator;
      TableScanOperator ts = (TableScanOperator) columnOrigin.op;
      tabAlias.append(ts.getConf().getAlias());
      colName.append(
              ExprNodeDescUtils.getColumnExpr(columnOrigin.col).getColumn());
      return true;
    }

    Operator<? extends OperatorDesc> parentOfRS = ctx.generator.getParentOperators().get(0);
    if (!(parentOfRS instanceof SelectOperator)) {
      colName.append(internalColName.toString());
      return true;
    }

    exprNodeDesc = parentOfRS.getColumnExprMap().get(internalColName.toString());
    colExpr = ExprNodeDescUtils.getColumnExpr(exprNodeDesc);

    if (colExpr == null) {
      return false;
    }

    colName.append(ExprNodeDescUtils.extractColName(colExpr));
    return true;
  }

  // Handle hint based semijoin
  private boolean processSemiJoinHints(
          ParseContext pCtx, DynamicListContext ctx,
          Map<String, List<SemiJoinHint>> hints, String tableAlias,
          String internalColName, String colName, TableScanOperator ts,
          String keyBaseAlias) throws SemanticException {
    if (hints.size() == 0) {
      return false;
    }

    List<SemiJoinHint> hintList = hints.get(tableAlias);
    if (hintList == null) {
      return false;
    }

    // Iterate through the list
    for (SemiJoinHint sjHint : hintList) {
      if (!colName.equals(sjHint.getColName())) {
        continue;
      }
      if (!ts.getConf().getAlias().equals(sjHint.getTarget())) {
        continue;
      }

      // match!
      LOG.info("Creating runtime filter due to user hint: column = " + colName);
      if (generateSemiJoinOperatorPlan(ctx, pCtx, ts, keyBaseAlias,
              internalColName, colName, sjHint)) {
        return true;
      }
      throw new SemanticException("The user hint to enforce semijoin failed required conditions");
    }
    return false;
  }

  private void replaceExprNode(DynamicListContext ctx, FilterDesc desc, ExprNodeDesc node) {
    if (ctx.grandParent == null) {
      desc.setPredicate(node);
    } else {
      int i = ctx.grandParent.getChildren().indexOf(ctx.parent);
      ctx.grandParent.getChildren().remove(i);
      ctx.grandParent.getChildren().add(i, node);
    }
  }

  private void cleanTableScanFilters(TableScanOperator ts) throws SemanticException {

    if (ts == null || ts.getConf() == null || ts.getConf().getFilterExpr() == null) {
      // nothing to do
      return;
    }

    DynamicPartitionPrunerContext removerContext = new DynamicPartitionPrunerContext();

    // collect the dynamic pruning conditions
    removerContext.dynLists.clear();
    GenTezUtils.collectDynamicPruningConditions(ts.getConf().getFilterExpr(), removerContext);

    for (DynamicListContext ctx : removerContext) {
      // remove the condition by replacing it with "true"
      ExprNodeDesc constNode = new ExprNodeConstantDesc(ctx.parent.getTypeInfo(), true);
      if (ctx.grandParent == null) {
        // we're the only node, just clear out the expression
        ts.getConf().setFilterExpr(null);
      } else {
        int i = ctx.grandParent.getChildren().indexOf(ctx.parent);
        ctx.grandParent.getChildren().remove(i);
        ctx.grandParent.getChildren().add(i, constNode);
      }
    }
  }

  private boolean isNonEquiJoin(ExprNodeDesc predicate)  {
    Preconditions.checkArgument(predicate instanceof ExprNodeGenericFuncDesc);

    ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) predicate;
    if (funcDesc.getGenericUDF() instanceof GenericUDFIn) {
      return false;
    }

    return true;
  }

  private void generateEventOperatorPlan(DynamicListContext ctx, ParseContext parseContext,
      TableScanOperator ts, String column, String columnType) {

    // we will put a fork in the plan at the source of the reduce sink
    Operator<? extends OperatorDesc> parentOfRS = ctx.generator.getParentOperators().get(0);

    // we need the expr that generated the key of the reduce sink
    ExprNodeDesc key = ctx.getKeyCol();

    // we also need the expr for the partitioned table
    ExprNodeDesc partKey = ctx.parent.getChildren().get(0);

    if (LOG.isDebugEnabled()) {
      LOG.debug("key expr: " + key);
      LOG.debug("partition key expr: " + partKey);
    }

    List<ExprNodeDesc> keyExprs = new ArrayList<ExprNodeDesc>();
    keyExprs.add(key);

    // group by requires "ArrayList", don't ask.
    ArrayList<String> outputNames = new ArrayList<String>();
    outputNames.add(HiveConf.getColumnInternalName(0));

    ArrayList<ColumnInfo> selectColInfos = new ArrayList<ColumnInfo>();
    selectColInfos.add(new ColumnInfo(outputNames.get(0), key.getTypeInfo(), "", false));

    // project the relevant key column
    SelectDesc select = new SelectDesc(keyExprs, outputNames);
    SelectOperator selectOp =
        (SelectOperator) OperatorFactory.getAndMakeChild(select, new RowSchema(selectColInfos), parentOfRS);
    Map<String, ExprNodeDesc> selectColumnExprMap = new HashMap<>();
    selectColumnExprMap.put(outputNames.get(0), key);
    selectOp.setColumnExprMap(selectColumnExprMap);

    // do a group by on the list to dedup
    float groupByMemoryUsage =
        HiveConf.getFloatVar(parseContext.getConf(), HiveConf.ConfVars.HIVEMAPAGGRHASHMEMORY);
    float memoryThreshold =
        HiveConf.getFloatVar(parseContext.getConf(),
            HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD);
    float minReductionHashAggr =
        HiveConf.getFloatVar(parseContext.getConf(),
            ConfVars.HIVEMAPAGGRHASHMINREDUCTION);

    ArrayList<ExprNodeDesc> groupByExprs = new ArrayList<ExprNodeDesc>();
    ExprNodeDesc groupByExpr =
        new ExprNodeColumnDesc(key.getTypeInfo(), outputNames.get(0), null, false);
    groupByExprs.add(groupByExpr);

    GroupByDesc groupBy =
        new GroupByDesc(GroupByDesc.Mode.HASH, outputNames, groupByExprs,
            new ArrayList<AggregationDesc>(), false, groupByMemoryUsage, memoryThreshold,
            minReductionHashAggr, null, false, -1, true);

    ArrayList<ColumnInfo> groupbyColInfos = new ArrayList<ColumnInfo>();
    groupbyColInfos.add(new ColumnInfo(outputNames.get(0), key.getTypeInfo(), "", false));

    GroupByOperator groupByOp = (GroupByOperator) OperatorFactory.getAndMakeChild(
        groupBy, new RowSchema(groupbyColInfos), selectOp);

    Map<String, ExprNodeDesc> colMap = new HashMap<String, ExprNodeDesc>();
    colMap.put(outputNames.get(0), groupByExpr);
    groupByOp.setColumnExprMap(colMap);

    // finally add the event broadcast operator
    if (HiveConf.getVar(parseContext.getConf(),
        ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
      DynamicPruningEventDesc eventDesc = new DynamicPruningEventDesc();
      eventDesc.setTableScan(ts);
      eventDesc.setGenerator(ctx.generator);
      eventDesc.setTable(PlanUtils.getReduceValueTableDesc(PlanUtils
          .getFieldSchemasFromColumnList(keyExprs, "key")));
      eventDesc.setTargetColumnName(column);
      eventDesc.setTargetColumnType(columnType);
      eventDesc.setPartKey(partKey);
      OperatorFactory.getAndMakeChild(eventDesc, groupByOp);
    } else {
      // Must be spark branch
      SparkPartitionPruningSinkDesc desc = new SparkPartitionPruningSinkDesc();
      desc.setTable(PlanUtils.getReduceValueTableDesc(PlanUtils
          .getFieldSchemasFromColumnList(keyExprs, "key")));
      desc.addTarget(column, columnType, partKey, null, ts);
      SparkPartitionPruningSinkOperator dppSink = (SparkPartitionPruningSinkOperator)
          OperatorFactory.getAndMakeChild(desc, groupByOp);
      if (HiveConf.getBoolVar(parseContext.getConf(),
          ConfVars.HIVE_COMBINE_EQUIVALENT_WORK_OPTIMIZATION)) {
        mayReuseExistingDPPSink(parentOfRS, Arrays.asList(selectOp, groupByOp, dppSink));
      }
    }
  }

  private void mayReuseExistingDPPSink(Operator<? extends OperatorDesc> branchingOP,
      List<Operator<? extends OperatorDesc>> newDPPBranch) {
    SparkPartitionPruningSinkOperator reusableDPP = SparkUtilities.findReusableDPPSink(branchingOP,
        newDPPBranch);
    if (reusableDPP != null) {
      CombineEquivalentWorkResolver.combineEquivalentDPPSinks(reusableDPP,
          (SparkPartitionPruningSinkOperator) newDPPBranch.get(newDPPBranch.size() - 1));
      branchingOP.removeChild(newDPPBranch.get(0));
    }
  }

  // Generates plan for min/max when dynamic partition pruning is ruled out.
  private boolean generateSemiJoinOperatorPlan(DynamicListContext ctx, ParseContext parseContext,
      TableScanOperator ts, String keyBaseAlias, String internalColName,
      String colName, SemiJoinHint sjHint) throws SemanticException {

    // we will put a fork in the plan at the source of the reduce sink
    Operator<? extends OperatorDesc> parentOfRS = ctx.generator.getParentOperators().get(0);

    // we need the expr that generated the key of the reduce sink
    ExprNodeDesc key = ctx.getKeyCol();

    assert colName != null;
    // Fetch the TableScan Operator.
    Operator<?> op = parentOfRS;
    while (!(op == null || op instanceof TableScanOperator ||
             op instanceof ReduceSinkOperator)) {
      op = op.getParentOperators().get(0);
    }
    Preconditions.checkNotNull(op);

    if (op instanceof TableScanOperator) {
      Table table = ((TableScanOperator) op).getConf().getTableMetadata();
      if (table.isPartitionKey(colName)) {
        // The column is partition column, skip the optimization.
        return false;
      }
    }

    // Check if there already exists a semijoin branch
    GroupByOperator gb = parseContext.getColExprToGBMap().get(key);
    if (gb != null) {
      // Already an existing semijoin branch, reuse it
      createFinalRsForSemiJoinOp(parseContext, ts, gb, key, keyBaseAlias,
              ctx.parent.getChildren().get(0), sjHint != null);
      // done!
      return true;
    }

    List<ExprNodeDesc> keyExprs = new ArrayList<ExprNodeDesc>();
    keyExprs.add(key);

    // group by requires "ArrayList", don't ask.
    ArrayList<String> outputNames = new ArrayList<String>();
    outputNames.add(HiveConf.getColumnInternalName(0));

    // project the relevant key column
    SelectDesc select = new SelectDesc(keyExprs, outputNames);

    // Create the new RowSchema for the projected column
    ColumnInfo columnInfo = parentOfRS.getSchema().getColumnInfo(internalColName);
    ArrayList<ColumnInfo> signature = new ArrayList<ColumnInfo>();
    signature.add(columnInfo);
    RowSchema rowSchema = new RowSchema(signature);

    // Create the column expr map
    Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
    ExprNodeDesc exprNode = null;
    if (columnInfo == null) {
      LOG.debug("No ColumnInfo found in {} for {}", parentOfRS.getOperatorId(), internalColName);
      return false;
    }
    exprNode = new ExprNodeColumnDesc(columnInfo);
    colExprMap.put(internalColName, exprNode);

    // Create the Select Operator
    SelectOperator selectOp =
            (SelectOperator) OperatorFactory.getAndMakeChild(select,
                    rowSchema, colExprMap, parentOfRS);

    // do a group by to aggregate min,max and bloom filter.
    float groupByMemoryUsage =
            HiveConf.getFloatVar(parseContext.getConf(), HiveConf.ConfVars.HIVEMAPAGGRHASHMEMORY);
    float memoryThreshold =
            HiveConf.getFloatVar(parseContext.getConf(),
                    HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD);
    float minReductionHashAggr =
        HiveConf.getFloatVar(parseContext.getConf(),
            ConfVars.HIVEMAPAGGRHASHMINREDUCTION);

    // Add min/max and bloom filter aggregations
    List<ObjectInspector> aggFnOIs = new ArrayList<ObjectInspector>();
    aggFnOIs.add(key.getWritableObjectInspector());
    ArrayList<ExprNodeDesc> params = new ArrayList<ExprNodeDesc>();
    params.add(
            new ExprNodeColumnDesc(key.getTypeInfo(), outputNames.get(0),
                    "", false));

    ArrayList<AggregationDesc> aggs = new ArrayList<AggregationDesc>();
    try {
      AggregationDesc min = new AggregationDesc("min",
              FunctionRegistry.getGenericUDAFEvaluator("min", aggFnOIs, false, false),
              params, false, Mode.PARTIAL1);
      AggregationDesc max = new AggregationDesc("max",
              FunctionRegistry.getGenericUDAFEvaluator("max", aggFnOIs, false, false),
              params, false, Mode.PARTIAL1);
      AggregationDesc bloomFilter = new AggregationDesc("bloom_filter",
              FunctionRegistry.getGenericUDAFEvaluator("bloom_filter", aggFnOIs, false, false),
              params, false, Mode.PARTIAL1);
      GenericUDAFBloomFilterEvaluator bloomFilterEval =
          (GenericUDAFBloomFilterEvaluator) bloomFilter.getGenericUDAFEvaluator();
      bloomFilterEval.setSourceOperator(selectOp);

      if (sjHint != null && sjHint.getNumEntries() > 0) {
        LOG.debug("Setting size for " + keyBaseAlias + " to " + sjHint.getNumEntries() + " based on the hint");
        bloomFilterEval.setHintEntries(sjHint.getNumEntries());
      }
      bloomFilterEval.setMaxEntries(parseContext.getConf().getLongVar(ConfVars.TEZ_MAX_BLOOM_FILTER_ENTRIES));
      bloomFilterEval.setMinEntries(parseContext.getConf().getLongVar(ConfVars.TEZ_MIN_BLOOM_FILTER_ENTRIES));
      bloomFilterEval.setFactor(parseContext.getConf().getFloatVar(ConfVars.TEZ_BLOOM_FILTER_FACTOR));
      bloomFilter.setGenericUDAFWritableEvaluator(bloomFilterEval);
      aggs.add(min);
      aggs.add(max);
      aggs.add(bloomFilter);
    } catch (SemanticException e) {
      LOG.error("Error creating min/max aggregations on key", e);
      throw new IllegalStateException("Error creating min/max aggregations on key", e);
    }

    // Create the Group by Operator
    ArrayList<String> gbOutputNames = new ArrayList<String>();
    gbOutputNames.add(SemanticAnalyzer.getColumnInternalName(0));
    gbOutputNames.add(SemanticAnalyzer.getColumnInternalName(1));
    gbOutputNames.add(SemanticAnalyzer.getColumnInternalName(2));
    GroupByDesc groupBy = new GroupByDesc(GroupByDesc.Mode.HASH,
        gbOutputNames, new ArrayList<ExprNodeDesc>(), aggs, false,
        groupByMemoryUsage, memoryThreshold, minReductionHashAggr, null, false, -1, false);

    ArrayList<ColumnInfo> groupbyColInfos = new ArrayList<ColumnInfo>();
    groupbyColInfos.add(new ColumnInfo(gbOutputNames.get(0), key.getTypeInfo(), "", false));
    groupbyColInfos.add(new ColumnInfo(gbOutputNames.get(1), key.getTypeInfo(), "", false));
    groupbyColInfos.add(new ColumnInfo(gbOutputNames.get(2), key.getTypeInfo(), "", false));

    GroupByOperator groupByOp = (GroupByOperator)OperatorFactory.getAndMakeChild(
            groupBy, new RowSchema(groupbyColInfos), selectOp);

    groupByOp.setColumnExprMap(new HashMap<String, ExprNodeDesc>());

    // Get the column names of the aggregations for reduce sink
    int colPos = 0;
    ArrayList<ExprNodeDesc> rsValueCols = new ArrayList<ExprNodeDesc>();
    for (int i = 0; i < aggs.size() - 1; i++) {
      ExprNodeColumnDesc colExpr = new ExprNodeColumnDesc(key.getTypeInfo(),
              gbOutputNames.get(colPos++), "", false);
      rsValueCols.add(colExpr);
    }

    // Bloom Filter uses binary
    ExprNodeColumnDesc colExpr = new ExprNodeColumnDesc(TypeInfoFactory.binaryTypeInfo,
            gbOutputNames.get(colPos++), "", false);
    rsValueCols.add(colExpr);

    // Create the reduce sink operator
    ReduceSinkDesc rsDesc = PlanUtils.getReduceSinkDesc(
            new ArrayList<ExprNodeDesc>(), rsValueCols, gbOutputNames, false,
            -1, 0, 1, Operation.NOT_ACID, NullOrdering.defaultNullOrder(parseContext.getConf()));
    ReduceSinkOperator rsOp = (ReduceSinkOperator)OperatorFactory.getAndMakeChild(
            rsDesc, new RowSchema(groupByOp.getSchema()), groupByOp);
    Map<String, ExprNodeDesc> columnExprMap = new HashMap<String, ExprNodeDesc>();
    rsOp.setColumnExprMap(columnExprMap);

    rsOp.getConf().setReducerTraits(EnumSet.of(ReduceSinkDesc.ReducerTraits.QUICKSTART));

    // Create the final Group By Operator
    ArrayList<AggregationDesc> aggsFinal = new ArrayList<AggregationDesc>();
    try {
      List<ObjectInspector> minFinalFnOIs = new ArrayList<ObjectInspector>();
      List<ObjectInspector> maxFinalFnOIs = new ArrayList<ObjectInspector>();
      List<ObjectInspector> bloomFilterFinalFnOIs = new ArrayList<ObjectInspector>();
      ArrayList<ExprNodeDesc> minFinalParams = new ArrayList<ExprNodeDesc>();
      ArrayList<ExprNodeDesc> maxFinalParams = new ArrayList<ExprNodeDesc>();
      ArrayList<ExprNodeDesc> bloomFilterFinalParams = new ArrayList<ExprNodeDesc>();
      // Use the expressions from Reduce Sink.
      minFinalFnOIs.add(rsValueCols.get(0).getWritableObjectInspector());
      maxFinalFnOIs.add(rsValueCols.get(1).getWritableObjectInspector());
      bloomFilterFinalFnOIs.add(rsValueCols.get(2).getWritableObjectInspector());
      // Coming from a ReduceSink the aggregations would be in the form VALUE._col0, VALUE._col1
      minFinalParams.add(
              new ExprNodeColumnDesc(
                      rsValueCols.get(0).getTypeInfo(),
                      Utilities.ReduceField.VALUE + "." +
                              gbOutputNames.get(0), "", false));
      maxFinalParams.add(
              new ExprNodeColumnDesc(
                      rsValueCols.get(1).getTypeInfo(),
                      Utilities.ReduceField.VALUE + "." +
                              gbOutputNames.get(1), "", false));
      bloomFilterFinalParams.add(
              new ExprNodeColumnDesc(
                      rsValueCols.get(2).getTypeInfo(),
                      Utilities.ReduceField.VALUE + "." +
                              gbOutputNames.get(2), "", false));

      AggregationDesc min = new AggregationDesc("min",
              FunctionRegistry.getGenericUDAFEvaluator("min", minFinalFnOIs,
                      false, false),
              minFinalParams, false, Mode.FINAL);
      AggregationDesc max = new AggregationDesc("max",
              FunctionRegistry.getGenericUDAFEvaluator("max", maxFinalFnOIs,
                      false, false),
              maxFinalParams, false, Mode.FINAL);
      AggregationDesc bloomFilter = new AggregationDesc("bloom_filter",
              FunctionRegistry.getGenericUDAFEvaluator("bloom_filter", bloomFilterFinalFnOIs,
                      false, false),
              bloomFilterFinalParams, false, Mode.FINAL);
      GenericUDAFBloomFilterEvaluator bloomFilterEval = (GenericUDAFBloomFilterEvaluator) bloomFilter.getGenericUDAFEvaluator();
      bloomFilterEval.setSourceOperator(selectOp);
      if (sjHint != null && sjHint.getNumEntries() > 0) {
        bloomFilterEval.setHintEntries(sjHint.getNumEntries());
      }
      bloomFilterEval.setMaxEntries(parseContext.getConf().getLongVar(ConfVars.TEZ_MAX_BLOOM_FILTER_ENTRIES));
      bloomFilterEval.setMinEntries(parseContext.getConf().getLongVar(ConfVars.TEZ_MIN_BLOOM_FILTER_ENTRIES));
      bloomFilterEval.setFactor(parseContext.getConf().getFloatVar(ConfVars.TEZ_BLOOM_FILTER_FACTOR));
      bloomFilter.setGenericUDAFWritableEvaluator(bloomFilterEval);

      aggsFinal.add(min);
      aggsFinal.add(max);
      aggsFinal.add(bloomFilter);
    } catch (SemanticException e) {
      LOG.error("Error creating min/max aggregations on key", e);
      throw new IllegalStateException("Error creating min/max aggregations on key", e);
    }

    GroupByDesc groupByDescFinal = new GroupByDesc(GroupByDesc.Mode.FINAL,
            gbOutputNames, new ArrayList<ExprNodeDesc>(), aggsFinal, false,
            groupByMemoryUsage, memoryThreshold, minReductionHashAggr, null, false, 0, false);
    GroupByOperator groupByOpFinal = (GroupByOperator)OperatorFactory.getAndMakeChild(
            groupByDescFinal, new RowSchema(rsOp.getSchema()), rsOp);
    groupByOpFinal.setColumnExprMap(new HashMap<String, ExprNodeDesc>());

    createFinalRsForSemiJoinOp(parseContext, ts, groupByOpFinal, key,
            keyBaseAlias, ctx.parent.getChildren().get(0), sjHint != null);

    return true;
  }

  private void createFinalRsForSemiJoinOp(
          ParseContext parseContext, TableScanOperator ts, GroupByOperator gb,
          ExprNodeDesc key, String keyBaseAlias, ExprNodeDesc colExpr,
          boolean isHint) throws SemanticException {
    ArrayList<String> gbOutputNames = new ArrayList<>();
    // One each for min, max and bloom filter
    gbOutputNames.add(SemanticAnalyzer.getColumnInternalName(0));
    gbOutputNames.add(SemanticAnalyzer.getColumnInternalName(1));
    gbOutputNames.add(SemanticAnalyzer.getColumnInternalName(2));

    int colPos = 0;
    ArrayList<ExprNodeDesc> rsValueCols = new ArrayList<ExprNodeDesc>();
    for (int i = 0; i < gbOutputNames.size() - 1; i++) {
      ExprNodeColumnDesc expr = new ExprNodeColumnDesc(key.getTypeInfo(),
              gbOutputNames.get(colPos++), "", false);
      rsValueCols.add(expr);
    }

    // Bloom Filter uses binary
    ExprNodeColumnDesc colBFExpr = new ExprNodeColumnDesc(TypeInfoFactory.binaryTypeInfo,
            gbOutputNames.get(colPos++), "", false);
    rsValueCols.add(colBFExpr);

    // Create the final Reduce Sink Operator
    ReduceSinkDesc rsDescFinal = PlanUtils.getReduceSinkDesc(
            new ArrayList<ExprNodeDesc>(), rsValueCols, gbOutputNames, false,
            -1, 0, 1, Operation.NOT_ACID, NullOrdering.defaultNullOrder(parseContext.getConf()));
    ReduceSinkOperator rsOpFinal = (ReduceSinkOperator)OperatorFactory.getAndMakeChild(
            rsDescFinal, new RowSchema(gb.getSchema()), gb);
    Map<String, ExprNodeDesc> columnExprMap = new HashMap<>();
    rsOpFinal.setColumnExprMap(columnExprMap);

    LOG.debug("DynamicSemiJoinPushdown: Saving RS to TS mapping: " + rsOpFinal + ": " + ts);
    SemiJoinBranchInfo sjInfo = new SemiJoinBranchInfo(ts, isHint);
    parseContext.getRsToSemiJoinBranchInfo().put(rsOpFinal, sjInfo);

    // Save the info that is required at query time to resolve dynamic/runtime values.
    RuntimeValuesInfo runtimeValuesInfo = new RuntimeValuesInfo();
    TableDesc rsFinalTableDesc = PlanUtils.getReduceValueTableDesc(
            PlanUtils.getFieldSchemasFromColumnList(rsValueCols, "_col"));
    List<String> dynamicValueIDs = new ArrayList<String>();
    dynamicValueIDs.add(keyBaseAlias + "_min");
    dynamicValueIDs.add(keyBaseAlias + "_max");
    dynamicValueIDs.add(keyBaseAlias + "_bloom_filter");

    runtimeValuesInfo.setTableDesc(rsFinalTableDesc);
    runtimeValuesInfo.setDynamicValueIDs(dynamicValueIDs);
    runtimeValuesInfo.setColExprs(rsValueCols);
    runtimeValuesInfo.setTargetColumns(Collections.singletonList(colExpr));
    parseContext.getRsToRuntimeValuesInfoMap().put(rsOpFinal, runtimeValuesInfo);
    parseContext.getColExprToGBMap().put(key, gb);
  }

}


text:
package co.cask.tigon.internal.app;

import co.cask.tigon.api.ResourceSpecification;
import co.cask.tigon.api.Resources;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonSerializationContext;

import java.lang.reflect.Type;

/**
 *
 */
final class ResourceSpecificationCodec extends AbstractSpecificationCodec<ResourceSpecification> {

  @Override
  public JsonElement serialize(ResourceSpecification src, Type typeOfSrc, JsonSerializationContext context) {
    JsonObject jsonObj = new JsonObject();

    jsonObj.add("virtualCores", new JsonPrimitive(src.getVirtualCores()));
    jsonObj.add("memoryMB", new JsonPrimitive(src.getMemoryMB()));

    return jsonObj;
  }

  @Override
  public ResourceSpecification deserialize(JsonElement json, Type typeOfT,
                                          JsonDeserializationContext context) throws JsonParseException {
    JsonObject jsonObj = json.getAsJsonObject();

    int cores = jsonObj.get("virtualCores").getAsInt();
    int memorySize = jsonObj.get("memoryMB").getAsInt();

    return new Resources(memorySize, cores);
  }
}

redpajama_set_name: RedPajamaGithub


text:
namespace Fluent.Localization.Languages
{
    [RibbonLocalization("Greek", "el")]
    public class Greek : RibbonLocalizationBase
    {
        public override string Automatic { get; } = "Αυτόματη";
        public override string BackstageButtonKeyTip { get; } = FallbackLocalization.BackstageButtonKeyTip;
        public override string BackstageButtonText { get; } = FallbackLocalization.BackstageButtonText;
        public override string CustomizeStatusBar { get; } = FallbackLocalization.CustomizeStatusBar;
        public override string ExpandButtonScreenTipText { get; } = FallbackLocalization.ExpandButtonScreenTipText;
        public override string ExpandButtonScreenTipTitle { get; } = FallbackLocalization.ExpandButtonScreenTipTitle;
        public override string MinimizeButtonScreenTipText { get; } = FallbackLocalization.MinimizeButtonScreenTipText;
        public override string MinimizeButtonScreenTipTitle { get; } = FallbackLocalization.MinimizeButtonScreenTipTitle;
        public override string MoreColors { get; } = "Περισσότερα χρώματα...";
        public override string NoColor { get; } = "Κανένα χρώμα";
        public override string QuickAccessToolBarDropDownButtonTooltip { get; } = FallbackLocalization.QuickAccessToolBarDropDownButtonTooltip;
        public override string QuickAccessToolBarMenuHeader { get; } = FallbackLocalization.QuickAccessToolBarMenuHeader;
        public override string QuickAccessToolBarMenuShowAbove { get; } = FallbackLocalization.QuickAccessToolBarMenuShowAbove;
        public override string QuickAccessToolBarMenuShowBelow { get; } = FallbackLocalization.QuickAccessToolBarMenuShowBelow;
        public override string QuickAccessToolBarMoreControlsButtonTooltip { get; } = FallbackLocalization.QuickAccessToolBarMoreControlsButtonTooltip;
        public override string RibbonContextMenuAddGallery { get; } = FallbackLocalization.RibbonContextMenuAddGallery;
        public override string RibbonContextMenuAddGroup { get; } = FallbackLocalization.RibbonContextMenuAddGroup;
        public override string RibbonContextMenuAddItem { get; } = FallbackLocalization.RibbonContextMenuAddItem;
        public override string RibbonContextMenuAddMenu { get; } = FallbackLocalization.RibbonContextMenuAddMenu;
        public override string RibbonContextMenuCustomizeQuickAccessToolBar { get; } = FallbackLocalization.RibbonContextMenuCustomizeQuickAccessToolBar;
        public override string RibbonContextMenuCustomizeRibbon { get; } = FallbackLocalization.RibbonContextMenuCustomizeRibbon;
        public override string RibbonContextMenuMinimizeRibbon { get; } = FallbackLocalization.RibbonContextMenuMinimizeRibbon;
        public override string RibbonContextMenuRemoveItem { get; } = FallbackLocalization.RibbonContextMenuRemoveItem;
        public override string RibbonContextMenuShowAbove { get; } = FallbackLocalization.RibbonContextMenuShowAbove;
        public override string RibbonContextMenuShowBelow { get; } = FallbackLocalization.RibbonContextMenuShowBelow;
        public override string ScreenTipDisableReasonHeader { get; } = FallbackLocalization.ScreenTipDisableReasonHeader;
        public override string ScreenTipF1LabelHeader { get; } = FallbackLocalization.ScreenTipF1LabelHeader;
    }
}


text:
function [B,x0,U] = chloro_i(chloro_file)
% [B,x0,U] = chloro_i(chloro_file)
%   initialisation of model chloro
%   B   Model size
%   x0  default initial state
%   U   measuerement list
%
% Stimela, 2004

% © Kim van Schagen,

U = st_Varia('LongNames'); % default with long Names

% Get Model parameter file for parameter specific sizes
chloro_fileEcht = chckFile(chloro_file);
P = st_getPdata(chloro_fileEcht, 'chloro');

% Parameters in P can now be used to determine B and x0

%%% MODEL-SPECIFIC => %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Fill B and x0

% initialisation
B.CStates=3;        % number of continous states
B.DStates=0;        % number of discrete states
B.Setpoints=1;      % number of extra setpoints
B.Measurements= 0;  % number of extra measurements
B.WaterIn  = 1;     % Incoming water Flow (yes=1, no=0)
B.WaterOut  = 1;    % Outgoing water Flow (yes=1, no=0)
B.Direct=1;         % direct feedthrough (yes=1, no=0);
B.SampleTime=0;     % SampleTime for discrete states (-1 = inherited, -2 = next hot in flag=4)

x0=[1 0 0];			% collumn vector with the same length as B_CStates

%%% <= MODEL-SPECIFIC  %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


text:
import { MuSchema } from './schema';
import { MuWriteStream, MuReadStream } from '../stream';

export class MuSchemaLogger<BaseSchema extends MuSchema<any>> {
    public name:string;
    public allocCount:number = 0;
    public freeCount:number = 0;

    private _schema:MuSchemaTrace<BaseSchema>;
    private _savedAlloc:number = 0;
    private _savedFree:number = 0;

    constructor (schema:MuSchemaTrace<BaseSchema>, name:string) {
        this._schema = schema;
        this.name = name;
    }

    public begin () {
        this._savedAlloc = this._schema.allocCount;
        this._savedFree = this._schema.freeCount;
    }

    public end () {
        this.allocCount += this._schema.allocCount - this._savedAlloc;
        this.freeCount += this._schema.freeCount - this._savedFree;
    }
}

export class MuSchemaTrace<BaseSchema extends MuSchema<any>>
    implements MuSchema<BaseSchema['identity']> {
    public schema:BaseSchema;
    public identity:BaseSchema['identity'];
    public muType:BaseSchema['muType'];
    public muData:BaseSchema['muData'];
    public json;

    public logs:MuSchemaLogger<BaseSchema>[] = [];

    public allocCount:number = 0;
    public freeCount:number = 0;

    public createLog (name:string) : MuSchemaLogger<BaseSchema> {
        const x = new MuSchemaLogger(this, name);
        this.logs.push(x);
        return x;
    }

    public printLog () {
        const x = this.logs.map(({ name, allocCount, freeCount }) => {
            return {
                name,
                alloc: allocCount,
                free: freeCount,
                object: allocCount - freeCount,
            };
        });
        x.push({
            name: 'total',
            alloc: this.allocCount,
            free: this.freeCount,
            object: this.allocCount - this.freeCount,
        });
        if (console.table) {
            console.table(x);
        } else {
            console.log(x.map((y) => JSON.stringify(y)).join('\n'));
        }
    }

    constructor (base:BaseSchema) {
        this.schema = base;
        this.identity = base.identity;
        this.muType = base.muType;
        this.muData = base.muData;
        this.json = base.json;
    }

    public alloc () : BaseSchema['identity'] {
        this.allocCount++;
        return this.schema.alloc();
    }

    public free (x:BaseSchema['identity']) : void {
        this.freeCount++;
        return this.schema.free(x);
    }

    public equal(
        a:BaseSchema['identity'],
        b:BaseSchema['identity'],
    ) : boolean {
        return this.schema.equal(a, b);
    }

    public clone (x:BaseSchema['identity']) : BaseSchema['identity'] {
        this.allocCount++;
        return this.schema.clone(x);
    }

    public assign (
        dst:BaseSchema['identity'],
        src:BaseSchema['identity'],
    ) : BaseSchema['identity'] {
        return this.schema.assign(dst, src);
    }

    public diff (
        base:BaseSchema['identity'],
        target:BaseSchema['identity'],
        out:MuWriteStream,
    ) : boolean {
        return this.schema.diff(base, target, out);
    }

    public patch (
        base:BaseSchema['identity'],
        inp:MuReadStream,
    ) : BaseSchema['identity'] {
        this.allocCount++;
        return this.schema.patch(base, inp);
    }

    public toJSON (x:BaseSchema['identity']) : any {
        return this.schema.toJSON(x);
    }

    public fromJSON (json:any) : BaseSchema['identity'] {
        return this.schema.fromJSON(json);
    }
}


text:
using NUnit.Framework;

namespace Shuttle.Recall.Tests
{
    [TestFixture]
    public class EventStoreConfigurationFixture
    {
        [Test]
        public void Should_be_able_to_add_and_check_for_active_projection_names()
        {
            var eventStoreOptions = new EventStoreOptions();

            Assert.That(eventStoreOptions.HasActiveProjection("not-registered"), Is.True);

            eventStoreOptions.ActiveProjections.Add("projection-1");

            Assert.That(eventStoreOptions.HasActiveProjection("not-registered"), Is.False);
            Assert.That(eventStoreOptions.HasActiveProjection("projection-1"), Is.True);
        }
    }
}


text:
Adds a selected config setting to the params object for Symphony CMS.

## Installation

1. Upload the `config_to_params` folder in this archive to your Symphony `/extensions` folder.

2. Enable it by selecting the "Config to Params", choose Enable from the with-selected menu, then click Apply.

3. You can now edit the `CONFIG_TO_PARAMS` config section in your `manifest/config.php` file an example has been added for you.

4. The values you add here will be available in the params object.

For more information, see <http://getsymphony.com/learn/tasks/view/install-an-extension/>


text:
from ._fluid_relay_operations_operations import FluidRelayOperationsOperations
from ._fluid_relay_servers_operations import FluidRelayServersOperations
from ._fluid_relay_containers_operations import FluidRelayContainersOperations

from ._patch import __all__ as _patch_all
from ._patch import *  # type: ignore # pylint: disable=unused-wildcard-import
from ._patch import patch_sdk as _patch_sdk
__all__ = [
    'FluidRelayOperationsOperations',
    'FluidRelayServersOperations',
    'FluidRelayContainersOperations',
]
__all__.extend([p for p in _patch_all if p not in __all__])
_patch_sdk()


text:
copyright:
 years: 2015, 2016

---

# Acerca de las notificaciones push
{: #overview-push}

Las notificaciones push son un servicio que puede utilizar para enviar notificaciones al dispositivo iOS y Android. Las notificaciones se pueden destinar a todos los usuarios de la aplicación o a un conjunto específico de usuarios y dispositivos mediante etiquetas. Puede
        administrar dispositivos, etiquetas y suscripciones. También puede utilizar un SDK (software development kit, kit de desarrollo de software) y API (application program interface, interfaz de programa de aplicaciones) REST (Representational State Transfer) para desarrollar más las aplicaciones de cliente. Para obtener información sobre el servicio push dedicado, consulte [Servicios dedicados](../../dedicated/index.html).

## Proceso de servicio de notificaciones push
{: #overview_push_process}

Los clientes móviles pueden suscribirse y registrarse para el servicio de notificaciones Push. Al
                arrancar, las aplicaciones móviles se registrarán y se suscribirán al servicio de notificaciones
                Push. Las notificaciones se asignan
                al servicio de APN (Apple Push Notification Service) o al servidor de GCM (Google Cloud Messaging)
                y, a continuación, se enviarán a los clientes móviles registrados.

![Visión general de push](images/overview.jpg)

###Aplicaciones móviles

Al arrancar, las aplicaciones móviles se registrarán
                        y se suscribirán al servicio de notificaciones Push para recibir
                        notificaciones.

###Aplicaciones de fondo

Las aplicaciones de fondo pueden estar de forma local o en una nube pública. Las aplicaciones de fondo utilizan el Servicio de notificaciones Push
                        para enviar notificaciones según el contexto a usuarios
                        móviles. No es necesario que las aplicaciones de fondo mantengan ni gestionen
                        dispositivos móviles ni información de usuarios para enviar notificaciones push. En su lugar,
                        las aplicaciones de fondo pueden utilizar el Servicio de notificaciones Push.

###Propietario de back-end de app

El rol que ha creado la aplicación de fondo móvil que empaqueta una
                        instancia del Servicio de notificaciones Push. Esta persona configura
                        el Servicio de notificaciones Push para que se ajuste a las aplicaciones de fondo que utilizan
                        el Servicio de notificaciones Push y las aplicaciones móviles que son el destino de notificaciones
                        push.

###Servicio de notificaciones Push

El servicio de notificaciones Push gestiona toda la información relacionada con los dispositivos que se registran para notificaciones. El servicio mantiene sus aplicaciones transparentes a los detalles tecnológicos de enviar notificaciones a estas plataformas móviles heterogéneas y lo maneja todo.

###Pasarelas

Los servicios de nube específicos de plataformas de dispositivos móviles tales como Google Cloud Messaging (GCM) o Apple Push Notification Service(APNs) utilizan el Servicio de notificaciones Push
                        para asignar notificaciones a las aplicaciones móviles.

## Tipos de notificación de envío
{: #overview-push-types}

###Difusión

Cuando una aplicación móvil se registra con el servicio de notificaciones Push, puede comenzar a recibir notificaciones. Las notificaciones de difusión son mensajes de notificación destinados a todos
                        los dispositivos que tienen la aplicación instalada y configurada para el Servicio de
                        notificaciones Push. Las notificaciones de difusión están
                        habilitadas de forma predeterminada con cualquier aplicación que está habilitada para las notificaciones
                        push. Cualquier aplicación que esté habilitada para el Servicio de notificaciones push tiene una suscripción predefinida a la etiqueta Push.ALL, que utiliza el servidor para difundir mensajes de notificación a todos los dispositivos. Para enviar una notificación de difusión
                        que utiliza la API Push REST, asegúrese de que el "destino" sea un archivo JSON vacío
                        al publicar en recursos de mensajes.

###Notificaciones basadas en etiquetas

Las notificaciones de código son mensajes de notificación que están pensados para todos los
                        dispositivos suscritos a un código determinado. Las notificaciones basadas en código
                        permiten la segmentación de notificaciones en función de los temas o de las áreas de temas. Los destinatarios de la notificación pueden elegir recibir notificaciones sólo si es sobre
                        un asunto o tema de su interés. Por lo tanto, la notificación basada en etiquetas
                        proporciona un medio de segmentar destinatarios. Esta característica habilita
                        la función de definir etiquetas y, a continuación, enviar y recibir mensajes por etiquetas. Un
                        mensaje sólo se ha dirigido a los dispositivos que están suscritos a una etiqueta. Primero debe
                        crear las etiquetas para la aplicación, configurar las suscripciones de etiquetas
                        y, a continuación, iniciar las notificaciones basadas en etiquetas. Para enviar una notificación basada en código
                        que utiliza la API REST, asegúrese de que los "tagNames" se proporcionen
                        al publicar en el recurso de mensajes.

###Notificaciones de difusión única

Las notificaciones de difusión única son mensajes de notificación que están pensados para un dispositivo concreto. Las notificaciones de difusión única no requieren ninguna configuración adicional y están habilitadas de forma predeterminada cuando se habilita la aplicación para notificaciones push. Para enviar una notificación de difusión única que utilice la API REST, asegúrese de que se proporcionan ID de dispositivo al publicar en un recurso de mensajes.

###Notificaciones basadas en plataforma

Las notificaciones se pueden definir para alcanzar una plataforma de dispositivo determinada. Por
                            ejemplo, se puede enviar una notificación únicamente a todos usuarios de Android. Para
                            enviar una notificación basada en plataforma que utilice la API REST, asegúrese
                            de que se proporcionan plataformas de destino al publicar en un recurso de
                            mensajes. Especifique las plataformas como una matriz. Las plataformas
                            soportadas son las siguientes:
* A (Apple)
* G (Google)


text:
// Utilities for working with XLA layout and shapes.

#ifndef TENSORFLOW_COMPILER_TF2XLA_LAYOUT_UTIL_H_
#define TENSORFLOW_COMPILER_TF2XLA_LAYOUT_UTIL_H_

#include <vector>

#include "tensorflow/compiler/tf2xla/xla_argument.h"
#include "tensorflow/compiler/tf2xla/xla_helpers.h"
#include "tensorflow/compiler/xla/shape.h"
#include "tensorflow/compiler/xla/statusor.h"
#include "tensorflow/compiler/xla/xla_data.pb.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/types.pb.h"

namespace tensorflow {

class XlaShapeLayoutHelpers {
 public:
  // The following defines the layout preference of an xla tensor.
  // The return value of LayoutPreferenceFn can be used in
  // XlaHelper::ShapeRepresentationFn.
  typedef std::function<XlaLayoutPreference(const TensorShape&, DataType,
                                            absl::optional<XlaArgument::Kind>)>
      LayoutPreferenceFn;

  // A bundle of LayoutPreferenceFn and ShapeRepresentationFn.
  struct ShapeDeterminationFns {
    // Use no preference function, and identity shape representation function,
    // as default value.
    ShapeDeterminationFns();

    ShapeDeterminationFns(
        LayoutPreferenceFn layout_preference_fn,
        XlaHelpers::ShapeRepresentationFn shape_representation_fn)
        : layout_preference_fn(layout_preference_fn),
          shape_representation_fn(shape_representation_fn) {}

    LayoutPreferenceFn layout_preference_fn;
    XlaHelpers::ShapeRepresentationFn shape_representation_fn;
  };
};

// Return a LayoutPreferenceFn that always uses kNoPreference layout.
XlaShapeLayoutHelpers::LayoutPreferenceFn UseNoPreferenceLayoutFn();

// Rewrites the layout of xla_shape if there is tiled sharding.
Status RewriteLayoutWithShardedShape(
    const absl::optional<xla::HloSharding>& sharding, bool use_fast_memory,
    XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns,
    xla::Shape* xla_shape);

// Adds reshapes to fix the layout of an output, if a shape_representation_fn or
// sharding is present.
StatusOr<xla::XlaOp> ReshapeWithCorrectRepresentationAndSharding(
    xla::XlaBuilder* builder, xla::XlaOp original, xla::Shape original_shape,
    XlaShapeLayoutHelpers::ShapeDeterminationFns shape_determination_fns,
    absl::optional<xla::OpSharding> sharding, bool fast_mem);

}  // namespace tensorflow

#endif  // TENSORFLOW_COMPILER_TF2XLA_SHAPE_UTIL_H_


text:
package com.treode.disk

import com.treode.async.{Async, Scheduler}, Async.supply

/** Aggregates results from LogReaders of individual disks. */
private class LogCollector (implicit
  files: FileSystem,
  scheduler: Scheduler,
  config: DiskConfig,
  events: DiskEvents
) {

  /** All the drives collected. */
  private var drives = Map.empty [Int, BootstrapDrive]

  /** The largest batch number seen. */
  private var batch = 0L

  /** The total bytes of log entries replayed. */
  private var bytes = 0L

  /** The total count of log entries replayed. */
  private var entries = 0L

  /** Close replay of a drive.
    *
    * @param drive The drive, with its LogWriter ready to append where reading finished.
    * @param batch The last batch replayed from the log this drive.
    * @param bytes The count of bytes replayed from the log on this drive.
    * @param entries The count of entries replayed from the log on this drive.
    */
  def reattach (drive: BootstrapDrive, batch: Long, bytes: Long, entries: Long): Unit =
    synchronized {
      drives += drive.id -> drive
      this.batch = math.max (this.batch, batch)
      this.bytes += bytes
      this.entries += entries
    }

  def result (common: SuperBlock.Common): BootstrapLaunch = {
    implicit val disk = new BootstrapDisk (common, drives, batch, bytes, entries)
    new BootstrapLaunch
  }}


text:
> A native Vue.js select component that provides similar functionality to Select2 without the overhead of jQuery.

#### Features
- AJAX Support
- Tagging
- List Filtering/Searching
- Supports Vuex
- Select Single/Multiple Options
- Tested with Bootstrap 3/4, Bulma, Foundation
- +95% Test Coverage
- ~33kb minified with CSS
- Zero dependencies

## Documentation
- **[Demo & Docs](http://sagalbot.github.io/vue-select/)**
- **[Example on JSBin](http://jsbin.com/saxaru/8/edit?html,js,output)**
- **[CodePen Template](http://codepen.io/sagalbot/pen/NpwrQO)**
- **[Trello Roadmap](https://trello.com/b/vWvITNzS/vue-select)**

## Install

###### Vue Compatibility
-  `vue ~2.0` use `vue-select ~2.0`
-  `vue ~1.0` use `vue-select ~1.0`

#### NPM
Install the package. _You should install `vue-select@1.3.3` for use with vue `~1.0`._

```bash
$ npm install vue-select
```

Register the component

```js
import Vue from 'vue'
import vSelect from 'vue-select'
Vue.component('v-select', vSelect)
```

You may now use the component in your markup

```html
<v-select v-model="selected" :options="['foo','bar']"></v-select>
```

#### CDN

Just include `vue` & `vue-select.js` - I recommend using [unpkg](https://unpkg.com/#/).

```html
<script scr="https://unpkg.com/vue@latest"></script>
<!-- use the latest release -->
<script src="https://unpkg.com/vue-select@latest"></script>
<!-- or point to a specific release -->
<script src="https://unpkg.com/vue-select@1.3.3"></script>
```

Then register the component in your javascript:

```js
Vue.component('v-select', VueSelect.VueSelect);
```

You may now use the component in your markup

```html
<v-select v-model="selected" :options="['foo','bar']"></v-select>
```

Here's an [example on JSBin](http://jsbin.com/saxaru/5/edit?html,js,output).

## Basic Usage

#### Syncing a Selected Value

The most common use case for `vue-select` is to have the chosen value synced with a parent component. `vue-select` takes advantage of the `v-model` syntax to sync values with a parent.

```html
<v-select v-model="selected"></v-select>
```
```js
new Vue({
  data: {
    selected: null
  }
})
```

#### Setting Options

`vue-select` accepts arrays of strings and objects to use as options through the `options` prop.

```html
<v-select :options="['foo','bar']"></v-select>
```

When provided an array of objects, `vue-select` will display a single value of the object. By default, `vue-select` will look for a key named 'label' on the object to use as display text.

```html
<v-select :options="[{label: 'foo', value: 'Foo'}]"></v-select>
```

### For more information, please visit the [vue-select documentation.](https://sagalbot.github.io/vue-select)


text:
/**
 * Clang frontend plugin to export an AST of clang into Json and Yojson (and ultimately Biniou)
 * while conforming to the inlined ATD specifications.
 */

#include "ASTExporter.h"

//===----------------------------------------------------------------------===//
// ASTExporter Plugin Main
//===----------------------------------------------------------------------===//

namespace {

  using namespace ASTLib;
  using namespace ASTPluginLib;

  template <class ATDWriter=YojsonWriter>
  class ExporterASTConsumer : public ASTConsumer {
  private:
    ASTExporterOptions Options;
    raw_ostream &OS;

  public:
    ExporterASTConsumer(const CompilerInstance &CI,
                        std::unique_ptr<ASTExporterOptions> &&Opts,
                        raw_ostream &OS)
    : Options(std::move(*Opts)), OS(OS)
    { }

    virtual void HandleTranslationUnit(ASTContext &Context) {
      TranslationUnitDecl *D = Context.getTranslationUnitDecl();
      ASTExporter<ATDWriter> P(OS, Context, Options);
      P.dumpDecl(D);
    }
  };

}

typedef ASTPluginLib::SimplePluginASTAction<ExporterASTConsumer<JsonWriter>, ASTExporterOptions> JsonExporterASTAction;
typedef ASTPluginLib::SimplePluginASTAction<ExporterASTConsumer<YojsonWriter>, ASTExporterOptions> YojsonExporterASTAction;

static FrontendPluginRegistry::Add<JsonExporterASTAction>
X("JsonASTExporter", "Export the AST of source files into ATD-specified Json data");

static FrontendPluginRegistry::Add<YojsonExporterASTAction>
Y("YojsonASTExporter", "Export the AST of source files into ATD-specified Yojson data");


text:
<?xml version="1.0" encoding="utf-8"?>
<resources>
    <color name="colorPrimary">#3F51B5</color>
    <color name="colorPrimaryDark">#303F9F</color>
    <color name="colorAccent">#FF4081</color>
    <color name="half_black">#E1E1E1</color>
</resources>


text:
layout: post
title: "Creating Survival Curves in R"
author: "Abraham Nunes"
date: "February 14, 2015"
published: true
status: publish
draft: false
tags: Suicide Psychiatry RStudio WHO
---

Survival analysis is a common procedure in biostatistics. Fortunately, `R` has extensive packages supporting these methods, including `survival` and `KMsurv`.

## Generating Data

First, let's load the packages and generate a dataframe with two dates, one for a hypothetical intervention date, and a second for an event or censoring date. The number of patients will be `n = 2000`.

{% highlight r %}
library(survival)
library(KMsurv)

#set origin date
originDate = as.Date("2000-01-01", format = "%Y-%m-%d")

#generate random number of days to add to the origin for each patient
nDays = rnorm(2000, mean = 60, sd = 100)

#generate vector of intervention dates by adding the nDays to the origin
interventionDate = originDate + nDays

#generate vector of event dates by adding nDays to the interventionDate
#   we will add the absolute number of nDays since event must occur after intervention
eventDate = interventionDate + abs(nDays)

#generate vector of events (TRUE or FALSE)
#   TRUE represents the event of interest
#   FALSE represents a censored value
#   We will set the probability of a TRUE event (vs. censoring) to 0.95
eventType = (rbinom(2000, 1, prob = 0.95) == 1)

#put these into a dataframe
kmdf = data.frame(intervention_date = interventionDate,
                  event_date = eventDate,
                  event = eventType)
{% endhighlight %}

The resulting data frame will appear as follows:

{% highlight text %}
##   intervention_date event_date event
## 1        1999-07-13 2000-01-01  TRUE
## 2        2000-06-03 2000-11-05  TRUE
## 3        2000-01-20 2000-02-09 FALSE
## 4        2000-02-20 2000-04-10  TRUE
## 5        2000-05-21 2000-10-09  TRUE
## 6        2000-04-26 2000-08-20  TRUE
{% endhighlight %}

To generate a survival object, though, we need to take the difference between the `event_date` and the `origin_date`. (We merely generated the dates above to facilitate learning how to take the difference between dates). The difference between dates can be taken as follows:

{% highlight r %}
kmdf$time_to_event = as.numeric(difftime(kmdf$event_date,
                                         kmdf$intervention_date,
                                         units = "days"))
{% endhighlight %}

## Fitting and Plotting the Survival Model

Now we will fit a survival model with a constant.

{% highlight r %}
sfit = survfit(Surv(kmdf$time_to_event, kmdf$event) ~ 0)

#plot sfit

plot(sfit,
     main = "Survival Curve for Randomly Generated Data",
     xlab = "Time to Event (Days)",
     ylab = "Proportion Surviving")
{% endhighlight %}

![plot of chunk unnamed-chunk-4](/figures/unnamed-chunk-4-1.png)

## Plotting Survival Curves for Multiple Groups

What about the (common) case in which there are multiple groups?

Let's start by generating another group of `n = 2000` patients whose survival is longer, but with a greater standard deviation.

{% highlight r %}
#generate random number of days to add to the origin for each patient
nDaysGroup2 = rnorm(2000, mean = 200, sd = 200)

#generate vector of intervention dates by adding the nDays to the origin
interventionDateGroup2 = originDate + nDays

#generate vector of event dates by adding nDays to the interventionDate
#   we will add the absolute number of nDays since event must occur after intervention
eventDateGroup2 = interventionDateGroup2 + abs(nDaysGroup2)

#generate vector of events (TRUE or FALSE)
#   TRUE represents the event of interest
#   FALSE represents a censored value
#   We will set the probability of a TRUE event (vs. censoring) to 0.95
eventTypeGroup2 = (rbinom(2000, 1, prob = 0.95) == 1)

#put these into a dataframe
kmdfGroup2 = data.frame(intervention_date = interventionDateGroup2,
                        event_date = eventDateGroup2,
                        event = eventTypeGroup2)

#compute time to event for new data frame
kmdfGroup2$time_to_event = as.numeric(difftime(kmdfGroup2$event_date,
                                               kmdfGroup2$intervention_date,
                                               units = "days"))
{% endhighlight %}

We'll need to bind the two data frames together, so we'll add a column to indicate the group names to each data frame before we stick them together.

{% highlight r %}
kmdf$group       = "Control"
kmdfGroup2$group = "Treatment"

kmdf_2groups = rbind(kmdf, kmdfGroup2)
{% endhighlight %}

We'll now generate and plot the survival model as before, with the new data frame `kmdf_2groups`. Note, however, that the term which was formerly constant is now set to the group variable of the new data frame, such that we get two survival curves (one for each group).

{% highlight r %}
sfit2 = survfit(Surv(kmdf_2groups$time_to_event, kmdf_2groups$event) ~ kmdf_2groups$group)

#plot sfit

plot(sfit2,
     main = "Survival Curve for Two Groups of Randomly Generated Data",
     xlab = "Time to Event (Days)",
     ylab = "Proportion Surviving",
     col  = c(1, 2))

legend("topright",
       levels(factor(kmdf_2groups$group)),
       col = c(1,2),
       lty = 1)
{% endhighlight %}

![plot of chunk unnamed-chunk-7](/figures/unnamed-chunk-7-1.png)

## Conclusion

Generating survival curves in `R` is very simple, and can be quite rewarding (particularly if you are using a *real* data set).


text:
// ----------------------------------------------------------------------------------
//
// Copyright Microsoft Corporation
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------------------------------------------------------------------------

using System;
using System.Linq;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
using Microsoft.Azure.Test.HttpRecorder;

namespace Microsoft.WindowsAzure.Commands.ScenarioTest
{
    // Excludes api version when matching mocked records.
    // If alternate api version is provided, uses that to match records else removes the api-version matching.
    public class PermissiveRecordMatcherWithApiExclusion : IRecordMatcher
    {
        private bool _ignoreGenericResource;
        private Dictionary<string, string> _providersToIgnore;
        private Dictionary<string, string> _userAgentsToIgnore;

        public PermissiveRecordMatcherWithApiExclusion(bool ignoreResourcesClient, Dictionary<string, string> providers)
        {
            _ignoreGenericResource = ignoreResourcesClient;
            _providersToIgnore = providers;
        }

        public PermissiveRecordMatcherWithApiExclusion(
            bool ignoreResourcesClient,
            Dictionary<string, string> providers,
            Dictionary<string, string> userAgents)
        {
            _ignoreGenericResource = ignoreResourcesClient;
            _providersToIgnore = providers;
            _userAgentsToIgnore = userAgents;
        }

        public string GetMatchingKey(System.Net.Http.HttpRequestMessage request)
        {
            var path = request.RequestUri.PathAndQuery;
            if (path.Contains("?&"))
            {
                path = path.Replace("?&", "?");
            }

            string version;
            if (ContainsIgnoredProvider(path, out version))
            {
                path = RemoveOrReplaceApiVersion(path, version);
            }
            else if (_userAgentsToIgnore != null && _userAgentsToIgnore.Any())
            {
                var agent = request.Headers.FirstOrDefault(h => h.Key.Equals("User-Agent"));
                if (agent.Key != null)
                {
                    foreach (var userAgnet in _userAgentsToIgnore)
                    {
                        if (agent.Value.Any(v => v.StartsWith(userAgnet.Key, StringComparison.OrdinalIgnoreCase)))
                        {
                            path = RemoveOrReplaceApiVersion(path, userAgnet.Value);
                            break;
                        }
                    }
                }
            }

            var encodedPath = Convert.ToBase64String(Encoding.UTF8.GetBytes(path));
            return string.Format("{0} {1}", request.Method, encodedPath);
        }

        public string GetMatchingKey(RecordEntry recordEntry)
        {
            var encodedPath = recordEntry.EncodedRequestUri;
            var path = recordEntry.RequestUri;
            var changed = false;
            if (path.Contains("?&"))
            {
                path = recordEntry.RequestUri.Replace("?&", "?");
                changed = true;
            }

            string version;
            if (ContainsIgnoredProvider(path, out version))
            {
                path = RemoveOrReplaceApiVersion(path, version);
                changed = true;
            }

            if (changed)
            {
                encodedPath = Convert.ToBase64String(Encoding.UTF8.GetBytes(path));
            }

            return string.Format("{0} {1}", recordEntry.RequestMethod, encodedPath);
        }

        private bool ContainsIgnoredProvider(string requestUri, out string version)
        {
            if (_ignoreGenericResource && !requestUri.Contains("providers"))
            {
                version = String.Empty;
                return true;
            }

            foreach (var provider in _providersToIgnore)
            {
                var providerString = string.Format("providers/{0}", provider.Key);
                if (requestUri.Contains(providerString))
                {
                    version = provider.Value;
                    return true;
                }
            }

            version = string.Empty;
            return false;
        }

        private string RemoveOrReplaceApiVersion(string requestUri, string version)
        {
            if (!string.IsNullOrWhiteSpace(version))
            {
                return Regex.Replace(requestUri, @"([\?&])api-version=[^&]+", string.Format("$1api-version={0}", version));
            }
            else
            {
                var result = Regex.Replace(requestUri, @"&api-version=[^&]+", string.Empty);
                return Regex.Replace(result, @"\?api-version=[^&]+[&]*", "?");
            }
        }
    }
}


text:
package com.midi_automator.view.windows.ImportFileChooser;

import javax.swing.JFileChooser;
import javax.swing.filechooser.FileFilter;
import javax.swing.filechooser.FileNameExtensionFilter;

import org.springframework.stereotype.Component;

import com.midi_automator.presenter.services.ImportExportService;

/**
 * The file chooser for choosing the import file
 *
 * @author aguelle
 *
 */
@Component
public class ImportFileChooser extends JFileChooser {

	private static final long serialVersionUID = 1L;

	public static final String NAME = "import file chooser";

	public void init() {

		FileFilter filter = new FileNameExtensionFilter(
				ImportExportService.MIDI_AUTOMATOR_FILE_TYPE,
				ImportExportService.MIDI_AUTOMATOR_FILE_EXTENSIONS);
		setAcceptAllFileFilterUsed(false);
		setFileFilter(filter);
		setName(NAME);
	}

}


text:
package org.algorithmdb.exceptions;

public class ImaginaryNumberException extends Exception {

	private static final long serialVersionUID = -3194402024083646792L;
	private String errorMessage;

	ImaginaryNumberException(String strMessage){
	    errorMessage = strMessage;
	}

	public ImaginaryNumberException(){}

	public String toString() {
		if(errorMessage != null) {
			return errorMessage;
		}
		return "ImaginaryNumberException: Number is Imaginary!";
	}
}


text:
<?php
namespace Core\Form;

use Zend\Form\Form;

class SettingsForm extends Form
{
    protected $_fields;

    protected $_country;

    public function __construct($name = null)
    {
        parent::__construct($name);
    }

    public function addFields($fields)
    {
        $this->_fields = $fields;
        return $this;
    }

    public function load()
    {
        if(!isset($this->_fields['groups']))
        {
            return $this;
        }
        foreach($this->_fields['groups'] as $group => $groupval)
        {
            foreach($groupval['fields'] as $element)
            {
                $value = null;
                if(isset($element['path']))
                {
                   $value = \Ething::getSiteConfig($element['path']);
                   if(strpos($value ,",")!== false)
                   {
                        $value = explode(",",\Ething::getSiteConfig($element['path']));
                   }
                }

                 $default = array(
                        'name' => isset($element['name'])?$element['name']:"",
                        'type' => isset($element['type'])?$element['type']:'text'
                    );
                $db = array('attributes' => array('value' => $value));
                $additionalParams = array();
                if(isset($element['attributes']))
                {
                    $finalattribute = array_merge($db['attributes'],$element['attributes']);

                    $additionalParams += array('attributes' => $finalattribute);
                }

                if(isset($element['backendmodel']))
                {
                    if(class_exists($element['backendmodel']))
                        $additionalParams += array('type' => $element['backendmodel']);
                }

                switch($element['type'])
                {
                    case "radio":
                    case "select":
                        $additionalParams += array('options' => array('label' => $element['options']['label'],
                                                'value_options' => isset($element['options']['sourcemodel'])?
                                                call_user_func(array(new $element['options']['sourcemodel'],(isset($element['options']['sourcemodel_method'])?$element['options']['sourcemodel_method']:"toOptionArray")))
                                                :$this->formOptionvalues(isset($element['options']['values'])?$element['options']['values']:array())));
                        break;
                }

                if(isset($element['name']))
                {
                    $finalset = array_merge($default,$additionalParams);
                    $this->add($finalset);
                }
            }
        }
        return $this;
    }

    public function addCountry($country)
    {
        $this->_country = $country;
        return $this;
    }

    public function formOptionvalues($options)
    {
        $values = array();
        foreach($options as $key =>  $option)
        {
            $values[$option['value']]= isset($option['label'])?$option['label']:$key;
        }
        return $values;
    }
}


text:
import { Component, OnInit } from '@angular/core';
import { Router } from '@angular/router';

import { Event } from '../../shared/backend.service';;
import { BackendService } from '../../shared/backend.service';
import { BackendMockService } from '../../shared/backendmock.service';
import { LoggerService } from '../../shared/logger.service';
import { AuthService } from '../../shared/auth.service';

@Component({
    moduleId: module.id,
    selector: 'app-event-list',
    templateUrl: 'event-list.component.html',
    styleUrls: ['event-list.component.css'],
    providers: [LoggerService, BackendService, BackendMockService]
})
export class EventListComponent implements OnInit {
    selectedEvent: Event = null;
    events: Event[] = null;

    constructor(
        private _router: Router,
        private _authService: AuthService,
        private _backendService: BackendMockService,
        private _logger: LoggerService) {
        console.log('events ctor');
    }

    ngOnInit() {
        console.log('event-list init');
        this.events = this._backendService.getEvents();
    }

    select(event: Event) {
        console.log('event selected ', event.id);
        this.selectedEvent = event;
        this._router.navigate(['/events', event.id]).then(_ => {
            //navigation done
        });
    };
}


text:
package com.android.tools.build.bundletool.device.activitymanager;

import static com.google.common.base.Preconditions.checkNotNull;

import com.android.tools.build.bundletool.device.activitymanager.ResourceConfigParser.ConfigQualifierParser;
import com.android.tools.build.bundletool.device.activitymanager.ResourceConfigParser.ResourceConfigHandler;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import java.util.Iterator;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Parser for locale qualifiers in the resource config string.
 *
 *
Parsing locales is trickier because the qualifier separator '-' is reused as the language
 * region separator, so we sometimes have to advance the iterator here. At the same time, we have to
 * roll back the iterator progress if the parsing fails.
 */
class LocaleParser<T> implements ConfigQualifierParser<T> {

  // "car" is a UiModeType value. It's not a valid language even though it matches the pattern.
  private static final String LANGUAGE_PATTERN = "(?!car)[A-Za-z]{2,3}";
  private static final String BCP47_PREFIX = "b+";
  // A resource qualifier might start with region continuation of the previous language qualifier
  // (e.g. en-rGB) and optionally followed by more locale qualifiers (e.g. en-rGB,fr-rFR).
  private static final String REGION_CONTINUATION_PATTERN = "r[A-Z]{2}($|,.+)";

  private static final Pattern LANGUAGE_REGION_PATTERN =
      Pattern.compile("(?<language>[A-Za-z]{2,3})(-r(?<region>[A-Za-z]{2}))?");
  private static final Splitter BCP47_SPLITTER = Splitter.on('+');
  private static final Splitter COMMA_SPLITTER = Splitter.on(',');
  private static final Joiner HYPHEN_JOINER = Joiner.on('-');
  private static final Joiner PLUS_JOINER = Joiner.on('+');

  /** Parses all locale qualifiers. */
  @Override
  public boolean parse(ConfigStringIterator iterator, ResourceConfigHandler<T> handler) {
    iterator.savePosition();
    try {
      parseInternal(iterator.getValue(), handler, iterator);
      return true;
    } catch (ParseException parseException) {
      iterator.restorePosition();
      return false;
    }
  }

  /**
   * Based on LocaleValue::InitFromParts from Aapt2. Added support for comma separated locales.
   *
   * @throws ParseException if any locale resource qualifier is invalid. The standard semantic is to
   *     rollback the iterator and continue parsing using the next parser in the list. See {@link
   *     ResourceConfigParser}.
   */
  private void parseInternal(
      String nextPart, ResourceConfigHandler<T> handler, Iterator<String> iterator)
      throws ParseException {

    // In order to support comma separated locales and parse them nicely, we try to concatenate
    // all locales (they can use the resource config string separator "-"). And then parse them
    // separately.
    Iterable<String> locales = COMMA_SPLITTER.split(getAllLocaleSegments(nextPart, iterator));

    String language;
    String region;

    for (String locale : locales) {
      if (locale.startsWith(BCP47_PREFIX)) {
        initFromBcp47Tag(locale.substring(BCP47_PREFIX.length()), handler);
      } else {
        Matcher languageRegionMatcher = LANGUAGE_REGION_PATTERN.matcher(locale);
        if (!languageRegionMatcher.matches()) {
          throw new ParseException(
              String.format("Unexpected language/region locale string '%s'.", locale));
        } else {
          language = languageRegionMatcher.group("language");
          region = languageRegionMatcher.group("region");
          checkNotNull(language);
          if (region != null) {
            handler.onLocale(language + "-" + region);
          } else {
            handler.onLocale(language);
          }
        }
      }
    }
  }

  /** Moves the iterator when necessary to read all locales encoded in the config string. */
  private String getAllLocaleSegments(String nextPart, Iterator<String> iterator) {
    ImmutableList.Builder<String> localeParts = ImmutableList.builder();

    // The first part can be just a language (not 'car') or contain ',' or BCP-47 expression.
    if (nextPart.contains(",")
        || nextPart.matches(LANGUAGE_PATTERN)
        || nextPart.startsWith(BCP47_PREFIX)) {
      localeParts.add(nextPart);
      // The next part must contain ',' or be just a region. We will validate against region
      // regex properly when we get to parse the specific locale part.
      while (iterator.hasNext()) {
        nextPart = iterator.next();
        if (nextPart.matches(REGION_CONTINUATION_PATTERN) || nextPart.contains(",")) {
          localeParts.add(nextPart);
        }
      }
    }
    return HYPHEN_JOINER.join(localeParts.build());
  }

  /**
   * Parses BCP-47 tag delimited by '+'.
   *
   *
Doesn't support scripts or variants, they are ignored.
   */
  private void initFromBcp47Tag(String bcp47Tag, ResourceConfigHandler<T> handler)
      throws ParseException {
    ImmutableList<String> subtags = ImmutableList.copyOf(BCP47_SPLITTER.split(bcp47Tag));

    String language = subtags.get(0);
    Optional<String> region = extractRegion(subtags);

    if (region.isPresent()) {
      handler.onLocale(language + "-" + region.get());
    } else {
      handler.onLocale(language);
    }
  }

  /** Implementation based on LocaleValue::InitFromBcp47TagImpl from Aapt2. */
  private Optional<String> extractRegion(ImmutableList<String> bcp47SubTags) throws ParseException {
    if (bcp47SubTags.size() == 1) {
      return Optional.empty();
    } else if (bcp47SubTags.size() == 2) {
      String secondSubTag = bcp47SubTags.get(1);
      switch (secondSubTag.length()) {
        case 2:
        case 3:
          return Optional.of(secondSubTag);
        case 4:
        case 5:
        case 6:
        case 7:
        case 8:
          // It's variant or script. Bundle Tool doesn't support it.
          return Optional.empty();
        default:
          break;
      }
    } else if (bcp47SubTags.size() == 3) {
      // The second subtag can be a script or region. If it's size 2 or 3 it's region.
      String secondSubTag = bcp47SubTags.get(1);
      if (secondSubTag.length() == 2 || secondSubTag.length() == 3) {
        return Optional.of(secondSubTag);
      }
      String thirdSubTag = bcp47SubTags.get(2);
      if (thirdSubTag.length() == 2 || thirdSubTag.length() == 3) {
        return Optional.of(thirdSubTag);
      } else if (thirdSubTag.length() >= 4) {
        // It's variant.
        return Optional.empty();
      }
    } else if (bcp47SubTags.size() == 4) {
      // If we have all 4 subtags they represent: language-script-region-variant.
      return Optional.of(bcp47SubTags.get(2));
    }

    // Unsupported BCP-47 string if we reach here.

    throw new ParseException(
        String.format(
            "Encountered unsupported or invalid BCP-47 string '%s'.",
            PLUS_JOINER.join(bcp47SubTags)));
  }

  private static class ParseException extends Exception {
    public ParseException(String cause) {
      super(cause);
    }
  }
}


text:
#### 0.0.0.47 - 27.05.2015
* Fixed a bug in counting the number of active worker threads, which is only used by `Scheduler.wait`.

#### 0.0.0.46 - 24.05.2015
* Changed to use `FSharp.Core` from NuGet.
* Removed PCL profiles.
* Fixed potential leak with `Job.tryIn` and `Job.tryInDelay`.

#### 0.0.0.45 - 10.04.2015
* Added basic `Stream.buffer` combinator.
* Renamed `Stream.subscribe*` to `Stream.consume*` to better describe semantics.
* Doc refinements.
* Added `Stream.pullOn`, `Stream.skipWhileJob`, `Stream.skipWhileFun`, `Stream.pullOn`, `Stream.foldBack`, `Stream.mapIgnore`,
`Stream.ambAll`, `Stream.appendAll`, `Stream.mergeAll` and `Stream.switchAll`
* Fixed to start reading the serialized variable immediately as documented.
* Added `Stream.subscribe*` as a shorthand for `Stream.iter* |> queue`.
* Added mutable `Stream.Property<'x>` that generates property change notifications for e.g. WPF data binding.
* Added `doFinalizeJob` and `doFinalizeFun`.
* Experimental wrapper for streams that tracks space safety via phantom types.
* Changed `groupByJob` and `groupByFun` to take an additional function/job for forming new groups.
* Added `Stream.tailsMapFun` and `Stream.initsMapFun` these are useful for lifting the `Stream.tails` and `Stream.inits` functions.
* Added `keepFollowing1` and renamed `keepLatest` to `keepPreceding` to make the naming more symmetric although the concepts of "following" and "preceding" aren't really fully symmetric (unless you allow time travel).
* A slightly more performant implementation of `keepLatestFuns`.
* Renamed from `lazify` to `keepLatest`.
* Lazification of live streams.
* Rethinking timing and throttling operations.
* Generalized `joinWith` and `mapJoin`.
* There is no sleep, only `timeOut`.
* Added ability to directly bind observables in job computation expressions.
* Removed superfluous methods from `Async.OnWithSchedulerBuilder`.

#### 0.0.0.44 - 02.03.2015
* Fixed to properly remove waiter in case it timed out rather than was signalled.
* Signal when top timed changes to improve timer accuracy.
* Add a test of the job computation builder.
* Remove redundant builder methods.
* Added `Job.tryFinallyFunDelay`, `Job.tryFinallyJobDelay`, `Job.tryWithDelay` and `Job.whileDoDelay`.
* Added `Stream.mapConst`.
* Removed `Stream.subscribeOnFirst`, `Stream.subscribeDuring` and `Stream.subscribingTo`, because `Stream.ofObservable` easily covers all of those use cases.

#### 0.0.0.43 - 28.02.2015
* Scale work sharing by the number of cores.

#### 0.0.0.42 - 27.02.2015
* Distribute work more eagerly.
* Name worker threads.
* Added monadic composition operator `>=>` for jobs.
* Updated documentation.
* Tuning streams.
* Tweaked Promisesc.
* Starting to benchmark streams.
* Avoid an allocation when choosing over lazy promises.
* Added `Stream.ofObservableOnMain`.
* Added `Async.getMain`.
* Added `Stream.ofObservableOn` and `Stream.ofObservable`.
* Added `IObservable<_>.onceAlt`.
* Renamed `withNack` as `withNackJob` and `wrapAbort` as `wrapAbortJob`.
* Added `Alt.wrapAbortFun`.
* Added `Alt.choosy`, which is an optimized version of `Alt.choose` for arrays.
* Added `Job.Scheduler.bind` for wrapping external asynchronous events.
* Added `Alt.withNackFun`.
* Inlineable `withNack` avoids closure allocation.
* Added experimental support for running async comptations on the main synchronization context, which must be explicitly configured by application code.
* Added `IObservable<'x>.onceAltOn` extension method for conveniently interfacing Hopac with suitable observables.
* Added `TopLevel.startDelay` and `TopLevel.queueDelay` for convenience.
* Renamed `Builder.Join` to `Builder.Plus` and recognized that `Zero´ must also be abstract.

#### 0.0.0.41 - 11.02.2015
* Make sure `StaticData` is initialized, fix for #52.
* Don't capture context, fix for #53.
* Added `Promise.Now.never`.
* Optimized anonymous class initialization patterns: small time and space improvement across the board.
* Another MissingMethodException workaround.

#### 0.0.0.40 - 06.02.2015
* Changed the priority queue to a simple leftist heap and added some extra logic to purge abandoned timeouts while merging. See #50.
* Added `TopLevel.memo` as an alias for `Promise.Now.delay`.
* Added `Stream.takeAndSkipUntil`.
* Disable inlining of `Latch.Now.create` as an attempt to work around a `MissingMethodException`.
* Fixing `combineLatest` to properly discard unmatched elements.
* Add more reference implementations to the docs.
* Renamed `Stream.switchOn` to `Stream.switchTo`.
* Make `Job.lift` inlineable.

#### 0.0.0.39 - 01.02.2015
* Added internal class for `for` -style jobs.
* Added `Job.tryInDelay`.
* Added `distinctUntilChanged`.
* Extended `groupByFun` and `groupByJob` with ability to close substreams.
* Added plain `iter`.
* Tests for streams.
* Shift is tricky to implement correctly.
* Documenting and refining streams.
* `sleep` doesn't exist anymore.
* Note about timeouts and liveness.

#### 0.0.0.38 - 24.01.2015
* Move Streams, BoundedMb and some functions form Hopac.Extra library to Hopac.
* Documenting and refining choice streams.
* Put `Streams` under `TopLevel`.
* Expose `Stream<'x>` at `Hopac` namespace level.
* Experimental streams builder.
* Refining streams.
* Lazy memoizing promise operators.
* Fixed missing init in IVar and Nack.

#### 0.0.0.37 - 21.01.2015
* Merge Hopac.Extra package.
* Renamed `Streams.finallyFun` to `Streams.onCloseFun` to emphasize semantic differences.
* Added `Streams.subscribeDuring`.
* Added `Streams.subscribe` to easily convert observables to streams.
* Added `Streams.finallyJob`.
* Added `Job.finallyFun`.

#### 0.0.0.36 - 20.01.2015
* Added `sleepMillis` and `timeOutMillis`.
* Flexible typing.
* `select` and `pick` are not necessary.
* Added `Streams.scanFrom*` and `Streams.foldFrom*`.
* Reduced the synchronization properties of StreamVar and StreamSrc.
* Added `Alt.raises` and `Streams.error`.
* Added experimental `IAsyncDisposable` interface and associated `Job.usingAsync` combinator.
* Added `Streams.tails`.
* Added `Alt.Ignore`.
* Generalized `MVar.modifyFun` and `MVar.modifyJob` to alternatives.

#### 0.0.0.35 - 19.12.2014
* Fixed to refer to Xamarin.iOS rather than MonoTouch.

#### 0.0.0.34 - 19.12.2014
* Removed ThreadPool and WaitHandle extension methods, because they are not supported on PCL profiles.
* Removed Scheduler ThreadPriority option, because ThreadPriority is not supported on PCL profiles.
* Reorganized to PCL and platform libraries.

#### 0.0.0.33 - 11.12.2014
* Added Async.toAltOn, Async.toAlt.
* Added Alt.tryFinallyFun, Alt.tryFinallyJob.

#### 0.0.0.32 - 02.12.2014
* Attempt to work around an inlining issue with Alt.never calls.

#### 0.0.0.31 - 25.11.2014
* Fixed bug in delayed promises as selective operations.

#### 0.0.0.30 - 19.11.2014
* Fixed a couple of (exception handling) cases where nacks were not triggered correctly.
* Added non-operator versions of bind, map and wrap for convenience.
* Reintroduced lazy promises.
* Enhanced timeOut to work with zero and infinite time spans.
* Added some more TopLevel combinators for convenience.

#### 0.0.0.29 - 29.09.2014
* Print warning on Mono when not using SGen.

#### 0.0.0.28 - 29.09.2014
* MonoAndroid

#### 0.0.0.27 - 28.09.2014
* MonoTouch

#### 0.0.0.26 - 22.09.2014
* Fixed not to rely on tail calls on Mono.

#### 0.0.0.25 - 20.09.2014
* Minor tweaks to make Hopac work more nicely on Mono 3.6.0+.

#### 0.0.0.24 - 27.07.2014
* Switched to .Net framework 4.5 (was 4.5.1).
* Removed For array overload to avoid typing issue.  Array.iterJob should now be used in performance critical cases.
* Added experimental Async <-> Job interop support.
* Renamed <|> to <|>? and added <|> with result type restricted to Job.
* IVar is now inherited from Promise and both now have low level polling ops.

#### 0.0.0.23 - 22.07.2014
* Fixed bug in Ch.Try.give introduced in previous version.</releaseNotes>