test_class.file: contrib/src/test/java/macrobase/analysis/contextualoutlier/ContextualOutlierDetectorTest.java
test_class.identifier: ContextualOutlierDetectorTest
test_case.body: @Test
      public void testContextualDiscreteAttribute() throws Exception {
          //construct a contextual outlier detector
          MacroBaseConf conf = new MacroBaseConf();
          conf.set(MacroBaseConf.METRICS, Arrays.asList("A1"));
          conf.set(MacroBaseConf.TRANSFORM_TYPE, "MAD");
          List<String> contextualDiscreteAttributes = new ArrayList<String>();
          contextualDiscreteAttributes.add("C1_Discrete");
          List<String> contextualDoubleAttributes = new ArrayList<String>();
  
          List<String> attributes = new ArrayList<>();
          attributes.addAll(contextualDiscreteAttributes);
          attributes.addAll(contextualDoubleAttributes);
  
          conf.set(MacroBaseConf.ATTRIBUTES, attributes);
          conf.set(ContextualConf.CONTEXTUAL_DISCRETE_ATTRIBUTES, contextualDiscreteAttributes);
          conf.set(ContextualConf.CONTEXTUAL_DOUBLE_ATTRIBUTES, contextualDoubleAttributes);
          conf.set(ContextualConf.CONTEXTUAL_DENSECONTEXTTAU, 0.4);
          conf.set(ContextualConf.CONTEXTUAL_NUMINTERVALS, 10);
          conf.set(MacroBaseConf.OUTLIER_STATIC_THRESHOLD, 3.0);
          conf.getEncoder().recordAttributeName(1, "A1");
          conf.getEncoder().recordAttributeName(2, "C1_Discrete");
          ContextualOutlierDetector contextualDetector = new ContextualOutlierDetector(conf);
          List<Datum> data = new ArrayList<>();
          for (int i = 0; i < 100; ++i) {
              double[] sample = new double[1];
              sample[0] = i;
              Integer[] c1 = new Integer[1];
              if (i < 5) {
                  c1[0] = 1;
              } else if (i >= 5 && i < 50) {
                  c1[0] = 2;
              } else {
                  c1[0] = 1;
              }
              data.add(makeDatum(conf, new ArrayList<>(), new ArrayRealVector(sample),
                                 new ArrayList<>(Arrays.asList(c1)),
                                 new ArrayRealVector()));
          }
  
          ContextualTransformer transformer = new ContextualTransformer(conf);
          transformer.consume(data);
          List<ContextualDatum> cdata = transformer.getStream().drain();
  
          Map<Context, List<OutlierClassificationResult>> context2Outliers = contextualDetector.searchContextualOutliers(cdata);
          TestCase.assertEquals(context2Outliers.size(), 1);
          for (Context context : context2Outliers.keySet()) {
              List<Interval> intervals = context.getIntervals();
              TestCase.assertEquals(intervals.size(), 1);
              assertEquals(intervals.get(0).getColumnName(), "C1_Discrete");
              TestCase.assertEquals(intervals.get(0) instanceof IntervalDiscrete, true);
              assertEquals("1",
                           conf.getEncoder().getAttribute(((IntervalDiscrete) intervals.get(0)).getValue()).getValue());
          }
      }
test_case.class_method_signature: ContextualOutlierDetectorTest.testContextualDiscreteAttribute()
test_case.full_signature: @Test public void testContextualDiscreteAttribute()
test_case.identifier: testContextualDiscreteAttribute
test_case.invocations: set asList set add addAll addAll set set set set set set recordAttributeName getEncoder recordAttributeName getEncoder add makeDatum asList consume drain getStream searchContextualOutliers assertEquals size keySet getIntervals assertEquals size assertEquals getColumnName get assertEquals get assertEquals getValue getAttribute getEncoder getValue get
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testContextualDiscreteAttribute()
focal_class.declarator: log = LoggerFactory.getLogger(ContextualOutlierDetector.class)
focal_class.modifier: private static final
focal_class.original_string: private static final Logger log = LoggerFactory.getLogger(ContextualOutlierDetector.class);
focal_class.type: Logger
focal_class.var_name: log
focal_class.declarator: conf
focal_class.modifier: private
focal_class.original_string: private MacroBaseConf conf;
focal_class.type: MacroBaseConf
focal_class.var_name: conf
focal_class.declarator: contextualDiscreteAttributes
focal_class.modifier: private
focal_class.original_string: private List<String> contextualDiscreteAttributes;
focal_class.type: List<String>
focal_class.var_name: contextualDiscreteAttributes
focal_class.declarator: contextualDoubleAttributes
focal_class.modifier: private
focal_class.original_string: private List<String> contextualDoubleAttributes;
focal_class.type: List<String>
focal_class.var_name: contextualDoubleAttributes
focal_class.declarator: totalContextualDimensions
focal_class.modifier: private
focal_class.original_string: private int totalContextualDimensions;
focal_class.type: int
focal_class.var_name: totalContextualDimensions
focal_class.declarator: globalContext
focal_class.modifier: private
focal_class.original_string: private Context globalContext;
focal_class.type: Context
focal_class.var_name: globalContext
focal_class.declarator: denseContextTau
focal_class.modifier: private
focal_class.original_string: private double denseContextTau;
focal_class.type: double
focal_class.var_name: denseContextTau
focal_class.declarator: numIntervals
focal_class.modifier: private
focal_class.original_string: private int numIntervals;
focal_class.type: int
focal_class.var_name: numIntervals
focal_class.declarator: maxPredicates
focal_class.modifier: private
focal_class.original_string: private int maxPredicates;
focal_class.type: int
focal_class.var_name: maxPredicates
focal_class.declarator: encoder
focal_class.modifier: private
focal_class.original_string: private DatumEncoder encoder;
focal_class.type: DatumEncoder
focal_class.var_name: encoder
focal_class.declarator: contextualOutputFile
focal_class.modifier: private
focal_class.original_string: private String contextualOutputFile;
focal_class.type: String
focal_class.var_name: contextualOutputFile
focal_class.declarator: densityPruning
focal_class.modifier: private
focal_class.original_string: private boolean densityPruning;
focal_class.type: boolean
focal_class.var_name: densityPruning
focal_class.declarator: dependencyPruning
focal_class.modifier: private
focal_class.original_string: private boolean dependencyPruning;
focal_class.type: boolean
focal_class.var_name: dependencyPruning
focal_class.declarator: distributionPruningForTraining
focal_class.modifier: private
focal_class.original_string: private boolean distributionPruningForTraining;
focal_class.type: boolean
focal_class.var_name: distributionPruningForTraining
focal_class.declarator: distributionPruningForScoring
focal_class.modifier: private
focal_class.original_string: private boolean distributionPruningForScoring;
focal_class.type: boolean
focal_class.var_name: distributionPruningForScoring
focal_class.declarator: alpha = 0.05
focal_class.modifier: private
focal_class.original_string: private double alpha = 0.05;
focal_class.type: double
focal_class.var_name: alpha
focal_class.declarator: context2Outliers = new HashMap<>()
focal_class.modifier: private
focal_class.original_string: private Map<Context, List<OutlierClassificationResult>> context2Outliers = new HashMap<>();
focal_class.type: Map<Context, List<OutlierClassificationResult>>
focal_class.var_name: context2Outliers
focal_class.declarator: densityPruning2 = 0
focal_class.modifier: private
focal_class.original_string: private int densityPruning2 = 0;
focal_class.type: int
focal_class.var_name: densityPruning2
focal_class.declarator: numOutlierDetectionRunsWithoutTrainingWithoutScoring = 0
focal_class.modifier: private
focal_class.original_string: private int numOutlierDetectionRunsWithoutTrainingWithoutScoring = 0;
focal_class.type: int
focal_class.var_name: numOutlierDetectionRunsWithoutTrainingWithoutScoring
focal_class.declarator: numOutlierDetectionRunsWithoutTrainingWithScoring = 0
focal_class.modifier: private
focal_class.original_string: private int numOutlierDetectionRunsWithoutTrainingWithScoring = 0;
focal_class.type: int
focal_class.var_name: numOutlierDetectionRunsWithoutTrainingWithScoring
focal_class.declarator: numOutlierDetectionRunsWithTrainingWithScoring = 0
focal_class.modifier: private
focal_class.original_string: private int numOutlierDetectionRunsWithTrainingWithScoring = 0;
focal_class.type: int
focal_class.var_name: numOutlierDetectionRunsWithTrainingWithScoring
focal_class.declarator: context2BitSet = new HashMap<Context, BitSet>()
focal_class.modifier: private
focal_class.original_string: private Map<Context, BitSet> context2BitSet = new HashMap<Context, BitSet>();
focal_class.type: Map<Context, BitSet>
focal_class.var_name: context2BitSet
focal_class.file: contrib/src/main/java/macrobase/analysis/contextualoutlier/ContextualOutlierDetector.java
focal_class.identifier: ContextualOutlierDetector
focal_class.class_method_signature: ContextualOutlierDetector.ContextualOutlierDetector(MacroBaseConf conf)
focal_class.full_signature: public  ContextualOutlierDetector(MacroBaseConf conf)
focal_class.identifier: ContextualOutlierDetector
focal_class.modifiers: public
focal_class.parameters: (MacroBaseConf conf)
focal_class.signature:  ContextualOutlierDetector(MacroBaseConf conf)
focal_class.class_method_signature: ContextualOutlierDetector.searchContextualOutliers(List<ContextualDatum> data)
focal_class.full_signature: public Map<Context, List<OutlierClassificationResult>> searchContextualOutliers(List<ContextualDatum> data)
focal_class.identifier: searchContextualOutliers
focal_class.modifiers: public
focal_class.parameters: (List<ContextualDatum> data)
focal_class.return: Map<Context, List<OutlierClassificationResult>>
focal_class.signature: Map<Context, List<OutlierClassificationResult>> searchContextualOutliers(List<ContextualDatum> data)
focal_class.class_method_signature: ContextualOutlierDetector.findInputOutliers(List<ContextualDatum> data)
focal_class.full_signature: private List<ContextualDatum> findInputOutliers(List<ContextualDatum> data)
focal_class.identifier: findInputOutliers
focal_class.modifiers: private
focal_class.parameters: (List<ContextualDatum> data)
focal_class.return: List<ContextualDatum>
focal_class.signature: List<ContextualDatum> findInputOutliers(List<ContextualDatum> data)
focal_class.class_method_signature: ContextualOutlierDetector.searchContextGivenOutliers(List<ContextualDatum> data)
focal_class.full_signature: public Map<Context, List<OutlierClassificationResult>> searchContextGivenOutliers(List<ContextualDatum> data)
focal_class.identifier: searchContextGivenOutliers
focal_class.modifiers: public
focal_class.parameters: (List<ContextualDatum> data)
focal_class.return: Map<Context, List<OutlierClassificationResult>>
focal_class.signature: Map<Context, List<OutlierClassificationResult>> searchContextGivenOutliers(List<ContextualDatum> data)
focal_class.class_method_signature: ContextualOutlierDetector.searchContextGivenOutliers(List<ContextualDatum> data, List<ContextualDatum> inputOutliers)
focal_class.full_signature: public Map<Context, List<OutlierClassificationResult>> searchContextGivenOutliers(List<ContextualDatum> data, List<ContextualDatum> inputOutliers)
focal_class.identifier: searchContextGivenOutliers
focal_class.modifiers: public
focal_class.parameters: (List<ContextualDatum> data, List<ContextualDatum> inputOutliers)
focal_class.return: Map<Context, List<OutlierClassificationResult>>
focal_class.signature: Map<Context, List<OutlierClassificationResult>> searchContextGivenOutliers(List<ContextualDatum> data, List<ContextualDatum> inputOutliers)
focal_class.class_method_signature: ContextualOutlierDetector.randomSampling(List<ContextualDatum> data, int minSampleSize)
focal_class.full_signature: private HashSet<ContextualDatum> randomSampling(List<ContextualDatum> data, int minSampleSize)
focal_class.identifier: randomSampling
focal_class.modifiers: private
focal_class.parameters: (List<ContextualDatum> data, int minSampleSize)
focal_class.return: HashSet<ContextualDatum>
focal_class.signature: HashSet<ContextualDatum> randomSampling(List<ContextualDatum> data, int minSampleSize)
focal_class.class_method_signature: ContextualOutlierDetector.levelUpLattice(List<LatticeNode> latticeNodes, List<ContextualDatum> data)
focal_class.full_signature: private List<LatticeNode> levelUpLattice(List<LatticeNode> latticeNodes, List<ContextualDatum> data)
focal_class.identifier: levelUpLattice
focal_class.modifiers: private
focal_class.parameters: (List<LatticeNode> latticeNodes, List<ContextualDatum> data)
focal_class.return: List<LatticeNode>
focal_class.signature: List<LatticeNode> levelUpLattice(List<LatticeNode> latticeNodes, List<ContextualDatum> data)
focal_class.class_method_signature: ContextualOutlierDetector.contextualOutlierDetection(List<ContextualDatum> data, Context context)
focal_class.full_signature: public List<Datum> contextualOutlierDetection(List<ContextualDatum> data, Context context)
focal_class.identifier: contextualOutlierDetection
focal_class.modifiers: public
focal_class.parameters: (List<ContextualDatum> data, Context context)
focal_class.return: List<Datum>
focal_class.signature: List<Datum> contextualOutlierDetection(List<ContextualDatum> data, Context context)
focal_class.class_method_signature: ContextualOutlierDetector.sameDistribution(Context p1, Context p2)
focal_class.full_signature: private boolean sameDistribution(Context p1, Context p2)
focal_class.identifier: sameDistribution
focal_class.modifiers: private
focal_class.parameters: (Context p1, Context p2)
focal_class.return: boolean
focal_class.signature: boolean sameDistribution(Context p1, Context p2)
focal_class.class_method_signature: ContextualOutlierDetector.constructDetector()
focal_class.full_signature: private BatchTrainScore constructDetector()
focal_class.identifier: constructDetector
focal_class.modifiers: private
focal_class.parameters: ()
focal_class.return: BatchTrainScore
focal_class.signature: BatchTrainScore constructDetector()
focal_class.class_method_signature: ContextualOutlierDetector.buildOneDimensionalLatticeNodes(List<ContextualDatum> data)
focal_class.full_signature: private List<LatticeNode> buildOneDimensionalLatticeNodes(List<ContextualDatum> data)
focal_class.identifier: buildOneDimensionalLatticeNodes
focal_class.modifiers: private
focal_class.parameters: (List<ContextualDatum> data)
focal_class.return: List<LatticeNode>
focal_class.signature: List<LatticeNode> buildOneDimensionalLatticeNodes(List<ContextualDatum> data)
focal_class.class_method_signature: ContextualOutlierDetector.buildOneDimensionalLatticeNodesGivenOutliers(List<ContextualDatum> data, List<ContextualDatum> inputOutliers)
focal_class.full_signature: private List<LatticeNode> buildOneDimensionalLatticeNodesGivenOutliers(List<ContextualDatum> data, List<ContextualDatum> inputOutliers)
focal_class.identifier: buildOneDimensionalLatticeNodesGivenOutliers
focal_class.modifiers: private
focal_class.parameters: (List<ContextualDatum> data, List<ContextualDatum> inputOutliers)
focal_class.return: List<LatticeNode>
focal_class.signature: List<LatticeNode> buildOneDimensionalLatticeNodesGivenOutliers(List<ContextualDatum> data, List<ContextualDatum> inputOutliers)
focal_class.class_method_signature: ContextualOutlierDetector.isEncoderSetup()
focal_class.full_signature: private boolean isEncoderSetup()
focal_class.identifier: isEncoderSetup
focal_class.modifiers: private
focal_class.parameters: ()
focal_class.return: boolean
focal_class.signature: boolean isEncoderSetup()
focal_class.class_method_signature: ContextualOutlierDetector.isInterestingInterval(Interval interval)
focal_class.full_signature: private boolean isInterestingInterval(Interval interval)
focal_class.identifier: isInterestingInterval
focal_class.modifiers: private
focal_class.parameters: (Interval interval)
focal_class.return: boolean
focal_class.signature: boolean isInterestingInterval(Interval interval)
focal_class.class_method_signature: ContextualOutlierDetector.initOneDimensionalDenseContextsAndContext2Data(List<ContextualDatum> data, int dimension, double curDensityThreshold)
focal_class.full_signature: private List<Context> initOneDimensionalDenseContextsAndContext2Data(List<ContextualDatum> data, int dimension, double curDensityThreshold)
focal_class.identifier: initOneDimensionalDenseContextsAndContext2Data
focal_class.modifiers: private
focal_class.parameters: (List<ContextualDatum> data, int dimension, double curDensityThreshold)
focal_class.return: List<Context>
focal_class.signature: List<Context> initOneDimensionalDenseContextsAndContext2Data(List<ContextualDatum> data, int dimension, double curDensityThreshold)
focal_class.class_method_signature: ContextualOutlierDetector.initOneDimensionalDenseContextsAndContext2DataGivenOutliers(List<ContextualDatum> data, int dimension, List<ContextualDatum> inputOutliers)
focal_class.full_signature: private List<Context> initOneDimensionalDenseContextsAndContext2DataGivenOutliers(List<ContextualDatum> data, int dimension, List<ContextualDatum> inputOutliers)
focal_class.identifier: initOneDimensionalDenseContextsAndContext2DataGivenOutliers
focal_class.modifiers: private
focal_class.parameters: (List<ContextualDatum> data, int dimension, List<ContextualDatum> inputOutliers)
focal_class.return: List<Context>
focal_class.signature: List<Context> initOneDimensionalDenseContextsAndContext2DataGivenOutliers(List<ContextualDatum> data, int dimension, List<ContextualDatum> inputOutliers)
focal_class.class_method_signature: ContextualOutlierDetector.indexes2BitSet(List<Integer> indexes, int total)
focal_class.full_signature: private BitSet indexes2BitSet(List<Integer> indexes, int total)
focal_class.identifier: indexes2BitSet
focal_class.modifiers: private
focal_class.parameters: (List<Integer> indexes, int total)
focal_class.return: BitSet
focal_class.signature: BitSet indexes2BitSet(List<Integer> indexes, int total)
focal_class.class_method_signature: ContextualOutlierDetector.bitSet2Indexes(BitSet bs)
focal_class.full_signature: private List<Integer> bitSet2Indexes(BitSet bs)
focal_class.identifier: bitSet2Indexes
focal_class.modifiers: private
focal_class.parameters: (BitSet bs)
focal_class.return: List<Integer>
focal_class.signature: List<Integer> bitSet2Indexes(BitSet bs)
focal_method.body: public Map<Context, List<OutlierClassificationResult>> searchContextualOutliers(List<ContextualDatum> data) throws Exception {
          Stopwatch sw = Stopwatch.createUnstarted();
          log.debug("Find global context outliers on data num tuples: {} , MBs {} ", data.size());
          sw.start();
          HashSet<ContextualDatum> sample = randomSampling(data, 100);
          globalContext = new Context(sample, densityPruning, dependencyPruning, alpha);
          contextualOutlierDetection(data, globalContext);
          sw.stop();
          long globalOutlierDetecionTime = sw.elapsed(TimeUnit.MILLISECONDS);
          sw.reset();
          log.debug("Done global context outlier remaining data size {} : (duration: {}ms)", data.size(),
                    globalOutlierDetecionTime);
          List<LatticeNode> preLatticeNodes = new ArrayList<LatticeNode>();
          List<LatticeNode> curLatticeNodes = new ArrayList<LatticeNode>();
          for (int level = 1; level <= totalContextualDimensions; level++) {
              if (level > maxPredicates)
                  break;
              log.debug("Build {}-dimensional contexts on all attributes", level);
              sw.start();
              if (level == 1) {
                  curLatticeNodes = buildOneDimensionalLatticeNodes(data);
              } else {
                  curLatticeNodes = levelUpLattice(preLatticeNodes, data);
              }
              sw.stop();
              long latticeNodesBuildTimeCurLevel = sw.elapsed(TimeUnit.MILLISECONDS);
              sw.reset();
              log.debug("Done building {}-dimensional contexts on all attributes (duration: {}ms)", level,
                        latticeNodesBuildTimeCurLevel);
              log.debug("Memory Usage: {}", MemoryUtil.checkMemoryUsage());
              if (curLatticeNodes.size() == 0) {
                  log.debug("No more dense contexts, thus no need to level up anymore");
                  break;
              }
              log.debug("Find {}-dimensional contextual outliers", level);
              sw.start();
              int numDenseContextsCurLevel = 0;
              //run contextual outlier detection
              for (LatticeNode node : curLatticeNodes) {
                  for (Context context : node.getDenseContexts()) {
                      contextualOutlierDetection(data, context);
                      numDenseContextsCurLevel++;
                  }
              }
              sw.stop();
              long contextualOutlierDetectionTimeCurLevel = sw.elapsed(TimeUnit.MILLISECONDS);
              sw.reset();
              log.debug("Done Find {}-dimensional contextual outliers (duration: {}ms)", level,
                        contextualOutlierDetectionTimeCurLevel);
              log.debug(
                      "Done Find {}-dimensional contextual outliers, there are {} dense contexts(average duration per context: {}ms)",
                      level, numDenseContextsCurLevel,
                      (numDenseContextsCurLevel == 0) ? 0 : contextualOutlierDetectionTimeCurLevel / numDenseContextsCurLevel);
              log.debug("Done Find {}-dimensional contextual outliers, densityPruning2: {}, "
                        + "numOutlierDetectionRunsWithoutTrainingWithoutScoring: {},  "
                        + "numOutlierDetectionRunsWithoutTrainingWithScoring: {},  "
                        + "numOutlierDetectionRunsWithTrainingWithScoring: {}",
                        level, densityPruning2,
                        numOutlierDetectionRunsWithoutTrainingWithoutScoring,
                        numOutlierDetectionRunsWithoutTrainingWithScoring,
                        numOutlierDetectionRunsWithTrainingWithScoring);
              log.debug("----------------------------------------------------------");
              //free up memory
              if (level >= 2) {
                  for (LatticeNode node : preLatticeNodes) {
                      for (Context context : node.getDenseContexts()) {
                          context2BitSet.remove(context);
                      }
                  }
              }
              preLatticeNodes = curLatticeNodes;
          }
          return context2Outliers;
      }
focal_method.class_method_signature: ContextualOutlierDetector.searchContextualOutliers(List<ContextualDatum> data)
focal_method.full_signature: public Map<Context, List<OutlierClassificationResult>> searchContextualOutliers(List<ContextualDatum> data)
focal_method.identifier: searchContextualOutliers
focal_method.invocations: createUnstarted debug size start randomSampling contextualOutlierDetection stop elapsed reset debug size debug start buildOneDimensionalLatticeNodes levelUpLattice stop elapsed reset debug debug checkMemoryUsage size debug debug start getDenseContexts contextualOutlierDetection stop elapsed reset debug debug debug debug getDenseContexts remove
focal_method.modifiers: public
focal_method.parameters: (List<ContextualDatum> data)
focal_method.return: Map<Context, List<OutlierClassificationResult>>
focal_method.signature: Map<Context, List<OutlierClassificationResult>> searchContextualOutliers(List<ContextualDatum> data)