test_class.declarator: log = LoggerFactory.getLogger(CSVIngesterTest.class)
test_class.modifier: private static final
test_class.original_string: private static final Logger log = LoggerFactory.getLogger(CSVIngesterTest.class);
test_class.type: Logger
test_class.var_name: log
test_class.file: legacy/src/test/java/macrobase/ingest/CSVIngesterTest.java
test_class.identifier: CSVIngesterTest
test_case.body: @Test
      public void testGetRows() throws Exception {
          MacroBaseConf conf = new MacroBaseConf();
          conf.set(MacroBaseConf.CSV_INPUT_FILE, "src/test/resources/data/simple.csv");
          conf.set(MacroBaseConf.ATTRIBUTES, Lists.newArrayList("A2","A5"));
          conf.set(MacroBaseConf.METRICS, Lists.newArrayList("A1","A3","A4"));
          conf.set(MacroBaseConf.BASE_QUERY, "csv:///src/test/resources/data/simple.csv");
  
          CSVIngester ingester = new CSVIngester(conf);
  
          Map<String, String> preds = new HashMap<>();
          preds.put("A2", "1");
  
          RowSet rs = ingester.getRows(conf.getString(MacroBaseConf.BASE_QUERY),
                  preds,
                  10,
                  0);
  
          assertEquals(1, rs.getRows().size());
          System.out.println(rs);
      }
test_case.class_method_signature: CSVIngesterTest.testGetRows()
test_case.full_signature: @Test public void testGetRows()
test_case.identifier: testGetRows
test_case.invocations: set set newArrayList set newArrayList set put getRows getString assertEquals size getRows println
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testGetRows()
focal_class.declarator: log = LoggerFactory.getLogger(CSVIngester.class)
focal_class.modifier: private static final
focal_class.original_string: private static final Logger log = LoggerFactory.getLogger(CSVIngester.class);
focal_class.type: Logger
focal_class.var_name: log
focal_class.declarator: csvParser
focal_class.modifier: private
focal_class.original_string: private CSVParser csvParser;
focal_class.type: CSVParser
focal_class.var_name: csvParser
focal_class.declarator: schema
focal_class.modifier: private
focal_class.original_string: private Map<String, Integer> schema;
focal_class.type: Map<String, Integer>
focal_class.var_name: schema
focal_class.declarator: filename
focal_class.modifier: private
focal_class.original_string: private String filename;
focal_class.type: String
focal_class.var_name: filename
focal_class.declarator: dataStream = new MBStream<>()
focal_class.modifier: private
focal_class.original_string: private MBStream<Datum> dataStream = new MBStream<>();
focal_class.type: MBStream<Datum>
focal_class.var_name: dataStream
focal_class.declarator: loaded = false
focal_class.modifier: private
focal_class.original_string: private boolean loaded = false;
focal_class.type: boolean
focal_class.var_name: loaded
focal_class.declarator: badRows = 0
focal_class.modifier: private
focal_class.original_string: private int badRows = 0;
focal_class.type: int
focal_class.var_name: badRows
focal_class.file: legacy/src/main/java/macrobase/ingest/CSVIngester.java
focal_class.identifier: CSVIngester
focal_class.class_method_signature: CSVIngester.CSVIngester(MacroBaseConf conf)
focal_class.full_signature: public  CSVIngester(MacroBaseConf conf)
focal_class.identifier: CSVIngester
focal_class.modifiers: public
focal_class.parameters: (MacroBaseConf conf)
focal_class.signature:  CSVIngester(MacroBaseConf conf)
focal_class.class_method_signature: CSVIngester.parseRecord(CSVRecord record)
focal_class.full_signature: private Datum parseRecord(CSVRecord record)
focal_class.identifier: parseRecord
focal_class.modifiers: private
focal_class.parameters: (CSVRecord record)
focal_class.return: Datum
focal_class.signature: Datum parseRecord(CSVRecord record)
focal_class.class_method_signature: CSVIngester.getStream()
focal_class.full_signature: @Override public MBStream<Datum> getStream()
focal_class.identifier: getStream
focal_class.modifiers: @Override public
focal_class.parameters: ()
focal_class.return: MBStream<Datum>
focal_class.signature: MBStream<Datum> getStream()
focal_class.class_method_signature: CSVIngester.getRows(String baseQuery,
                            Map<String, String> preds,
                            int limit,
                            int offset)
focal_class.full_signature: @Override public RowSet getRows(String baseQuery,
                            Map<String, String> preds,
                            int limit,
                            int offset)
focal_class.identifier: getRows
focal_class.modifiers: @Override public
focal_class.parameters: (String baseQuery,
                            Map<String, String> preds,
                            int limit,
                            int offset)
focal_class.return: RowSet
focal_class.signature: RowSet getRows(String baseQuery,
                            Map<String, String> preds,
                            int limit,
                            int offset)
focal_class.class_method_signature: CSVIngester.includeRow(CSVRecord record, Map<String, String> preds)
focal_class.full_signature: private boolean includeRow(CSVRecord record, Map<String, String> preds)
focal_class.identifier: includeRow
focal_class.modifiers: private
focal_class.parameters: (CSVRecord record, Map<String, String> preds)
focal_class.return: boolean
focal_class.signature: boolean includeRow(CSVRecord record, Map<String, String> preds)
focal_class.superclass: extends DataIngester
focal_method.body: @Override
      public RowSet getRows(String baseQuery,
                            Map<String, String> preds,
                            int limit,
                            int offset) throws Exception{
  
          filename = conf.getString(MacroBaseConf.CSV_INPUT_FILE);
          Compression compression = conf.getCsvCompression();
  
          if (compression == Compression.GZIP) {
              InputStream fileStream = new FileInputStream(filename);
              InputStream gzipStream = new GZIPInputStream(fileStream);
              Reader decoder = new InputStreamReader(gzipStream);
              csvParser = new CSVParser(decoder, CSVFormat.DEFAULT.withHeader());
          } else {
              File csvFile = new File(conf.getString(MacroBaseConf.CSV_INPUT_FILE));
              csvParser = CSVParser.parse(csvFile, Charset.defaultCharset(), CSVFormat.DEFAULT.withHeader());
          }
          schema = csvParser.getHeaderMap();
          Iterator<CSVRecord> rawIterator = csvParser.iterator();
          int rowCount = 0;
  
          List<RowSet.Row> rows = Lists.newArrayList();
          while (rawIterator.hasNext() && rowCount < limit) {
              CSVRecord record = rawIterator.next();
              List<ColumnValue> columnValues = Lists.newArrayList();
  
              if (includeRow(record, preds)) {
                  for (Map.Entry<String, Integer> se : schema.entrySet()) {
                      columnValues.add(new ColumnValue(se.getKey(),record.get(se.getValue())));
                  }
  
                  rows.add(new RowSet.Row(columnValues));
                  rowCount++;
              }
          }
           return new RowSet(rows);
      }
focal_method.class_method_signature: CSVIngester.getRows(String baseQuery,
                            Map<String, String> preds,
                            int limit,
                            int offset)
focal_method.full_signature: @Override public RowSet getRows(String baseQuery,
                            Map<String, String> preds,
                            int limit,
                            int offset)
focal_method.identifier: getRows
focal_method.invocations: getString getCsvCompression withHeader getString parse defaultCharset withHeader getHeaderMap iterator newArrayList hasNext next newArrayList includeRow entrySet add getKey get getValue add
focal_method.modifiers: @Override public
focal_method.parameters: (String baseQuery,
                            Map<String, String> preds,
                            int limit,
                            int offset)
focal_method.return: RowSet
focal_method.signature: RowSet getRows(String baseQuery,
                            Map<String, String> preds,
                            int limit,
                            int offset)