text:
package org.apache.metron.indexing.dao;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.GroupRequest;
import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.GroupResult;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.integration.InMemoryComponent;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;

public abstract class SearchIntegrationTest {
  /**
   * [
   * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "duplicate_name_field": "data 1", "guid":"bro_1"},
   * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "bro_field": "bro data 2", "duplicate_name_field": "data 2", "guid":"bro_2"},
   * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "bro_field": "bro data 3", "duplicate_name_field": "data 3", "guid":"bro_3"},
   * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 4", "duplicate_name_field": "data 4", "guid":"bro_4"},
   * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 5", "duplicate_name_field": "data 5", "guid":"bro_5"}
   * ]
   */
  @Multiline
  public static String broData;

  /**
   * [
   * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "snort_field": 10, "duplicate_name_field": 1, "guid":"snort_1", "threat:triage:score":"10"},
   * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 20, "duplicate_name_field": 2, "guid":"snort_2", "threat:triage:score":"20"},
   * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 30, "duplicate_name_field": 3, "guid":"snort_3"},
   * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 40, "duplicate_name_field": 4, "guid":"snort_4"},
   * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 50, "duplicate_name_field": 5, "guid":"snort_5"}
   * ]
   */
  @Multiline
  public static String snortData;

  /**
   * {
   * "indices": ["bro", "snort"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String allQuery;

  /**
   * {
   * "guid": "bro_3",
   * "sensorType": "bro"
   * }
   */
  @Multiline
  public static String findOneGuidQuery;

  /**
   * [
   * {
   * "guid": "bro_1",
   * "sensorType": "bro"
   * },
   * {
   * "guid": "snort_2",
   * "sensorType": "snort"
   * }
   * ]
   */
  @Multiline
  public static String getAllLatestQuery;

  /**
   * {
   * "indices": ["bro", "snort"],
   * "query": "ip_src_addr:192.168.1.1",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String filterQuery;

  /**
   * {
   * "indices": ["bro", "snort"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "ip_src_port",
   *     "sortOrder": "asc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String sortQuery;

  /**
   * {
   *  "indices": [
   *    "snort",
   *    "bro"
   *  ],
   * "query": "*",
   * "from": 0,
   * "size": 25,
   * "sort": [
   *    {
   *      "field": "threat:triage:score",
   *      "sortOrder": "asc"
   *    }
   *  ]
   * }
   */
  @Multiline
  public static String sortAscendingWithMissingFields;

  /**
   * {
   *  "indices": [
   *    "snort",
   *    "bro"
   *  ],
   * "query": "*",
   * "from": 0,
   * "size": 25,
   * "sort": [
   *    {
   *      "field": "threat:triage:score",
   *      "sortOrder": "desc"
   *    }
   *  ]
   * }
   */
  @Multiline
  public static String sortDescendingWithMissingFields;

  /**
   * {
   * "indices": ["bro", "snort"],
   * "query": "*",
   * "from": 4,
   * "size": 3,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String paginationQuery;

  /**
   * {
   * "indices": ["bro"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String indexQuery;

  /**
   * {
   * "facetFields": ["source:type", "ip_src_addr", "ip_src_port", "long_field", "timestamp", "latitude", "score", "is_alert"],
   * "indices": ["bro", "snort"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String facetQuery;

  /**
   * {
   * "facetFields": ["location_point"],
   * "indices": ["bro", "snort"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String badFacetQuery;

  /**
   * {
   * "indices": ["bro", "snort"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String disabledFacetQuery;

  /**
   * {
   * "indices": ["bro", "snort"],
   * "query": "*",
   * "from": 0,
   * "size": 101,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String exceededMaxResultsQuery;

  /**
   * {
   * "fields": ["ip_src_addr"],
   * "indices": ["bro", "snort"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String fieldsQuery;

  /**
   * {
   * "fields": ["ip_src_addr"],
   * "indices": ["bro", "snort"],
   * "query": "ip_src_addr:192.168.1.9",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String noResultsFieldsQuery;

  /**
   * {
   * "fields": ["guid"],
   * "indices": ["bro"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "guid",
   *     "sortOrder": "asc"
   *   }
   * ]
   * }
   * }
   */
  @Multiline
  public static String sortByGuidQuery;

  /**
   * {
   * "groups": [
   *   {
   *     "field":"is_alert"
   *   },
   *   {
   *     "field":"latitude"
   *   }
   * ],
   * "scoreField":"score",
   * "indices": ["bro", "snort"],
   * "query": "*"
   * }
   */
  @Multiline
  public static String groupByQuery;

  /**
   * {
   * "groups": [
   *   {
   *     "field":"is_alert",
   *     "order": {
   *       "groupOrderType": "count",
   *       "sortOrder": "ASC"
   *     }
   *   },
   *   {
   *     "field":"ip_src_addr",
   *     "order": {
   *       "groupOrderType": "term",
   *       "sortOrder": "DESC"
   *     }
   *   }
   * ],
   * "indices": ["bro", "snort"],
   * "query": "*"
   * }
   */
  @Multiline
  public static String sortedGroupByQuery;

  /**
   * {
   * "groups": [
   *   {
   *     "field":"location_point"
   *   }
   * ],
   * "indices": ["bro", "snort"],
   * "query": "*"
   * }
   */
  @Multiline
  public static String badGroupQuery;

  /**
   * {
   * "groups": [
   *   {
   *     "field":"ip_src_addr",
   *     "order": {
   *       "groupOrderType": "term",
   *       "sortOrder": "DESC"
   *     }
   *   }
   * ],
   * "indices": ["bro", "snort"],
   * "query": "*"
   * }
   */
  @Multiline
  public static String groupByIpQuery;

  protected static IndexDao dao;
  protected static InMemoryComponent indexComponent;

  @Before
  public synchronized void setup() throws Exception {
    if(dao == null && indexComponent == null) {
      indexComponent = startIndex();
      loadTestData();
      dao = createDao();
    }
  }

  @Rule
  public ExpectedException thrown = ExpectedException.none();

  @Test
  public void all_query_returns_all_results() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class);
    SearchResponse response = dao.search(request);
    Assert.assertEquals(10, response.getTotal());
    List<SearchResult> results = response.getResults();
    Assert.assertEquals(10, results.size());
    for(int i = 0;i < 5;++i) {
      Assert.assertEquals("snort", results.get(i).getSource().get("source:type"));
      Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp"));
    }
    for (int i = 5; i < 10; ++i) {
      Assert.assertEquals("bro", results.get(i).getSource().get("source:type"));
      Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp"));
    }
  }

  @Test
  public void find_one_guid() throws Exception {
    GetRequest request = JSONUtils.INSTANCE.load(findOneGuidQuery, GetRequest.class);
    Optional<Map<String, Object>> response = dao.getLatestResult(request);
    Assert.assertTrue(response.isPresent());
    Map<String, Object> doc = response.get();
    Assert.assertEquals("bro", doc.get("source:type"));
    Assert.assertEquals(3, doc.get("timestamp"));
  }

  @Test
  public void get_all_latest_guid() throws Exception {
    List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new JSONUtils.ReferenceSupplier<List<GetRequest>>(){});
    Map<String, Document> docs = new HashMap<>();

    for(Document doc : dao.getAllLatest(request)) {
      docs.put(doc.getGuid(), doc);
    }
    Assert.assertEquals(2, docs.size());
    Assert.assertTrue(docs.keySet().contains("bro_1"));
    Assert.assertTrue(docs.keySet().contains("snort_2"));
    Assert.assertEquals("bro", docs.get("bro_1").getDocument().get("source:type"));
    Assert.assertEquals("snort", docs.get("snort_2").getDocument().get("source:type"));
  }

  @Test
  public void filter_query_filters_results() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class);
    SearchResponse response = dao.search(request);
    Assert.assertEquals(3, response.getTotal());
    List<SearchResult> results = response.getResults();
    Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
    Assert.assertEquals(9, results.get(0).getSource().get("timestamp"));
    Assert.assertEquals("snort", results.get(1).getSource().get("source:type"));
    Assert.assertEquals(7, results.get(1).getSource().get("timestamp"));
    Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
    Assert.assertEquals(1, results.get(2).getSource().get("timestamp"));
  }

  @Test
  public void sort_query_sorts_results_ascending() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class);
    SearchResponse response = dao.search(request);
    Assert.assertEquals(10, response.getTotal());
    List<SearchResult> results = response.getResults();
    for (int i = 8001; i < 8011; ++i) {
      Assert.assertEquals(i, results.get(i - 8001).getSource().get("ip_src_port"));
    }
  }

  @Test
  public void sort_ascending_with_missing_fields() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(sortAscendingWithMissingFields, SearchRequest.class);
    SearchResponse response = dao.search(request);
    Assert.assertEquals(10, response.getTotal());
    List<SearchResult> results = response.getResults();
    Assert.assertEquals(10, results.size());

    // the remaining are missing the 'threat:triage:score' and should be sorted last
    for (int i = 0; i < 8; i++) {
      Assert.assertFalse(results.get(i).getSource().containsKey("threat:triage:score"));
    }

    // validate sorted order - there are only 2 with a 'threat:triage:score'
    Assert.assertEquals("10", results.get(8).getSource().get("threat:triage:score"));
    Assert.assertEquals("20", results.get(9).getSource().get("threat:triage:score"));
  }

  @Test
  public void sort_descending_with_missing_fields() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(sortDescendingWithMissingFields, SearchRequest.class);
    SearchResponse response = dao.search(request);
    Assert.assertEquals(10, response.getTotal());
    List<SearchResult> results = response.getResults();
    Assert.assertEquals(10, results.size());

    // validate sorted order - there are only 2 with a 'threat:triage:score'
    Assert.assertEquals("20", results.get(0).getSource().get("threat:triage:score"));
    Assert.assertEquals("10", results.get(1).getSource().get("threat:triage:score"));

    // the remaining are missing the 'threat:triage:score' and should be sorted last
    for (int i = 2; i < 10; i++) {
      Assert.assertFalse(results.get(i).getSource().containsKey("threat:triage:score"));
    }
  }

  @Test
  public void results_are_paginated() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class);
    SearchResponse response = dao.search(request);
    Assert.assertEquals(10, response.getTotal());
    List<SearchResult> results = response.getResults();
    Assert.assertEquals(3, results.size());
    Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
    Assert.assertEquals(6, results.get(0).getSource().get("timestamp"));
    Assert.assertEquals("bro", results.get(1).getSource().get("source:type"));
    Assert.assertEquals(5, results.get(1).getSource().get("timestamp"));
    Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
    Assert.assertEquals(4, results.get(2).getSource().get("timestamp"));
  }

  @Test
  public void returns_results_only_for_specified_indices() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class);
    SearchResponse response = dao.search(request);
    Assert.assertEquals(5, response.getTotal());
    List<SearchResult> results = response.getResults();
    for (int i = 5, j = 0; i > 0; i--, j++) {
      Assert.assertEquals("bro", results.get(j).getSource().get("source:type"));
      Assert.assertEquals(i, results.get(j).getSource().get("timestamp"));
    }
  }

  @Test
  public void facet_query_yields_field_types() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class);
    SearchResponse response = dao.search(request);
    Assert.assertEquals(10, response.getTotal());
    Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
    Assert.assertEquals(8, facetCounts.size());
    Map<String, Long> sourceTypeCounts = facetCounts.get("source:type");
    Assert.assertEquals(2, sourceTypeCounts.size());
    Assert.assertEquals(new Long(5), sourceTypeCounts.get("bro"));
    Assert.assertEquals(new Long(5), sourceTypeCounts.get("snort"));
    Map<String, Long> ipSrcAddrCounts = facetCounts.get("ip_src_addr");
    Assert.assertEquals(8, ipSrcAddrCounts.size());
    Assert.assertEquals(new Long(3), ipSrcAddrCounts.get("192.168.1.1"));
    Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.2"));
    Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.3"));
    Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.4"));
    Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.5"));
    Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.6"));
    Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.7"));
    Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.8"));
    Map<String, Long> ipSrcPortCounts = facetCounts.get("ip_src_port");
    Assert.assertEquals(10, ipSrcPortCounts.size());
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8001"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8002"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8003"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8004"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8005"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8006"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8007"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8008"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8009"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8010"));
    Map<String, Long> longFieldCounts = facetCounts.get("long_field");
    Assert.assertEquals(2, longFieldCounts.size());
    Assert.assertEquals(new Long(8), longFieldCounts.get("10000"));
    Assert.assertEquals(new Long(2), longFieldCounts.get("20000"));
    Map<String, Long> timestampCounts = facetCounts.get("timestamp");
    Assert.assertEquals(10, timestampCounts.size());
    Assert.assertEquals(new Long(1), timestampCounts.get("1"));
    Assert.assertEquals(new Long(1), timestampCounts.get("2"));
    Assert.assertEquals(new Long(1), timestampCounts.get("3"));
    Assert.assertEquals(new Long(1), timestampCounts.get("4"));
    Assert.assertEquals(new Long(1), timestampCounts.get("5"));
    Assert.assertEquals(new Long(1), timestampCounts.get("6"));
    Assert.assertEquals(new Long(1), timestampCounts.get("7"));
    Assert.assertEquals(new Long(1), timestampCounts.get("8"));
    Assert.assertEquals(new Long(1), timestampCounts.get("9"));
    Assert.assertEquals(new Long(1), timestampCounts.get("10"));
    Map<String, Long> latitudeCounts = facetCounts.get("latitude");
    Assert.assertEquals(2, latitudeCounts.size());
    List<String> latitudeKeys = new ArrayList<>(latitudeCounts.keySet());
    Collections.sort(latitudeKeys);
    Assert.assertEquals(48.0001, Double.parseDouble(latitudeKeys.get(0)), 0.00001);
    Assert.assertEquals(48.5839, Double.parseDouble(latitudeKeys.get(1)), 0.00001);
    Assert.assertEquals(new Long(2), latitudeCounts.get(latitudeKeys.get(0)));
    Assert.assertEquals(new Long(8), latitudeCounts.get(latitudeKeys.get(1)));
    Map<String, Long> scoreFieldCounts = facetCounts.get("score");
    Assert.assertEquals(4, scoreFieldCounts.size());
    List<String> scoreFieldKeys = new ArrayList<>(scoreFieldCounts.keySet());
    Collections.sort(scoreFieldKeys);
    Assert.assertEquals(10.0, Double.parseDouble(scoreFieldKeys.get(0)), 0.00001);
    Assert.assertEquals(20.0, Double.parseDouble(scoreFieldKeys.get(1)), 0.00001);
    Assert.assertEquals(50.0, Double.parseDouble(scoreFieldKeys.get(2)), 0.00001);
    Assert.assertEquals(98.0, Double.parseDouble(scoreFieldKeys.get(3)), 0.00001);
    Assert.assertEquals(new Long(4), scoreFieldCounts.get(scoreFieldKeys.get(0)));
    Assert.assertEquals(new Long(2), scoreFieldCounts.get(scoreFieldKeys.get(1)));
    Assert.assertEquals(new Long(3), scoreFieldCounts.get(scoreFieldKeys.get(2)));
    Assert.assertEquals(new Long(1), scoreFieldCounts.get(scoreFieldKeys.get(3)));
    Map<String, Long> isAlertCounts = facetCounts.get("is_alert");
    Assert.assertEquals(2, isAlertCounts.size());
    Assert.assertEquals(new Long(6), isAlertCounts.get("true"));
    Assert.assertEquals(new Long(4), isAlertCounts.get("false"));
  }

  @Test
  public void bad_facet_query_throws_exception() throws Exception {
    thrown.expect(InvalidSearchException.class);
    thrown.expectMessage("Failed to execute search");
    SearchRequest request = JSONUtils.INSTANCE.load(badFacetQuery, SearchRequest.class);
    dao.search(request);
  }

  @Test
  public void disabled_facet_query_returns_null_count() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(disabledFacetQuery, SearchRequest.class);
    SearchResponse response = dao.search(request);
    Assert.assertNull(response.getFacetCounts());
  }

  @Test
  public void exceeding_max_resulsts_throws_exception() throws Exception {
    thrown.expect(InvalidSearchException.class);
    thrown.expectMessage("Search result size must be less than 100");
    SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class);
    dao.search(request);
  }

  @Test
  public void returns_column_data_for_multiple_indices() throws Exception {
    Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort"));
    Assert.assertEquals(15, fieldTypes.size());
    Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
    Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
    Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
    Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
    Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
    Assert.assertEquals(FieldType.DATE, fieldTypes.get("timestamp"));
    Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude"));
    Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
    Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
    Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
    Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
    Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
    //NOTE: This is because the field is in both bro and snort and they have different types.
    Assert.assertEquals(FieldType.OTHER, fieldTypes.get("duplicate_name_field"));
    Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score"));
    Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert"));
  }

  @Test
  public void returns_column_metadata_for_specified_indices() throws Exception {
    // getColumnMetadata with only bro
    {
      Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro"));
      Assert.assertEquals(13, fieldTypes.size());
      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field"));
      Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
      Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
      Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
      Assert.assertEquals(FieldType.DATE, fieldTypes.get("timestamp"));
      Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude"));
      Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
      Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
      Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field"));
      Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert"));
    }
    // getColumnMetadata with only snort
    {
      Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort"));
      Assert.assertEquals(14, fieldTypes.size());
      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field"));
      Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
      Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
      Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
      Assert.assertEquals(FieldType.DATE, fieldTypes.get("timestamp"));
      Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude"));
      Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
      Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
      Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field"));
      Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert"));
    }
    // getColumnMetadata with an index that doesn't exist
    {
      Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("someindex"));
      Assert.assertEquals(0, fieldTypes.size());
    }
  }

  @Test
  public void group_by_ip_query() throws Exception {
    GroupRequest request = JSONUtils.INSTANCE.load(groupByIpQuery, GroupRequest.class);
    GroupResponse response = dao.group(request);

    // expect only 1 group for 'ip_src_addr'
    Assert.assertEquals("ip_src_addr", response.getGroupedBy());

    // there are 8 different 'ip_src_addr' values
    List<GroupResult> groups = response.getGroupResults();
    Assert.assertEquals(8, groups.size());

    // expect dotted-decimal notation in descending order
    Assert.assertEquals("192.168.1.8", groups.get(0).getKey());
    Assert.assertEquals("192.168.1.7", groups.get(1).getKey());
    Assert.assertEquals("192.168.1.6", groups.get(2).getKey());
    Assert.assertEquals("192.168.1.5", groups.get(3).getKey());
    Assert.assertEquals("192.168.1.4", groups.get(4).getKey());
    Assert.assertEquals("192.168.1.3", groups.get(5).getKey());
    Assert.assertEquals("192.168.1.2", groups.get(6).getKey());
    Assert.assertEquals("192.168.1.1", groups.get(7).getKey());
  }

  @Test
  public void no_results_returned_when_query_does_not_match() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class);
    SearchResponse response = dao.search(request);
    Assert.assertEquals(0, response.getTotal());
  }

  @Test
  public void group_by_returns_results_in_groups() throws Exception {
    // Group by test case, default order is count descending
    GroupRequest request = JSONUtils.INSTANCE.load(groupByQuery, GroupRequest.class);
    GroupResponse response = dao.group(request);
    Assert.assertEquals("is_alert", response.getGroupedBy());
    List<GroupResult> isAlertGroups = response.getGroupResults();
    Assert.assertEquals(2, isAlertGroups.size());

    // isAlert == true group
    GroupResult trueGroup = isAlertGroups.get(0);
    Assert.assertEquals("true", trueGroup.getKey());
    Assert.assertEquals(6, trueGroup.getTotal());
    Assert.assertEquals("latitude", trueGroup.getGroupedBy());
    Assert.assertEquals(198.0, trueGroup.getScore(), 0.00001);
    List<GroupResult> trueLatitudeGroups = trueGroup.getGroupResults();
    Assert.assertEquals(2, trueLatitudeGroups.size());

    // isAlert == true && latitude == 48.5839 group
    GroupResult trueLatitudeGroup2 = trueLatitudeGroups.get(0);
    Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001);
    Assert.assertEquals(5, trueLatitudeGroup2.getTotal());
    Assert.assertEquals(148.0, trueLatitudeGroup2.getScore(), 0.00001);

    // isAlert == true && latitude == 48.0001 group
    GroupResult trueLatitudeGroup1 = trueLatitudeGroups.get(1);
    Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001);
    Assert.assertEquals(1, trueLatitudeGroup1.getTotal());
    Assert.assertEquals(50.0, trueLatitudeGroup1.getScore(), 0.00001);

    // isAlert == false group
    GroupResult falseGroup = isAlertGroups.get(1);
    Assert.assertEquals("false", falseGroup.getKey());
    Assert.assertEquals("latitude", falseGroup.getGroupedBy());
    Assert.assertEquals(130.0, falseGroup.getScore(), 0.00001);
    List<GroupResult> falseLatitudeGroups = falseGroup.getGroupResults();
    Assert.assertEquals(2, falseLatitudeGroups.size());

    // isAlert == false && latitude == 48.5839 group
    GroupResult falseLatitudeGroup2 = falseLatitudeGroups.get(0);
    Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001);
    Assert.assertEquals(3, falseLatitudeGroup2.getTotal());
    Assert.assertEquals(80.0, falseLatitudeGroup2.getScore(), 0.00001);

    // isAlert == false && latitude == 48.0001 group
    GroupResult falseLatitudeGroup1 = falseLatitudeGroups.get(1);
    Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001);
    Assert.assertEquals(1, falseLatitudeGroup1.getTotal());
    Assert.assertEquals(50.0, falseLatitudeGroup1.getScore(), 0.00001);
  }

  @Test
  public void group_by_returns_results_in_sorted_groups() throws Exception {
    // Group by with sorting test case where is_alert is sorted by count ascending and ip_src_addr is sorted by term descending
    GroupRequest request = JSONUtils.INSTANCE.load(sortedGroupByQuery, GroupRequest.class);
    GroupResponse response = dao.group(request);
    Assert.assertEquals("is_alert", response.getGroupedBy());
    List<GroupResult> isAlertGroups = response.getGroupResults();
    Assert.assertEquals(2, isAlertGroups.size());

    // isAlert == false group
    GroupResult falseGroup = isAlertGroups.get(0);
    Assert.assertEquals(4, falseGroup.getTotal());
    Assert.assertEquals("ip_src_addr", falseGroup.getGroupedBy());
    List<GroupResult> falseIpSrcAddrGroups = falseGroup.getGroupResults();
    Assert.assertEquals(4, falseIpSrcAddrGroups.size());

    // isAlert == false && ip_src_addr == 192.168.1.8 group
    GroupResult falseIpSrcAddrGroup1 = falseIpSrcAddrGroups.get(0);
    Assert.assertEquals("192.168.1.8", falseIpSrcAddrGroup1.getKey());
    Assert.assertEquals(1, falseIpSrcAddrGroup1.getTotal());
    Assert.assertNull(falseIpSrcAddrGroup1.getGroupedBy());
    Assert.assertNull(falseIpSrcAddrGroup1.getGroupResults());

    // isAlert == false && ip_src_addr == 192.168.1.7 group
    GroupResult falseIpSrcAddrGroup2 = falseIpSrcAddrGroups.get(1);
    Assert.assertEquals("192.168.1.7", falseIpSrcAddrGroup2.getKey());
    Assert.assertEquals(1, falseIpSrcAddrGroup2.getTotal());
    Assert.assertNull(falseIpSrcAddrGroup2.getGroupedBy());
    Assert.assertNull(falseIpSrcAddrGroup2.getGroupResults());

    // isAlert == false && ip_src_addr == 192.168.1.6 group
    GroupResult falseIpSrcAddrGroup3 = falseIpSrcAddrGroups.get(2);
    Assert.assertEquals("192.168.1.6", falseIpSrcAddrGroup3.getKey());
    Assert.assertEquals(1, falseIpSrcAddrGroup3.getTotal());
    Assert.assertNull(falseIpSrcAddrGroup3.getGroupedBy());
    Assert.assertNull(falseIpSrcAddrGroup3.getGroupResults());

    // isAlert == false && ip_src_addr == 192.168.1.2 group
    GroupResult falseIpSrcAddrGroup4 = falseIpSrcAddrGroups.get(3);
    Assert.assertEquals("192.168.1.2", falseIpSrcAddrGroup4.getKey());
    Assert.assertEquals(1, falseIpSrcAddrGroup4.getTotal());
    Assert.assertNull(falseIpSrcAddrGroup4.getGroupedBy());
    Assert.assertNull(falseIpSrcAddrGroup4.getGroupResults());

    // isAlert == false group
    GroupResult trueGroup = isAlertGroups.get(1);
    Assert.assertEquals(6, trueGroup.getTotal());
    Assert.assertEquals("ip_src_addr", trueGroup.getGroupedBy());
    List<GroupResult> trueIpSrcAddrGroups = trueGroup.getGroupResults();
    Assert.assertEquals(4, trueIpSrcAddrGroups.size());

    // isAlert == false && ip_src_addr == 192.168.1.5 group
    GroupResult trueIpSrcAddrGroup1 = trueIpSrcAddrGroups.get(0);
    Assert.assertEquals("192.168.1.5", trueIpSrcAddrGroup1.getKey());
    Assert.assertEquals(1, trueIpSrcAddrGroup1.getTotal());
    Assert.assertNull(trueIpSrcAddrGroup1.getGroupedBy());
    Assert.assertNull(trueIpSrcAddrGroup1.getGroupResults());

    // isAlert == false && ip_src_addr == 192.168.1.4 group
    GroupResult trueIpSrcAddrGroup2 = trueIpSrcAddrGroups.get(1);
    Assert.assertEquals("192.168.1.4", trueIpSrcAddrGroup2.getKey());
    Assert.assertEquals(1, trueIpSrcAddrGroup2.getTotal());
    Assert.assertNull(trueIpSrcAddrGroup2.getGroupedBy());
    Assert.assertNull(trueIpSrcAddrGroup2.getGroupResults());

    // isAlert == false && ip_src_addr == 192.168.1.3 group
    GroupResult trueIpSrcAddrGroup3 = trueIpSrcAddrGroups.get(2);
    Assert.assertEquals("192.168.1.3", trueIpSrcAddrGroup3.getKey());
    Assert.assertEquals(1, trueIpSrcAddrGroup3.getTotal());
    Assert.assertNull(trueIpSrcAddrGroup3.getGroupedBy());
    Assert.assertNull(trueIpSrcAddrGroup3.getGroupResults());

    // isAlert == false && ip_src_addr == 192.168.1.1 group
    GroupResult trueIpSrcAddrGroup4 = trueIpSrcAddrGroups.get(3);
    Assert.assertEquals("192.168.1.1", trueIpSrcAddrGroup4.getKey());
    Assert.assertEquals(3, trueIpSrcAddrGroup4.getTotal());
    Assert.assertNull(trueIpSrcAddrGroup4.getGroupedBy());
    Assert.assertNull(trueIpSrcAddrGroup4.getGroupResults());
  }

  @Test
  public void throws_exception_on_aggregation_queries_on_non_string_non_numeric_fields()
          throws Exception {
    thrown.expect(InvalidSearchException.class);
    thrown.expectMessage("Failed to execute search");
    GroupRequest request = JSONUtils.INSTANCE.load(badGroupQuery, GroupRequest.class);
    dao.group(request);
  }

  @Test
  public void queries_fields() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class);
    SearchResponse response = dao.search(request);
    Assert.assertEquals(10, response.getTotal());
    List<SearchResult> results = response.getResults();
    for (int i = 0; i < 5; ++i) {
      Map<String, Object> source = results.get(i).getSource();
      Assert.assertEquals(1, source.size());
      Assert.assertNotNull(source.get("ip_src_addr"));
    }
    for (int i = 5; i < 10; ++i) {
      Map<String, Object> source = results.get(i).getSource();
      Assert.assertEquals(1, source.size());
      Assert.assertNotNull(source.get("ip_src_addr"));
    }
  }

  @Test
  public void sort_by_guid() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(sortByGuidQuery, SearchRequest.class);
    SearchResponse response = dao.search(request);
    Assert.assertEquals(5, response.getTotal());
    List<SearchResult> results = response.getResults();
    for (int i = 0; i < 5; ++i) {
      Map<String, Object> source = results.get(i).getSource();
      Assert.assertEquals(1, source.size());
      Assert.assertEquals(source.get("guid"), "bro_" + (i + 1));
    }
  }

  @AfterClass
  public static void stop() throws Exception {
    indexComponent.stop();
  }

  protected abstract IndexDao createDao() throws Exception;
  protected abstract InMemoryComponent startIndex() throws Exception;
  protected abstract void loadTestData() throws Exception;
}


text:
package org.cancergrid.ws.query;

import org.cancergrid.schema.query.Query;
import org.cancergrid.ws.query.QueryServiceManager;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

public class QueryServiceManagerEVSTest
{
	private static org.cancergrid.ws.query.QueryServiceManager qsm;
	private static String EVS ="EVS-DescLogicConcept";

	@Before
	public void createQueryServiceManager()
	{
		 qsm = new QueryServiceManager();
	}

            @Test
	public void queryEVSValidResponse()
	{
		org.cancergrid.services.query.QueryDocument queryDoc = org.cancergrid.services.query.QueryDocument.Factory.newInstance();
		Query query = queryDoc.addNewQuery().addNewQuery();
		query.setResource(EVS);
		query.setTerm("*blood*");
		query.setNumResults(5);
		org.cancergrid.services.query.QueryResponseDocument response = qsm.query(queryDoc);
		Assert.assertTrue("Invalid response format: " + response.toString(), response.validate());
	}

	@Test
	public void queryStringEVSValidResponse() throws Exception
	{
		org.cancergrid.services.query.QueryStringDocument queryDoc = org.cancergrid.services.query.QueryStringDocument.Factory.newInstance();
		Query query = queryDoc.addNewQueryString().addNewQuery();
		query.setResource(EVS);
		query.setTerm("*blood*");
		query.setNumResults(5);
		org.cancergrid.services.query.QueryStringResponseDocument response = qsm.queryString(queryDoc);
		org.cancergrid.schema.result_set.ResultSetDocument results = org.cancergrid.schema.result_set.ResultSetDocument.Factory.parse(response.getQueryStringResponse().getReturn());

		Assert.assertTrue("Invalid response format: " + response.toString(), results.validate());
	}
}

redpajama_set_name: RedPajamaGithub


text:
'use strict';

var fs = require('fs');

exports.get     = get;
exports.getSync = getSync;

// Define line type patterns and their identifiers
var linetypes = {
  dbannotest          : { pattern: /\/\*+?\s*@?test\s*\*\//, returns: { name:1 } },
  dbannoreportingtest : { pattern: /\/\*+?\s*@?reporting-test\s*\*\//, returns: { name:1 } },
  dbstart             : { pattern: /^\s?\/\*\*/ },
  dbanno              : { pattern: /^\s*\*\s*@?([\w:-_\(\)]+)(\s*(.*))?/, returns: { name:1, value:3 } },
  dbend               : { pattern: /^\s?\*\/$/ },
  fnsig1              : { pattern: /^\s?function\s+(\w+)\s?/, returns: { name:1 } },
  fnsig2              : { pattern: /^\s?exports\.(\w+)\s?=\s?function/, returns: { name:1 } },
  fnsig3              : { pattern: /^\s?module\.exports\s?=\s?function/, returns: { name:'anonymous' } },
  fnsig4              : { pattern: /exports\[['"](\w+)['"]\]\s*=\s*function/, returns: { name:1 } },
  fnsig5              : { pattern: /^\s*(var|let|const)\s+(\w+)\s*=\s*function/, returns: { name:2 } },
  fnsig6              : { pattern: /^\s*(var|let|const)\s+(\w+)\s*=\s*new Function/, returns: { name:2 } },
  fnsig7              : { pattern: /^\s*(var|let|const)\s+(\w+)\s*=\s*\(.*?\)\s*=>/, returns: { name:2 } },
  fnsig8              : { pattern: /^\s?module.exports\s*=\s*{/, returns: { name:'module' } },
  es6asyncfn          : { pattern: /async\s(\w+)\s?/, returns: { name:1 } },
  es6fn               : { pattern: /\s?(\w+)\s?/, returns: { name:1 } },
};

function trim(str) {
  return 'string' == typeof str ? str.replace(/^\s+/,'').replace(/\s+$/,'') : str;
}

/**
 * Runs through linetypes, identifying str
 */
function identify(str) {

  // Try all predefined line type patterns
  for (var i in linetypes) {

    // Check for a match
    var match = str.match(linetypes[i].pattern);
    if (null !== match) {

      // Default returns type and regexp match results
      var returns = { id: i, match: match };

      // If specific values are requested, fill those in
      if ('undefined' != typeof linetypes[i].returns) {
        for (var r in linetypes[i].returns) {
          returns[r] = 'string' == typeof linetypes[i].returns[r] ?
              linetypes[i].returns[r] : trim(match[linetypes[i].returns[r]]);
        }
      }

      return returns;
    }
  }

  return false;
}

function analyze(data) {

  // Object for holding annotation data
  var annotations = {};

  // Iterate over lines
  var lines = data.split('\n');
  var current = {};
  for (var num in lines) {

    // Get line type
    var type = identify(lines[num]);

    if (type.name == 'get') {
      continue;
    }

    switch (type.id) {
      case 'dbannotest':
        current['test'] = true;
        break;

      case 'dbannoreportingtest':
        current['reporting-test'] = true;
        break;

      case 'dbstart':
        // Reset current annotation object
        current = {};
        break;

      case 'dbanno':
        // Store whatever was annotated, bucketing values of duplicate keys
        var annotationValue = undefined === type.value || '' === type.value ? true : type.value;
          // Create array to hold items
        if (undefined !== current[type.name]) {
          if ('string' == typeof current[type.name])
            current[type.name] = [current[type.name]];
          current[type.name].push(annotationValue);
        } else {
          current[type.name] = annotationValue;
        }
        break;
      case 'fnsig1':
      case 'fnsig2':
      case 'fnsig3':
      case 'fnsig4':
      case 'fnsig5':
      case 'fnsig6':
      case 'fnsig7':
      case 'fnsig8':
      case 'es6asyncfn':
      case 'es6fn':
        // Store current annotations under function name
        annotations[type.name] = current;
        current = {};
        break;

      default: break;
    }
  }

  return annotations;
}

/**
 * Get an object representation of all annotations in file
 *
 * @param string filename Path to file
 * @param function callback
 *
 */
function get(filename, callback) {

  // Read file
  fs.readFile(filename, 'utf8', function(err, data) {
    if (!err) {
      var annotations = analyze(data);
      callback(null, annotations);
    } else {
      callback(err, null);
    }
  });
}

function getSync(filename) {

  // Read file
  var data = fs.readFileSync(filename, 'utf8');

  // Analyze if possible
  if ('string' == typeof data) {
    return analyze(data);
  } else {
    return false;
  }
}


text: search_result['2704']=["topic_0000000000000678.html","BadgesSubCategoryDto Class",""];


text:
/*INTERMEDIATE_PATH contains  ASSOCIATION(ASSOCIATION_ID) connected by underscore */
CREATE TABLE PATH(
     PATH_ID           NUMBER(38,0)         NOT NULL,
     FIRST_ENTITY_ID   NUMBER(38,0)         NULL,
     INTERMEDIATE_PATH VARCHAR2(1000)  NULL,
     LAST_ENTITY_ID    NUMBER(38,0)         NULL,
     PRIMARY KEY (PATH_ID)
);
/* Possible values for ASSOCIATION_TYPE are 1 and 2
ASSOCIATION_TYPE = 1 represents INTER_MODEL_ASSOCIATION.
ASSOCIATION_TYPE = 2 represents INTRA_MODEL_ASSOCIATION.
*/
CREATE TABLE ASSOCIATION(
    ASSOCIATION_ID    NUMBER(38,0)    NOT NULL,
    ASSOCIATION_TYPE  NUMBER(8,0)    NOT NULL ,
    PRIMARY KEY (ASSOCIATION_ID)
);

CREATE TABLE INTER_MODEL_ASSOCIATION(
    ASSOCIATION_ID      NUMBER(38,0)  NOT NULL REFERENCES ASSOCIATION(ASSOCIATION_ID),
    LEFT_ENTITY_ID      NUMBER(38,0)  NOT NULL,
    LEFT_ATTRIBUTE_ID   NUMBER(38,0)  NOT NULL,
    RIGHT_ENTITY_ID     NUMBER(38,0)  NOT NULL,
    RIGHT_ATTRIBUTE_ID  NUMBER(38,0)  NOT NULL,
    PRIMARY KEY (ASSOCIATION_ID)
);
CREATE TABLE INTRA_MODEL_ASSOCIATION(
    ASSOCIATION_ID    NUMBER(38,0)    NOT NULL REFERENCES ASSOCIATION(ASSOCIATION_ID),
    DE_ASSOCIATION_ID NUMBER(38,0)    NOT NULL,
    PRIMARY KEY (ASSOCIATION_ID)
);
CREATE TABLE CAB2B_ID_TABLE(
    NEXT_ASSOCIATION_ID    NUMBER(38,0)    NOT NULL,
    PRIMARY KEY (NEXT_ASSOCIATION_ID)
);
CREATE TABLE CURATED_PATH (
	curated_path_Id NUMBER(38,0),
	entity_ids VARCHAR2(1000),
	selected NUMBER(1,0),
	PRIMARY KEY (curated_path_Id)
);

/*This is mapping table for many-to-many relationship between tables PATH and CURATED_PATH */
CREATE TABLE CURATED_PATH_TO_PATH (
	curated_path_Id NUMBER(38,0) REFERENCES CURATED_PATH (curated_path_Id),
	path_id NUMBER(38,0)  REFERENCES PATH (path_id),
	PRIMARY KEY (curated_path_Id,path_id)
);

CREATE INDEX FK9651EF32D8D56A78 ON PATH (FIRST_ENTITY_ID,LAST_ENTITY_ID);
INSERT INTO CAB2B_ID_TABLE(NEXT_ASSOCIATION_ID) VALUES(1);

CREATE TABLE DATA_CATEGORIAL_ATTRIBUTE (
   ID NUMBER(19,0) NOT NULL,
   PRIMARY KEY (ID)
);
CREATE TABLE DATA_CATEGORIAL_CLASS (
   ID NUMBER(19,0) NOT NULL,
   PRIMARY KEY (ID)
);
CREATE TABLE CAB2B_DATA_CATEGORY (
   ID NUMBER(19,0) NOT NULL,
   description VARCHAR2(255),
   name VARCHAR2(255) UNIQUE,
   ROOT_CLASS_ID NUMBER(19,0) UNIQUE,
   PRIMARY KEY (ID)
);
CREATE TABLE CAB2B_QUERY (
   IDENTIFIER NUMBER(19,0) NOT NULL,
   ENTITY_ID NUMBER(19,0) NOT NULL,
   PRIMARY KEY (IDENTIFIER)
);
CREATE TABLE OUTPUT_CLASS_URLS (
   CAB2B_QUERY_ID NUMBER(19,0) NOT NULL,
   OUTPUT_CLASS_URL VARCHAR2(255),
   POSITION NUMBER(10,0) NOT NULL,
   PRIMARY KEY (CAB2B_QUERY_ID, POSITION)
);

ALTER TABLE CAB2B_DATA_CATEGORY ADD CONSTRAINT FKFA70BDE8DF75106F FOREIGN KEY (ROOT_CLASS_ID) REFERENCES DATA_CATEGORIAL_CLASS;

ALTER TABLE OUTPUT_CLASS_URLS ADD CONSTRAINT FKE131CD697ADD4524 FOREIGN KEY (CAB2B_QUERY_ID) REFERENCES CAB2B_QUERY;

COMMIT;


text:
package org.apache.drill.exec.vector.complex.writer;

import static org.apache.drill.TestBuilder.listOf;
import static org.apache.drill.TestBuilder.mapOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.List;
import java.util.zip.GZIPOutputStream;

import com.google.common.base.Joiner;

import org.apache.drill.BaseTestQuery;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.store.easy.json.JSONRecordReader;
import org.apache.drill.exec.vector.IntVector;
import org.apache.drill.exec.vector.RepeatedBigIntVector;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;

import com.google.common.base.Charsets;
import com.google.common.io.Files;

import org.junit.rules.TemporaryFolder;

public class TestJsonReader extends BaseTestQuery {
//  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJsonReader.class);

  private static final boolean VERBOSE_DEBUG = false;

  @Rule
  public TemporaryFolder folder = new TemporaryFolder();

  @Test
  public void testEmptyList() throws Exception {
    String root = FileUtils.getResourceAsFile("/store/json/emptyLists").toURI().toString();
    String query = String.format("select count(a[0]) as ct from dfs_test.`%s`", root, root);

    testBuilder()
        .sqlQuery(query)
        .ordered()
        .baselineColumns("ct")
        .baselineValues(6l)
        .build()
        .run();
  }

  @Test
  public void schemaChange() throws Exception {
    test("select b from dfs.`${WORKING_PATH}/src/test/resources/vector/complex/writer/schemaChange/`");
  }

  @Test
  public void testFieldSelectionBug() throws Exception {
    try {
      testBuilder()
          .sqlQuery("select t.field_4.inner_3 as col_1, t.field_4 as col_2 from cp.`store/json/schema_change_int_to_string.json` t")
          .unOrdered()
          .optionSettingQueriesForTestQuery("alter session set `store.json.all_text_mode` = true")
          .baselineColumns("col_1", "col_2")
          .baselineValues(
              mapOf(),
              mapOf(
                  "inner_1", listOf(),
                  "inner_3", mapOf()))
          .baselineValues(
              mapOf("inner_object_field_1", "2"),
              mapOf(
                  "inner_1", listOf("1", "2", "3"),
                  "inner_2", "3",
                  "inner_3", mapOf("inner_object_field_1", "2")))
          .baselineValues(
              mapOf(),
              mapOf(
                  "inner_1", listOf("4", "5", "6"),
                  "inner_2", "3",
                  "inner_3", mapOf()))
          .go();
    } finally {
      test("alter session set `store.json.all_text_mode` = false");
    }
  }

  @Test
  public void testSplitAndTransferFailure() throws Exception {
    final String testVal = "a string";
    testBuilder()
        .sqlQuery("select flatten(config) as flat from cp.`/store/json/null_list.json`")
        .ordered()
        .baselineColumns("flat")
        .baselineValues(listOf())
        .baselineValues(listOf(testVal))
        .go();

    test("select flatten(config) as flat from cp.`/store/json/null_list_v2.json`");
    testBuilder()
        .sqlQuery("select flatten(config) as flat from cp.`/store/json/null_list_v2.json`")
        .ordered()
        .baselineColumns("flat")
        .baselineValues(mapOf("repeated_varchar", listOf()))
        .baselineValues(mapOf("repeated_varchar", listOf(testVal)))
        .go();

    testBuilder()
        .sqlQuery("select flatten(config) as flat from cp.`/store/json/null_list_v3.json`")
        .ordered()
        .baselineColumns("flat")
        .baselineValues(mapOf("repeated_map", listOf(mapOf("repeated_varchar", listOf()))))
        .baselineValues(mapOf("repeated_map", listOf(mapOf("repeated_varchar", listOf(testVal)))))
        .go();
  }

  @Test
  @Ignore("DRILL-1824")
  public void schemaChangeValidate() throws Exception {
    testBuilder() //
      .sqlQuery("select b from dfs.`${WORKING_PATH}/src/test/resources/vector/complex/writer/schemaChange/`") //
      .unOrdered() //
      .jsonBaselineFile("/vector/complex/writer/expected.json") //
      .build()
      .run();
  }

  public void runTestsOnFile(String filename, UserBitShared.QueryType queryType, String[] queries, long[] rowCounts) throws Exception {
    if (VERBOSE_DEBUG) {
      System.out.println("===================");
      System.out.println("source data in json");
      System.out.println("===================");
      System.out.println(Files.toString(FileUtils.getResourceAsFile(filename), Charsets.UTF_8));
    }

    int i = 0;
    for (String query : queries) {
      if (VERBOSE_DEBUG) {
        System.out.println("=====");
        System.out.println("query");
        System.out.println("=====");
        System.out.println(query);
        System.out.println("======");
        System.out.println("result");
        System.out.println("======");
      }
      int rowCount = testRunAndPrint(queryType, query);
      assertEquals(rowCounts[i], rowCount);
      System.out.println();
      i++;
    }
  }

  @Test
  public void testReadCompressed() throws Exception {
    String filepath = "compressed_json.json";
    File f = folder.newFile(filepath);
    PrintWriter out = new PrintWriter(f);
    out.println("{\"a\" :5}");
    out.close();

    gzipIt(f);
    testBuilder()
        .sqlQuery("select * from dfs.`" + f.getPath() + ".gz" + "`")
        .unOrdered()
        .baselineColumns("a")
        .baselineValues(5l)
        .build().run();

    // test reading the uncompressed version as well
    testBuilder()
        .sqlQuery("select * from dfs.`" + f.getPath() + "`")
        .unOrdered()
        .baselineColumns("a")
        .baselineValues(5l)
        .build().run();
  }

  public static void gzipIt(File sourceFile) throws IOException {

    // modified from: http://www.mkyong.com/java/how-to-compress-a-file-in-gzip-format/
    byte[] buffer = new byte[1024];
    GZIPOutputStream gzos =
        new GZIPOutputStream(new FileOutputStream(sourceFile.getPath() + ".gz"));

    FileInputStream in =
        new FileInputStream(sourceFile);

    int len;
    while ((len = in.read(buffer)) > 0) {
      gzos.write(buffer, 0, len);
    }
    in.close();
    gzos.finish();
    gzos.close();
  }

  @Test
  public void testDrill_1419() throws Exception {
    String[] queries = {"select t.trans_id, t.trans_info.prod_id[0],t.trans_info.prod_id[1] from cp.`/store/json/clicks.json` t limit 5"};
    long[] rowCounts = {5};
    String filename = "/store/json/clicks.json";
    runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
  }

  @Test
  public void testRepeatedCount() throws Exception {
    test("select repeated_count(str_list) from cp.`/store/json/json_basic_repeated_varchar.json`");
    test("select repeated_count(INT_col) from cp.`/parquet/alltypes_repeated.json`");
    test("select repeated_count(FLOAT4_col) from cp.`/parquet/alltypes_repeated.json`");
    test("select repeated_count(VARCHAR_col) from cp.`/parquet/alltypes_repeated.json`");
    test("select repeated_count(BIT_col) from cp.`/parquet/alltypes_repeated.json`");
  }

  @Test
  public void testRepeatedContains() throws Exception {
    test("select repeated_contains(str_list, 'asdf') from cp.`/store/json/json_basic_repeated_varchar.json`");
    test("select repeated_contains(INT_col, -2147483648) from cp.`/parquet/alltypes_repeated.json`");
    test("select repeated_contains(FLOAT4_col, -1000000000000.0) from cp.`/parquet/alltypes_repeated.json`");
    test("select repeated_contains(VARCHAR_col, 'qwerty' ) from cp.`/parquet/alltypes_repeated.json`");
    test("select repeated_contains(BIT_col, true) from cp.`/parquet/alltypes_repeated.json`");
    test("select repeated_contains(BIT_col, false) from cp.`/parquet/alltypes_repeated.json`");
  }

  @Test
  public void testSingleColumnRead_vector_fill_bug() throws Exception {
    String[] queries = {"select * from cp.`/store/json/single_column_long_file.json`"};
    long[] rowCounts = {13512};
    String filename = "/store/json/single_column_long_file.json";
    runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
  }

  @Test
  public void testNonExistentColumnReadAlone() throws Exception {
    String[] queries = {"select non_existent_column from cp.`/store/json/single_column_long_file.json`"};
    long[] rowCounts = {13512};
    String filename = "/store/json/single_column_long_file.json";
    runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
  }

  @Test
  public void testAllTextMode() throws Exception {
    test("alter system set `store.json.all_text_mode` = true");
    String[] queries = {"select * from cp.`/store/json/schema_change_int_to_string.json`"};
    long[] rowCounts = {3};
    String filename = "/store/json/schema_change_int_to_string.json";
    runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
    test("alter system set `store.json.all_text_mode` = false");
  }

  @Test
  public void readComplexWithStar() throws Exception {
    List<QueryDataBatch> results = testSqlWithResults("select * from cp.`/store/json/test_complex_read_with_star.json`");
    assertEquals(1, results.size());

    RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());
    QueryDataBatch batch = results.get(0);

    assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData()));
    assertEquals(3, batchLoader.getSchema().getFieldCount());
    testExistentColumns(batchLoader);

    batch.release();
    batchLoader.clear();
  }

  @Test
  public void testNullWhereListExpected() throws Exception {
    test("alter system set `store.json.all_text_mode` = true");
    String[] queries = {"select * from cp.`/store/json/null_where_list_expected.json`"};
    long[] rowCounts = {3};
    String filename = "/store/json/null_where_list_expected.json";
    runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
    test("alter system set `store.json.all_text_mode` = false");
  }

  @Test
  public void testNullWhereMapExpected() throws Exception {
    test("alter system set `store.json.all_text_mode` = true");
    String[] queries = {"select * from cp.`/store/json/null_where_map_expected.json`"};
    long[] rowCounts = {3};
    String filename = "/store/json/null_where_map_expected.json";
    runTestsOnFile(filename, UserBitShared.QueryType.SQL, queries, rowCounts);
    test("alter system set `store.json.all_text_mode` = false");
  }

  @Test
  public void ensureProjectionPushdown() throws Exception {
    // Tests to make sure that we are correctly eliminating schema changing columns.  If completes, means that the projection pushdown was successful.
    test("alter system set `store.json.all_text_mode` = false; "
        + "select  t.field_1, t.field_3.inner_1, t.field_3.inner_2, t.field_4.inner_1 "
        + "from cp.`store/json/schema_change_int_to_string.json` t");
  }

  // The project pushdown rule is correctly adding the projected columns to the scan, however it is not removing
  // the redundant project operator after the scan, this tests runs a physical plan generated from one of the tests to
  // ensure that the project is filtering out the correct data in the scan alone
  @Test
  public void testProjectPushdown() throws Exception {
    String[] queries = {Files.toString(FileUtils.getResourceAsFile("/store/json/project_pushdown_json_physical_plan.json"), Charsets.UTF_8)};
    long[] rowCounts = {3};
    String filename = "/store/json/schema_change_int_to_string.json";
    test("alter system set `store.json.all_text_mode` = false");
    runTestsOnFile(filename, UserBitShared.QueryType.PHYSICAL, queries, rowCounts);

    List<QueryDataBatch> results = testPhysicalWithResults(queries[0]);
    assertEquals(1, results.size());
    // "`field_1`", "`field_3`.`inner_1`", "`field_3`.`inner_2`", "`field_4`.`inner_1`"

    RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());
    QueryDataBatch batch = results.get(0);
    assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData()));

    // this used to be five.  It is now three.  This is because the plan doesn't have a project.
    // Scanners are not responsible for projecting non-existent columns (as long as they project one column)
    assertEquals(3, batchLoader.getSchema().getFieldCount());
    testExistentColumns(batchLoader);

    batch.release();
    batchLoader.clear();
  }

  @Test
  public void testJsonDirectoryWithEmptyFile() throws Exception {
    String root = FileUtils.getResourceAsFile("/store/json/jsonDirectoryWithEmpyFile").toURI().toString();

    String queryRightEmpty = String.format(
        "select * from dfs_test.`%s`", root);

    testBuilder()
        .sqlQuery(queryRightEmpty)
        .unOrdered()
        .baselineColumns("a")
        .baselineValues(1l)
        .build()
        .run();
  }

  private void testExistentColumns(RecordBatchLoader batchLoader) throws SchemaChangeException {
    VectorWrapper<?> vw = batchLoader.getValueAccessorById(
        RepeatedBigIntVector.class, //
        batchLoader.getValueVectorId(SchemaPath.getCompoundPath("field_1")).getFieldIds() //
    );
    assertEquals("[1]", vw.getValueVector().getAccessor().getObject(0).toString());
    assertEquals("[5]", vw.getValueVector().getAccessor().getObject(1).toString());
    assertEquals("[5,10,15]", vw.getValueVector().getAccessor().getObject(2).toString());

    vw = batchLoader.getValueAccessorById(
        IntVector.class, //
        batchLoader.getValueVectorId(SchemaPath.getCompoundPath("field_3", "inner_1")).getFieldIds() //
    );
    assertNull(vw.getValueVector().getAccessor().getObject(0));
    assertEquals(2l, vw.getValueVector().getAccessor().getObject(1));
    assertEquals(5l, vw.getValueVector().getAccessor().getObject(2));

    vw = batchLoader.getValueAccessorById(
        IntVector.class, //
        batchLoader.getValueVectorId(SchemaPath.getCompoundPath("field_3", "inner_2")).getFieldIds() //
    );
    assertNull(vw.getValueVector().getAccessor().getObject(0));
    assertNull(vw.getValueVector().getAccessor().getObject(1));
    assertEquals(3l, vw.getValueVector().getAccessor().getObject(2));

    vw = batchLoader.getValueAccessorById(
        RepeatedBigIntVector.class, //
        batchLoader.getValueVectorId(SchemaPath.getCompoundPath("field_4", "inner_1")).getFieldIds() //
    );
    assertEquals("[]", vw.getValueVector().getAccessor().getObject(0).toString());
    assertEquals("[1,2,3]", vw.getValueVector().getAccessor().getObject(1).toString());
    assertEquals("[4,5,6]", vw.getValueVector().getAccessor().getObject(2).toString());
  }

  @Test
  public void testSelectStarWithUnionType() throws Exception {
    try {
      String query = "select * from cp.`jsoninput/union/a.json`";
      testBuilder()
              .sqlQuery(query)
              .ordered()
              .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
              .baselineColumns("field1", "field2")
              .baselineValues(
                      1L, 1.2
              )
              .baselineValues(
                      listOf(2L), 1.2
              )
              .baselineValues(
                      mapOf("inner1", 3L, "inner2", 4L), listOf(3L, 4.0, "5")
              )
              .baselineValues(
                      mapOf("inner1", 3L,
                              "inner2", listOf(
                                      mapOf(
                                              "innerInner1", 1L,
                                              "innerInner2",
                                              listOf(
                                                      3L,
                                                      "a"
                                              )
                                      )
                              )
                      ),
                      listOf(
                              mapOf("inner3", 7L),
                              4.0,
                              "5",
                              mapOf("inner4", 9L),
                              listOf(
                                      mapOf(
                                              "inner5", 10L,
                                              "inner6", 11L
                                      ),
                                      mapOf(
                                              "inner5", 12L,
                                              "inner7", 13L
                                      )
                              )
                      )
              ).go();
    } finally {
      testNoResult("alter session set `exec.enable_union_type` = false");
    }
  }

  @Test
  public void testSelectFromListWithCase() throws Exception {
    String query = "select a, typeOf(a) `type` from " +
            "(select case when is_list(field2) then field2[4][1].inner7 end a " +
            "from cp.`jsoninput/union/a.json`) where a is not null";
    try {
      testBuilder()
              .sqlQuery(query)
              .ordered()
              .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
              .baselineColumns("a", "type")
              .baselineValues(13L, "BIGINT")
              .go();
    } finally {
      testNoResult("alter session set `exec.enable_union_type` = false");
    }
  }

  @Test
  public void testTypeCase() throws Exception {
    String query = "select case when is_bigint(field1) " +
            "then field1 when is_list(field1) then field1[0] " +
            "when is_map(field1) then t.field1.inner1 end f1 from cp.`jsoninput/union/a.json` t";
    try {
      testBuilder()
              .sqlQuery(query)
              .ordered()
              .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
              .baselineColumns("f1")
              .baselineValues(1L)
              .baselineValues(2L)
              .baselineValues(3L)
              .baselineValues(3L)
              .go();
    } finally {
      testNoResult("alter session set `exec.enable_union_type` = false");
    }
  }

  @Test
  public void testSumWithTypeCase() throws Exception {
    String query = "select sum(cast(f1 as bigint)) sum_f1 from " +
            "(select case when is_bigint(field1) then field1 " +
            "when is_list(field1) then field1[0] when is_map(field1) then t.field1.inner1 end f1 " +
            "from cp.`jsoninput/union/a.json` t)";
    try {
      testBuilder()
              .sqlQuery(query)
              .ordered()
              .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
              .baselineColumns("sum_f1")
              .baselineValues(9L)
              .go();
    } finally {
      testNoResult("alter session set `exec.enable_union_type` = false");
    }
  }

  @Test
  public void testUnionExpressionMaterialization() throws Exception {
    String query = "select a + b c from cp.`jsoninput/union/b.json`";
    try {
      testBuilder()
              .sqlQuery(query)
              .ordered()
              .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
              .baselineColumns("c")
              .baselineValues(3L)
              .baselineValues(7.0)
              .baselineValues(11.0)
              .go();
    } finally {
      testNoResult("alter session set `exec.enable_union_type` = false");
    }
  }

  @Test
  public void testNestedUnionExpression() throws Exception {
    String query = "select typeof(t.a.b) c from cp.`jsoninput/union/nestedUnion.json` t";
    try {
      testBuilder()
              .sqlQuery(query)
              .ordered()
              .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
              .baselineColumns("c")
              .baselineValues("BIGINT")
              .baselineValues("VARCHAR")
              .baselineValues("MAP")
              .go();

    } finally {
      testNoResult("alter session set `exec.enable_union_type` = false");
    }
  }

  @Test
  public void testSumMultipleBatches() throws Exception {
    String dfs_temp = getDfsTestTmpSchemaLocation();
    File table_dir = new File(dfs_temp, "multi_batch");
    table_dir.mkdir();
    BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "a.json")));
    for (int i = 0; i < 10000; i++) {
      os.write("{ type : \"map\", data : { a : 1 } }\n".getBytes());
      os.write("{ type : \"bigint\", data : 1 }\n".getBytes());
    }
    os.flush();
    os.close();
    String query = "select sum(cast(case when `type` = 'map' then t.data.a else data end as bigint)) `sum` from dfs_test.tmp.multi_batch t";
    try {
      testBuilder()
              .sqlQuery(query)
              .ordered()
              .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
              .baselineColumns("sum")
              .baselineValues(20000L)
              .go();
    } finally {
      testNoResult("alter session set `exec.enable_union_type` = false");
    }
  }

  @Test
  public void testSumFilesWithDifferentSchema() throws Exception {
    String dfs_temp = getDfsTestTmpSchemaLocation();
    File table_dir = new File(dfs_temp, "multi_file");
    table_dir.mkdir();
    BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "a.json")));
    for (int i = 0; i < 10000; i++) {
      os.write("{ type : \"map\", data : { a : 1 } }\n".getBytes());
    }
    os.flush();
    os.close();
    os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "b.json")));
    for (int i = 0; i < 10000; i++) {
      os.write("{ type : \"bigint\", data : 1 }\n".getBytes());
    }
    os.flush();
    os.close();
    String query = "select sum(cast(case when `type` = 'map' then t.data.a else data end as bigint)) `sum` from dfs_test.tmp.multi_file t";
    try {
      testBuilder()
              .sqlQuery(query)
              .ordered()
              .optionSettingQueriesForTestQuery("alter session set `exec.enable_union_type` = true")
              .baselineColumns("sum")
              .baselineValues(20000L)
              .go();
    } finally {
      testNoResult("alter session set `exec.enable_union_type` = false");
    }
  }

  @Test
  public void drill_4032() throws Exception {
    String dfs_temp = getDfsTestTmpSchemaLocation();
    File table_dir = new File(dfs_temp, "drill_4032");
    table_dir.mkdir();
    BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "a.json")));
    os.write("{\"col1\": \"val1\",\"col2\": null}".getBytes());
    os.write("{\"col1\": \"val1\",\"col2\": {\"col3\":\"abc\", \"col4\":\"xyz\"}}".getBytes());
    os.flush();
    os.close();
    os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "b.json")));
    os.write("{\"col1\": \"val1\",\"col2\": null}".getBytes());
    os.write("{\"col1\": \"val1\",\"col2\": null}".getBytes());
    os.flush();
    os.close();
    testNoResult("select t.col2.col3 from dfs_test.tmp.drill_4032 t");
  }

  @Test
  public void drill_4479() throws Exception {
    try {
      String dfs_temp = getDfsTestTmpSchemaLocation();
      File table_dir = new File(dfs_temp, "drill_4479");
      table_dir.mkdir();
      BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(new File(table_dir, "mostlynulls.json")));
      // Create an entire batch of null values for 3 columns
      for (int i = 0 ; i < JSONRecordReader.DEFAULT_ROWS_PER_BATCH; i++) {
        os.write("{\"a\": null, \"b\": null, \"c\": null}".getBytes());
      }
      // Add a row with {bigint,  float, string} values
      os.write("{\"a\": 123456789123, \"b\": 99.999, \"c\": \"Hello World\"}".getBytes());
      os.flush();
      os.close();

      String query1 = "select c, count(*) as cnt from dfs_test.tmp.drill_4479 t group by c";
      String query2 = "select a, b, c, count(*) as cnt from dfs_test.tmp.drill_4479 t group by a, b, c";
      String query3 = "select max(a) as x, max(b) as y, max(c) as z from dfs_test.tmp.drill_4479 t";

      testBuilder()
        .sqlQuery(query1)
        .ordered()
        .optionSettingQueriesForTestQuery("alter session set `store.json.all_text_mode` = true")
        .baselineColumns("c", "cnt")
        .baselineValues(null, 4096L)
        .baselineValues("Hello World", 1L)
        .go();

      testBuilder()
        .sqlQuery(query2)
        .ordered()
        .optionSettingQueriesForTestQuery("alter session set `store.json.all_text_mode` = true")
        .baselineColumns("a", "b", "c", "cnt")
        .baselineValues(null, null, null, 4096L)
        .baselineValues("123456789123", "99.999", "Hello World", 1L)
        .go();

      testBuilder()
        .sqlQuery(query3)
        .ordered()
        .optionSettingQueriesForTestQuery("alter session set `store.json.all_text_mode` = true")
        .baselineColumns("x", "y", "z")
        .baselineValues("123456789123", "99.999", "Hello World")
        .go();

    } finally {
      testNoResult("alter session set `store.json.all_text_mode` = false");
    }
  }

}


text:
game.NewProfile = me.ScreenObject.extend({
	/**
	 *  action to perform on state change
	 */
	onResetEvent: function() {
		me.game.world.addChild(new me.Sprite(0, 0, me.loader.getImage('new-screen')), -10); // TODO
                document.getElementById("input").style.visibility="visible";
                document.getElementById("register").style.visibility="visible";
                me.input.unbindKey(me.input.KEY.B);
                me.input.unbindKey(me.input.KEY.Q);
                me.input.unbindKey(me.input.KEY.W);
                me.input.unbindKey(me.input.KEY.E);
                me.input.unbindKey(me.input.KEY.A);
                var exp1cost = ((game.data.exp1 +1));
                me.game.world.addChild(new (me.Renderable.extend({
                    init:function(){
                        this._super(me.Renderable,'init', [10,10,300,50]);
                        this.font= new me.Font("Arial", 26, "white");

                    },
                    draw:function(renderer){

                        this.font.draw(renderer.getContext(), "PICK A USERNAME AND PASSWORD", this.pos.x , this.pos.y);
                    }
                })));

         },
	/**
	 *  action to perform when leaving this screen (state change)
	 */
	onDestroyEvent: function() {
            document.getElementById("input").style.visibility="hidden";
                document.getElementById("register").style.visibility="hidden";
            }
});


text:
void WriteToFile(const char* path, char* value);

#endif // FILE_H_


text:
package com.github.scaronthesky.eternalwinterwars.view.constants;

import org.andengine.util.color.Color;

/**
 * @author Manuel Seiche
 * @since 21.10.2014
 *
 */
public abstract class Constants {
	public static final String ANIMATION_KEY_MOVE_LEFT = "move_left";
	public static final String ANIMATION_KEY_MOVE_RIGHT = "move_right";
	public static final String ANIMATION_KEY_MOVE_UP = "move_up";
	public static final String ANIMATION_KEY_MOVE_DOWN = "move_down";
	public static final String ANIMATION_KEY_ATTACK_LEFT = "attack_left";
	public static final String ANIMATION_KEY_ATTACK_RIGHT = "attack_right";
	public static final String ANIMATION_KEY_ATTACK_UP = "attack_up";
	public static final String ANIMATION_KEY_ATTACK_DOWN = "attack_down";
	public static final String ANIMATION_KEY_IDLE = "idle";
	public static final String ANIMATION_KEY_DIE = "die";

	public static final float UNIT_MOVE_DURATION_PER_CELL = 0.5f;
	public static final float UNIT_CREATE_DURATION = 0.5f;

	public static final float PARTICLE_CALC_PIXELS_PER_SNOWFLAKE = 4608f;

	public static final float INCOME_MOVE_TO_DEST_DURATION = 2f;
	public static final float DAMAGE_MOVE_TO_DEST_DURATION = 1.5f;

	public static final float CAMERA_CONTROL_MOVE_MULTIPLICATOR = 50f;
	public static final float CAMERA_CONTROL_BASE_APPEAR_DURATION = 0.5f;
	public static final float RANGED_ATTACK_FLY_DURATION_PER_PIXEL = 0.003f;

	public static final int GAME_BASE_ENTITY_UNIT_KNIGHT_HEALTH = 100;
	public static final int GAME_BASE_ENTITY_UNIT_MARKSMAN_HEALTH = 70;
	public static final int GAME_BASE_ENTITY_UNIT_ARTILLERY_HEALTH = 150;
	public static final int GAME_BASE_ENTITY_UNIT_CAVALLERY_HEALTH = 120;
	public static final int GAME_BASE_ENTITY_BUILDING_CASTLE_HEALTH = 600;

	public static final float GAME_BASE_ENTITY_HEALTHBAR_HEIGHT_PERC = 0.1f;

	public static final Color FOG_OF_WAR_COLOR = Color.BLACK;
	public static final float FOG_OF_WAR_ALPHA_LIGHTENED = 0.4f;
}


text:
@interface STFromStoryboardViewController : UIViewController

@end


text:
import os

import pytest

from marvin import config, marvindb
from marvin.api.api import Interaction
from marvin.tools.maps import _get_bintemps

# class MarvinTest
# TODO Replace skipTest and skipBrian with skipif
# TODO use monkeypatch to set initial config variables
# TODO replace _reset_the_config with monkeypatch
# TODO reimplement set_sasurl (use function-level fixture?)

releases = ['MPL-4', 'MPL-5']
plateifus = ['8485-1901']

bintypes = {}
for release in releases:
    __, dapver = config.lookUpVersions(release)
    bintypes[release] = [bintemp.split('-')[0] for bintemp in _get_bintemps(dapver)]

class Galaxy:
    """An example galaxy for Marvin-tools testing."""

    sasbasedir = os.getenv("$SAS_BASE_DIR")
    mangaredux = os.getenv("MANGA_SPECTRO_REDUX")
    mangaanalysis = os.getenv("MANGA_SPECTRO_ANALYSIS")
    dir3d = 'stack'

    def __init__(self, plateifu):
        self.plateifu = plateifu
        self.plate, self.ifu = self.plateifu.split('-')
        self.plate = int(self.plate)
        self.release = config.release
        self.drpver, self.dapver = config.lookUpVersions(self.release)

    # TODO move to a mock data file/directory
    def get_data(self):
        self.galaxies = {'8485-1901': {'mangaid': '1-209232',
                                       'cubepk': 10179,
                                       'ra': 232.544703894,
                                       'dec': 48.6902009334,
                                       'redshift': 0.0407447}
                         }

    def set_galaxy_data(self):
        # Grab data from mock file
        data = self.galaxies[self.plateifu]

        self.mangaid = data['mangaid']
        self.cubepk = data['cubepk']
        self.ra = data['ra']
        self.dec = data['dec']
        self.redshift = data['redshift']

    def set_filenames(self, bintype=None, template=None):
        default_bintemp = _get_bintemps(self.dapver, default=True)
        default_bin, default_temp = default_bintemp.split('-', 1)

        self.bintype = bintype if bintype is not None else default_bin
        self.template = template if template is not None else default_temp
        self.bintemp = '{0}-{1}'.format(self.bintype, self.template)

        self.cubename = 'manga-{0}-LOGCUBE.fits.gz'.format(self.plateifu)
        self.rssname = 'manga-{0}-LOGRSS.fits.gz'.format(self.plateifu)
        self.imgname = '{0}.png'.format(self.ifu)
        self.mapsname = 'manga-{0}-MAPS-{1}.fits.gz'.format(self.plateifu, self.bintemp)
        self.modelname = 'manga-{0}-LOGCUBE-{1}.fits.gz'.format(self.plateifu, self.bintemp)

    def set_filepaths(self):
        # Paths
        self.drppath = os.path.join(self.mangaredux, self.drpver)
        self.dappath = os.path.join(self.mangaanalysis, self.drpver, self.dapver)
        self.imgpath = os.path.join(self.mangaredux, self.drpver, str(self.plate), self.dir3d,
                                    'images')

        # DRP filename paths
        self.cubepath = os.path.join(self.drppath, str(self.plate), self.dir3d, self.cubename)
        self.rsspath = os.path.join(self.drppath, str(self.plate), self.dir3d, self.rssname)

        # DAP filename paths
        self.analysispath = os.path.join(self.dappath, self.bintemp, str(self.plate), self.ifu)
        self.mapspath = os.path.join(self.analysispath, self.mapsname)
        self.modelpath = os.path.join(self.analysispath, self.modelname)

class DB:
    def __init__(self):
        self._marvindb = marvindb
        self.session = marvindb.session

@pytest.fixture(scope='session')
def set_config():
    config.use_sentry = False
    config.add_github_message = False

@pytest.fixture(scope='session')
def set_sasurl(loc='local', port=5000):
    istest = True if loc == 'utah' else False
    config.switchSasUrl(loc, test=istest, port=port)
    response = Interaction('api/general/getroutemap', request_type='get')
    config.urlmap = response.getRouteMap()

@pytest.fixture(scope='session', params=releases)
def set_release(request):
    config.setMPL(request.param)

@pytest.fixture(scope='session')
def start_marvin_session(set_config):
    yield DB()

@pytest.fixture(scope='session', params=plateifus)
def get_plateifu(request):
    return request.param

@pytest.fixture(scope='session', params=bintypes[config.release])
def get_bintype(request):
    return request.param

@pytest.fixture(scope='session')
def galaxy(start_marvin_session, set_release, get_plateifu, get_bintype):
    gal = Galaxy(plateifu=get_plateifu)
    gal.get_data()
    gal.set_galaxy_data()
    gal.set_filenames(bintype=get_bintype)
    gal.set_filepaths()
    gal.maps_filename = gal.mapspath
    gal.modelcube_filename = gal.modelpath
    yield gal


text:
package androidx.media3.extractor.metadata.vorbis;

import static com.google.common.truth.Truth.assertThat;

import android.os.Parcel;
import androidx.media3.common.MediaMetadata;
import androidx.media3.common.Metadata;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;

/** Test for {@link VorbisComment}. */
@RunWith(AndroidJUnit4.class)
public final class VorbisCommentTest {

  @Test
  public void parcelable() {
    VorbisComment vorbisCommentFrameToParcel = new VorbisComment("key", "value");

    Parcel parcel = Parcel.obtain();
    vorbisCommentFrameToParcel.writeToParcel(parcel, 0);
    parcel.setDataPosition(0);

    VorbisComment vorbisCommentFrameFromParcel = VorbisComment.CREATOR.createFromParcel(parcel);
    assertThat(vorbisCommentFrameFromParcel).isEqualTo(vorbisCommentFrameToParcel);

    parcel.recycle();
  }

  @Test
  public void populateMediaMetadata_setsMediaMetadataValues() {
    String title = "the title";
    String artist = "artist";
    String albumTitle = "album title";
    String albumArtist = "album Artist";
    String description = "a description about the audio.";
    List<Metadata.Entry> entries =
        ImmutableList.of(
            new VorbisComment("TITLE", title),
            new VorbisComment("ARTIST", artist),
            new VorbisComment("ALBUM", albumTitle),
            new VorbisComment("ALBUMARTIST", albumArtist),
            new VorbisComment("DESCRIPTION", description));
    MediaMetadata.Builder builder = MediaMetadata.EMPTY.buildUpon();

    for (Metadata.Entry entry : entries) {
      entry.populateMediaMetadata(builder);
    }
    MediaMetadata mediaMetadata = builder.build();

    assertThat(mediaMetadata.title.toString()).isEqualTo(title);
    assertThat(mediaMetadata.artist.toString()).isEqualTo(artist);
    assertThat(mediaMetadata.albumTitle.toString()).isEqualTo(albumTitle);
    assertThat(mediaMetadata.albumArtist.toString()).isEqualTo(albumArtist);
    assertThat(mediaMetadata.description.toString()).isEqualTo(description);
  }
}


text:
@interface PodsDummy_Pods_EmpowerApp_tvOSTests : NSObject
@end
@implementation PodsDummy_Pods_EmpowerApp_tvOSTests
@end


text:
using EventSourcing.Poc.EventSourcing.Mutex;

namespace EventSourcing.Poc.Processing.Mutex {
    public interface IMutexCollector {
        void Register(IMutex mutex);
        void Register(IMutexAsync mutex);
        void Collect();
    }
}


text:
/**
 * The WebGLPipeline After Flush Event.
 *
 * This event is dispatched by a WebGLPipeline right after it has issued a drawArrays command
 * and cleared its vertex count.
 *
 * @event Phaser.Renderer.WebGL.Pipelines.Events#AFTER_FLUSH
 * @since 3.50.0
 *
 * @param {Phaser.Renderer.WebGL.WebGLPipeline} pipeline - The pipeline that has flushed.
 * @param {boolean} isPostFlush - Was this flush invoked as part of a post-process, or not?
 */
module.exports = 'pipelineafterflush';