test_class.file: imhotep-server/src/test/java/com/indeed/imhotep/local/TestIndexReWriter.java
test_class.identifier: TestIndexReWriter
test_case.body: @Test
      public void testMergingStringTermDocIterator() throws ImhotepOutOfMemoryException, IOException {
          /* make session 1 */
          MockFlamdexReader r1 =
                  new MockFlamdexReader(Arrays.asList("if1"), Arrays.asList("sf1", "sf2", "sf3"),
                                        Arrays.asList("if1"), 10);
  
          r1.addStringTerm("sf1", "0", 0);
          r1.addStringTerm("sf1", "1", 1);
          r1.addStringTerm("sf1", "2", 2);
          r1.addStringTerm("sf1", "3", 3);
          r1.addStringTerm("sf1", "4", 4);
  
          r1.addStringTerm("sf2", "5", 5);
          r1.addStringTerm("sf2", "6", 6);
          r1.addStringTerm("sf2", "7", 7);
          r1.addStringTerm("sf2", "8", 8);
          r1.addStringTerm("sf2", "9", 9);
  
          r1.addStringTerm("sf3", "0", 0);
          r1.addStringTerm("sf3", "1", 1);
          r1.addStringTerm("sf3", "2", 2);
          r1.addStringTerm("sf3", "3", 3);
          r1.addStringTerm("sf3", "4", 4);
  
          r1.addIntTerm("if1", 3, Arrays.asList(1, 4, 5, 6, 7));
  
          ImhotepLocalSession session1 = new ImhotepLocalSession(r1);
  
          /* make session 2 */
          MockFlamdexReader r2 =
                  new MockFlamdexReader(Arrays.asList("if1"), Arrays.asList("sf1", "sf2"),
                                        Arrays.asList("if1"), 10);
          r2.addStringTerm("sf1", "0", 5);
          r2.addStringTerm("sf1", "1", 6);
          r2.addStringTerm("sf1", "2", 7);
          r2.addStringTerm("sf1", "3", 8);
          r2.addStringTerm("sf1", "4", 9);
  
          r2.addStringTerm("sf2", "0", 0);
          r2.addStringTerm("sf2", "1", 1);
          r2.addStringTerm("sf2", "2", 2);
          r2.addStringTerm("sf2", "3", 3);
          r2.addStringTerm("sf2", "4", 4);
  
          r2.addIntTerm("if1", 3, Arrays.asList(1, 4, 5, 6, 7));
  
          ImhotepLocalSession session2 = new ImhotepLocalSession(r2);
  
          /* make session 3 */
          MockFlamdexReader r3 =
                  new MockFlamdexReader(Arrays.asList("if1"), Arrays.asList("sf2"),
                                        Arrays.asList("if1"), 5);
          r3.addStringTerm("sf2", "0", 0);
          r3.addStringTerm("sf2", "1", 1);
          r3.addStringTerm("sf2", "2", 2);
          r3.addStringTerm("sf2", "3", 3);
          r3.addStringTerm("sf2", "4", 4);
  
          r2.addIntTerm("if1", 3, Arrays.asList(1, 4, 5, 6, 7));
  
          ImhotepLocalSession session3 = new ImhotepLocalSession(r3);
  
          /* merge sessions */
          MockFlamdexWriter w = new MockFlamdexWriter("foo");
          IndexReWriter irw =
                  new IndexReWriter(
                                    Arrays.asList(session3, session1, session2),
                                    session1,
                                    new MemoryReservationContext(
                                                                 new ImhotepMemoryPool(Long.MAX_VALUE)));
          irw.optimizeIndecies(new ArrayList<String>(), Arrays.asList("sf1", "sf2"), w);
  
          assertEquals("Merged String Fields are wrong",
                       Arrays.asList("sf1", "sf2"),
                       w.getStringFields());
          assertEquals("Merged String Terms are wrong", Arrays.asList(5, 20), w.getStringTerms()
                                                                               .get("sf1").get("0"));
          assertEquals("Merged String Terms are wrong", Arrays.asList(6, 21), w.getStringTerms()
                                                                               .get("sf1").get("1"));
          assertEquals("Merged String Terms are wrong", Arrays.asList(7, 22), w.getStringTerms()
                                                                               .get("sf1").get("2"));
          assertEquals("Merged String Terms are wrong", Arrays.asList(8, 23), w.getStringTerms()
                                                                               .get("sf1").get("3"));
          assertEquals("Merged String Terms are wrong", Arrays.asList(9, 24), w.getStringTerms()
                                                                               .get("sf1").get("4"));
  
          assertEquals("Merged String Terms are wrong", Arrays.asList(0, 15), w.getStringTerms()
                                                                               .get("sf2").get("0"));
          assertEquals("Merged String Terms are wrong", Arrays.asList(1, 16), w.getStringTerms()
                                                                               .get("sf2").get("1"));
          assertEquals("Merged String Terms are wrong", Arrays.asList(2, 17), w.getStringTerms()
                                                                               .get("sf2").get("2"));
          assertEquals("Merged String Terms are wrong", Arrays.asList(3, 18), w.getStringTerms()
                                                                               .get("sf2").get("3"));
          assertEquals("Merged String Terms are wrong", Arrays.asList(4, 19), w.getStringTerms()
                                                                               .get("sf2").get("4"));
          assertEquals("Merged String Terms are wrong", Arrays.asList(10), w.getStringTerms()
                                                                            .get("sf2").get("5"));
          assertEquals("Merged String Terms are wrong", Arrays.asList(11), w.getStringTerms()
                                                                            .get("sf2").get("6"));
          assertEquals("Merged String Terms are wrong", Arrays.asList(12), w.getStringTerms()
                                                                            .get("sf2").get("7"));
          assertEquals("Merged String Terms are wrong", Arrays.asList(13), w.getStringTerms()
                                                                            .get("sf2").get("8"));
          assertEquals("Merged String Terms are wrong", Arrays.asList(14), w.getStringTerms()
                                                                            .get("sf2").get("9"));
  
          session1.close();
          session2.close();
          session3.close();
      }
test_case.class_method_signature: TestIndexReWriter.testMergingStringTermDocIterator()
test_case.full_signature: @Test public void testMergingStringTermDocIterator()
test_case.identifier: testMergingStringTermDocIterator
test_case.invocations: asList asList asList addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addIntTerm asList asList asList asList addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addIntTerm asList asList asList asList addStringTerm addStringTerm addStringTerm addStringTerm addStringTerm addIntTerm asList asList optimizeIndecies asList assertEquals asList getStringFields assertEquals asList get get getStringTerms assertEquals asList get get getStringTerms assertEquals asList get get getStringTerms assertEquals asList get get getStringTerms assertEquals asList get get getStringTerms assertEquals asList get get getStringTerms assertEquals asList get get getStringTerms assertEquals asList get get getStringTerms assertEquals asList get get getStringTerms assertEquals asList get get getStringTerms assertEquals asList get get getStringTerms assertEquals asList get get getStringTerms assertEquals asList get get getStringTerms assertEquals asList get get getStringTerms assertEquals asList get get getStringTerms close close close
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testMergingStringTermDocIterator()
focal_class.declarator: sessions
focal_class.modifier: private final
focal_class.original_string: private final List<ImhotepLocalSession> sessions;
focal_class.type: List<ImhotepLocalSession>
focal_class.var_name: sessions
focal_class.declarator: newSession
focal_class.modifier: private final
focal_class.original_string: private final ImhotepLocalSession newSession;
focal_class.type: ImhotepLocalSession
focal_class.var_name: newSession
focal_class.declarator: sessionDocIdOffsets
focal_class.modifier: private
focal_class.original_string: private int[] sessionDocIdOffsets;
focal_class.type: int[]
focal_class.var_name: sessionDocIdOffsets
focal_class.declarator: memory
focal_class.modifier: private final
focal_class.original_string: private final MemoryReservationContext memory;
focal_class.type: MemoryReservationContext
focal_class.var_name: memory
focal_class.declarator: newGroupLookup
focal_class.modifier: private
focal_class.original_string: private GroupLookup newGroupLookup;
focal_class.type: GroupLookup
focal_class.var_name: newGroupLookup
focal_class.declarator: perSessionMappings
focal_class.modifier: private
focal_class.original_string: private List<int[]> perSessionMappings;
focal_class.type: List<int[]>
focal_class.var_name: perSessionMappings
focal_class.declarator: dynamicMetrics
focal_class.modifier: private
focal_class.original_string: private Map<String, DynamicMetric> dynamicMetrics;
focal_class.type: Map<String, DynamicMetric>
focal_class.var_name: dynamicMetrics
focal_class.declarator: newMaxDocs
focal_class.modifier: private
focal_class.original_string: private long newMaxDocs;
focal_class.type: long
focal_class.var_name: newMaxDocs
focal_class.file: imhotep-server/src/main/java/com/indeed/imhotep/local/IndexReWriter.java
focal_class.identifier: IndexReWriter
focal_class.class_method_signature: IndexReWriter.IndexReWriter(List<ImhotepLocalSession> localSessions, 
                           ImhotepLocalSession newSession,
                           MemoryReservationContext memory)
focal_class.full_signature: public  IndexReWriter(List<ImhotepLocalSession> localSessions, 
                           ImhotepLocalSession newSession,
                           MemoryReservationContext memory)
focal_class.identifier: IndexReWriter
focal_class.modifiers: public
focal_class.parameters: (List<ImhotepLocalSession> localSessions, 
                           ImhotepLocalSession newSession,
                           MemoryReservationContext memory)
focal_class.signature:  IndexReWriter(List<ImhotepLocalSession> localSessions, 
                           ImhotepLocalSession newSession,
                           MemoryReservationContext memory)
focal_class.class_method_signature: IndexReWriter.getNewGroupLookup()
focal_class.full_signature: public GroupLookup getNewGroupLookup()
focal_class.identifier: getNewGroupLookup
focal_class.modifiers: public
focal_class.parameters: ()
focal_class.return: GroupLookup
focal_class.signature: GroupLookup getNewGroupLookup()
focal_class.class_method_signature: IndexReWriter.getDynamicMetrics()
focal_class.full_signature: public Map<String, DynamicMetric> getDynamicMetrics()
focal_class.identifier: getDynamicMetrics
focal_class.modifiers: public
focal_class.parameters: ()
focal_class.return: Map<String, DynamicMetric>
focal_class.signature: Map<String, DynamicMetric> getDynamicMetrics()
focal_class.class_method_signature: IndexReWriter.getPerSessionMappings()
focal_class.full_signature: public List<int[]> getPerSessionMappings()
focal_class.identifier: getPerSessionMappings
focal_class.modifiers: public
focal_class.parameters: ()
focal_class.return: List<int[]>
focal_class.signature: List<int[]> getPerSessionMappings()
focal_class.class_method_signature: IndexReWriter.getNumSessionsMerged()
focal_class.full_signature: public int getNumSessionsMerged()
focal_class.identifier: getNumSessionsMerged
focal_class.modifiers: public
focal_class.parameters: ()
focal_class.return: int
focal_class.signature: int getNumSessionsMerged()
focal_class.class_method_signature: IndexReWriter.optimizeIndecies(@Nonnull final List<String> intFields,
                                   @Nonnull final List<String> stringFields,
                                   @Nonnull final FlamdexWriter w)
focal_class.full_signature: public void optimizeIndecies(@Nonnull final List<String> intFields,
                                   @Nonnull final List<String> stringFields,
                                   @Nonnull final FlamdexWriter w)
focal_class.identifier: optimizeIndecies
focal_class.modifiers: public
focal_class.parameters: (@Nonnull final List<String> intFields,
                                   @Nonnull final List<String> stringFields,
                                   @Nonnull final FlamdexWriter w)
focal_class.return: void
focal_class.signature: void optimizeIndecies(@Nonnull final List<String> intFields,
                                   @Nonnull final List<String> stringFields,
                                   @Nonnull final FlamdexWriter w)
focal_class.class_method_signature: IndexReWriter.constructPerSessionNewToOldIdMappings(int[] oldToNewDocIdMapping)
focal_class.full_signature: private List<int[]> constructPerSessionNewToOldIdMappings(int[] oldToNewDocIdMapping)
focal_class.identifier: constructPerSessionNewToOldIdMappings
focal_class.modifiers: private
focal_class.parameters: (int[] oldToNewDocIdMapping)
focal_class.return: List<int[]>
focal_class.signature: List<int[]> constructPerSessionNewToOldIdMappings(int[] oldToNewDocIdMapping)
focal_class.class_method_signature: IndexReWriter.remapDocIds(List<ImhotepLocalSession> sessions)
focal_class.full_signature: private int[] remapDocIds(List<ImhotepLocalSession> sessions)
focal_class.identifier: remapDocIds
focal_class.modifiers: private
focal_class.parameters: (List<ImhotepLocalSession> sessions)
focal_class.return: int[]
focal_class.signature: int[] remapDocIds(List<ImhotepLocalSession> sessions)
focal_method.body: public void optimizeIndecies(@Nonnull final List<String> intFields,
                                   @Nonnull final List<String> stringFields,
                                   @Nonnull final FlamdexWriter w) throws IOException,
                                                                  ImhotepOutOfMemoryException {
          final int[] docIdBuffer = new int[128];
          List<IntTermDocIterator> intIters = new ArrayList<IntTermDocIterator>(sessions.size());
          List<StringTermDocIterator> stringIters =
                  new ArrayList<StringTermDocIterator>(sessions.size());
          List<Integer> sessionOffsets = new ArrayList<Integer>(this.sessionDocIdOffsets.length);
          int[] oldToNewDocIdMapping;
  
          oldToNewDocIdMapping = remapDocIds(this.sessions);
          w.resetMaxDocs(this.newMaxDocs);
  
          for (final String intField : intFields) {
              intIters.clear();
              sessionOffsets.clear();
              for (int i = 0; i < sessions.size(); i++) {
                  ImhotepLocalSession session = sessions.get(i);
                  IntTermDocIterator iter = session.getReader().getIntTermDocIterator(intField);
                  if (iter == null) {
                      continue;
                  }
                  intIters.add(iter);
                  sessionOffsets.add(this.sessionDocIdOffsets[i]);
              }
              final MergingIntTermDocIterator iter =
                      new MergingIntTermDocIterator(intIters, oldToNewDocIdMapping, sessionOffsets);
              final IntFieldWriter ifw = w.getIntFieldWriter(intField);
              while (iter.nextTerm()) {
                  ifw.nextTerm(iter.term());
                  /*
                   * Write all the terms and groups to the new index, skipping
                   * those in group 0
                   */
                  int n;
                  do {
                      n = iter.fillDocIdBuffer(docIdBuffer);
                      for (int i = 0; i < n; ++i) {
                          final int docId = docIdBuffer[i];
                          if (docId == -1) {
                              /* doc was in group 0 */
                              continue;
                          }
                          ifw.nextDoc(docId);
                      }
                  } while (n == docIdBuffer.length);
              }
              iter.close();
              ifw.close();
          }
  
          for (final String stringField : stringFields) {
              stringIters.clear();
              sessionOffsets.clear();
              for (int i = 0; i < sessions.size(); i++) {
                  ImhotepLocalSession session = sessions.get(i);
                  StringTermDocIterator iter =
                          session.getReader().getStringTermDocIterator(stringField);
                  if (iter == null) {
                      continue;
                  }
                  stringIters.add(iter);
                  sessionOffsets.add(this.sessionDocIdOffsets[i]);
              }
              final MergingStringTermDocIterator iter =
                      new MergingStringTermDocIterator(stringIters, oldToNewDocIdMapping,
                                                       sessionOffsets);
              final StringFieldWriter sfw = w.getStringFieldWriter(stringField);
              while (iter.nextTerm()) {
                  sfw.nextTerm(iter.term());
                  /*
                   * Write all the terms and groups to the new index, skipping
                   * those in group 0
                   */
                  int n;
                  do {
                      n = iter.fillDocIdBuffer(docIdBuffer);
                      for (int i = 0; i < n; ++i) {
                          final int docId = docIdBuffer[i];
                          if (docId == -1) {
                              /* doc was in group 0 */
                              continue;
                          }
                          sfw.nextDoc(docId);
                      }
                  } while (n == docIdBuffer.length);
              }
              iter.close();
              sfw.close();
          }
  
          this.perSessionMappings = constructPerSessionNewToOldIdMappings(oldToNewDocIdMapping);
  
          memory.releaseMemory(oldToNewDocIdMapping.length * 4L);
      }
focal_method.class_method_signature: IndexReWriter.optimizeIndecies(@Nonnull final List<String> intFields,
                                   @Nonnull final List<String> stringFields,
                                   @Nonnull final FlamdexWriter w)
focal_method.full_signature: public void optimizeIndecies(@Nonnull final List<String> intFields,
                                   @Nonnull final List<String> stringFields,
                                   @Nonnull final FlamdexWriter w)
focal_method.identifier: optimizeIndecies
focal_method.invocations: size size remapDocIds resetMaxDocs clear clear size get getIntTermDocIterator getReader add add getIntFieldWriter nextTerm nextTerm term fillDocIdBuffer nextDoc close close clear clear size get getStringTermDocIterator getReader add add getStringFieldWriter nextTerm nextTerm term fillDocIdBuffer nextDoc close close constructPerSessionNewToOldIdMappings releaseMemory
focal_method.modifiers: public
focal_method.parameters: (@Nonnull final List<String> intFields,
                                   @Nonnull final List<String> stringFields,
                                   @Nonnull final FlamdexWriter w)
focal_method.return: void
focal_method.signature: void optimizeIndecies(@Nonnull final List<String> intFields,
                                   @Nonnull final List<String> stringFields,
                                   @Nonnull final FlamdexWriter w)