test_class.declarator: RECORD_0 = "{'day':'Sunday','letters':'6'}"
test_class.modifier: private static final
test_class.original_string: private static final String RECORD_0 = "{'day':'Sunday','letters':'6'}";
test_class.type: String
test_class.var_name: RECORD_0
test_class.declarator: RECORD_1 = "{'day':'Monday','letters':'6'}"
test_class.modifier: private static final
test_class.original_string: private static final String RECORD_1 = "{'day':'Monday','letters':'6'}";
test_class.type: String
test_class.var_name: RECORD_1
test_class.declarator: RECORD_2 = "{'day':'Tuesday','letters':'7'}"
test_class.modifier: private static final
test_class.original_string: private static final String RECORD_2 = "{'day':'Tuesday','letters':'7'}";
test_class.type: String
test_class.var_name: RECORD_2
test_class.declarator: SLEEP_ID = "test-sleep-id-12345"
test_class.modifier: private static final
test_class.original_string: private static final String SLEEP_ID = "test-sleep-id-12345";
test_class.type: String
test_class.var_name: SLEEP_ID
test_class.declarator: jsonParser = new JsonParser()
test_class.modifier: private
test_class.original_string: private JsonParser jsonParser = new JsonParser();
test_class.type: JsonParser
test_class.var_name: jsonParser
test_class.declarator: config
test_class.modifier: private
test_class.original_string: private Configuration config;
test_class.type: Configuration
test_class.var_name: config
test_class.declarator: mockTaskContext
test_class.modifier: @Mock private
test_class.original_string: @Mock private TaskAttemptContext mockTaskContext;
test_class.type: TaskAttemptContext
test_class.var_name: mockTaskContext
test_class.declarator: mockSleeper
test_class.modifier: @Mock private
test_class.original_string: @Mock private Sleeper mockSleeper;
test_class.type: Sleeper
test_class.var_name: mockSleeper
test_class.declarator: basePath
test_class.modifier: private
test_class.original_string: private Path basePath;
test_class.type: Path
test_class.var_name: basePath
test_class.declarator: shardPath
test_class.modifier: private
test_class.original_string: private Path shardPath;
test_class.type: Path
test_class.var_name: shardPath
test_class.declarator: estimatedNumRecords
test_class.modifier: private
test_class.original_string: private long estimatedNumRecords;
test_class.type: long
test_class.var_name: estimatedNumRecords
test_class.declarator: inputSplit
test_class.modifier: private
test_class.original_string: private ShardedInputSplit inputSplit;
test_class.type: ShardedInputSplit
test_class.var_name: inputSplit
test_class.declarator: recordReader
test_class.modifier: private
test_class.original_string: private DynamicFileListRecordReader<LongWritable, JsonObject> recordReader;
test_class.type: DynamicFileListRecordReader<LongWritable, JsonObject>
test_class.var_name: recordReader
test_class.declarator: fileSystem
test_class.modifier: private
test_class.original_string: private FileSystem fileSystem;
test_class.type: FileSystem
test_class.var_name: fileSystem
test_class.file: bigquery/src/test/java/com/google/cloud/hadoop/io/bigquery/DynamicFileListRecordReaderTest.java
test_class.identifier: DynamicFileListRecordReaderTest
test_case.body: @Test
    public void testEmptyFileIsOnlyFileAndNotZeroIndex() throws IOException {
      fileSystem.createNewFile(new Path(shardPath.getParent(), "data-001.json"));
      checkNextKeyValueWouldBlock();
      fileSystem.createNewFile(new Path(shardPath.getParent(), "data-002.json"));
      // Second file-marker with different index causes IllegalStateException.
      assertThrows(IllegalStateException.class, () -> recordReader.nextKeyValue());
    }
test_case.class_method_signature: DynamicFileListRecordReaderTest.testEmptyFileIsOnlyFileAndNotZeroIndex()
test_case.full_signature: @Test public void testEmptyFileIsOnlyFileAndNotZeroIndex()
test_case.identifier: testEmptyFileIsOnlyFileAndNotZeroIndex
test_case.invocations: createNewFile getParent checkNextKeyValueWouldBlock createNewFile getParent assertThrows nextKeyValue
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testEmptyFileIsOnlyFileAndNotZeroIndex()
focal_class.declarator: logger = GoogleLogger.forEnclosingClass()
focal_class.modifier: private static final
focal_class.original_string: private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
focal_class.type: GoogleLogger
focal_class.var_name: logger
focal_class.declarator: inputDirectoryAndPattern
focal_class.modifier: private
focal_class.original_string: private Path inputDirectoryAndPattern;
focal_class.type: Path
focal_class.var_name: inputDirectoryAndPattern
focal_class.declarator: estimatedNumRecords
focal_class.modifier: private
focal_class.original_string: private long estimatedNumRecords;
focal_class.type: long
focal_class.var_name: estimatedNumRecords
focal_class.declarator: maxPollAttempts
focal_class.modifier: private
focal_class.original_string: private int maxPollAttempts;
focal_class.type: int
focal_class.var_name: maxPollAttempts
focal_class.declarator: pollIntervalMs
focal_class.modifier: private
focal_class.original_string: private int pollIntervalMs;
focal_class.type: int
focal_class.var_name: pollIntervalMs
focal_class.declarator: context
focal_class.modifier: private
focal_class.original_string: private TaskAttemptContext context;
focal_class.type: TaskAttemptContext
focal_class.var_name: context
focal_class.declarator: fileSystem
focal_class.modifier: private
focal_class.original_string: private FileSystem fileSystem;
focal_class.type: FileSystem
focal_class.var_name: fileSystem
focal_class.declarator: exportPattern
focal_class.modifier: private
focal_class.original_string: private Pattern exportPattern;
focal_class.type: Pattern
focal_class.var_name: exportPattern
focal_class.declarator: recordsRead = 0
focal_class.modifier: private
focal_class.original_string: private long recordsRead = 0;
focal_class.type: long
focal_class.var_name: recordsRead
focal_class.declarator: delegateRecordReaderFactory
focal_class.modifier: private
focal_class.original_string: private DelegateRecordReaderFactory<K, V> delegateRecordReaderFactory;
focal_class.type: DelegateRecordReaderFactory<K, V>
focal_class.var_name: delegateRecordReaderFactory
focal_class.declarator: delegateReader = null
focal_class.modifier: private
focal_class.original_string: private RecordReader<K, V> delegateReader = null;
focal_class.type: RecordReader<K, V>
focal_class.var_name: delegateReader
focal_class.declarator: knownFileSet = new HashSet<>()
focal_class.modifier: private
focal_class.original_string: private Set<String> knownFileSet = new HashSet<>();
focal_class.type: Set<String>
focal_class.var_name: knownFileSet
focal_class.declarator: fileQueue = new ArrayDeque<>()
focal_class.modifier: private
focal_class.original_string: private Queue<FileStatus> fileQueue = new ArrayDeque<>();
focal_class.type: Queue<FileStatus>
focal_class.var_name: fileQueue
focal_class.declarator: endFileNumber = -1
focal_class.modifier: private
focal_class.original_string: private int endFileNumber = -1;
focal_class.type: int
focal_class.var_name: endFileNumber
focal_class.declarator: sleeper = Sleeper.DEFAULT
focal_class.modifier: private
focal_class.original_string: private Sleeper sleeper = Sleeper.DEFAULT;
focal_class.type: Sleeper
focal_class.var_name: sleeper
focal_class.declarator: currentKey = null
focal_class.modifier: private
focal_class.original_string: private K currentKey = null;
focal_class.type: K
focal_class.var_name: currentKey
focal_class.declarator: currentValue = null
focal_class.modifier: private
focal_class.original_string: private V currentValue = null;
focal_class.type: V
focal_class.var_name: currentValue
focal_class.file: bigquery/src/main/java/com/google/cloud/hadoop/io/bigquery/DynamicFileListRecordReader.java
focal_class.identifier: DynamicFileListRecordReader
focal_class.class_method_signature: DynamicFileListRecordReader.DynamicFileListRecordReader(
        DelegateRecordReaderFactory<K, V> delegateRecordReaderFactory)
focal_class.full_signature: public  DynamicFileListRecordReader(
        DelegateRecordReaderFactory<K, V> delegateRecordReaderFactory)
focal_class.identifier: DynamicFileListRecordReader
focal_class.modifiers: public
focal_class.parameters: (
        DelegateRecordReaderFactory<K, V> delegateRecordReaderFactory)
focal_class.signature:  DynamicFileListRecordReader(
        DelegateRecordReaderFactory<K, V> delegateRecordReaderFactory)
focal_class.class_method_signature: DynamicFileListRecordReader.initialize(InputSplit genericSplit, TaskAttemptContext context)
focal_class.full_signature: @Override public void initialize(InputSplit genericSplit, TaskAttemptContext context)
focal_class.identifier: initialize
focal_class.modifiers: @Override public
focal_class.parameters: (InputSplit genericSplit, TaskAttemptContext context)
focal_class.return: void
focal_class.signature: void initialize(InputSplit genericSplit, TaskAttemptContext context)
focal_class.class_method_signature: DynamicFileListRecordReader.nextKeyValue()
focal_class.full_signature: @Override public boolean nextKeyValue()
focal_class.identifier: nextKeyValue
focal_class.modifiers: @Override public
focal_class.parameters: ()
focal_class.return: boolean
focal_class.signature: boolean nextKeyValue()
focal_class.class_method_signature: DynamicFileListRecordReader.getCurrentKey()
focal_class.full_signature: @Override public K getCurrentKey()
focal_class.identifier: getCurrentKey
focal_class.modifiers: @Override public
focal_class.parameters: ()
focal_class.return: K
focal_class.signature: K getCurrentKey()
focal_class.class_method_signature: DynamicFileListRecordReader.getCurrentValue()
focal_class.full_signature: @Override public V getCurrentValue()
focal_class.identifier: getCurrentValue
focal_class.modifiers: @Override public
focal_class.parameters: ()
focal_class.return: V
focal_class.signature: V getCurrentValue()
focal_class.class_method_signature: DynamicFileListRecordReader.getProgress()
focal_class.full_signature: @Override public float getProgress()
focal_class.identifier: getProgress
focal_class.modifiers: @Override public
focal_class.parameters: ()
focal_class.return: float
focal_class.signature: float getProgress()
focal_class.class_method_signature: DynamicFileListRecordReader.close()
focal_class.full_signature: @Override public void close()
focal_class.identifier: close
focal_class.modifiers: @Override public
focal_class.parameters: ()
focal_class.return: void
focal_class.signature: void close()
focal_class.class_method_signature: DynamicFileListRecordReader.setSleeper(Sleeper sleeper)
focal_class.full_signature: @VisibleForTesting void setSleeper(Sleeper sleeper)
focal_class.identifier: setSleeper
focal_class.modifiers: @VisibleForTesting
focal_class.parameters: (Sleeper sleeper)
focal_class.return: void
focal_class.signature: void setSleeper(Sleeper sleeper)
focal_class.class_method_signature: DynamicFileListRecordReader.populateCurrentKeyValue()
focal_class.full_signature: private void populateCurrentKeyValue()
focal_class.identifier: populateCurrentKeyValue
focal_class.modifiers: private
focal_class.parameters: ()
focal_class.return: void
focal_class.signature: void populateCurrentKeyValue()
focal_class.class_method_signature: DynamicFileListRecordReader.isNextFileReady()
focal_class.full_signature: private boolean isNextFileReady()
focal_class.identifier: isNextFileReady
focal_class.modifiers: private
focal_class.parameters: ()
focal_class.return: boolean
focal_class.signature: boolean isNextFileReady()
focal_class.class_method_signature: DynamicFileListRecordReader.moveToNextFile()
focal_class.full_signature: private FileStatus moveToNextFile()
focal_class.identifier: moveToNextFile
focal_class.modifiers: private
focal_class.parameters: ()
focal_class.return: FileStatus
focal_class.signature: FileStatus moveToNextFile()
focal_class.class_method_signature: DynamicFileListRecordReader.shouldExpectMoreFiles()
focal_class.full_signature: private boolean shouldExpectMoreFiles()
focal_class.identifier: shouldExpectMoreFiles
focal_class.modifiers: private
focal_class.parameters: ()
focal_class.return: boolean
focal_class.signature: boolean shouldExpectMoreFiles()
focal_class.class_method_signature: DynamicFileListRecordReader.parseFileIndex(String fileName)
focal_class.full_signature: private int parseFileIndex(String fileName)
focal_class.identifier: parseFileIndex
focal_class.modifiers: private
focal_class.parameters: (String fileName)
focal_class.return: int
focal_class.signature: int parseFileIndex(String fileName)
focal_class.class_method_signature: DynamicFileListRecordReader.setEndFileMarkerFile(String fileName)
focal_class.full_signature: private void setEndFileMarkerFile(String fileName)
focal_class.identifier: setEndFileMarkerFile
focal_class.modifiers: private
focal_class.parameters: (String fileName)
focal_class.return: void
focal_class.signature: void setEndFileMarkerFile(String fileName)
focal_class.class_method_signature: DynamicFileListRecordReader.refreshFileList()
focal_class.full_signature: private void refreshFileList()
focal_class.identifier: refreshFileList
focal_class.modifiers: private
focal_class.parameters: ()
focal_class.return: void
focal_class.signature: void refreshFileList()
focal_class.superclass: extends RecordReader<K, V>
focal_method.body: @Override
    public boolean nextKeyValue() throws IOException, InterruptedException {
      currentValue = null;
  
      // Check if we already have a reader in-progress.
      if (delegateReader != null) {
        if (delegateReader.nextKeyValue()) {
          populateCurrentKeyValue();
          return true;
        } else {
          delegateReader.close();
          delegateReader = null;
        }
      }
  
      boolean needRefresh = !isNextFileReady() && shouldExpectMoreFiles();
      int pollAttempt = 0;
      while (needRefresh && (maxPollAttempts < 0 || pollAttempt < maxPollAttempts)) {
        logger.atFine().log(
            "No files available after %d attempt(s), but more are expected; refreshing ...",
            pollAttempt + 1);
        refreshFileList();
        needRefresh = !isNextFileReady() && shouldExpectMoreFiles();
        if (needRefresh) {
          logger.atFine().log("No new files found, sleeping before trying again...");
          sleeper.sleep(pollIntervalMs);
          context.progress();
          pollAttempt++;
        }
      }
  
      if (needRefresh) {
        throw new IllegalStateException(
            String.format(
                "Couldn't obtain any files after %d attempt(s). This could happen if in the first"
                    + " failed task attempt BigQuery returned 0 records, but didn't create 0-record"
                    + " file, in this case the second task attempt record reader will wait"
                    + " indefinitely, but no files will appear.",
                pollAttempt));
      }
  
      if (isNextFileReady()) {
        // Open the file and see if it's the 0-record end of dataset marker:
        FileStatus newFile = moveToNextFile();
        logger.atInfo().log(
            "Moving to next file '%s' which has %s bytes. Records read so far: %s",
            newFile.getPath(), newFile.getLen(), recordsRead);
  
        InputSplit split = new FileSplit(newFile.getPath(), 0, newFile.getLen(), new String[0]);
        delegateReader = delegateRecordReaderFactory.createDelegateRecordReader(
            split, context.getConfiguration());
        delegateReader.initialize(split, context);
        if (!delegateReader.nextKeyValue()) {
          // we found the end of dataset marker.
          setEndFileMarkerFile(newFile.getPath().getName());
          return nextKeyValue();
        } else {
          populateCurrentKeyValue();
          return true;
        }
      }
  
      Preconditions.checkState(
          !shouldExpectMoreFiles(),
          "Should not have exited the refresh loop shouldExpectMoreFiles = true "
              + "and no files ready to read.");
  
      // No files ready and we shouldn't expect any more.
      return false;
    }
focal_method.class_method_signature: DynamicFileListRecordReader.nextKeyValue()
focal_method.full_signature: @Override public boolean nextKeyValue()
focal_method.identifier: nextKeyValue
focal_method.invocations: nextKeyValue populateCurrentKeyValue close isNextFileReady shouldExpectMoreFiles log atFine refreshFileList isNextFileReady shouldExpectMoreFiles log atFine sleep progress format isNextFileReady moveToNextFile log atInfo getPath getLen getPath getLen createDelegateRecordReader getConfiguration initialize nextKeyValue setEndFileMarkerFile getName getPath nextKeyValue populateCurrentKeyValue checkState shouldExpectMoreFiles
focal_method.modifiers: @Override public
focal_method.parameters: ()
focal_method.return: boolean
focal_method.signature: boolean nextKeyValue()