test_class.declarator: mockBigquery
test_class.modifier: @Mock private
test_class.original_string: @Mock private Bigquery mockBigquery;
test_class.type: Bigquery
test_class.var_name: mockBigquery
test_class.declarator: mockBigqueryJobs
test_class.modifier: @Mock private
test_class.original_string: @Mock private Bigquery.Jobs mockBigqueryJobs;
test_class.type: Bigquery.Jobs
test_class.var_name: mockBigqueryJobs
test_class.declarator: mockBigqueryJobsGet
test_class.modifier: @Mock private
test_class.original_string: @Mock private Bigquery.Jobs.Get mockBigqueryJobsGet;
test_class.type: Bigquery.Jobs.Get
test_class.var_name: mockBigqueryJobsGet
test_class.declarator: mockBigqueryJobsInsert
test_class.modifier: @Mock private
test_class.original_string: @Mock private Bigquery.Jobs.Insert mockBigqueryJobsInsert;
test_class.type: Bigquery.Jobs.Insert
test_class.var_name: mockBigqueryJobsInsert
test_class.declarator: mockBigqueryTables
test_class.modifier: @Mock private
test_class.original_string: @Mock private Bigquery.Tables mockBigqueryTables;
test_class.type: Bigquery.Tables
test_class.var_name: mockBigqueryTables
test_class.declarator: mockBigqueryTablesGet
test_class.modifier: @Mock private
test_class.original_string: @Mock private Bigquery.Tables.Get mockBigqueryTablesGet;
test_class.type: Bigquery.Tables.Get
test_class.var_name: mockBigqueryTablesGet
test_class.declarator: mockErrorExtractor
test_class.modifier: @Mock private
test_class.original_string: @Mock private ApiErrorExtractor mockErrorExtractor;
test_class.type: ApiErrorExtractor
test_class.var_name: mockErrorExtractor
test_class.declarator: jobStatus
test_class.modifier: private
test_class.original_string: private JobStatus jobStatus;
test_class.type: JobStatus
test_class.var_name: jobStatus
test_class.declarator: jobHandle
test_class.modifier: private
test_class.original_string: private Job jobHandle;
test_class.type: Job
test_class.var_name: jobHandle
test_class.declarator: fakeTable
test_class.modifier: private
test_class.original_string: private Table fakeTable;
test_class.type: Table
test_class.var_name: fakeTable
test_class.declarator: fakeTableSchema
test_class.modifier: private
test_class.original_string: private TableSchema fakeTableSchema;
test_class.type: TableSchema
test_class.var_name: fakeTableSchema
test_class.declarator: jobProjectId = "google.com:foo-project"
test_class.modifier: private
test_class.original_string: private String jobProjectId = "google.com:foo-project";
test_class.type: String
test_class.var_name: jobProjectId
test_class.declarator: kmsKeyName =
        "projects/google.com:foo-project/locations/us-west1/keyRings/ring-1/cryptoKeys/key-1"
test_class.modifier: private final
test_class.original_string: private final String kmsKeyName =
        "projects/google.com:foo-project/locations/us-west1/keyRings/ring-1/cryptoKeys/key-1";
test_class.type: String
test_class.var_name: kmsKeyName
test_class.declarator: tableRef
test_class.modifier: private
test_class.original_string: private TableReference tableRef;
test_class.type: TableReference
test_class.var_name: tableRef
test_class.declarator: projectId = "google.com:bar-project"
test_class.modifier: private
test_class.original_string: private String projectId = "google.com:bar-project";
test_class.type: String
test_class.var_name: projectId
test_class.declarator: datasetId = "test_dataset"
test_class.modifier: private
test_class.original_string: private String datasetId = "test_dataset";
test_class.type: String
test_class.var_name: datasetId
test_class.declarator: tableId = "test_table"
test_class.modifier: private
test_class.original_string: private String tableId = "test_table";
test_class.type: String
test_class.var_name: tableId
test_class.declarator: jobId = "bigquery-job-1234"
test_class.modifier: private
test_class.original_string: private String jobId = "bigquery-job-1234";
test_class.type: String
test_class.var_name: jobId
test_class.declarator: helper
test_class.modifier: private
test_class.original_string: private BigQueryHelper helper;
test_class.type: BigQueryHelper
test_class.var_name: helper
test_class.file: bigquery/src/test/java/com/google/cloud/hadoop/io/bigquery/BigQueryHelperTest.java
test_class.identifier: BigQueryHelperTest
test_case.body: @Test
    public void testImportBigQueryFromGcs() throws Exception {
      when(mockBigqueryTablesGet.execute()).thenReturn(fakeTable);
  
      final ArgumentCaptor<Job> jobCaptor = ArgumentCaptor.forClass(Job.class);
      doAnswer(
              new Answer<Job>() {
                @Override
                public Job answer(InvocationOnMock invocationOnMock) throws Throwable {
                  verify(mockBigqueryJobs, times(1)).insert(eq(jobProjectId), jobCaptor.capture());
                  return jobCaptor.getValue();
                }
              })
          .when(mockBigqueryJobsInsert)
          .execute();
      when(mockBigqueryJobsGet.execute()).thenReturn(jobHandle);
  
      // Run importBigQueryFromGcs method.
      helper.importFromGcs(
          jobProjectId,
          tableRef,
          fakeTableSchema,
          /* timePartitioning= */ null,
          kmsKeyName,
          BigQueryFileFormat.NEWLINE_DELIMITED_JSON,
          BigQueryConfiguration.OUTPUT_TABLE_CREATE_DISPOSITION.getDefault(),
          BigQueryConfiguration.OUTPUT_TABLE_WRITE_DISPOSITION.getDefault(),
          ImmutableList.of("test-import-path"),
          true);
  
      // Verify correct calls to BigQuery are made.
      verify(mockBigquery, times(2)).jobs();
  
      // Verify correct calls to BigQuery.Jobs are made.
      verify(mockBigqueryJobs, times(1))
          .get(eq(jobProjectId), eq(jobCaptor.getValue().getJobReference().getJobId()));
      Job job = jobCaptor.getValue();
      assertThat(job.getConfiguration().getLoad().getSourceUris()).contains("test-import-path");
      assertThat(job.getConfiguration().getLoad().getDestinationTable()).isEqualTo(tableRef);
      assertThat(job.getJobReference().getLocation()).isEqualTo("test_location");
  
      // Verify we poll for job in correct location
      verify(mockBigqueryJobsGet).setLocation(eq("test_location"));
  
      // Verify correct calls to BigQuery.Jobs.Get are made.
      verify(mockBigqueryJobsGet, times(1)).execute();
  
      // Verify correct calls to BigQuery.Jobs.Insert are made.
      verify(mockBigqueryJobsInsert, times(1)).execute();
  
      verify(mockBigquery).datasets();
    }
test_case.class_method_signature: BigQueryHelperTest.testImportBigQueryFromGcs()
test_case.full_signature: @Test public void testImportBigQueryFromGcs()
test_case.identifier: testImportBigQueryFromGcs
test_case.invocations: thenReturn when execute forClass execute when doAnswer insert verify times eq capture getValue thenReturn when execute importFromGcs getDefault getDefault of jobs verify times get verify times eq eq getJobId getJobReference getValue getValue contains assertThat getSourceUris getLoad getConfiguration isEqualTo assertThat getDestinationTable getLoad getConfiguration isEqualTo assertThat getLocation getJobReference setLocation verify eq execute verify times execute verify times datasets verify
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testImportBigQueryFromGcs()
focal_class.declarator: BIGQUERY_JOB_ID_PATTERN = "[a-zA-Z0-9_-]+"
focal_class.modifier: public static final
focal_class.original_string: public static final String BIGQUERY_JOB_ID_PATTERN = "[a-zA-Z0-9_-]+";
focal_class.type: String
focal_class.var_name: BIGQUERY_JOB_ID_PATTERN
focal_class.declarator: BIGQUERY_JOB_ID_MAX_LENGTH = 1024
focal_class.modifier: public static final
focal_class.original_string: public static final int BIGQUERY_JOB_ID_MAX_LENGTH = 1024;
focal_class.type: int
focal_class.var_name: BIGQUERY_JOB_ID_MAX_LENGTH
focal_class.declarator: logger = GoogleLogger.forEnclosingClass()
focal_class.modifier: private static final
focal_class.original_string: private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
focal_class.type: GoogleLogger
focal_class.var_name: logger
focal_class.declarator: errorExtractor = ApiErrorExtractor.INSTANCE
focal_class.modifier: private
focal_class.original_string: private ApiErrorExtractor errorExtractor = ApiErrorExtractor.INSTANCE;
focal_class.type: ApiErrorExtractor
focal_class.var_name: errorExtractor
focal_class.declarator: service
focal_class.modifier: private
focal_class.original_string: private Bigquery service;
focal_class.type: Bigquery
focal_class.var_name: service
focal_class.file: bigquery/src/main/java/com/google/cloud/hadoop/io/bigquery/BigQueryHelper.java
focal_class.identifier: BigQueryHelper
focal_class.class_method_signature: BigQueryHelper.BigQueryHelper(Bigquery service)
focal_class.full_signature: public  BigQueryHelper(Bigquery service)
focal_class.identifier: BigQueryHelper
focal_class.modifiers: public
focal_class.parameters: (Bigquery service)
focal_class.signature:  BigQueryHelper(Bigquery service)
focal_class.class_method_signature: BigQueryHelper.getRawBigquery()
focal_class.full_signature: public Bigquery getRawBigquery()
focal_class.identifier: getRawBigquery
focal_class.modifiers: public
focal_class.parameters: ()
focal_class.return: Bigquery
focal_class.signature: Bigquery getRawBigquery()
focal_class.class_method_signature: BigQueryHelper.importFederatedFromGcs(
        String projectId,
        TableReference tableRef,
        @Nullable TableSchema schema,
        BigQueryFileFormat sourceFormat,
        List<String> gcsPaths)
focal_class.full_signature: public void importFederatedFromGcs(
        String projectId,
        TableReference tableRef,
        @Nullable TableSchema schema,
        BigQueryFileFormat sourceFormat,
        List<String> gcsPaths)
focal_class.identifier: importFederatedFromGcs
focal_class.modifiers: public
focal_class.parameters: (
        String projectId,
        TableReference tableRef,
        @Nullable TableSchema schema,
        BigQueryFileFormat sourceFormat,
        List<String> gcsPaths)
focal_class.return: void
focal_class.signature: void importFederatedFromGcs(
        String projectId,
        TableReference tableRef,
        @Nullable TableSchema schema,
        BigQueryFileFormat sourceFormat,
        List<String> gcsPaths)
focal_class.class_method_signature: BigQueryHelper.importFromGcs(
        String projectId,
        TableReference tableRef,
        @Nullable TableSchema schema,
        @Nullable TimePartitioning timePartitioning,
        @Nullable String kmsKeyName,
        BigQueryFileFormat sourceFormat,
        String createDisposition,
        String writeDisposition,
        List<String> gcsPaths,
        boolean awaitCompletion)
focal_class.full_signature: public void importFromGcs(
        String projectId,
        TableReference tableRef,
        @Nullable TableSchema schema,
        @Nullable TimePartitioning timePartitioning,
        @Nullable String kmsKeyName,
        BigQueryFileFormat sourceFormat,
        String createDisposition,
        String writeDisposition,
        List<String> gcsPaths,
        boolean awaitCompletion)
focal_class.identifier: importFromGcs
focal_class.modifiers: public
focal_class.parameters: (
        String projectId,
        TableReference tableRef,
        @Nullable TableSchema schema,
        @Nullable TimePartitioning timePartitioning,
        @Nullable String kmsKeyName,
        BigQueryFileFormat sourceFormat,
        String createDisposition,
        String writeDisposition,
        List<String> gcsPaths,
        boolean awaitCompletion)
focal_class.return: void
focal_class.signature: void importFromGcs(
        String projectId,
        TableReference tableRef,
        @Nullable TableSchema schema,
        @Nullable TimePartitioning timePartitioning,
        @Nullable String kmsKeyName,
        BigQueryFileFormat sourceFormat,
        String createDisposition,
        String writeDisposition,
        List<String> gcsPaths,
        boolean awaitCompletion)
focal_class.class_method_signature: BigQueryHelper.exportBigQueryToGcs(
        String projectId, TableReference tableRef, List<String> gcsPaths, boolean awaitCompletion)
focal_class.full_signature: public void exportBigQueryToGcs(
        String projectId, TableReference tableRef, List<String> gcsPaths, boolean awaitCompletion)
focal_class.identifier: exportBigQueryToGcs
focal_class.modifiers: public
focal_class.parameters: (
        String projectId, TableReference tableRef, List<String> gcsPaths, boolean awaitCompletion)
focal_class.return: void
focal_class.signature: void exportBigQueryToGcs(
        String projectId, TableReference tableRef, List<String> gcsPaths, boolean awaitCompletion)
focal_class.class_method_signature: BigQueryHelper.tableExists(TableReference tableRef)
focal_class.full_signature: public boolean tableExists(TableReference tableRef)
focal_class.identifier: tableExists
focal_class.modifiers: public
focal_class.parameters: (TableReference tableRef)
focal_class.return: boolean
focal_class.signature: boolean tableExists(TableReference tableRef)
focal_class.class_method_signature: BigQueryHelper.getTable(TableReference tableRef)
focal_class.full_signature: public Table getTable(TableReference tableRef)
focal_class.identifier: getTable
focal_class.modifiers: public
focal_class.parameters: (TableReference tableRef)
focal_class.return: Table
focal_class.signature: Table getTable(TableReference tableRef)
focal_class.class_method_signature: BigQueryHelper.createJobReference(
        String projectId, String jobIdPrefix, @Nullable String location)
focal_class.full_signature: public JobReference createJobReference(
        String projectId, String jobIdPrefix, @Nullable String location)
focal_class.identifier: createJobReference
focal_class.modifiers: public
focal_class.parameters: (
        String projectId, String jobIdPrefix, @Nullable String location)
focal_class.return: JobReference
focal_class.signature: JobReference createJobReference(
        String projectId, String jobIdPrefix, @Nullable String location)
focal_class.class_method_signature: BigQueryHelper.checkJobIdEquality(Job expected, Job actual)
focal_class.full_signature: public void checkJobIdEquality(Job expected, Job actual)
focal_class.identifier: checkJobIdEquality
focal_class.modifiers: public
focal_class.parameters: (Job expected, Job actual)
focal_class.return: void
focal_class.signature: void checkJobIdEquality(Job expected, Job actual)
focal_class.class_method_signature: BigQueryHelper.insertJobOrFetchDuplicate(String projectId, Job job)
focal_class.full_signature: public Job insertJobOrFetchDuplicate(String projectId, Job job)
focal_class.identifier: insertJobOrFetchDuplicate
focal_class.modifiers: public
focal_class.parameters: (String projectId, Job job)
focal_class.return: Job
focal_class.signature: Job insertJobOrFetchDuplicate(String projectId, Job job)
focal_class.class_method_signature: BigQueryHelper.setErrorExtractor(ApiErrorExtractor errorExtractor)
focal_class.full_signature: @VisibleForTesting void setErrorExtractor(ApiErrorExtractor errorExtractor)
focal_class.identifier: setErrorExtractor
focal_class.modifiers: @VisibleForTesting
focal_class.parameters: (ApiErrorExtractor errorExtractor)
focal_class.return: void
focal_class.signature: void setErrorExtractor(ApiErrorExtractor errorExtractor)
focal_method.body: public void importFromGcs(
        String projectId,
        TableReference tableRef,
        @Nullable TableSchema schema,
        @Nullable TimePartitioning timePartitioning,
        @Nullable String kmsKeyName,
        BigQueryFileFormat sourceFormat,
        String createDisposition,
        String writeDisposition,
        List<String> gcsPaths,
        boolean awaitCompletion)
        throws IOException, InterruptedException {
      logger.atInfo().log(
          "Importing into table '%s' from %s paths; path[0] is '%s'; awaitCompletion: %s;"
              + " timePartitioning: %s",
          lazy(() -> BigQueryStrings.toString(tableRef)),
          gcsPaths.size(),
          gcsPaths.isEmpty() ? "(empty)" : gcsPaths.get(0),
          awaitCompletion,
          timePartitioning);
  
      // Create load conf with minimal requirements.
      JobConfigurationLoad loadConfig = new JobConfigurationLoad();
      loadConfig.setSchema(schema);
      loadConfig.setSourceFormat(sourceFormat.getFormatIdentifier());
      loadConfig.setSourceUris(gcsPaths);
      loadConfig.setDestinationTable(tableRef);
      loadConfig.setTimePartitioning(timePartitioning);
      loadConfig.setCreateDisposition(createDisposition);
      loadConfig.setWriteDisposition(writeDisposition);
      if (!Strings.isNullOrEmpty(kmsKeyName)) {
        loadConfig.setDestinationEncryptionConfiguration(
            new EncryptionConfiguration().setKmsKeyName(kmsKeyName));
      }
      // Auto detect the schema if we're not given one, otherwise use the passed schema.
      if (schema == null) {
        logger.atInfo().log("No import schema provided, auto detecting schema.");
        loadConfig.setAutodetect(true);
      } else {
        logger.atInfo().log("Using provided import schema '%s'.", schema);
      }
  
      JobConfiguration config = new JobConfiguration();
      config.setLoad(loadConfig);
  
      // Get the dataset to determine the location
      Dataset dataset;
      try {
        dataset = service.datasets().get(tableRef.getProjectId(), tableRef.getDatasetId()).execute();
      } catch (IOException ioe) {
        throw new IOException(
            String.format(
                "Failed to get dataset '%s' in project '%s' for table '%s'",
                tableRef.getDatasetId(), tableRef.getProjectId(), tableRef),
            ioe);
      }
  
      JobReference jobReference =
          createJobReference(projectId, "direct-bigqueryhelper-import", dataset.getLocation());
      Job job = new Job();
      job.setConfiguration(config);
      job.setJobReference(jobReference);
  
      // Insert and run job.
      insertJobOrFetchDuplicate(projectId, job);
  
      if (awaitCompletion) {
        // Poll until job is complete.
        BigQueryUtils.waitForJobCompletion(getRawBigquery(), projectId, jobReference, () -> {});
      }
    }
focal_method.class_method_signature: BigQueryHelper.importFromGcs(
        String projectId,
        TableReference tableRef,
        @Nullable TableSchema schema,
        @Nullable TimePartitioning timePartitioning,
        @Nullable String kmsKeyName,
        BigQueryFileFormat sourceFormat,
        String createDisposition,
        String writeDisposition,
        List<String> gcsPaths,
        boolean awaitCompletion)
focal_method.full_signature: public void importFromGcs(
        String projectId,
        TableReference tableRef,
        @Nullable TableSchema schema,
        @Nullable TimePartitioning timePartitioning,
        @Nullable String kmsKeyName,
        BigQueryFileFormat sourceFormat,
        String createDisposition,
        String writeDisposition,
        List<String> gcsPaths,
        boolean awaitCompletion)
focal_method.identifier: importFromGcs
focal_method.invocations: log atInfo lazy toString size isEmpty get setSchema setSourceFormat getFormatIdentifier setSourceUris setDestinationTable setTimePartitioning setCreateDisposition setWriteDisposition isNullOrEmpty setDestinationEncryptionConfiguration setKmsKeyName log atInfo setAutodetect log atInfo setLoad execute get datasets getProjectId getDatasetId format getDatasetId getProjectId createJobReference getLocation setConfiguration setJobReference insertJobOrFetchDuplicate waitForJobCompletion getRawBigquery
focal_method.modifiers: public
focal_method.parameters: (
        String projectId,
        TableReference tableRef,
        @Nullable TableSchema schema,
        @Nullable TimePartitioning timePartitioning,
        @Nullable String kmsKeyName,
        BigQueryFileFormat sourceFormat,
        String createDisposition,
        String writeDisposition,
        List<String> gcsPaths,
        boolean awaitCompletion)
focal_method.return: void
focal_method.signature: void importFromGcs(
        String projectId,
        TableReference tableRef,
        @Nullable TableSchema schema,
        @Nullable TimePartitioning timePartitioning,
        @Nullable String kmsKeyName,
        BigQueryFileFormat sourceFormat,
        String createDisposition,
        String writeDisposition,
        List<String> gcsPaths,
        boolean awaitCompletion)