test_class.declarator: LOGGER = LoggerFactory.getLogger(UploaderControllerTest.class)
test_class.modifier: private static final
test_class.original_string: private static final Logger LOGGER = LoggerFactory.getLogger(UploaderControllerTest.class);
test_class.type: Logger
test_class.var_name: LOGGER
test_class.file: herd-code/herd-tools/herd-uploader/src/test/java/org/finra/herd/tools/uploader/UploaderControllerTest.java
test_class.identifier: UploaderControllerTest
test_class.superclass: extends AbstractUploaderTest
test_case.body: @Test
      public void testPerformUploadInvalidLocalDir() throws Exception
      {
          // Create uploader input manifest file in LOCAL_TEMP_PATH_INPUT directory
          File manifestFile = createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), getTestUploaderInputManifestDto());
          Assert.assertTrue(manifestFile.isFile());
  
          // Try to run upload by specifying a non-existing local directory.
          S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto();
          s3FileTransferRequestParamsDto.setLocalPath(Paths.get(LOCAL_TEMP_PATH_INPUT.toString(), "I_DO_NOT_EXIST").toString());
          RegServerAccessParamsDto regServerAccessParamsDto =
              RegServerAccessParamsDto.builder().withRegServerHost(WEB_SERVICE_HOSTNAME).withRegServerPort(WEB_SERVICE_HTTPS_PORT).withUseSsl(true)
                  .withUsername(WEB_SERVICE_HTTPS_USERNAME).withPassword(WEB_SERVICE_HTTPS_PASSWORD).build();
          try
          {
              uploaderController.performUpload(regServerAccessParamsDto, manifestFile, s3FileTransferRequestParamsDto, false, false, TEST_RETRY_ATTEMPTS,
                  TEST_RETRY_DELAY_SECS);
              fail("Should throw an IllegalArgumentException when local directory does not exist.");
          }
          catch (IllegalArgumentException e)
          {
              assertTrue(e.getMessage().startsWith("Invalid local base directory"));
          }
      }
test_case.class_method_signature: UploaderControllerTest.testPerformUploadInvalidLocalDir()
test_case.full_signature: @Test public void testPerformUploadInvalidLocalDir()
test_case.identifier: testPerformUploadInvalidLocalDir
test_case.invocations: createManifestFile toString getTestUploaderInputManifestDto assertTrue isFile getTestS3FileTransferRequestParamsDto setLocalPath toString get toString build withPassword withUsername withUseSsl withRegServerPort withRegServerHost builder performUpload fail assertTrue startsWith getMessage
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testPerformUploadInvalidLocalDir()
focal_class.declarator: LOGGER = LoggerFactory.getLogger(UploaderController.class)
focal_class.modifier: private static final
focal_class.original_string: private static final Logger LOGGER = LoggerFactory.getLogger(UploaderController.class);
focal_class.type: Logger
focal_class.var_name: LOGGER
focal_class.declarator: businessObjectDataHelper
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private BusinessObjectDataHelper businessObjectDataHelper;
focal_class.type: BusinessObjectDataHelper
focal_class.var_name: businessObjectDataHelper
focal_class.declarator: configurationHelper
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private ConfigurationHelper configurationHelper;
focal_class.type: ConfigurationHelper
focal_class.var_name: configurationHelper
focal_class.declarator: herdThreadHelper
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private HerdThreadHelper herdThreadHelper;
focal_class.type: HerdThreadHelper
focal_class.var_name: herdThreadHelper
focal_class.declarator: jsonHelper
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private JsonHelper jsonHelper;
focal_class.type: JsonHelper
focal_class.var_name: jsonHelper
focal_class.declarator: manifestReader
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private UploaderManifestReader manifestReader;
focal_class.type: UploaderManifestReader
focal_class.var_name: manifestReader
focal_class.declarator: storageHelper
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private StorageHelper storageHelper;
focal_class.type: StorageHelper
focal_class.var_name: storageHelper
focal_class.declarator: uploaderWebClient
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private UploaderWebClient uploaderWebClient;
focal_class.type: UploaderWebClient
focal_class.var_name: uploaderWebClient
focal_class.file: herd-code/herd-tools/herd-uploader/src/main/java/org/finra/herd/tools/uploader/UploaderController.java
focal_class.identifier: UploaderController
focal_class.class_method_signature: UploaderController.performUpload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto params,
          Boolean createNewVersion, Boolean force, Integer maxRetryAttempts, Integer retryDelaySecs)
focal_class.full_signature: @SuppressFBWarnings(value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", justification = "manifestReader.readJsonManifest will always return an UploaderInputManifestDto object.") public void performUpload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto params,
          Boolean createNewVersion, Boolean force, Integer maxRetryAttempts, Integer retryDelaySecs)
focal_class.identifier: performUpload
focal_class.modifiers: @SuppressFBWarnings(value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", justification = "manifestReader.readJsonManifest will always return an UploaderInputManifestDto object.") public
focal_class.parameters: (RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto params,
          Boolean createNewVersion, Boolean force, Integer maxRetryAttempts, Integer retryDelaySecs)
focal_class.return: void
focal_class.signature: void performUpload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto params,
          Boolean createNewVersion, Boolean force, Integer maxRetryAttempts, Integer retryDelaySecs)
focal_class.class_method_signature: UploaderController.addStorageFilesWithRetry(BusinessObjectDataKey businessObjectDataKey, UploaderInputManifestDto manifest, S3FileTransferRequestParamsDto params,
          String storageName, Integer maxRetryAttempts, Integer retryDelaySecs)
focal_class.full_signature: private void addStorageFilesWithRetry(BusinessObjectDataKey businessObjectDataKey, UploaderInputManifestDto manifest, S3FileTransferRequestParamsDto params,
          String storageName, Integer maxRetryAttempts, Integer retryDelaySecs)
focal_class.identifier: addStorageFilesWithRetry
focal_class.modifiers: private
focal_class.parameters: (BusinessObjectDataKey businessObjectDataKey, UploaderInputManifestDto manifest, S3FileTransferRequestParamsDto params,
          String storageName, Integer maxRetryAttempts, Integer retryDelaySecs)
focal_class.return: void
focal_class.signature: void addStorageFilesWithRetry(BusinessObjectDataKey businessObjectDataKey, UploaderInputManifestDto manifest, S3FileTransferRequestParamsDto params,
          String storageName, Integer maxRetryAttempts, Integer retryDelaySecs)
focal_class.class_method_signature: UploaderController.checkLatestBusinessObjectDataVersion(UploaderInputManifestDto manifest, Boolean force)
focal_class.full_signature: private void checkLatestBusinessObjectDataVersion(UploaderInputManifestDto manifest, Boolean force)
focal_class.identifier: checkLatestBusinessObjectDataVersion
focal_class.modifiers: private
focal_class.parameters: (UploaderInputManifestDto manifest, Boolean force)
focal_class.return: void
focal_class.signature: void checkLatestBusinessObjectDataVersion(UploaderInputManifestDto manifest, Boolean force)
focal_class.class_method_signature: UploaderController.findDuplicateFiles(List<File> files)
focal_class.full_signature: private List<File> findDuplicateFiles(List<File> files)
focal_class.identifier: findDuplicateFiles
focal_class.modifiers: private
focal_class.parameters: (List<File> files)
focal_class.return: List<File>
focal_class.signature: List<File> findDuplicateFiles(List<File> files)
focal_class.class_method_signature: UploaderController.getValidatedLocalFiles(String localDir, List<ManifestFile> manifestFiles)
focal_class.full_signature: private List<File> getValidatedLocalFiles(String localDir, List<ManifestFile> manifestFiles)
focal_class.identifier: getValidatedLocalFiles
focal_class.modifiers: private
focal_class.parameters: (String localDir, List<ManifestFile> manifestFiles)
focal_class.return: List<File>
focal_class.signature: List<File> getValidatedLocalFiles(String localDir, List<ManifestFile> manifestFiles)
focal_class.class_method_signature: UploaderController.logS3KeyPrefixContents(S3FileTransferRequestParamsDto params)
focal_class.full_signature: private void logS3KeyPrefixContents(S3FileTransferRequestParamsDto params)
focal_class.identifier: logS3KeyPrefixContents
focal_class.modifiers: private
focal_class.parameters: (S3FileTransferRequestParamsDto params)
focal_class.return: void
focal_class.signature: void logS3KeyPrefixContents(S3FileTransferRequestParamsDto params)
focal_class.superclass: extends DataBridgeController
focal_method.body: @SuppressFBWarnings(value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE",
          justification = "manifestReader.readJsonManifest will always return an UploaderInputManifestDto object.")
      public void performUpload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto params,
          Boolean createNewVersion, Boolean force, Integer maxRetryAttempts, Integer retryDelaySecs)
          throws InterruptedException, JAXBException, IOException, URISyntaxException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException
      {
          boolean cleanUpS3KeyPrefixOnFailure = false;
          BusinessObjectDataKey businessObjectDataKey = null;
  
          try
          {
              // Process manifest file
              UploaderInputManifestDto manifest = manifestReader.readJsonManifest(manifestPath);
  
              String storageName = getStorageNameFromManifest(manifest);
              manifest.setStorageName(storageName);
  
              // Validate local files and prepare a list of source files to copy to S3.
              List<File> sourceFiles = getValidatedLocalFiles(params.getLocalPath(), manifest.getManifestFiles());
  
              // Validate that we do not have duplicate files listed in the manifest file.
              List<File> duplicateFiles = findDuplicateFiles(sourceFiles);
  
              if (!duplicateFiles.isEmpty())
              {
                  throw new IllegalArgumentException(
                      String.format("Manifest contains duplicate file names. Duplicates: [\"%s\"]", StringUtils.join(duplicateFiles, "\", \"")));
              }
  
              // Initialize uploader web client.
              uploaderWebClient.setRegServerAccessParamsDto(regServerAccessParamsDto);
  
              // Handle the latest business object data version if one exists.
              checkLatestBusinessObjectDataVersion(manifest, force);
  
              // Pre-register a new version of business object data in UPLOADING state with the registration server.
              BusinessObjectData businessObjectData = uploaderWebClient.preRegisterBusinessObjectData(manifest, storageName, createNewVersion);
  
              // Get business object data key.
              businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectData);
  
              // Get the business object data version.
              Integer businessObjectDataVersion = businessObjectDataKey.getBusinessObjectDataVersion();
  
              // Add credential provider.
              params.getAdditionalAwsCredentialsProviders().add(new AutoRefreshCredentialProvider()
              {
                  @Override
                  public AwsCredential getNewAwsCredential() throws Exception
                  {
                      return uploaderWebClient.getBusinessObjectDataUploadCredential(manifest, storageName, businessObjectDataVersion, null).getAwsCredential();
                  }
              });
  
              // Get S3 key prefix from the business object data pre-registration response.
              String s3KeyPrefix = IterableUtils.get(businessObjectData.getStorageUnits(), 0).getStorageDirectory().getDirectoryPath();
  
              // Get S3 bucket information.
              Storage storage = uploaderWebClient.getStorage(storageName);
  
              // Get S3 bucket name.  Please note that since this value is required we pass a "true" flag.
              String s3BucketName =
                  storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storage, true);
  
              // Set the KMS ID, if available
              String kmsKeyId =
                  storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KMS_KEY_ID), storage, false);
              params.setKmsKeyId(kmsKeyId);
  
              // Special handling for the maxThreads command line option.
              params.setMaxThreads(adjustIntegerValue(params.getMaxThreads(), MIN_THREADS, MAX_THREADS));
  
              // Populate several missing fields in the S3 file transfer request parameters DTO.
              params.setS3BucketName(s3BucketName);
              // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
              params.setS3KeyPrefix(s3KeyPrefix + "/");
              params.setFiles(sourceFiles);
  
              // Check if the destination S3 key prefix is empty.
              // When listing S3 files, by default, we do not ignore 0 byte objects that represent S3 directories.
              if (s3Service.listDirectory(params).isEmpty())
              {
                  cleanUpS3KeyPrefixOnFailure = true;
              }
              else
              {
                  throw new IllegalStateException(String
                      .format("The destination S3 folder is not empty. S3 Bucket Name: \"%s\". S3 key prefix: \"%s\".", params.getS3BucketName(),
                          params.getS3KeyPrefix()));
              }
  
              // Upload files.
              s3Service.uploadFileList(params);
  
              // Get the list of files uploaded to S3 key prefix.
              if (LOGGER.isInfoEnabled())
              {
                  logS3KeyPrefixContents(params);
              }
  
              // Add storage files to the business object data.
              addStorageFilesWithRetry(businessObjectDataKey, manifest, params, storage.getName(), maxRetryAttempts, retryDelaySecs);
  
              // Change status of the business object data to VALID.
              uploaderWebClient.updateBusinessObjectDataStatus(businessObjectDataKey, BusinessObjectDataStatusEntity.VALID);
          }
          catch (InterruptedException | JAXBException | IOException | URISyntaxException e)
          {
              // If we got to the point of checking the target S3 key prefix before this failure
              // occurred, let's rollback the data transfer (clean up the S3 key prefix).
              if (cleanUpS3KeyPrefixOnFailure)
              {
                  LOGGER.info(String
                      .format("Rolling back the S3 data transfer by deleting keys/objects with prefix \"%s\" from bucket \"%s\".", params.getS3KeyPrefix(),
                          params.getS3BucketName()));
                  s3Service.deleteDirectoryIgnoreException(params);
              }
  
              // If a new business object data version got pre-registered, update it's status to INVALID.
              if (businessObjectDataKey != null)
              {
                  uploaderWebClient.updateBusinessObjectDataStatusIgnoreException(businessObjectDataKey, BusinessObjectDataStatusEntity.INVALID);
              }
  
              throw e;
          }
      }
focal_method.class_method_signature: UploaderController.performUpload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto params,
          Boolean createNewVersion, Boolean force, Integer maxRetryAttempts, Integer retryDelaySecs)
focal_method.full_signature: @SuppressFBWarnings(value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", justification = "manifestReader.readJsonManifest will always return an UploaderInputManifestDto object.") public void performUpload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto params,
          Boolean createNewVersion, Boolean force, Integer maxRetryAttempts, Integer retryDelaySecs)
focal_method.identifier: performUpload
focal_method.invocations: readJsonManifest getStorageNameFromManifest setStorageName getValidatedLocalFiles getLocalPath getManifestFiles findDuplicateFiles isEmpty format join setRegServerAccessParamsDto checkLatestBusinessObjectDataVersion preRegisterBusinessObjectData getBusinessObjectDataKey getBusinessObjectDataVersion add getAdditionalAwsCredentialsProviders getAwsCredential getBusinessObjectDataUploadCredential getDirectoryPath getStorageDirectory get getStorageUnits getStorage getStorageAttributeValueByName getProperty getStorageAttributeValueByName getProperty setKmsKeyId setMaxThreads adjustIntegerValue getMaxThreads setS3BucketName setS3KeyPrefix setFiles isEmpty listDirectory format getS3BucketName getS3KeyPrefix uploadFileList isInfoEnabled logS3KeyPrefixContents addStorageFilesWithRetry getName updateBusinessObjectDataStatus info format getS3KeyPrefix getS3BucketName deleteDirectoryIgnoreException updateBusinessObjectDataStatusIgnoreException
focal_method.modifiers: @SuppressFBWarnings(value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", justification = "manifestReader.readJsonManifest will always return an UploaderInputManifestDto object.") public
focal_method.parameters: (RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto params,
          Boolean createNewVersion, Boolean force, Integer maxRetryAttempts, Integer retryDelaySecs)
focal_method.return: void
focal_method.signature: void performUpload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto params,
          Boolean createNewVersion, Boolean force, Integer maxRetryAttempts, Integer retryDelaySecs)