test_class.file: herd-code/herd-tools/herd-downloader/src/test/java/org/finra/herd/tools/downloader/DownloaderControllerTest.java
test_class.identifier: DownloaderControllerTest
test_class.superclass: extends AbstractDownloaderTest
test_case.body: @Test
      public void testPerformDownload() throws Exception
      {
          runDownload();
      }
test_case.class_method_signature: DownloaderControllerTest.testPerformDownload()
test_case.full_signature: @Test public void testPerformDownload()
test_case.identifier: testPerformDownload
test_case.invocations: runDownload
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testPerformDownload()
focal_class.declarator: LOGGER = LoggerFactory.getLogger(DownloaderController.class)
focal_class.modifier: private static final
focal_class.original_string: private static final Logger LOGGER = LoggerFactory.getLogger(DownloaderController.class);
focal_class.type: Logger
focal_class.var_name: LOGGER
focal_class.declarator: businessObjectDataHelper
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private BusinessObjectDataHelper businessObjectDataHelper;
focal_class.type: BusinessObjectDataHelper
focal_class.var_name: businessObjectDataHelper
focal_class.declarator: configurationHelper
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private ConfigurationHelper configurationHelper;
focal_class.type: ConfigurationHelper
focal_class.var_name: configurationHelper
focal_class.declarator: manifestReader
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private DownloaderManifestReader manifestReader;
focal_class.type: DownloaderManifestReader
focal_class.var_name: manifestReader
focal_class.declarator: manifestWriter
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private DownloaderManifestWriter manifestWriter;
focal_class.type: DownloaderManifestWriter
focal_class.var_name: manifestWriter
focal_class.declarator: downloaderWebClient
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private DownloaderWebClient downloaderWebClient;
focal_class.type: DownloaderWebClient
focal_class.var_name: downloaderWebClient
focal_class.declarator: storageFileHelper
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private StorageFileHelper storageFileHelper;
focal_class.type: StorageFileHelper
focal_class.var_name: storageFileHelper
focal_class.declarator: storageHelper
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private StorageHelper storageHelper;
focal_class.type: StorageHelper
focal_class.var_name: storageHelper
focal_class.declarator: OUTPUT_MANIFEST_FILE_NAME = "manifest.json"
focal_class.modifier: private static final
focal_class.original_string: private static final String OUTPUT_MANIFEST_FILE_NAME = "manifest.json";
focal_class.type: String
focal_class.var_name: OUTPUT_MANIFEST_FILE_NAME
focal_class.file: herd-code/herd-tools/herd-downloader/src/main/java/org/finra/herd/tools/downloader/DownloaderController.java
focal_class.identifier: DownloaderController
focal_class.class_method_signature: DownloaderController.performDownload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath,
          S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto)
focal_class.full_signature: @SuppressFBWarnings(value = {"BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", "NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}, justification = "manifestReader.readJsonManifest will always return an DownloaderInputManifestDto object. targetLocalDirectory.list().length will not" + " return a NullPointerException.") public void performDownload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath,
          S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto)
focal_class.identifier: performDownload
focal_class.modifiers: @SuppressFBWarnings(value = {"BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", "NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}, justification = "manifestReader.readJsonManifest will always return an DownloaderInputManifestDto object. targetLocalDirectory.list().length will not" + " return a NullPointerException.") public
focal_class.parameters: (RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath,
          S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto)
focal_class.return: void
focal_class.signature: void performDownload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath,
          S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto)
focal_class.class_method_signature: DownloaderController.logLocalDirectoryContents(File directory)
focal_class.full_signature: private void logLocalDirectoryContents(File directory)
focal_class.identifier: logLocalDirectoryContents
focal_class.modifiers: private
focal_class.parameters: (File directory)
focal_class.return: void
focal_class.signature: void logLocalDirectoryContents(File directory)
focal_class.class_method_signature: DownloaderController.createDownloaderOutputManifestDto(BusinessObjectData businessObjectData, StorageUnit storageUnit, String s3KeyPrefix)
focal_class.full_signature: protected DownloaderOutputManifestDto createDownloaderOutputManifestDto(BusinessObjectData businessObjectData, StorageUnit storageUnit, String s3KeyPrefix)
focal_class.identifier: createDownloaderOutputManifestDto
focal_class.modifiers: protected
focal_class.parameters: (BusinessObjectData businessObjectData, StorageUnit storageUnit, String s3KeyPrefix)
focal_class.return: DownloaderOutputManifestDto
focal_class.signature: DownloaderOutputManifestDto createDownloaderOutputManifestDto(BusinessObjectData businessObjectData, StorageUnit storageUnit, String s3KeyPrefix)
focal_class.superclass: extends DataBridgeController
focal_method.body: @SuppressFBWarnings(value = {"BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", "NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}, justification =
          "manifestReader.readJsonManifest will always return an DownloaderInputManifestDto object. targetLocalDirectory.list().length will not" +
              " return a NullPointerException.")
      public void performDownload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath,
          S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto)
          throws InterruptedException, JAXBException, IOException, URISyntaxException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException
      {
          boolean cleanUpTargetLocalDirectoryOnFailure = false;
          File targetLocalDirectory = null;
  
          try
          {
              // Process manifest file.
              DownloaderInputManifestDto manifest = manifestReader.readJsonManifest(manifestPath);
  
              String storageName = getStorageNameFromManifest(manifest);
  
              // Get business object data from the herd registration server.
              downloaderWebClient.setRegServerAccessParamsDto(regServerAccessParamsDto);
              BusinessObjectData businessObjectData = downloaderWebClient.getBusinessObjectData(manifest);
  
              manifest.setBusinessObjectDataVersion(String.valueOf(businessObjectData.getVersion()));
              manifest.setBusinessObjectFormatVersion(String.valueOf(businessObjectData.getBusinessObjectFormatVersion()));
              s3FileTransferRequestParamsDto.getAdditionalAwsCredentialsProviders().add(new AutoRefreshCredentialProvider()
              {
                  @Override
                  public AwsCredential getNewAwsCredential() throws Exception
                  {
                      return downloaderWebClient.getStorageUnitDownloadCredential(manifest, storageName).getAwsCredential();
                  }
              });
  
              // Get a storage unit that belongs to the S3 storage.
              StorageUnit storageUnit = businessObjectDataHelper.getStorageUnitByStorageName(businessObjectData, storageName);
  
              // Get the expected S3 key prefix and S3 bucket name.
              S3KeyPrefixInformation s3KeyPrefixInformation = downloaderWebClient.getS3KeyPrefix(businessObjectData);
  
              // Check if the target folder (local directory + S3 key prefix) exists and try to create it if it does not.
              targetLocalDirectory = Paths.get(s3FileTransferRequestParamsDto.getLocalPath(), s3KeyPrefixInformation.getS3KeyPrefix()).toFile();
              if (!targetLocalDirectory.isDirectory())
              {
                  // Create the local directory including any necessary but nonexistent parent directories.
                  if (!targetLocalDirectory.mkdirs())
                  {
                      throw new IllegalArgumentException(String.format("Failed to create target local directory \"%s\".", targetLocalDirectory.getPath()));
                  }
              }
              else
              {
                  // Check if the target local directory is empty.
                  if (targetLocalDirectory.list().length > 0)
                  {
                      throw new IllegalArgumentException(String.format("The target local directory \"%s\" is not empty.", targetLocalDirectory.getPath()));
                  }
              }
  
              // Get S3 bucket information.
              Storage storage = downloaderWebClient.getStorage(storageName);
  
              // Get S3 bucket name.  Please note that since this value is required we pass a "true" flag.
              String s3BucketName =
                  storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storage, true);
  
              // Get the list of S3 files matching the expected S3 key prefix.
              s3FileTransferRequestParamsDto.setS3BucketName(s3BucketName);
              // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
              s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefixInformation.getS3KeyPrefix() + "/");
              // When listing S3 files, we ignore 0 byte objects that represent S3 directories.
              List<String> actualS3Files = storageFileHelper.getFilePathsFromS3ObjectSummaries(s3Service.listDirectory(s3FileTransferRequestParamsDto, true));
  
              // Validate S3 files before we start the download.
              storageFileHelper.validateStorageUnitS3Files(storageUnit, actualS3Files, s3KeyPrefixInformation.getS3KeyPrefix());
  
              // Special handling for the maxThreads command line option.
              s3FileTransferRequestParamsDto.setMaxThreads(adjustIntegerValue(s3FileTransferRequestParamsDto.getMaxThreads(), MIN_THREADS, MAX_THREADS));
  
              // Download S3 files to the target local directory.
              s3FileTransferRequestParamsDto.setRecursive(true);
              cleanUpTargetLocalDirectoryOnFailure = true;
              s3Service.downloadDirectory(s3FileTransferRequestParamsDto);
  
              // Validate the downloaded files.
              storageFileHelper.validateDownloadedS3Files(s3FileTransferRequestParamsDto.getLocalPath(), s3KeyPrefixInformation.getS3KeyPrefix(), storageUnit);
  
              // Log a list of files downloaded to the target local directory.
              if (LOGGER.isInfoEnabled())
              {
                  logLocalDirectoryContents(targetLocalDirectory);
              }
  
              // Create a downloader output manifest file.
              DownloaderOutputManifestDto downloaderOutputManifestDto =
                  createDownloaderOutputManifestDto(businessObjectData, storageUnit, s3KeyPrefixInformation.getS3KeyPrefix());
              manifestWriter.writeJsonManifest(targetLocalDirectory, OUTPUT_MANIFEST_FILE_NAME, downloaderOutputManifestDto);
          }
          catch (InterruptedException | JAXBException | IOException | URISyntaxException e)
          {
              // If we got to the point of validating the target local directory being empty before this failure
              // occurred, let's rollback the data transfer by cleaning up the local target directory.
              if (cleanUpTargetLocalDirectoryOnFailure)
              {
                  LOGGER.info(String.format("Rolling back the S3 data transfer by cleaning up \"%s\" target local directory.", targetLocalDirectory));
                  HerdFileUtils.cleanDirectoryIgnoreException(targetLocalDirectory);
              }
  
              throw e;
          }
      }
focal_method.class_method_signature: DownloaderController.performDownload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath,
          S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto)
focal_method.full_signature: @SuppressFBWarnings(value = {"BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", "NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}, justification = "manifestReader.readJsonManifest will always return an DownloaderInputManifestDto object. targetLocalDirectory.list().length will not" + " return a NullPointerException.") public void performDownload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath,
          S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto)
focal_method.identifier: performDownload
focal_method.invocations: readJsonManifest getStorageNameFromManifest setRegServerAccessParamsDto getBusinessObjectData setBusinessObjectDataVersion valueOf getVersion setBusinessObjectFormatVersion valueOf getBusinessObjectFormatVersion add getAdditionalAwsCredentialsProviders getAwsCredential getStorageUnitDownloadCredential getStorageUnitByStorageName getS3KeyPrefix toFile get getLocalPath getS3KeyPrefix isDirectory mkdirs format getPath list format getPath getStorage getStorageAttributeValueByName getProperty setS3BucketName setS3KeyPrefix getS3KeyPrefix getFilePathsFromS3ObjectSummaries listDirectory validateStorageUnitS3Files getS3KeyPrefix setMaxThreads adjustIntegerValue getMaxThreads setRecursive downloadDirectory validateDownloadedS3Files getLocalPath getS3KeyPrefix isInfoEnabled logLocalDirectoryContents createDownloaderOutputManifestDto getS3KeyPrefix writeJsonManifest info format cleanDirectoryIgnoreException
focal_method.modifiers: @SuppressFBWarnings(value = {"BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", "NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}, justification = "manifestReader.readJsonManifest will always return an DownloaderInputManifestDto object. targetLocalDirectory.list().length will not" + " return a NullPointerException.") public
focal_method.parameters: (RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath,
          S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto)
focal_method.return: void
focal_method.signature: void performDownload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath,
          S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto)