test_class.declarator: formatAnalysis
test_class.modifier: @Autowired
test_class.original_string: @Autowired
      FormatAnalysis formatAnalysis;
test_class.type: FormatAnalysis
test_class.var_name: formatAnalysis
test_class.declarator: qualityAnalysis
test_class.modifier: @Autowired
test_class.original_string: @Autowired
      QualityAnalysis qualityAnalysis;
test_class.type: QualityAnalysis
test_class.var_name: qualityAnalysis
test_class.declarator: schemaAnalysis
test_class.modifier: @Autowired
test_class.original_string: @Autowired
      SchemaAnalysis schemaAnalysis;
test_class.type: SchemaAnalysis
test_class.var_name: schemaAnalysis
test_class.declarator: contentAnalysis
test_class.modifier: @Autowired
test_class.original_string: @Autowired
      ContentAnalysis contentAnalysis;
test_class.type: ContentAnalysis
test_class.var_name: contentAnalysis
test_class.declarator: random = new Random()
test_class.modifier: private
test_class.original_string: private Random random = new Random();
test_class.type: Random
test_class.var_name: random
test_class.file: dataprep-dataset/src/test/java/org/talend/dataprep/dataset/service/analysis/synchronous/QualityAnalysisTest.java
test_class.identifier: QualityAnalysisTest
test_class.superclass: extends DataSetBaseTest
test_case.body: @Test
      public void testAnalysisWithInvalidValues() {
          String dsId = UUID.randomUUID().toString();
          final DataSetMetadata metadata = metadataBuilder.metadata().id(dsId).build();
          dataSetMetadataRepository.save(metadata);
          contentStore.storeAsRaw(metadata,
                  DataSetServiceTest.class.getResourceAsStream("../dataset_with_invalid_records.csv"));
          formatAnalysis.analyze(dsId);
          contentAnalysis.analyze(dsId);
          schemaAnalysis.analyze(dsId);
          // Analyze quality
          qualityAnalysis.analyze(dsId);
          final DataSetMetadata actual = dataSetMetadataRepository.get(dsId);
          assertThat(actual.getLifecycle().qualityAnalyzed(), is(true));
          assertThat(actual.getContent().getNbRecords(), is(9L));
          assertThat(actual.getRowMetadata().getColumns().size(), is(2));
          ColumnMetadata secondColumn = actual.getRowMetadata().getColumns().get(1);
          Quality quality = secondColumn.getQuality();
          assertThat(quality.getValid(), is(6));
          assertThat(quality.getInvalid(), is(2));
          assertThat(quality.getEmpty(), is(1));
  
      }
test_case.class_method_signature: QualityAnalysisTest.testAnalysisWithInvalidValues()
test_case.full_signature: @Test public void testAnalysisWithInvalidValues()
test_case.identifier: testAnalysisWithInvalidValues
test_case.invocations: toString randomUUID build id metadata save storeAsRaw getResourceAsStream analyze analyze analyze analyze get assertThat qualityAnalyzed getLifecycle is assertThat getNbRecords getContent is assertThat size getColumns getRowMetadata is get getColumns getRowMetadata getQuality assertThat getValid is assertThat getInvalid is assertThat getEmpty is
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testAnalysisWithInvalidValues()
focal_class.declarator: LOGGER = LoggerFactory.getLogger(QualityAnalysis.class)
focal_class.modifier: private static final
focal_class.original_string: private static final Logger LOGGER = LoggerFactory.getLogger(QualityAnalysis.class);
focal_class.type: Logger
focal_class.var_name: LOGGER
focal_class.declarator: repository
focal_class.modifier: @Autowired
focal_class.original_string: @Autowired
      DataSetMetadataRepository repository;
focal_class.type: DataSetMetadataRepository
focal_class.var_name: repository
focal_class.declarator: store
focal_class.modifier: @Autowired
focal_class.original_string: @Autowired
      ContentStoreRouter store;
focal_class.type: ContentStoreRouter
focal_class.var_name: store
focal_class.declarator: adapter
focal_class.modifier: @Autowired
focal_class.original_string: @Autowired
      StatisticsAdapter adapter;
focal_class.type: StatisticsAdapter
focal_class.var_name: adapter
focal_class.declarator: analyzerService
focal_class.modifier: @Autowired
focal_class.original_string: @Autowired
      AnalyzerService analyzerService;
focal_class.type: AnalyzerService
focal_class.var_name: analyzerService
focal_class.declarator: maxRecord = 2000
focal_class.modifier: @Value("#{'${max_records:2000}'}")
      private final
focal_class.original_string: @Value("#{'${max_records:2000}'}")
      private final int maxRecord = 2000;
focal_class.type: int
focal_class.var_name: maxRecord
focal_class.file: dataprep-dataset/src/main/java/org/talend/dataprep/dataset/service/analysis/synchronous/QualityAnalysis.java
focal_class.identifier: QualityAnalysis
focal_class.interfaces: implements SynchronousDataSetAnalyzer
focal_class.class_method_signature: QualityAnalysis.analyze(String dataSetId)
focal_class.full_signature: @Override public void analyze(String dataSetId)
focal_class.identifier: analyze
focal_class.modifiers: @Override public
focal_class.parameters: (String dataSetId)
focal_class.return: void
focal_class.signature: void analyze(String dataSetId)
focal_class.class_method_signature: QualityAnalysis.order()
focal_class.full_signature: @Override public int order()
focal_class.identifier: order
focal_class.modifiers: @Override public
focal_class.parameters: ()
focal_class.return: int
focal_class.signature: int order()
focal_class.class_method_signature: QualityAnalysis.computeQuality(DataSetMetadata dataset, Stream<DataSetRow> records, long limit)
focal_class.full_signature: public void computeQuality(DataSetMetadata dataset, Stream<DataSetRow> records, long limit)
focal_class.identifier: computeQuality
focal_class.modifiers: public
focal_class.parameters: (DataSetMetadata dataset, Stream<DataSetRow> records, long limit)
focal_class.return: void
focal_class.signature: void computeQuality(DataSetMetadata dataset, Stream<DataSetRow> records, long limit)
focal_method.body: @Override
      public void analyze(String dataSetId) {
          if (StringUtils.isEmpty(dataSetId)) {
              throw new IllegalArgumentException("Data set id cannot be null or empty.");
          }
          DistributedLock datasetLock = repository.createDatasetMetadataLock(dataSetId);
          datasetLock.lock();
          try {
              DataSetMetadata metadata = repository.get(dataSetId);
              if (metadata == null) {
                  LOGGER.info("Unable to analyze quality of data set #{}: seems to be removed.", dataSetId);
                  return;
              }
              // e.g. excel multi sheet dataset when user has not choose the sheet yet
              if (!metadata.getLifecycle().isInProgress()) {
                  LOGGER.debug("No need to recompute quality of data set #{} (statistics are completed).", dataSetId);
                  return;
              }
              if (!metadata.getLifecycle().schemaAnalyzed()) {
                  LOGGER.debug(
                          "Schema information must be computed before quality analysis can be performed, ignoring message");
                  return; // no acknowledge to allow re-poll.
              }
  
              try (Stream<DataSetRow> stream = store.stream(metadata)) {
  
                  LOGGER.debug("Analyzing quality of dataset #{}...", metadata.getId());
                  // New data set, or reached the max limit of records for synchronous analysis, trigger a full scan (but
                  // async).
                  final long dataSetSize = metadata.getContent().getNbRecords();
                  final boolean isNewDataSet = dataSetSize == 0;
                  if (isNewDataSet || dataSetSize == maxRecord) {
                      // If data set size is maxRecord, performs a full scan, otherwise only take first maxRecord
                      // records.
                      computeQuality(metadata, stream, dataSetSize == maxRecord ? -1 : maxRecord);
                  }
                  // Turn on / off "in progress" flag
                  if (isNewDataSet && metadata.getContent().getNbRecords() >= maxRecord) {
                      metadata.getLifecycle().setInProgress(true);
                  } else {
                      metadata.getLifecycle().setInProgress(false);
                  }
                  // ... all quality is now analyzed, mark it so.
                  metadata.getLifecycle().qualityAnalyzed(true);
                  final DataSetMetadata savedDataSetMetadata = repository.get(metadata.getId());
                  // in order to check that the dataset was not deleted during analysis
                  if (savedDataSetMetadata != null) {
                      repository.save(metadata);
                      LOGGER.debug("Analyzed quality of dataset #{}.", dataSetId);
                  }
  
              } catch (Exception e) {
                  LOGGER.warn("dataset '{}' generate an error, message: {} ", dataSetId, e.getMessage());
                  throw new TDPException(DataSetErrorCodes.UNABLE_TO_ANALYZE_DATASET_QUALITY, e);
              }
          } finally {
              datasetLock.unlock();
          }
      }
focal_method.class_method_signature: QualityAnalysis.analyze(String dataSetId)
focal_method.full_signature: @Override public void analyze(String dataSetId)
focal_method.identifier: analyze
focal_method.invocations: isEmpty createDatasetMetadataLock lock get info isInProgress getLifecycle debug schemaAnalyzed getLifecycle debug stream debug getId getNbRecords getContent computeQuality getNbRecords getContent setInProgress getLifecycle setInProgress getLifecycle qualityAnalyzed getLifecycle get getId save debug warn getMessage unlock
focal_method.modifiers: @Override public
focal_method.parameters: (String dataSetId)
focal_method.return: void
focal_method.signature: void analyze(String dataSetId)