test_class.declarator: formatAnalysis
test_class.modifier: @Autowired
test_class.original_string: @Autowired
      FormatAnalysis formatAnalysis;
test_class.type: FormatAnalysis
test_class.var_name: formatAnalysis
test_class.file: dataprep-dataset/src/test/java/org/talend/dataprep/dataset/service/analysis/synchronous/FormatAnalysisTest.java
test_class.identifier: FormatAnalysisTest
test_class.superclass: extends DataSetBaseTest
test_case.body: @Test
      public void testEncodingDetection_UTF16LE_WithoutBOM() {
          String id = UUID.randomUUID().toString();
          final DataSetMetadata metadata = metadataBuilder.metadata().id(id).build();
          dataSetMetadataRepository.save(metadata);
          contentStore.storeAsRaw(metadata, DataSetServiceTest.class.getResourceAsStream("../utf16_LE_without_bom.txt"));
  
          formatAnalysis.analyze(id);
          final DataSetMetadata actual = dataSetMetadataRepository.get(id);
          assertThat(actual, notNullValue());
          assertThat(actual.getContent().getFormatFamilyId(), is(CSVFormatFamily.BEAN_ID));
          assertThat(actual.getContent().getMediaType(), is("text/csv"));
          assertThat(actual.getContent().getParameters().get("SEPARATOR"), is(","));
          assertThat(actual.getEncoding(), is("UTF-16LE"));
      }
test_case.class_method_signature: FormatAnalysisTest.testEncodingDetection_UTF16LE_WithoutBOM()
test_case.full_signature: @Test public void testEncodingDetection_UTF16LE_WithoutBOM()
test_case.identifier: testEncodingDetection_UTF16LE_WithoutBOM
test_case.invocations: toString randomUUID build id metadata save storeAsRaw getResourceAsStream analyze get assertThat notNullValue assertThat getFormatFamilyId getContent is assertThat getMediaType getContent is assertThat get getParameters getContent is assertThat getEncoding is
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testEncodingDetection_UTF16LE_WithoutBOM()
focal_class.declarator: NO_BOM = {}
focal_class.modifier: private static final
focal_class.original_string: private static final byte[] NO_BOM = {};
focal_class.type: byte[]
focal_class.var_name: NO_BOM
focal_class.declarator: UTF_16_LE_BOM = { (byte) 0xFF, (byte) 0xFE }
focal_class.modifier: private static final
focal_class.original_string: private static final byte[] UTF_16_LE_BOM = { (byte) 0xFF, (byte) 0xFE };
focal_class.type: byte[]
focal_class.var_name: UTF_16_LE_BOM
focal_class.declarator: UTF_16_BE_BOM = { (byte) 0xFE, (byte) 0xFF }
focal_class.modifier: private static final
focal_class.original_string: private static final byte[] UTF_16_BE_BOM = { (byte) 0xFE, (byte) 0xFF };
focal_class.type: byte[]
focal_class.var_name: UTF_16_BE_BOM
focal_class.declarator: UTF_32_LE_BOM = { (byte) 0xFE, (byte) 0xFF, (byte) 0x00, (byte) 0x00 }
focal_class.modifier: private static final
focal_class.original_string: private static final byte[] UTF_32_LE_BOM = { (byte) 0xFE, (byte) 0xFF, (byte) 0x00, (byte) 0x00 };
focal_class.type: byte[]
focal_class.var_name: UTF_32_LE_BOM
focal_class.declarator: UTF_32_BE_BOM = { (byte) 0x00, (byte) 0x00, (byte) 0xFE, (byte) 0xFF }
focal_class.modifier: private static final
focal_class.original_string: private static final byte[] UTF_32_BE_BOM = { (byte) 0x00, (byte) 0x00, (byte) 0xFE, (byte) 0xFF };
focal_class.type: byte[]
focal_class.var_name: UTF_32_BE_BOM
focal_class.declarator: UTF_8_BOM = { (byte) 0xEF, (byte) 0xBB, (byte) 0xBF }
focal_class.modifier: private static final
focal_class.original_string: private static final byte[] UTF_8_BOM = { (byte) 0xEF, (byte) 0xBB, (byte) 0xBF };
focal_class.type: byte[]
focal_class.var_name: UTF_8_BOM
focal_class.declarator: BOMS =
              { NO_BOM, UTF_16_LE_BOM, UTF_16_BE_BOM, UTF_32_LE_BOM, UTF_32_BE_BOM, UTF_8_BOM }
focal_class.modifier: private static final
focal_class.original_string: private static final byte[][] BOMS =
              { NO_BOM, UTF_16_LE_BOM, UTF_16_BE_BOM, UTF_32_LE_BOM, UTF_32_BE_BOM, UTF_8_BOM };
focal_class.type: byte[][]
focal_class.var_name: BOMS
focal_class.declarator: LOG = LoggerFactory.getLogger(FormatAnalysis.class)
focal_class.modifier: private static final
focal_class.original_string: private static final Logger LOG = LoggerFactory.getLogger(FormatAnalysis.class);
focal_class.type: Logger
focal_class.var_name: LOG
focal_class.declarator: repository
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private DataSetMetadataRepository repository;
focal_class.type: DataSetMetadataRepository
focal_class.var_name: repository
focal_class.declarator: store
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private ContentStoreRouter store;
focal_class.type: ContentStoreRouter
focal_class.var_name: store
focal_class.declarator: formatFamilyFactory
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private FormatFamilyFactory formatFamilyFactory;
focal_class.type: FormatFamilyFactory
focal_class.var_name: formatFamilyFactory
focal_class.declarator: detector
focal_class.modifier: @Autowired
      private
focal_class.original_string: @Autowired
      private CompositeFormatDetector detector;
focal_class.type: CompositeFormatDetector
focal_class.var_name: detector
focal_class.file: dataprep-dataset/src/main/java/org/talend/dataprep/dataset/service/analysis/synchronous/FormatAnalysis.java
focal_class.identifier: FormatAnalysis
focal_class.interfaces: implements SynchronousDataSetAnalyzer
focal_class.class_method_signature: FormatAnalysis.analyze(String dataSetId)
focal_class.full_signature: @Override public void analyze(String dataSetId)
focal_class.identifier: analyze
focal_class.modifiers: @Override public
focal_class.parameters: (String dataSetId)
focal_class.return: void
focal_class.signature: void analyze(String dataSetId)
focal_class.class_method_signature: FormatAnalysis.verifyFormat(Format detectedFormat)
focal_class.full_signature: private void verifyFormat(Format detectedFormat)
focal_class.identifier: verifyFormat
focal_class.modifiers: private
focal_class.parameters: (Format detectedFormat)
focal_class.return: void
focal_class.signature: void verifyFormat(Format detectedFormat)
focal_class.class_method_signature: FormatAnalysis.internalUpdateMetadata(DataSetMetadata updatedMetadata, Format format)
focal_class.full_signature: private void internalUpdateMetadata(DataSetMetadata updatedMetadata, Format format)
focal_class.identifier: internalUpdateMetadata
focal_class.modifiers: private
focal_class.parameters: (DataSetMetadata updatedMetadata, Format format)
focal_class.return: void
focal_class.signature: void internalUpdateMetadata(DataSetMetadata updatedMetadata, Format format)
focal_class.class_method_signature: FormatAnalysis.update(DataSetMetadata original, DataSetMetadata updated)
focal_class.full_signature: public void update(DataSetMetadata original, DataSetMetadata updated)
focal_class.identifier: update
focal_class.modifiers: public
focal_class.parameters: (DataSetMetadata original, DataSetMetadata updated)
focal_class.return: void
focal_class.signature: void update(DataSetMetadata original, DataSetMetadata updated)
focal_class.class_method_signature: FormatAnalysis.parseColumnNameInformation(String dataSetId, DataSetMetadata metadata, Format format)
focal_class.full_signature: private void parseColumnNameInformation(String dataSetId, DataSetMetadata metadata, Format format)
focal_class.identifier: parseColumnNameInformation
focal_class.modifiers: private
focal_class.parameters: (String dataSetId, DataSetMetadata metadata, Format format)
focal_class.return: void
focal_class.signature: void parseColumnNameInformation(String dataSetId, DataSetMetadata metadata, Format format)
focal_class.class_method_signature: FormatAnalysis.order()
focal_class.full_signature: @Override public int order()
focal_class.identifier: order
focal_class.modifiers: @Override public
focal_class.parameters: ()
focal_class.return: int
focal_class.signature: int order()
focal_class.class_method_signature: FormatAnalysis.addBOM(InputStream stream, byte[] bom)
focal_class.full_signature: private InputStream addBOM(InputStream stream, byte[] bom)
focal_class.identifier: addBOM
focal_class.modifiers: private
focal_class.parameters: (InputStream stream, byte[] bom)
focal_class.return: InputStream
focal_class.signature: InputStream addBOM(InputStream stream, byte[] bom)
focal_method.body: @Override
      public void analyze(String dataSetId) {
  
          if (StringUtils.isEmpty(dataSetId)) {
              throw new IllegalArgumentException("Data set id cannot be null or empty.");
          }
  
          final Marker marker = Markers.dataset(dataSetId);
  
          DistributedLock datasetLock = repository.createDatasetMetadataLock(dataSetId);
          datasetLock.lock();
          try {
              DataSetMetadata metadata = repository.get(dataSetId);
              if (metadata != null) {
  
                  Format detectedFormat = null;
                  for (byte[] bom : BOMS) {
                      try (InputStream content = store.getAsRaw(metadata, 10)) { // 10 line should be enough to detect format
  
                          detectedFormat = detector.detect(addBOM(content, bom));
                      } catch (IOException e) {
                          throw new TDPException(DataSetErrorCodes.UNABLE_TO_READ_DATASET_CONTENT, e);
                      }
                      if (detectedFormat != null
                              && !(detectedFormat.getFormatFamily() instanceof UnsupportedFormatFamily)) {
                          break;
                      }
                  }
  
                  LOG.debug(marker, "using {} to parse the dataset", detectedFormat);
  
                  verifyFormat(detectedFormat);
  
                  internalUpdateMetadata(metadata, detectedFormat);
  
                  LOG.debug(marker, "format analysed for dataset");
              } else {
                  LOG.info(marker, "Data set no longer exists.");
              }
          } finally {
              datasetLock.unlock();
          }
      }
focal_method.class_method_signature: FormatAnalysis.analyze(String dataSetId)
focal_method.full_signature: @Override public void analyze(String dataSetId)
focal_method.identifier: analyze
focal_method.invocations: isEmpty dataset createDatasetMetadataLock lock get getAsRaw detect addBOM getFormatFamily debug verifyFormat internalUpdateMetadata debug info unlock
focal_method.modifiers: @Override public
focal_method.parameters: (String dataSetId)
focal_method.return: void
focal_method.signature: void analyze(String dataSetId)