test_class.file: core/src/test/java/org/carrot2/clustering/lingo/LingoClusteringAlgorithmTest.java
test_class.identifier: LingoClusteringAlgorithmTest
test_class.superclass: extends ClusteringAlgorithmTestBase<LingoClusteringAlgorithm>
test_case.body: @Test
    public void testClusteringWithDfThreshold() {
      LingoClusteringAlgorithm algorithm = algorithm();
      algorithm.preprocessing.wordDfThreshold.set(100);
  
      List<Cluster<Document>> clusters =
          algorithm.cluster(
              SampleDocumentData.DOCUMENTS_DATA_MINING.stream(),
              CachedLangComponents.loadCached("English"));
  
      // Clustering with df threshold must not fail
      Assertions.assertThat(clusters).isEmpty();
    }
test_case.class_method_signature: LingoClusteringAlgorithmTest.testClusteringWithDfThreshold()
test_case.full_signature: @Test public void testClusteringWithDfThreshold()
test_case.identifier: testClusteringWithDfThreshold
test_case.invocations: algorithm set cluster stream loadCached isEmpty assertThat
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testClusteringWithDfThreshold()
focal_class.declarator: NAME = "Lingo"
focal_class.modifier: public static final
focal_class.original_string: public static final String NAME = "Lingo";
focal_class.type: String
focal_class.var_name: NAME
focal_class.declarator: REQUIRED_LANGUAGE_COMPONENTS =
        new HashSet<>(
            Arrays.asList(Stemmer.class, Tokenizer.class, LexicalData.class, LabelFormatter.class))
focal_class.modifier: private static final
focal_class.original_string: private static final Set<Class<?>> REQUIRED_LANGUAGE_COMPONENTS =
        new HashSet<>(
            Arrays.asList(Stemmer.class, Tokenizer.class, LexicalData.class, LabelFormatter.class));
focal_class.type: Set<Class<?>>
focal_class.var_name: REQUIRED_LANGUAGE_COMPONENTS
focal_class.declarator: scoreWeight =
        attributes.register(
            "scoreWeight",
            AttrDouble.builder().label("Size-score sorting ratio").min(0).max(1).defaultValue(0.))
focal_class.modifier: public
focal_class.original_string: public AttrDouble scoreWeight =
        attributes.register(
            "scoreWeight",
            AttrDouble.builder().label("Size-score sorting ratio").min(0).max(1).defaultValue(0.));
focal_class.type: AttrDouble
focal_class.var_name: scoreWeight
focal_class.declarator: desiredClusterCount =
        attributes.register(
            "desiredClusterCount",
            AttrInteger.builder().label("Desired cluster count").min(2).max(100).defaultValue(30))
focal_class.modifier: public
focal_class.original_string: public AttrInteger desiredClusterCount =
        attributes.register(
            "desiredClusterCount",
            AttrInteger.builder().label("Desired cluster count").min(2).max(100).defaultValue(30));
focal_class.type: AttrInteger
focal_class.var_name: desiredClusterCount
focal_class.declarator: preprocessing
focal_class.modifier: public
focal_class.original_string: public CompletePreprocessingPipeline preprocessing;
focal_class.type: CompletePreprocessingPipeline
focal_class.var_name: preprocessing
focal_class.declarator: matrixBuilder
focal_class.modifier: public
focal_class.original_string: public TermDocumentMatrixBuilder matrixBuilder;
focal_class.type: TermDocumentMatrixBuilder
focal_class.var_name: matrixBuilder
focal_class.declarator: matrixReducer
focal_class.modifier: public
focal_class.original_string: public TermDocumentMatrixReducer matrixReducer;
focal_class.type: TermDocumentMatrixReducer
focal_class.var_name: matrixReducer
focal_class.declarator: clusterBuilder
focal_class.modifier: public
focal_class.original_string: public ClusterBuilder clusterBuilder;
focal_class.type: ClusterBuilder
focal_class.var_name: clusterBuilder
focal_class.declarator: queryHint =
        attributes.register("queryHint", SharedInfrastructure.queryHintAttribute())
focal_class.modifier: public final
focal_class.original_string: public final AttrString queryHint =
        attributes.register("queryHint", SharedInfrastructure.queryHintAttribute());
focal_class.type: AttrString
focal_class.var_name: queryHint
focal_class.file: core/src/main/java/org/carrot2/clustering/lingo/LingoClusteringAlgorithm.java
focal_class.identifier: LingoClusteringAlgorithm
focal_class.interfaces: implements ClusteringAlgorithm
focal_class.class_method_signature: LingoClusteringAlgorithm.requiredLanguageComponents()
focal_class.full_signature: @Override public Set<Class<?>> requiredLanguageComponents()
focal_class.identifier: requiredLanguageComponents
focal_class.modifiers: @Override public
focal_class.parameters: ()
focal_class.return: Set<Class<?>>
focal_class.signature: Set<Class<?>> requiredLanguageComponents()
focal_class.class_method_signature: LingoClusteringAlgorithm.cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_class.full_signature: @Override public List<Cluster<T>> cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_class.identifier: cluster
focal_class.modifiers: @Override public
focal_class.parameters: (
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_class.return: List<Cluster<T>>
focal_class.signature: List<Cluster<T>> cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_class.class_method_signature: LingoClusteringAlgorithm.computeClusterCount(int desiredClusterCountBase, int documentCount)
focal_class.full_signature: static int computeClusterCount(int desiredClusterCountBase, int documentCount)
focal_class.identifier: computeClusterCount
focal_class.modifiers: static
focal_class.parameters: (int desiredClusterCountBase, int documentCount)
focal_class.return: int
focal_class.signature: int computeClusterCount(int desiredClusterCountBase, int documentCount)
focal_class.superclass: extends AttrComposite
focal_method.body: @Override
    public <T extends Document> List<Cluster<T>> cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents) {
      List<T> documents = docStream.collect(Collectors.toList());
  
      // Preprocessing of documents
      final PreprocessingContext context =
          preprocessing.preprocess(documents.stream(), queryHint.get(), languageComponents);
  
      // Further processing only if there are words to process
      List<Cluster<T>> clusters = new ArrayList<>();
      if (context.hasLabels()) {
        // Term-document matrix building and reduction
        final VectorSpaceModelContext vsmContext = new VectorSpaceModelContext(context);
        final ReducedVectorSpaceModelContext reducedVsmContext =
            new ReducedVectorSpaceModelContext(vsmContext);
        LingoProcessingContext lingoContext = new LingoProcessingContext(reducedVsmContext);
  
        TermDocumentMatrixBuilder matrixBuilder = this.matrixBuilder;
        matrixBuilder.buildTermDocumentMatrix(vsmContext);
        matrixBuilder.buildTermPhraseMatrix(vsmContext);
  
        matrixReducer.reduce(
            reducedVsmContext, computeClusterCount(desiredClusterCount.get(), documents.size()));
  
        // Cluster label building
        clusterBuilder.buildLabels(lingoContext, matrixBuilder.termWeighting);
  
        // Document assignment
        clusterBuilder.assignDocuments(lingoContext);
  
        // Cluster merging
        clusterBuilder.merge(lingoContext);
  
        // Format final clusters
        final LabelFormatter labelFormatter =
            lingoContext.preprocessingContext.languageComponents.get(LabelFormatter.class);
  
        final int[] clusterLabelIndex = lingoContext.clusterLabelFeatureIndex;
        final BitSet[] clusterDocuments = lingoContext.clusterDocuments;
        final double[] clusterLabelScore = lingoContext.clusterLabelScore;
        for (int i = 0; i < clusterLabelIndex.length; i++) {
          final Cluster<T> cluster = new Cluster<>();
  
          final int labelFeature = clusterLabelIndex[i];
          if (labelFeature < 0) {
            // Cluster removed during merging
            continue;
          }
  
          // Add label and score
          cluster.addLabel(context.format(labelFormatter, labelFeature));
          cluster.setScore(clusterLabelScore[i]);
  
          // Add documents
          final BitSet bs = clusterDocuments[i];
          for (int bit = bs.nextSetBit(0); bit >= 0; bit = bs.nextSetBit(bit + 1)) {
            cluster.addDocument(documents.get(bit));
          }
  
          // Add cluster
          clusters.add(cluster);
        }
      }
  
      clusters = SharedInfrastructure.reorderByWeightedScoreAndSize(clusters, this.scoreWeight.get());
      return clusters;
    }
focal_method.class_method_signature: LingoClusteringAlgorithm.cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_method.full_signature: @Override public List<Cluster<T>> cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_method.identifier: cluster
focal_method.invocations: collect toList preprocess stream get hasLabels buildTermDocumentMatrix buildTermPhraseMatrix reduce computeClusterCount get size buildLabels assignDocuments merge get addLabel format setScore nextSetBit nextSetBit addDocument get add reorderByWeightedScoreAndSize get
focal_method.modifiers: @Override public
focal_method.parameters: (
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_method.return: List<Cluster<T>>
focal_method.signature: List<Cluster<T>> cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)