test_class.file: core/src/test/java/org/carrot2/clustering/stc/STCClusteringAlgorithmTest.java
test_class.identifier: STCClusteringAlgorithmTest
test_class.superclass: extends ClusteringAlgorithmTestBase<STCClusteringAlgorithm>
test_case.body: @Test
    public void testMergingBaseClustersWithStemEquivalentPhrases() {
      List<String> titles =
          Arrays.asList(
              "good programs . foo1",
              "foo2 good programs . foo2",
              "good programs taste good",
              "good programs are good",
              "good programming . foo3",
              "foo4 good programming . foo4",
              "good programming makes you feel better");
  
      STCClusteringAlgorithm algorithm = algorithm();
      algorithm.minBaseClusterScore.set(0d);
  
      Stream<TestDocument> documentStream = titles.stream().map(title -> new TestDocument(title));
      List<Cluster<Document>> clusters =
          algorithm.cluster(documentStream, CachedLangComponents.loadCached("English"));
  
      Assertions.assertThat(clusters.stream().flatMap(cluster -> cluster.getLabels().stream()))
          .contains("Good Programs")
          .doesNotContain("Good Programming");
    }
test_case.class_method_signature: STCClusteringAlgorithmTest.testMergingBaseClustersWithStemEquivalentPhrases()
test_case.full_signature: @Test public void testMergingBaseClustersWithStemEquivalentPhrases()
test_case.identifier: testMergingBaseClustersWithStemEquivalentPhrases
test_case.invocations: asList algorithm set map stream cluster loadCached doesNotContain contains assertThat flatMap stream stream getLabels
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testMergingBaseClustersWithStemEquivalentPhrases()
focal_class.declarator: REQUIRED_LANGUAGE_COMPONENTS =
        new HashSet<>(
            Arrays.asList(Stemmer.class, Tokenizer.class, LexicalData.class, LabelFormatter.class))
focal_class.modifier: private static final
focal_class.original_string: private static final Set<Class<?>> REQUIRED_LANGUAGE_COMPONENTS =
        new HashSet<>(
            Arrays.asList(Stemmer.class, Tokenizer.class, LexicalData.class, LabelFormatter.class));
focal_class.type: Set<Class<?>>
focal_class.var_name: REQUIRED_LANGUAGE_COMPONENTS
focal_class.declarator: NAME = "STC"
focal_class.modifier: public static final
focal_class.original_string: public static final String NAME = "STC";
focal_class.type: String
focal_class.var_name: NAME
focal_class.declarator: queryHint =
        attributes.register("queryHint", SharedInfrastructure.queryHintAttribute())
focal_class.modifier: public final
focal_class.original_string: public final AttrString queryHint =
        attributes.register("queryHint", SharedInfrastructure.queryHintAttribute());
focal_class.type: AttrString
focal_class.var_name: queryHint
focal_class.declarator: ignoreWordIfInHigherDocsPercent =
        attributes.register(
            "ignoreWordIfInHigherDocsPercent",
            AttrDouble.builder()
                .label("Ignore words appearing in more than a given fraction of documents")
                .min(0)
                .max(1)
                .defaultValue(0.9))
focal_class.modifier: public
focal_class.original_string: public AttrDouble ignoreWordIfInHigherDocsPercent =
        attributes.register(
            "ignoreWordIfInHigherDocsPercent",
            AttrDouble.builder()
                .label("Ignore words appearing in more than a given fraction of documents")
                .min(0)
                .max(1)
                .defaultValue(0.9));
focal_class.type: AttrDouble
focal_class.var_name: ignoreWordIfInHigherDocsPercent
focal_class.declarator: minBaseClusterScore =
        attributes.register(
            "minBaseClusterScore",
            AttrDouble.builder().label("Minimum base cluster score").min(0).max(10).defaultValue(2.))
focal_class.modifier: public
focal_class.original_string: public AttrDouble minBaseClusterScore =
        attributes.register(
            "minBaseClusterScore",
            AttrDouble.builder().label("Minimum base cluster score").min(0).max(10).defaultValue(2.));
focal_class.type: AttrDouble
focal_class.var_name: minBaseClusterScore
focal_class.declarator: minBaseClusterSize =
        attributes.register(
            "minBaseClusterSize",
            AttrInteger.builder()
                .label("Minimum number of documents in a base cluster")
                .min(2)
                .max(20)
                .defaultValue(2))
focal_class.modifier: public
focal_class.original_string: public AttrInteger minBaseClusterSize =
        attributes.register(
            "minBaseClusterSize",
            AttrInteger.builder()
                .label("Minimum number of documents in a base cluster")
                .min(2)
                .max(20)
                .defaultValue(2));
focal_class.type: AttrInteger
focal_class.var_name: minBaseClusterSize
focal_class.declarator: maxBaseClusters =
        attributes.register(
            "maxBaseClusters",
            AttrInteger.builder().label("Maximum number of base cluster").min(2).defaultValue(300))
focal_class.modifier: public
focal_class.original_string: public AttrInteger maxBaseClusters =
        attributes.register(
            "maxBaseClusters",
            AttrInteger.builder().label("Maximum number of base cluster").min(2).defaultValue(300));
focal_class.type: AttrInteger
focal_class.var_name: maxBaseClusters
focal_class.declarator: maxClusters =
        attributes.register(
            "maxClusters",
            AttrInteger.builder().label("Maximum number of final clusters").min(1).defaultValue(15))
focal_class.modifier: public
focal_class.original_string: public AttrInteger maxClusters =
        attributes.register(
            "maxClusters",
            AttrInteger.builder().label("Maximum number of final clusters").min(1).defaultValue(15));
focal_class.type: AttrInteger
focal_class.var_name: maxClusters
focal_class.declarator: mergeThreshold =
        attributes.register(
            "mergeThreshold",
            AttrDouble.builder()
                .label("Base cluster merge threshold")
                .min(0)
                .max(1)
                .defaultValue(0.6))
focal_class.modifier: public
focal_class.original_string: public AttrDouble mergeThreshold =
        attributes.register(
            "mergeThreshold",
            AttrDouble.builder()
                .label("Base cluster merge threshold")
                .min(0)
                .max(1)
                .defaultValue(0.6));
focal_class.type: AttrDouble
focal_class.var_name: mergeThreshold
focal_class.declarator: maxPhraseOverlap =
        attributes.register(
            "maxPhraseOverlap",
            AttrDouble.builder()
                .label("Maximum cluster phrase overlap")
                .min(0)
                .max(1)
                .defaultValue(0.6))
focal_class.modifier: public
focal_class.original_string: public AttrDouble maxPhraseOverlap =
        attributes.register(
            "maxPhraseOverlap",
            AttrDouble.builder()
                .label("Maximum cluster phrase overlap")
                .min(0)
                .max(1)
                .defaultValue(0.6));
focal_class.type: AttrDouble
focal_class.var_name: maxPhraseOverlap
focal_class.declarator: mostGeneralPhraseCoverage =
        attributes.register(
            "mostGeneralPhraseCoverage",
            AttrDouble.builder()
                .label("Minimum general phrase coverage")
                .min(0)
                .max(1)
                .defaultValue(0.5))
focal_class.modifier: public
focal_class.original_string: public AttrDouble mostGeneralPhraseCoverage =
        attributes.register(
            "mostGeneralPhraseCoverage",
            AttrDouble.builder()
                .label("Minimum general phrase coverage")
                .min(0)
                .max(1)
                .defaultValue(0.5));
focal_class.type: AttrDouble
focal_class.var_name: mostGeneralPhraseCoverage
focal_class.declarator: maxWordsPerLabel =
        attributes.register(
            "maxWordsPerLabel",
            AttrInteger.builder().label("Maximum words per label").min(1).defaultValue(4))
focal_class.modifier: public
focal_class.original_string: public AttrInteger maxWordsPerLabel =
        attributes.register(
            "maxWordsPerLabel",
            AttrInteger.builder().label("Maximum words per label").min(1).defaultValue(4));
focal_class.type: AttrInteger
focal_class.var_name: maxWordsPerLabel
focal_class.declarator: maxPhrasesPerLabel =
        attributes.register(
            "maxPhrasesPerLabel",
            AttrInteger.builder().label("Maximum phrases per label").min(1).defaultValue(3))
focal_class.modifier: public
focal_class.original_string: public AttrInteger maxPhrasesPerLabel =
        attributes.register(
            "maxPhrasesPerLabel",
            AttrInteger.builder().label("Maximum phrases per label").min(1).defaultValue(3));
focal_class.type: AttrInteger
focal_class.var_name: maxPhrasesPerLabel
focal_class.declarator: singleTermBoost =
        attributes.register(
            "singleTermBoost",
            AttrDouble.builder().label("Boost single-term clusters").min(0).defaultValue(0.5))
focal_class.modifier: public
focal_class.original_string: public AttrDouble singleTermBoost =
        attributes.register(
            "singleTermBoost",
            AttrDouble.builder().label("Boost single-term clusters").min(0).defaultValue(0.5));
focal_class.type: AttrDouble
focal_class.var_name: singleTermBoost
focal_class.declarator: optimalPhraseLength =
        attributes.register(
            "optimalPhraseLength",
            AttrInteger.builder().label("Optimal cluster label length").min(1).defaultValue(3))
focal_class.modifier: public
focal_class.original_string: public AttrInteger optimalPhraseLength =
        attributes.register(
            "optimalPhraseLength",
            AttrInteger.builder().label("Optimal cluster label length").min(1).defaultValue(3));
focal_class.type: AttrInteger
focal_class.var_name: optimalPhraseLength
focal_class.declarator: optimalPhraseLengthDev =
        attributes.register(
            "optimalPhraseLengthDev",
            AttrDouble.builder()
                .label("Optimal cluster label length's tolerance")
                .min(0.5)
                .defaultValue(2.))
focal_class.modifier: public
focal_class.original_string: public AttrDouble optimalPhraseLengthDev =
        attributes.register(
            "optimalPhraseLengthDev",
            AttrDouble.builder()
                .label("Optimal cluster label length's tolerance")
                .min(0.5)
                .defaultValue(2.));
focal_class.type: AttrDouble
focal_class.var_name: optimalPhraseLengthDev
focal_class.declarator: documentCountBoost =
        attributes.register(
            "documentCountBoost",
            AttrDouble.builder().label("Base cluster document count boost").min(0).defaultValue(1.))
focal_class.modifier: public
focal_class.original_string: public AttrDouble documentCountBoost =
        attributes.register(
            "documentCountBoost",
            AttrDouble.builder().label("Base cluster document count boost").min(0).defaultValue(1.));
focal_class.type: AttrDouble
focal_class.var_name: documentCountBoost
focal_class.declarator: scoreWeight =
        attributes.register(
            "scoreWeight",
            AttrDouble.builder().label("Size-score sorting ratio").min(0).max(1).defaultValue(0.))
focal_class.modifier: public
focal_class.original_string: public AttrDouble scoreWeight =
        attributes.register(
            "scoreWeight",
            AttrDouble.builder().label("Size-score sorting ratio").min(0).max(1).defaultValue(0.));
focal_class.type: AttrDouble
focal_class.var_name: scoreWeight
focal_class.declarator: mergeStemEquivalentBaseClusters =
        attributes.register(
            "mergeStemEquivalentBaseClusters",
            AttrBoolean.builder()
                .label("Merge all stem-equivalent phrases when discovering base clusters")
                .defaultValue(true))
focal_class.modifier: public
focal_class.original_string: public AttrBoolean mergeStemEquivalentBaseClusters =
        attributes.register(
            "mergeStemEquivalentBaseClusters",
            AttrBoolean.builder()
                .label("Merge all stem-equivalent phrases when discovering base clusters")
                .defaultValue(true));
focal_class.type: AttrBoolean
focal_class.var_name: mergeStemEquivalentBaseClusters
focal_class.declarator: preprocessing
focal_class.modifier: public
focal_class.original_string: public BasicPreprocessingPipeline preprocessing;
focal_class.type: BasicPreprocessingPipeline
focal_class.var_name: preprocessing
focal_class.declarator: labelFormatter
focal_class.modifier: private
focal_class.original_string: private LabelFormatter labelFormatter;
focal_class.type: LabelFormatter
focal_class.var_name: labelFormatter
focal_class.declarator: NOT_SELECTED = (p) -> !p.selected
focal_class.modifier: private static final
focal_class.original_string: private static final Predicate<PhraseCandidate> NOT_SELECTED = (p) -> !p.selected;
focal_class.type: Predicate<PhraseCandidate>
focal_class.var_name: NOT_SELECTED
focal_class.declarator: sb
focal_class.modifier: private
focal_class.original_string: private GeneralizedSuffixTree.SequenceBuilder sb;
focal_class.type: GeneralizedSuffixTree.SequenceBuilder
focal_class.var_name: sb
focal_class.declarator: context
focal_class.modifier: private
focal_class.original_string: private PreprocessingContext context;
focal_class.type: PreprocessingContext
focal_class.var_name: context
focal_class.file: core/src/main/java/org/carrot2/clustering/stc/STCClusteringAlgorithm.java
focal_class.identifier: STCClusteringAlgorithm
focal_class.interfaces: implements ClusteringAlgorithm
focal_class.class_method_signature: STCClusteringAlgorithm.requiredLanguageComponents()
focal_class.full_signature: @Override public Set<Class<?>> requiredLanguageComponents()
focal_class.identifier: requiredLanguageComponents
focal_class.modifiers: @Override public
focal_class.parameters: ()
focal_class.return: Set<Class<?>>
focal_class.signature: Set<Class<?>> requiredLanguageComponents()
focal_class.class_method_signature: STCClusteringAlgorithm.cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_class.full_signature: @Override public List<Cluster<T>> cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_class.identifier: cluster
focal_class.modifiers: @Override public
focal_class.parameters: (
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_class.return: List<Cluster<T>>
focal_class.signature: List<Cluster<T>> cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_class.class_method_signature: STCClusteringAlgorithm.createBaseClusters(SequenceBuilder sb)
focal_class.full_signature: private List<ClusterCandidate> createBaseClusters(SequenceBuilder sb)
focal_class.identifier: createBaseClusters
focal_class.modifiers: private
focal_class.parameters: (SequenceBuilder sb)
focal_class.return: List<ClusterCandidate>
focal_class.signature: List<ClusterCandidate> createBaseClusters(SequenceBuilder sb)
focal_class.class_method_signature: STCClusteringAlgorithm.mergeStemEquivalentBaseClusters(
        SequenceBuilder sb, final List<ClusterCandidate> candidates)
focal_class.full_signature: private void mergeStemEquivalentBaseClusters(
        SequenceBuilder sb, final List<ClusterCandidate> candidates)
focal_class.identifier: mergeStemEquivalentBaseClusters
focal_class.modifiers: private
focal_class.parameters: (
        SequenceBuilder sb, final List<ClusterCandidate> candidates)
focal_class.return: void
focal_class.signature: void mergeStemEquivalentBaseClusters(
        SequenceBuilder sb, final List<ClusterCandidate> candidates)
focal_class.class_method_signature: STCClusteringAlgorithm.createMergedClusters(List<ClusterCandidate> baseClusters)
focal_class.full_signature: private ArrayList<ClusterCandidate> createMergedClusters(List<ClusterCandidate> baseClusters)
focal_class.identifier: createMergedClusters
focal_class.modifiers: private
focal_class.parameters: (List<ClusterCandidate> baseClusters)
focal_class.return: ArrayList<ClusterCandidate>
focal_class.signature: ArrayList<ClusterCandidate> createMergedClusters(List<ClusterCandidate> baseClusters)
focal_class.class_method_signature: STCClusteringAlgorithm.merge(
        PreprocessingContext context, IntStack mergeList, List<ClusterCandidate> baseClusters)
focal_class.full_signature: private ClusterCandidate merge(
        PreprocessingContext context, IntStack mergeList, List<ClusterCandidate> baseClusters)
focal_class.identifier: merge
focal_class.modifiers: private
focal_class.parameters: (
        PreprocessingContext context, IntStack mergeList, List<ClusterCandidate> baseClusters)
focal_class.return: ClusterCandidate
focal_class.signature: ClusterCandidate merge(
        PreprocessingContext context, IntStack mergeList, List<ClusterCandidate> baseClusters)
focal_class.class_method_signature: STCClusteringAlgorithm.markSubSuperPhrases(ArrayList<PhraseCandidate> phrases)
focal_class.full_signature: private void markSubSuperPhrases(ArrayList<PhraseCandidate> phrases)
focal_class.identifier: markSubSuperPhrases
focal_class.modifiers: private
focal_class.parameters: (ArrayList<PhraseCandidate> phrases)
focal_class.return: void
focal_class.signature: void markSubSuperPhrases(ArrayList<PhraseCandidate> phrases)
focal_class.class_method_signature: STCClusteringAlgorithm.markOverlappingPhrases(
        PreprocessingContext context, ArrayList<PhraseCandidate> phrases)
focal_class.full_signature: private void markOverlappingPhrases(
        PreprocessingContext context, ArrayList<PhraseCandidate> phrases)
focal_class.identifier: markOverlappingPhrases
focal_class.modifiers: private
focal_class.parameters: (
        PreprocessingContext context, ArrayList<PhraseCandidate> phrases)
focal_class.return: void
focal_class.signature: void markOverlappingPhrases(
        PreprocessingContext context, ArrayList<PhraseCandidate> phrases)
focal_class.class_method_signature: STCClusteringAlgorithm.computeIntersection(int[] a, int aPos, int aLength, int[] b, int bPos, int bLength)
focal_class.full_signature: static int computeIntersection(int[] a, int aPos, int aLength, int[] b, int bPos, int bLength)
focal_class.identifier: computeIntersection
focal_class.modifiers: static
focal_class.parameters: (int[] a, int aPos, int aLength, int[] b, int bPos, int bLength)
focal_class.return: int
focal_class.signature: int computeIntersection(int[] a, int aPos, int aLength, int[] b, int bPos, int bLength)
focal_class.class_method_signature: STCClusteringAlgorithm.appendUniqueWords(
        PreprocessingContext context, IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.full_signature: private void appendUniqueWords(
        PreprocessingContext context, IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.identifier: appendUniqueWords
focal_class.modifiers: private
focal_class.parameters: (
        PreprocessingContext context, IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.return: void
focal_class.signature: void appendUniqueWords(
        PreprocessingContext context, IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.class_method_signature: STCClusteringAlgorithm.appendWords(IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.full_signature: private void appendWords(IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.identifier: appendWords
focal_class.modifiers: private
focal_class.parameters: (IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.return: void
focal_class.signature: void appendWords(IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.class_method_signature: STCClusteringAlgorithm.postProcessing(
        List<T> documents, List<ClusterCandidate> candidates, List<Cluster<T>> clusters)
focal_class.full_signature: private void postProcessing(
        List<T> documents, List<ClusterCandidate> candidates, List<Cluster<T>> clusters)
focal_class.identifier: postProcessing
focal_class.modifiers: private
focal_class.parameters: (
        List<T> documents, List<ClusterCandidate> candidates, List<Cluster<T>> clusters)
focal_class.return: void
focal_class.signature: void postProcessing(
        List<T> documents, List<ClusterCandidate> candidates, List<Cluster<T>> clusters)
focal_class.class_method_signature: STCClusteringAlgorithm.collectPhrases(ClusterCandidate c, Cluster<?> cluster)
focal_class.full_signature: private void collectPhrases(ClusterCandidate c, Cluster<?> cluster)
focal_class.identifier: collectPhrases
focal_class.modifiers: private
focal_class.parameters: (ClusterCandidate c, Cluster<?> cluster)
focal_class.return: void
focal_class.signature: void collectPhrases(ClusterCandidate c, Cluster<?> cluster)
focal_class.class_method_signature: STCClusteringAlgorithm.collectDocuments(
        List<T> documents, List<T> l, BitSet bitset)
focal_class.full_signature: private List<T> collectDocuments(
        List<T> documents, List<T> l, BitSet bitset)
focal_class.identifier: collectDocuments
focal_class.modifiers: private
focal_class.parameters: (
        List<T> documents, List<T> l, BitSet bitset)
focal_class.return: List<T>
focal_class.signature: List<T> collectDocuments(
        List<T> documents, List<T> l, BitSet bitset)
focal_class.class_method_signature: STCClusteringAlgorithm.buildLabel(int[] phraseIndices)
focal_class.full_signature: private String buildLabel(int[] phraseIndices)
focal_class.identifier: buildLabel
focal_class.modifiers: private
focal_class.parameters: (int[] phraseIndices)
focal_class.return: String
focal_class.signature: String buildLabel(int[] phraseIndices)
focal_class.class_method_signature: STCClusteringAlgorithm.toString(PhraseCandidate c)
focal_class.full_signature: @SuppressWarnings("unused") private String toString(PhraseCandidate c)
focal_class.identifier: toString
focal_class.modifiers: @SuppressWarnings("unused") private
focal_class.parameters: (PhraseCandidate c)
focal_class.return: String
focal_class.signature: String toString(PhraseCandidate c)
focal_class.class_method_signature: STCClusteringAlgorithm.buildDebugLabel(int[] phraseIndices)
focal_class.full_signature: @SuppressWarnings("unused") private String buildDebugLabel(int[] phraseIndices)
focal_class.identifier: buildDebugLabel
focal_class.modifiers: @SuppressWarnings("unused") private
focal_class.parameters: (int[] phraseIndices)
focal_class.return: String
focal_class.signature: String buildDebugLabel(int[] phraseIndices)
focal_class.class_method_signature: STCClusteringAlgorithm.checkAcceptablePhrase(IntStack path)
focal_class.full_signature: final boolean checkAcceptablePhrase(IntStack path)
focal_class.identifier: checkAcceptablePhrase
focal_class.modifiers: final
focal_class.parameters: (IntStack path)
focal_class.return: boolean
focal_class.signature: boolean checkAcceptablePhrase(IntStack path)
focal_class.class_method_signature: STCClusteringAlgorithm.effectivePhraseLength(IntStack path)
focal_class.full_signature: final int effectivePhraseLength(IntStack path)
focal_class.identifier: effectivePhraseLength
focal_class.modifiers: final
focal_class.parameters: (IntStack path)
focal_class.return: int
focal_class.signature: int effectivePhraseLength(IntStack path)
focal_class.class_method_signature: STCClusteringAlgorithm.baseClusterScore(final int phraseLength, final int documentCount)
focal_class.full_signature: final float baseClusterScore(final int phraseLength, final int documentCount)
focal_class.identifier: baseClusterScore
focal_class.modifiers: final
focal_class.parameters: (final int phraseLength, final int documentCount)
focal_class.return: float
focal_class.signature: float baseClusterScore(final int phraseLength, final int documentCount)
focal_class.class_method_signature: STCClusteringAlgorithm.indexOf(
        int[] source,
        int sourceOffset,
        int sourceCount,
        int[] target,
        int targetOffset,
        int targetCount)
focal_class.full_signature: private static int indexOf(
        int[] source,
        int sourceOffset,
        int sourceCount,
        int[] target,
        int targetOffset,
        int targetCount)
focal_class.identifier: indexOf
focal_class.modifiers: private static
focal_class.parameters: (
        int[] source,
        int sourceOffset,
        int sourceCount,
        int[] target,
        int targetOffset,
        int targetCount)
focal_class.return: int
focal_class.signature: int indexOf(
        int[] source,
        int sourceOffset,
        int sourceCount,
        int[] target,
        int targetOffset,
        int targetCount)
focal_class.superclass: extends AttrComposite
focal_method.body: @Override
    public <T extends Document> List<Cluster<T>> cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents) {
      List<T> documents = docStream.collect(Collectors.toList());
      List<Cluster<T>> clusters = new ArrayList<>();
  
      /*
       * Step 1. Preprocessing: tokenization, stop word marking and stemming (if available).
       */
      context = preprocessing.preprocess(documents.stream(), queryHint.get(), languageComponents);
      labelFormatter = context.languageComponents.get(LabelFormatter.class);
  
      /*
       * Step 2: Create a generalized suffix tree from phrases in the input.
       */
      sb = new GeneralizedSuffixTree.SequenceBuilder();
  
      final int[] tokenIndex = context.allTokens.wordIndex;
      final short[] tokenType = context.allTokens.type;
      for (int i = 0; i < tokenIndex.length; i++) {
        /* Advance until the first real token. */
        if (tokenIndex[i] == -1) {
          if ((tokenType[i] & (Tokenizer.TF_SEPARATOR_DOCUMENT | Tokenizer.TF_TERMINATOR)) != 0) {
            sb.endDocument();
          }
          continue;
        }
  
        /* We have the first token. Advance until non-token. */
        final int s = i;
  
        while (tokenIndex[i + 1] != -1) i++;
        final int phraseLength = 1 + i - s;
        if (phraseLength >= 1) {
          /* We have a phrase. */
          sb.addPhrase(tokenIndex, s, phraseLength);
        }
      }
      sb.buildSuffixTree();
  
      /*
       * Step 3: Find "base" clusters by looking up frequently recurring phrases in the
       * generalized suffix tree.
       */
      List<ClusterCandidate> baseClusters = createBaseClusters(sb);
  
      /*
       * Step 4: Merge base clusters that overlap too much to form final clusters.
       */
      List<ClusterCandidate> mergedClusters = createMergedClusters(baseClusters);
  
      /*
       * Step 5: Create the junk (unassigned documents) cluster and create the final
       * set of clusters in Carrot2 format.
       */
      postProcessing(documents, mergedClusters, clusters);
  
      return SharedInfrastructure.reorderByWeightedScoreAndSize(clusters, this.scoreWeight.get());
    }
focal_method.class_method_signature: STCClusteringAlgorithm.cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_method.full_signature: @Override public List<Cluster<T>> cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_method.identifier: cluster
focal_method.invocations: collect toList preprocess stream get get endDocument addPhrase buildSuffixTree createBaseClusters createMergedClusters postProcessing reorderByWeightedScoreAndSize get
focal_method.modifiers: @Override public
focal_method.parameters: (
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_method.return: List<Cluster<T>>
focal_method.signature: List<Cluster<T>> cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)