test_class.file: core/src/test/java/org/carrot2/clustering/stc/STCClusteringAlgorithmTest.java
test_class.identifier: STCClusteringAlgorithmTest
test_class.superclass: extends ClusteringAlgorithmTestBase<STCClusteringAlgorithm>
test_case.body: @Test
    public void testCarrot1008() throws Exception {
      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
      DocumentBuilder db = dbf.newDocumentBuilder();
      Element root;
      try (InputStream is = getClass().getResourceAsStream("CARROT-1008.xml")) {
        root = db.parse(is).getDocumentElement();
      }
  
      Stream<TestDocument> docStream =
          elementStream(root, "document")
              .map(
                  docElement -> {
                    String title =
                        elementStream(docElement, "title")
                            .map(e -> e.getTextContent())
                            .collect(Collectors.joining(" . "));
                    String snippet =
                        elementStream(docElement, "snippet")
                            .map(e -> e.getTextContent())
                            .collect(Collectors.joining(" . "));
                    return new TestDocument(title, snippet);
                  });
  
      STCClusteringAlgorithm algorithm = algorithm();
      algorithm.maxClusters.set(30);
  
      List<Cluster<Document>> clusters =
          algorithm.cluster(docStream, CachedLangComponents.loadCached("English"));
  
      List<String> collect =
          clusters.stream().flatMap(c -> c.getLabels().stream()).collect(Collectors.toList());
      Assertions.assertThat(collect.contains("Guns") && collect.contains("Gun")).isFalse();
    }
test_case.class_method_signature: STCClusteringAlgorithmTest.testCarrot1008()
test_case.full_signature: @Test public void testCarrot1008()
test_case.identifier: testCarrot1008
test_case.invocations: newInstance newDocumentBuilder getResourceAsStream getClass getDocumentElement parse map elementStream collect map elementStream getTextContent joining collect map elementStream getTextContent joining algorithm set cluster loadCached collect flatMap stream stream getLabels toList isFalse assertThat contains contains
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testCarrot1008()
focal_class.declarator: REQUIRED_LANGUAGE_COMPONENTS =
        new HashSet<>(
            Arrays.asList(Stemmer.class, Tokenizer.class, LexicalData.class, LabelFormatter.class))
focal_class.modifier: private static final
focal_class.original_string: private static final Set<Class<?>> REQUIRED_LANGUAGE_COMPONENTS =
        new HashSet<>(
            Arrays.asList(Stemmer.class, Tokenizer.class, LexicalData.class, LabelFormatter.class));
focal_class.type: Set<Class<?>>
focal_class.var_name: REQUIRED_LANGUAGE_COMPONENTS
focal_class.declarator: NAME = "STC"
focal_class.modifier: public static final
focal_class.original_string: public static final String NAME = "STC";
focal_class.type: String
focal_class.var_name: NAME
focal_class.declarator: queryHint =
        attributes.register("queryHint", SharedInfrastructure.queryHintAttribute())
focal_class.modifier: public final
focal_class.original_string: public final AttrString queryHint =
        attributes.register("queryHint", SharedInfrastructure.queryHintAttribute());
focal_class.type: AttrString
focal_class.var_name: queryHint
focal_class.declarator: ignoreWordIfInHigherDocsPercent =
        attributes.register(
            "ignoreWordIfInHigherDocsPercent",
            AttrDouble.builder()
                .label("Ignore words appearing in more than a given fraction of documents")
                .min(0)
                .max(1)
                .defaultValue(0.9))
focal_class.modifier: public
focal_class.original_string: public AttrDouble ignoreWordIfInHigherDocsPercent =
        attributes.register(
            "ignoreWordIfInHigherDocsPercent",
            AttrDouble.builder()
                .label("Ignore words appearing in more than a given fraction of documents")
                .min(0)
                .max(1)
                .defaultValue(0.9));
focal_class.type: AttrDouble
focal_class.var_name: ignoreWordIfInHigherDocsPercent
focal_class.declarator: minBaseClusterScore =
        attributes.register(
            "minBaseClusterScore",
            AttrDouble.builder().label("Minimum base cluster score").min(0).max(10).defaultValue(2.))
focal_class.modifier: public
focal_class.original_string: public AttrDouble minBaseClusterScore =
        attributes.register(
            "minBaseClusterScore",
            AttrDouble.builder().label("Minimum base cluster score").min(0).max(10).defaultValue(2.));
focal_class.type: AttrDouble
focal_class.var_name: minBaseClusterScore
focal_class.declarator: minBaseClusterSize =
        attributes.register(
            "minBaseClusterSize",
            AttrInteger.builder()
                .label("Minimum number of documents in a base cluster")
                .min(2)
                .max(20)
                .defaultValue(2))
focal_class.modifier: public
focal_class.original_string: public AttrInteger minBaseClusterSize =
        attributes.register(
            "minBaseClusterSize",
            AttrInteger.builder()
                .label("Minimum number of documents in a base cluster")
                .min(2)
                .max(20)
                .defaultValue(2));
focal_class.type: AttrInteger
focal_class.var_name: minBaseClusterSize
focal_class.declarator: maxBaseClusters =
        attributes.register(
            "maxBaseClusters",
            AttrInteger.builder().label("Maximum number of base cluster").min(2).defaultValue(300))
focal_class.modifier: public
focal_class.original_string: public AttrInteger maxBaseClusters =
        attributes.register(
            "maxBaseClusters",
            AttrInteger.builder().label("Maximum number of base cluster").min(2).defaultValue(300));
focal_class.type: AttrInteger
focal_class.var_name: maxBaseClusters
focal_class.declarator: maxClusters =
        attributes.register(
            "maxClusters",
            AttrInteger.builder().label("Maximum number of final clusters").min(1).defaultValue(15))
focal_class.modifier: public
focal_class.original_string: public AttrInteger maxClusters =
        attributes.register(
            "maxClusters",
            AttrInteger.builder().label("Maximum number of final clusters").min(1).defaultValue(15));
focal_class.type: AttrInteger
focal_class.var_name: maxClusters
focal_class.declarator: mergeThreshold =
        attributes.register(
            "mergeThreshold",
            AttrDouble.builder()
                .label("Base cluster merge threshold")
                .min(0)
                .max(1)
                .defaultValue(0.6))
focal_class.modifier: public
focal_class.original_string: public AttrDouble mergeThreshold =
        attributes.register(
            "mergeThreshold",
            AttrDouble.builder()
                .label("Base cluster merge threshold")
                .min(0)
                .max(1)
                .defaultValue(0.6));
focal_class.type: AttrDouble
focal_class.var_name: mergeThreshold
focal_class.declarator: maxPhraseOverlap =
        attributes.register(
            "maxPhraseOverlap",
            AttrDouble.builder()
                .label("Maximum cluster phrase overlap")
                .min(0)
                .max(1)
                .defaultValue(0.6))
focal_class.modifier: public
focal_class.original_string: public AttrDouble maxPhraseOverlap =
        attributes.register(
            "maxPhraseOverlap",
            AttrDouble.builder()
                .label("Maximum cluster phrase overlap")
                .min(0)
                .max(1)
                .defaultValue(0.6));
focal_class.type: AttrDouble
focal_class.var_name: maxPhraseOverlap
focal_class.declarator: mostGeneralPhraseCoverage =
        attributes.register(
            "mostGeneralPhraseCoverage",
            AttrDouble.builder()
                .label("Minimum general phrase coverage")
                .min(0)
                .max(1)
                .defaultValue(0.5))
focal_class.modifier: public
focal_class.original_string: public AttrDouble mostGeneralPhraseCoverage =
        attributes.register(
            "mostGeneralPhraseCoverage",
            AttrDouble.builder()
                .label("Minimum general phrase coverage")
                .min(0)
                .max(1)
                .defaultValue(0.5));
focal_class.type: AttrDouble
focal_class.var_name: mostGeneralPhraseCoverage
focal_class.declarator: maxWordsPerLabel =
        attributes.register(
            "maxWordsPerLabel",
            AttrInteger.builder().label("Maximum words per label").min(1).defaultValue(4))
focal_class.modifier: public
focal_class.original_string: public AttrInteger maxWordsPerLabel =
        attributes.register(
            "maxWordsPerLabel",
            AttrInteger.builder().label("Maximum words per label").min(1).defaultValue(4));
focal_class.type: AttrInteger
focal_class.var_name: maxWordsPerLabel
focal_class.declarator: maxPhrasesPerLabel =
        attributes.register(
            "maxPhrasesPerLabel",
            AttrInteger.builder().label("Maximum phrases per label").min(1).defaultValue(3))
focal_class.modifier: public
focal_class.original_string: public AttrInteger maxPhrasesPerLabel =
        attributes.register(
            "maxPhrasesPerLabel",
            AttrInteger.builder().label("Maximum phrases per label").min(1).defaultValue(3));
focal_class.type: AttrInteger
focal_class.var_name: maxPhrasesPerLabel
focal_class.declarator: singleTermBoost =
        attributes.register(
            "singleTermBoost",
            AttrDouble.builder().label("Boost single-term clusters").min(0).defaultValue(0.5))
focal_class.modifier: public
focal_class.original_string: public AttrDouble singleTermBoost =
        attributes.register(
            "singleTermBoost",
            AttrDouble.builder().label("Boost single-term clusters").min(0).defaultValue(0.5));
focal_class.type: AttrDouble
focal_class.var_name: singleTermBoost
focal_class.declarator: optimalPhraseLength =
        attributes.register(
            "optimalPhraseLength",
            AttrInteger.builder().label("Optimal cluster label length").min(1).defaultValue(3))
focal_class.modifier: public
focal_class.original_string: public AttrInteger optimalPhraseLength =
        attributes.register(
            "optimalPhraseLength",
            AttrInteger.builder().label("Optimal cluster label length").min(1).defaultValue(3));
focal_class.type: AttrInteger
focal_class.var_name: optimalPhraseLength
focal_class.declarator: optimalPhraseLengthDev =
        attributes.register(
            "optimalPhraseLengthDev",
            AttrDouble.builder()
                .label("Optimal cluster label length's tolerance")
                .min(0.5)
                .defaultValue(2.))
focal_class.modifier: public
focal_class.original_string: public AttrDouble optimalPhraseLengthDev =
        attributes.register(
            "optimalPhraseLengthDev",
            AttrDouble.builder()
                .label("Optimal cluster label length's tolerance")
                .min(0.5)
                .defaultValue(2.));
focal_class.type: AttrDouble
focal_class.var_name: optimalPhraseLengthDev
focal_class.declarator: documentCountBoost =
        attributes.register(
            "documentCountBoost",
            AttrDouble.builder().label("Base cluster document count boost").min(0).defaultValue(1.))
focal_class.modifier: public
focal_class.original_string: public AttrDouble documentCountBoost =
        attributes.register(
            "documentCountBoost",
            AttrDouble.builder().label("Base cluster document count boost").min(0).defaultValue(1.));
focal_class.type: AttrDouble
focal_class.var_name: documentCountBoost
focal_class.declarator: scoreWeight =
        attributes.register(
            "scoreWeight",
            AttrDouble.builder().label("Size-score sorting ratio").min(0).max(1).defaultValue(0.))
focal_class.modifier: public
focal_class.original_string: public AttrDouble scoreWeight =
        attributes.register(
            "scoreWeight",
            AttrDouble.builder().label("Size-score sorting ratio").min(0).max(1).defaultValue(0.));
focal_class.type: AttrDouble
focal_class.var_name: scoreWeight
focal_class.declarator: mergeStemEquivalentBaseClusters =
        attributes.register(
            "mergeStemEquivalentBaseClusters",
            AttrBoolean.builder()
                .label("Merge all stem-equivalent phrases when discovering base clusters")
                .defaultValue(true))
focal_class.modifier: public
focal_class.original_string: public AttrBoolean mergeStemEquivalentBaseClusters =
        attributes.register(
            "mergeStemEquivalentBaseClusters",
            AttrBoolean.builder()
                .label("Merge all stem-equivalent phrases when discovering base clusters")
                .defaultValue(true));
focal_class.type: AttrBoolean
focal_class.var_name: mergeStemEquivalentBaseClusters
focal_class.declarator: preprocessing
focal_class.modifier: public
focal_class.original_string: public BasicPreprocessingPipeline preprocessing;
focal_class.type: BasicPreprocessingPipeline
focal_class.var_name: preprocessing
focal_class.declarator: labelFormatter
focal_class.modifier: private
focal_class.original_string: private LabelFormatter labelFormatter;
focal_class.type: LabelFormatter
focal_class.var_name: labelFormatter
focal_class.declarator: NOT_SELECTED = (p) -> !p.selected
focal_class.modifier: private static final
focal_class.original_string: private static final Predicate<PhraseCandidate> NOT_SELECTED = (p) -> !p.selected;
focal_class.type: Predicate<PhraseCandidate>
focal_class.var_name: NOT_SELECTED
focal_class.declarator: sb
focal_class.modifier: private
focal_class.original_string: private GeneralizedSuffixTree.SequenceBuilder sb;
focal_class.type: GeneralizedSuffixTree.SequenceBuilder
focal_class.var_name: sb
focal_class.declarator: context
focal_class.modifier: private
focal_class.original_string: private PreprocessingContext context;
focal_class.type: PreprocessingContext
focal_class.var_name: context
focal_class.file: core/src/main/java/org/carrot2/clustering/stc/STCClusteringAlgorithm.java
focal_class.identifier: STCClusteringAlgorithm
focal_class.interfaces: implements ClusteringAlgorithm
focal_class.class_method_signature: STCClusteringAlgorithm.requiredLanguageComponents()
focal_class.full_signature: @Override public Set<Class<?>> requiredLanguageComponents()
focal_class.identifier: requiredLanguageComponents
focal_class.modifiers: @Override public
focal_class.parameters: ()
focal_class.return: Set<Class<?>>
focal_class.signature: Set<Class<?>> requiredLanguageComponents()
focal_class.class_method_signature: STCClusteringAlgorithm.cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_class.full_signature: @Override public List<Cluster<T>> cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_class.identifier: cluster
focal_class.modifiers: @Override public
focal_class.parameters: (
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_class.return: List<Cluster<T>>
focal_class.signature: List<Cluster<T>> cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_class.class_method_signature: STCClusteringAlgorithm.createBaseClusters(SequenceBuilder sb)
focal_class.full_signature: private List<ClusterCandidate> createBaseClusters(SequenceBuilder sb)
focal_class.identifier: createBaseClusters
focal_class.modifiers: private
focal_class.parameters: (SequenceBuilder sb)
focal_class.return: List<ClusterCandidate>
focal_class.signature: List<ClusterCandidate> createBaseClusters(SequenceBuilder sb)
focal_class.class_method_signature: STCClusteringAlgorithm.mergeStemEquivalentBaseClusters(
        SequenceBuilder sb, final List<ClusterCandidate> candidates)
focal_class.full_signature: private void mergeStemEquivalentBaseClusters(
        SequenceBuilder sb, final List<ClusterCandidate> candidates)
focal_class.identifier: mergeStemEquivalentBaseClusters
focal_class.modifiers: private
focal_class.parameters: (
        SequenceBuilder sb, final List<ClusterCandidate> candidates)
focal_class.return: void
focal_class.signature: void mergeStemEquivalentBaseClusters(
        SequenceBuilder sb, final List<ClusterCandidate> candidates)
focal_class.class_method_signature: STCClusteringAlgorithm.createMergedClusters(List<ClusterCandidate> baseClusters)
focal_class.full_signature: private ArrayList<ClusterCandidate> createMergedClusters(List<ClusterCandidate> baseClusters)
focal_class.identifier: createMergedClusters
focal_class.modifiers: private
focal_class.parameters: (List<ClusterCandidate> baseClusters)
focal_class.return: ArrayList<ClusterCandidate>
focal_class.signature: ArrayList<ClusterCandidate> createMergedClusters(List<ClusterCandidate> baseClusters)
focal_class.class_method_signature: STCClusteringAlgorithm.merge(
        PreprocessingContext context, IntStack mergeList, List<ClusterCandidate> baseClusters)
focal_class.full_signature: private ClusterCandidate merge(
        PreprocessingContext context, IntStack mergeList, List<ClusterCandidate> baseClusters)
focal_class.identifier: merge
focal_class.modifiers: private
focal_class.parameters: (
        PreprocessingContext context, IntStack mergeList, List<ClusterCandidate> baseClusters)
focal_class.return: ClusterCandidate
focal_class.signature: ClusterCandidate merge(
        PreprocessingContext context, IntStack mergeList, List<ClusterCandidate> baseClusters)
focal_class.class_method_signature: STCClusteringAlgorithm.markSubSuperPhrases(ArrayList<PhraseCandidate> phrases)
focal_class.full_signature: private void markSubSuperPhrases(ArrayList<PhraseCandidate> phrases)
focal_class.identifier: markSubSuperPhrases
focal_class.modifiers: private
focal_class.parameters: (ArrayList<PhraseCandidate> phrases)
focal_class.return: void
focal_class.signature: void markSubSuperPhrases(ArrayList<PhraseCandidate> phrases)
focal_class.class_method_signature: STCClusteringAlgorithm.markOverlappingPhrases(
        PreprocessingContext context, ArrayList<PhraseCandidate> phrases)
focal_class.full_signature: private void markOverlappingPhrases(
        PreprocessingContext context, ArrayList<PhraseCandidate> phrases)
focal_class.identifier: markOverlappingPhrases
focal_class.modifiers: private
focal_class.parameters: (
        PreprocessingContext context, ArrayList<PhraseCandidate> phrases)
focal_class.return: void
focal_class.signature: void markOverlappingPhrases(
        PreprocessingContext context, ArrayList<PhraseCandidate> phrases)
focal_class.class_method_signature: STCClusteringAlgorithm.computeIntersection(int[] a, int aPos, int aLength, int[] b, int bPos, int bLength)
focal_class.full_signature: static int computeIntersection(int[] a, int aPos, int aLength, int[] b, int bPos, int bLength)
focal_class.identifier: computeIntersection
focal_class.modifiers: static
focal_class.parameters: (int[] a, int aPos, int aLength, int[] b, int bPos, int bLength)
focal_class.return: int
focal_class.signature: int computeIntersection(int[] a, int aPos, int aLength, int[] b, int bPos, int bLength)
focal_class.class_method_signature: STCClusteringAlgorithm.appendUniqueWords(
        PreprocessingContext context, IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.full_signature: private void appendUniqueWords(
        PreprocessingContext context, IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.identifier: appendUniqueWords
focal_class.modifiers: private
focal_class.parameters: (
        PreprocessingContext context, IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.return: void
focal_class.signature: void appendUniqueWords(
        PreprocessingContext context, IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.class_method_signature: STCClusteringAlgorithm.appendWords(IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.full_signature: private void appendWords(IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.identifier: appendWords
focal_class.modifiers: private
focal_class.parameters: (IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.return: void
focal_class.signature: void appendWords(IntStack words, IntStack offsets, PhraseCandidate p)
focal_class.class_method_signature: STCClusteringAlgorithm.postProcessing(
        List<T> documents, List<ClusterCandidate> candidates, List<Cluster<T>> clusters)
focal_class.full_signature: private void postProcessing(
        List<T> documents, List<ClusterCandidate> candidates, List<Cluster<T>> clusters)
focal_class.identifier: postProcessing
focal_class.modifiers: private
focal_class.parameters: (
        List<T> documents, List<ClusterCandidate> candidates, List<Cluster<T>> clusters)
focal_class.return: void
focal_class.signature: void postProcessing(
        List<T> documents, List<ClusterCandidate> candidates, List<Cluster<T>> clusters)
focal_class.class_method_signature: STCClusteringAlgorithm.collectPhrases(ClusterCandidate c, Cluster<?> cluster)
focal_class.full_signature: private void collectPhrases(ClusterCandidate c, Cluster<?> cluster)
focal_class.identifier: collectPhrases
focal_class.modifiers: private
focal_class.parameters: (ClusterCandidate c, Cluster<?> cluster)
focal_class.return: void
focal_class.signature: void collectPhrases(ClusterCandidate c, Cluster<?> cluster)
focal_class.class_method_signature: STCClusteringAlgorithm.collectDocuments(
        List<T> documents, List<T> l, BitSet bitset)
focal_class.full_signature: private List<T> collectDocuments(
        List<T> documents, List<T> l, BitSet bitset)
focal_class.identifier: collectDocuments
focal_class.modifiers: private
focal_class.parameters: (
        List<T> documents, List<T> l, BitSet bitset)
focal_class.return: List<T>
focal_class.signature: List<T> collectDocuments(
        List<T> documents, List<T> l, BitSet bitset)
focal_class.class_method_signature: STCClusteringAlgorithm.buildLabel(int[] phraseIndices)
focal_class.full_signature: private String buildLabel(int[] phraseIndices)
focal_class.identifier: buildLabel
focal_class.modifiers: private
focal_class.parameters: (int[] phraseIndices)
focal_class.return: String
focal_class.signature: String buildLabel(int[] phraseIndices)
focal_class.class_method_signature: STCClusteringAlgorithm.toString(PhraseCandidate c)
focal_class.full_signature: @SuppressWarnings("unused") private String toString(PhraseCandidate c)
focal_class.identifier: toString
focal_class.modifiers: @SuppressWarnings("unused") private
focal_class.parameters: (PhraseCandidate c)
focal_class.return: String
focal_class.signature: String toString(PhraseCandidate c)
focal_class.class_method_signature: STCClusteringAlgorithm.buildDebugLabel(int[] phraseIndices)
focal_class.full_signature: @SuppressWarnings("unused") private String buildDebugLabel(int[] phraseIndices)
focal_class.identifier: buildDebugLabel
focal_class.modifiers: @SuppressWarnings("unused") private
focal_class.parameters: (int[] phraseIndices)
focal_class.return: String
focal_class.signature: String buildDebugLabel(int[] phraseIndices)
focal_class.class_method_signature: STCClusteringAlgorithm.checkAcceptablePhrase(IntStack path)
focal_class.full_signature: final boolean checkAcceptablePhrase(IntStack path)
focal_class.identifier: checkAcceptablePhrase
focal_class.modifiers: final
focal_class.parameters: (IntStack path)
focal_class.return: boolean
focal_class.signature: boolean checkAcceptablePhrase(IntStack path)
focal_class.class_method_signature: STCClusteringAlgorithm.effectivePhraseLength(IntStack path)
focal_class.full_signature: final int effectivePhraseLength(IntStack path)
focal_class.identifier: effectivePhraseLength
focal_class.modifiers: final
focal_class.parameters: (IntStack path)
focal_class.return: int
focal_class.signature: int effectivePhraseLength(IntStack path)
focal_class.class_method_signature: STCClusteringAlgorithm.baseClusterScore(final int phraseLength, final int documentCount)
focal_class.full_signature: final float baseClusterScore(final int phraseLength, final int documentCount)
focal_class.identifier: baseClusterScore
focal_class.modifiers: final
focal_class.parameters: (final int phraseLength, final int documentCount)
focal_class.return: float
focal_class.signature: float baseClusterScore(final int phraseLength, final int documentCount)
focal_class.class_method_signature: STCClusteringAlgorithm.indexOf(
        int[] source,
        int sourceOffset,
        int sourceCount,
        int[] target,
        int targetOffset,
        int targetCount)
focal_class.full_signature: private static int indexOf(
        int[] source,
        int sourceOffset,
        int sourceCount,
        int[] target,
        int targetOffset,
        int targetCount)
focal_class.identifier: indexOf
focal_class.modifiers: private static
focal_class.parameters: (
        int[] source,
        int sourceOffset,
        int sourceCount,
        int[] target,
        int targetOffset,
        int targetCount)
focal_class.return: int
focal_class.signature: int indexOf(
        int[] source,
        int sourceOffset,
        int sourceCount,
        int[] target,
        int targetOffset,
        int targetCount)
focal_class.superclass: extends AttrComposite
focal_method.body: @Override
    public <T extends Document> List<Cluster<T>> cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents) {
      List<T> documents = docStream.collect(Collectors.toList());
      List<Cluster<T>> clusters = new ArrayList<>();
  
      /*
       * Step 1. Preprocessing: tokenization, stop word marking and stemming (if available).
       */
      context = preprocessing.preprocess(documents.stream(), queryHint.get(), languageComponents);
      labelFormatter = context.languageComponents.get(LabelFormatter.class);
  
      /*
       * Step 2: Create a generalized suffix tree from phrases in the input.
       */
      sb = new GeneralizedSuffixTree.SequenceBuilder();
  
      final int[] tokenIndex = context.allTokens.wordIndex;
      final short[] tokenType = context.allTokens.type;
      for (int i = 0; i < tokenIndex.length; i++) {
        /* Advance until the first real token. */
        if (tokenIndex[i] == -1) {
          if ((tokenType[i] & (Tokenizer.TF_SEPARATOR_DOCUMENT | Tokenizer.TF_TERMINATOR)) != 0) {
            sb.endDocument();
          }
          continue;
        }
  
        /* We have the first token. Advance until non-token. */
        final int s = i;
  
        while (tokenIndex[i + 1] != -1) i++;
        final int phraseLength = 1 + i - s;
        if (phraseLength >= 1) {
          /* We have a phrase. */
          sb.addPhrase(tokenIndex, s, phraseLength);
        }
      }
      sb.buildSuffixTree();
  
      /*
       * Step 3: Find "base" clusters by looking up frequently recurring phrases in the
       * generalized suffix tree.
       */
      List<ClusterCandidate> baseClusters = createBaseClusters(sb);
  
      /*
       * Step 4: Merge base clusters that overlap too much to form final clusters.
       */
      List<ClusterCandidate> mergedClusters = createMergedClusters(baseClusters);
  
      /*
       * Step 5: Create the junk (unassigned documents) cluster and create the final
       * set of clusters in Carrot2 format.
       */
      postProcessing(documents, mergedClusters, clusters);
  
      return SharedInfrastructure.reorderByWeightedScoreAndSize(clusters, this.scoreWeight.get());
    }
focal_method.class_method_signature: STCClusteringAlgorithm.cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_method.full_signature: @Override public List<Cluster<T>> cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_method.identifier: cluster
focal_method.invocations: collect toList preprocess stream get get endDocument addPhrase buildSuffixTree createBaseClusters createMergedClusters postProcessing reorderByWeightedScoreAndSize get
focal_method.modifiers: @Override public
focal_method.parameters: (
        Stream<? extends T> docStream, LanguageComponents languageComponents)
focal_method.return: List<Cluster<T>>
focal_method.signature: List<Cluster<T>> cluster(
        Stream<? extends T> docStream, LanguageComponents languageComponents)