test_class.file: storm/trident-ml/src/test/java/com/github/pmerienne/trident/ml/preprocessing/EnglishTokenizerTest.java
test_class.identifier: EnglishTokenizerTest
test_case.body: @Test
  	public void testTokenize() {
  		// Given
  		EnglishTokenizer tokenizer = new EnglishTokenizer();
  		String text = "I can't argue with some arguments on argus with argues";
  
  		// When
  		List<String> actualTokens = tokenizer.tokenize(text);
  
  		// Then
  		List<String> expectedTokens = Arrays.asList("i", "can't", "argu", "some", "argument", "argu", "argu");
  		assertEquals(expectedTokens, actualTokens);
  	}
test_case.class_method_signature: EnglishTokenizerTest.testTokenize()
test_case.full_signature: @Test public void testTokenize()
test_case.identifier: testTokenize
test_case.invocations: tokenize asList assertEquals
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testTokenize()
focal_class.declarator: serialVersionUID = -4307517944955017402L
focal_class.modifier: private static final
focal_class.original_string: private static final long serialVersionUID = -4307517944955017402L;
focal_class.type: long
focal_class.var_name: serialVersionUID
focal_class.declarator: LOGGER = LogFactory.getLog(EnglishTokenizer.class)
focal_class.modifier: private static final
focal_class.original_string: private static final Log LOGGER = LogFactory.getLog(EnglishTokenizer.class);
focal_class.type: Log
focal_class.var_name: LOGGER
focal_class.declarator: LUCENE_VERSION = Version.LUCENE_36
focal_class.modifier: private final transient static
focal_class.original_string: private final transient static Version LUCENE_VERSION = Version.LUCENE_36;
focal_class.type: Version
focal_class.var_name: LUCENE_VERSION
focal_class.declarator: stopWords = null
focal_class.modifier: protected
focal_class.original_string: protected List<String> stopWords = null;
focal_class.type: List<String>
focal_class.var_name: stopWords
focal_class.declarator: stemExclusionsSet = new HashSet<String>()
focal_class.modifier: protected
focal_class.original_string: protected Set<String> stemExclusionsSet = new HashSet<String>();
focal_class.type: Set<String>
focal_class.var_name: stemExclusionsSet
focal_class.declarator: nGram = false
focal_class.modifier: private
focal_class.original_string: private boolean nGram = false;
focal_class.type: boolean
focal_class.var_name: nGram
focal_class.declarator: minNGram = 2
focal_class.modifier: private
focal_class.original_string: private int minNGram = 2;
focal_class.type: int
focal_class.var_name: minNGram
focal_class.declarator: maxNGram = 2
focal_class.modifier: private
focal_class.original_string: private int maxNGram = 2;
focal_class.type: int
focal_class.var_name: maxNGram
focal_class.file: storm/trident-ml/src/main/java/com/github/pmerienne/trident/ml/preprocessing/EnglishTokenizer.java
focal_class.identifier: EnglishTokenizer
focal_class.interfaces: implements Serializable, TextTokenizer
focal_class.class_method_signature: EnglishTokenizer.EnglishTokenizer()
focal_class.full_signature: public  EnglishTokenizer()
focal_class.identifier: EnglishTokenizer
focal_class.modifiers: public
focal_class.parameters: ()
focal_class.signature:  EnglishTokenizer()
focal_class.class_method_signature: EnglishTokenizer.EnglishTokenizer(int minNGram, int maxNGram)
focal_class.full_signature: public  EnglishTokenizer(int minNGram, int maxNGram)
focal_class.identifier: EnglishTokenizer
focal_class.modifiers: public
focal_class.parameters: (int minNGram, int maxNGram)
focal_class.signature:  EnglishTokenizer(int minNGram, int maxNGram)
focal_class.class_method_signature: EnglishTokenizer.EnglishTokenizer(List<String> stopWords)
focal_class.full_signature: public  EnglishTokenizer(List<String> stopWords)
focal_class.identifier: EnglishTokenizer
focal_class.modifiers: public
focal_class.parameters: (List<String> stopWords)
focal_class.signature:  EnglishTokenizer(List<String> stopWords)
focal_class.class_method_signature: EnglishTokenizer.EnglishTokenizer(List<String> stopWords, Set<String> stemExclusionsSet)
focal_class.full_signature: public  EnglishTokenizer(List<String> stopWords, Set<String> stemExclusionsSet)
focal_class.identifier: EnglishTokenizer
focal_class.modifiers: public
focal_class.parameters: (List<String> stopWords, Set<String> stemExclusionsSet)
focal_class.signature:  EnglishTokenizer(List<String> stopWords, Set<String> stemExclusionsSet)
focal_class.class_method_signature: EnglishTokenizer.tokenize(String text)
focal_class.full_signature: public List<String> tokenize(String text)
focal_class.identifier: tokenize
focal_class.modifiers: public
focal_class.parameters: (String text)
focal_class.return: List<String>
focal_class.signature: List<String> tokenize(String text)
focal_class.class_method_signature: EnglishTokenizer.createTokenStream(String text)
focal_class.full_signature: protected TokenStream createTokenStream(String text)
focal_class.identifier: createTokenStream
focal_class.modifiers: protected
focal_class.parameters: (String text)
focal_class.return: TokenStream
focal_class.signature: TokenStream createTokenStream(String text)
focal_class.class_method_signature: EnglishTokenizer.setNGram(int minNGram, int maxNGram)
focal_class.full_signature: public void setNGram(int minNGram, int maxNGram)
focal_class.identifier: setNGram
focal_class.modifiers: public
focal_class.parameters: (int minNGram, int maxNGram)
focal_class.return: void
focal_class.signature: void setNGram(int minNGram, int maxNGram)
focal_class.class_method_signature: EnglishTokenizer.isnGram()
focal_class.full_signature: public boolean isnGram()
focal_class.identifier: isnGram
focal_class.modifiers: public
focal_class.parameters: ()
focal_class.return: boolean
focal_class.signature: boolean isnGram()
focal_class.class_method_signature: EnglishTokenizer.setnGram(boolean nGram)
focal_class.full_signature: public void setnGram(boolean nGram)
focal_class.identifier: setnGram
focal_class.modifiers: public
focal_class.parameters: (boolean nGram)
focal_class.return: void
focal_class.signature: void setnGram(boolean nGram)
focal_class.class_method_signature: EnglishTokenizer.getMinNGram()
focal_class.full_signature: public int getMinNGram()
focal_class.identifier: getMinNGram
focal_class.modifiers: public
focal_class.parameters: ()
focal_class.return: int
focal_class.signature: int getMinNGram()
focal_class.class_method_signature: EnglishTokenizer.setMinNGram(int minNGram)
focal_class.full_signature: public void setMinNGram(int minNGram)
focal_class.identifier: setMinNGram
focal_class.modifiers: public
focal_class.parameters: (int minNGram)
focal_class.return: void
focal_class.signature: void setMinNGram(int minNGram)
focal_class.class_method_signature: EnglishTokenizer.getMaxNGram()
focal_class.full_signature: public int getMaxNGram()
focal_class.identifier: getMaxNGram
focal_class.modifiers: public
focal_class.parameters: ()
focal_class.return: int
focal_class.signature: int getMaxNGram()
focal_class.class_method_signature: EnglishTokenizer.setMaxNGram(int maxNGram)
focal_class.full_signature: public void setMaxNGram(int maxNGram)
focal_class.identifier: setMaxNGram
focal_class.modifiers: public
focal_class.parameters: (int maxNGram)
focal_class.return: void
focal_class.signature: void setMaxNGram(int maxNGram)
focal_class.class_method_signature: EnglishTokenizer.getStopWords()
focal_class.full_signature: public List<String> getStopWords()
focal_class.identifier: getStopWords
focal_class.modifiers: public
focal_class.parameters: ()
focal_class.return: List<String>
focal_class.signature: List<String> getStopWords()
focal_class.class_method_signature: EnglishTokenizer.setStopWords(List<String> stopWords)
focal_class.full_signature: public void setStopWords(List<String> stopWords)
focal_class.identifier: setStopWords
focal_class.modifiers: public
focal_class.parameters: (List<String> stopWords)
focal_class.return: void
focal_class.signature: void setStopWords(List<String> stopWords)
focal_class.class_method_signature: EnglishTokenizer.setStemExclusionsSet(Set<String> stemExclusionsSet)
focal_class.full_signature: public void setStemExclusionsSet(Set<String> stemExclusionsSet)
focal_class.identifier: setStemExclusionsSet
focal_class.modifiers: public
focal_class.parameters: (Set<String> stemExclusionsSet)
focal_class.return: void
focal_class.signature: void setStemExclusionsSet(Set<String> stemExclusionsSet)
focal_class.class_method_signature: EnglishTokenizer.getStemExclusionsSet()
focal_class.full_signature: public Set<String> getStemExclusionsSet()
focal_class.identifier: getStemExclusionsSet
focal_class.modifiers: public
focal_class.parameters: ()
focal_class.return: Set<String>
focal_class.signature: Set<String> getStemExclusionsSet()
focal_method.body: public List<String> tokenize(String text) {
  		List<String> words = new ArrayList<String>();
  		if (text != null && !text.isEmpty()) {
  			TokenStream tokenStream = this.createTokenStream(text);
  			CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class);
  
  			try {
  				while (tokenStream.incrementToken()) {
  					String term = charTermAttribute.toString();
  					words.add(term);
  				}
  			} catch (IOException ioe) {
  				LOGGER.error("Unable to analyze text. Cause : " + ioe.getMessage(), ioe);
  			} finally {
  				try {
  					tokenStream.end();
  					tokenStream.close();
  				} catch (IOException e) {
  					// Can't do nothing!!
  					LOGGER.error("Unable to close token stream : " + e.getMessage());
  				}
  			}
  		}
  
  		return words;
  	}
focal_method.class_method_signature: EnglishTokenizer.tokenize(String text)
focal_method.full_signature: public List<String> tokenize(String text)
focal_method.identifier: tokenize
focal_method.invocations: isEmpty createTokenStream addAttribute incrementToken toString add error getMessage end close error getMessage
focal_method.modifiers: public
focal_method.parameters: (String text)
focal_method.return: List<String>
focal_method.signature: List<String> tokenize(String text)