test_class.file: dexter-core/src/test/java/it/cnr/isti/hpc/dexter/shingle/ShingleExtractorTest.java
test_class.identifier: ShingleExtractorTest
test_case.body: @Test
  	public void testAddText() throws Exception {
  		String text = null;
  		ShingleExtractor shingler = new ShingleExtractor(text);
  		Iterator<Shingle> it = shingler.iterator();
  		assertFalse(it.hasNext());
  		shingler = new ShingleExtractor("");
  		it = shingler.iterator();
  		assertFalse(it.hasNext());
  
  	}
test_case.class_method_signature: ShingleExtractorTest.testAddText()
test_case.full_signature: @Test public void testAddText()
test_case.identifier: testAddText
test_case.invocations: iterator assertFalse hasNext iterator assertFalse hasNext
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testAddText()
focal_class.declarator: maxShingleSize
focal_class.modifier: private
focal_class.original_string: private int maxShingleSize;
focal_class.type: int
focal_class.var_name: maxShingleSize
focal_class.declarator: DEFAULT_MAX_SHINGLE_SIZE = 6
focal_class.modifier: private static final
focal_class.original_string: private static final int DEFAULT_MAX_SHINGLE_SIZE = 6;
focal_class.type: int
focal_class.var_name: DEFAULT_MAX_SHINGLE_SIZE
focal_class.declarator: cleanedSentences
focal_class.modifier: private final
focal_class.original_string: private final List<List<Token>> cleanedSentences;
focal_class.type: List<List<Token>>
focal_class.var_name: cleanedSentences
focal_class.declarator: sm
focal_class.modifier: private final
focal_class.original_string: private final SpotManager sm;
focal_class.type: SpotManager
focal_class.var_name: sm
focal_class.declarator: ts
focal_class.modifier: private final
focal_class.original_string: private final TokenSegmenter ts;
focal_class.type: TokenSegmenter
focal_class.var_name: ts
focal_class.declarator: ss
focal_class.modifier: private final
focal_class.original_string: private final SentenceSegmenter ss;
focal_class.type: SentenceSegmenter
focal_class.var_name: ss
focal_class.file: dexter-core/src/main/java/it/cnr/isti/hpc/dexter/shingle/ShingleExtractor.java
focal_class.identifier: ShingleExtractor
focal_class.interfaces: implements Iterable<Shingle>
focal_class.class_method_signature: ShingleExtractor.ShingleExtractor()
focal_class.full_signature: private  ShingleExtractor()
focal_class.identifier: ShingleExtractor
focal_class.modifiers: private
focal_class.parameters: ()
focal_class.signature:  ShingleExtractor()
focal_class.class_method_signature: ShingleExtractor.ShingleExtractor(Article a)
focal_class.full_signature: public  ShingleExtractor(Article a)
focal_class.identifier: ShingleExtractor
focal_class.modifiers: public
focal_class.parameters: (Article a)
focal_class.signature:  ShingleExtractor(Article a)
focal_class.class_method_signature: ShingleExtractor.ShingleExtractor(String text)
focal_class.full_signature: public  ShingleExtractor(String text)
focal_class.identifier: ShingleExtractor
focal_class.modifiers: public
focal_class.parameters: (String text)
focal_class.signature:  ShingleExtractor(String text)
focal_class.class_method_signature: ShingleExtractor.process(String text)
focal_class.full_signature: public void process(String text)
focal_class.identifier: process
focal_class.modifiers: public
focal_class.parameters: (String text)
focal_class.return: void
focal_class.signature: void process(String text)
focal_class.class_method_signature: ShingleExtractor.addText(String text)
focal_class.full_signature: private void addText(String text)
focal_class.identifier: addText
focal_class.modifiers: private
focal_class.parameters: (String text)
focal_class.return: void
focal_class.signature: void addText(String text)
focal_class.class_method_signature: ShingleExtractor.getMaxShingleSize()
focal_class.full_signature: public int getMaxShingleSize()
focal_class.identifier: getMaxShingleSize
focal_class.modifiers: public
focal_class.parameters: ()
focal_class.return: int
focal_class.signature: int getMaxShingleSize()
focal_class.class_method_signature: ShingleExtractor.iterator()
focal_class.full_signature: @Override public Iterator<Shingle> iterator()
focal_class.identifier: iterator
focal_class.modifiers: @Override public
focal_class.parameters: ()
focal_class.return: Iterator<Shingle>
focal_class.signature: Iterator<Shingle> iterator()
focal_class.class_method_signature: ShingleExtractor.setMaxShingleSize(int maxShingleSize)
focal_class.full_signature: public void setMaxShingleSize(int maxShingleSize)
focal_class.identifier: setMaxShingleSize
focal_class.modifiers: public
focal_class.parameters: (int maxShingleSize)
focal_class.return: void
focal_class.signature: void setMaxShingleSize(int maxShingleSize)
focal_method.body: private void addText(String text) {
  		if (text == null || text.isEmpty())
  			return;
  		List<Sentence> sentences = ss.splitPos(text);
  
  		// int start = 0;
  		for (Sentence sentence : sentences) {
  			String currSentence = sentence.getText();
  			int startSentence = sentence.getStart();
  
  			// System.out.println("SENTENCE [" + currSentence + "]");
  			// //List<Token> textShingles = new LinkedList<Token>();
  			// FIXME CLEAN SHOULD NO CHANGE THE OFFSETS OF THE TOKENS
  			// currSentence = sm.clean(currSentence);
  			List<Token> tokens = ts.tokenizePos(currSentence);
  			List<Token> cleanTokens = new LinkedList<Token>();
  			// experimental
  			for (Token t : tokens) {
  
  				t.setStart(t.getStart() + startSentence);
  				t.setEnd(t.getEnd() + startSentence);
  				String token = text.substring(t.getStart(), t.getEnd());
  				String cleanToken = sm.clean(token);
  				if (cleanToken.isEmpty())
  					continue;
  
  				// System.out.println(token + "-> " + cleanToken);
  				// System.out.println("token in text: " + token);
  
  				// Skip empty token (or tokens made only of chars cleaned
  				// above)
  
  				t.setText(cleanToken);
  				cleanTokens.add(t);
  			}
  			cleanedSentences.add(cleanTokens);
  		}
  
  	}
focal_method.class_method_signature: ShingleExtractor.addText(String text)
focal_method.full_signature: private void addText(String text)
focal_method.identifier: addText
focal_method.invocations: isEmpty splitPos getText getStart tokenizePos setStart getStart setEnd getEnd substring getStart getEnd clean isEmpty setText add add
focal_method.modifiers: private
focal_method.parameters: (String text)
focal_method.return: void
focal_method.signature: void addText(String text)