gt: 			for (int i=0 ; i<matchIndexes.size() ; i++) {
context: /*
   *    TopicDetector.java
   *    Copyright (C) 2007 David Milne, d.n.milne@gmail.com
   *
   *    This program is free software; you can redistribute it and/or modify
   *    it under the terms of the GNU General Public License as published by
   *    the Free Software Foundation; either version 2 of the License, or
   *    (at your option) any later version.
   *
   *    This program is distributed in the hope that it will be useful,
   *    but WITHOUT ANY WARRANTY; without even the implied warranty of
   *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   *    GNU General Public License for more details.
   *
   *    You should have received a copy of the GNU General Public License
   *    along with this program; if not, write to the Free Software
   *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
   */
  
  
  package org.wikipedia.miner.annotation;
  
  import java.io.*;
  import java.util.*;
  import java.util.regex.*;
  
  import org.wikipedia.miner.model.*;
  import org.wikipedia.miner.model.Page.PageType;
  import org.wikipedia.miner.util.*;
  import org.wikipedia.miner.annotation.preprocessing.*;
  
  /**
   * This class detects topics that occur in plain text, using Disambiguator to resolve ambiguous terms and phrases. 
   * Many of the detected topics will be rubbish (extracted from unhelpful terms, such as <em>and</em> or <em>the</em>, so you will probably want to use either a LinkDetector or 
   * some simple heuristics to weed out the least useful ones (see Topic for the features that are available for separating important topics from less helpful ones). 
   * <p>
   * This also doesn't resolve collisions (e.g. "united states" collides with "states of america" in "united states of america"). 
   * The DocumentTagger has methods to resolve these.
   * 
   *  @author David Milne 
   */
  public class TopicDetector {
  	
  	private Wikipedia wikipedia ;
  	//private SentenceSplitter ss; 
  	private Disambiguator disambiguator ;
  	//private HashSet<String> stopwords ;
  	
  	
  	private boolean strictDisambiguation ;
  	private boolean allowDisambiguations ;
  	
  	private int maxTopicsForRelatedness = 25 ;
  	
  	
  	
  	
  	/**
  	 * Initializes a new topic detector.
  	 * 
  	 * @param wikipedia an initialized instance of Wikipedia
  	 * @param disambiguator a trained 
  	 * @param stopwordFile an optional (may be null) file containing 
  	 * @param strictDisambiguation
  	 * @param allowDisambiguations 
  	 * @throws IOException 
  	 */
  	public TopicDetector(Wikipedia wikipedia, Disambiguator disambiguator, boolean strictDisambiguation, boolean allowDisambiguations) throws IOException {
  		this.wikipedia = wikipedia ;
  		this.disambiguator = disambiguator ;
  		
  		this.strictDisambiguation = strictDisambiguation ;
  		this.allowDisambiguations = allowDisambiguations ;
  		
  		/*
  		this.stopwords = new HashSet<String>() ;
  		if (stopwordFile != null) {
  			BufferedReader input = new BufferedReader(new FileReader(stopwordFile)) ;
  			String line ;
  			while ((line=input.readLine()) != null) 
  				stopwords.add(line.trim()) ;			
  		}*/
  		
  		//TODO:Check caching 
  		/*
  		if (!wikipedia.getEnvironment().isGeneralityCached()) 
  			System.err.println("TopicDetector | Warning: generality has not been cached, so this will run significantly slower than it needs to.") ;
  		*/	
  			
  	}
  	
  	/**
  	 * Gathers a collection of topics from the given document. 
  	 * 
  	 * @param doc a document that has been preprocessed so that markup (html, mediawiki, etc) is safely ignored.
  	 * @param rc a cache in which relatedness measures will be saved so they aren't repeatedly calculated. This may be null.  
  	 * @return a vector of topics that were mined from the document.
  	 * @throws Exception
  	 */
  	public Vector<Topic> getTopics(PreprocessedDocument doc, RelatednessCache rc) throws Exception {
  		
  		if (rc == null)
  			rc = new RelatednessCache(disambiguator.getArticleComparer()) ;
  		
  
  		//Vector<String> sentences = ss.getSentences(doc.getPreprocessedText(), SentenceSplitter.MULTIPLE_NEWLINES) ;
  		Vector<TopicReference> references = getReferences(doc.getPreprocessedText()) ;
  		
  		Collection<Topic> temp = getTopics(references, doc.getContextText(), doc.getOriginalText().length(), rc).values() ;
  		calculateRelatedness(temp, rc) ;
  
  		Vector<Topic> topics = new Vector<Topic>() ;
  		for (Topic t:temp) {
  			if (!doc.isTopicBanned(t.getId())) 
  					topics.add(t) ;
  		}
  		
  		return topics ;
  	}
  	
  	/**
  	 * Gathers a collection of topics from the given document. 
  	 * 
  	 * @param text text to mine topics from. This must be plain text, without any form of markup. 
  	 * @param rc a cache in which relatedness measures will be saved so they aren't repeatedly calculated. This may be null. 
  	 * @return a collection of topics that were mined from the document.
  	 * @throws Exception
  	 */
  	public Collection<Topic> getTopics(String text, RelatednessCache rc) throws Exception {
  		
  		if (rc == null)
  			rc = new RelatednessCache(disambiguator.getArticleComparer()) ;
  			
  
  		//Vector<String> sentences = ss.getSentences(text, SentenceSplitter.MULTIPLE_NEWLINES) ;
  		Vector<TopicReference> references = getReferences(text) ;
  		
  		HashMap<Integer,Topic> topicsById = getTopics(references, "", text.length(), rc) ;
  
  		Collection<Topic> topics = topicsById.values() ;
  		calculateRelatedness(topics, rc) ;
  		
  		return topics ;
  	}
  	
  	private void calculateRelatedness(Collection<Topic> topics, RelatednessCache cache) throws Exception{
  		
  		TreeSet<Article> weightedTopics = new TreeSet<Article>() ;
  		
  		for (Topic t:topics) {
  			if (t.getType() != PageType.article)
  				continue ;
  			
  			Article art = (Article)wikipedia.getPageById(t.getId()) ;
  			
  			art.setWeight(t.getAverageLinkProbability() * t.getOccurances()) ;
  			weightedTopics.add(art) ;
  		}
  		
  		for (Topic topic: topics) {
  			
  			double totalWeight = 0 ;
  			double totalWeightedRelatedness = 0 ;
  			
  			int count = 0 ;
  			
  			for (Article art: weightedTopics) {
  				if (count++ > maxTopicsForRelatedness)
  					break ;
  				
  				double weightedRelatedness = art.getWeight() * cache.getRelatedness(topic, art) ;
  				
  				totalWeight = totalWeight + art.getWeight();
  				totalWeightedRelatedness = totalWeightedRelatedness + weightedRelatedness;
  				
  			}
  			
  			topic.setRelatednessToOtherTopics((float)(totalWeightedRelatedness/totalWeight)) ;
  		}
  	}
  	
  	
  	
  	
  	private Vector<TopicReference> getReferences(String text) {
  
  		Vector<TopicReference> references = new Vector<TopicReference>() ;
  		//int sentenceStart = 0 ;
  
  		//ProgressDisplayer pd = new ProgressDisplayer(" - gathering candidate links", sentences.length, 0.1) ;
  
  		//for (String sentence:sentences) {
  			String s = "$ " + text + " $" ;
  			//pd.update() ;
  
  			Pattern p = Pattern.compile("[\\s\\{\\}\\(\\)\"\'\\.\\,\\;\\:\\-\\_]") ;  //would just match all non-word chars, but we don't want to match utf chars
  			Matcher m = p.matcher(s) ;
  
  			Vector<Integer> matchIndexes = new Vector<Integer>() ;
  
  			while (m.find()) 
  				matchIndexes.add(m.start()) ;
  
  