gt:             if (args[i].equals("-aanfilter")) {
context: ///////////////////////////////////////////////////////////////////////////////
  // Copyright (C) 2003-9 Jason Baldridge and Michael White (University of Edinburgh / The Ohio State University)
  //
  // This library is free software; you can redistribute it and/or
  // modify it under the terms of the GNU Lesser General Public
  // License as published by the Free Software Foundation; either
  // version 2.1 of the License, or (at your option) any later version.
  //
  // This library is distributed in the hope that it will be useful,
  // but WITHOUT ANY WARRANTY; without even the implied warranty of
  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  // GNU Lesser General Public License for more details.
  //
  // You should have received a copy of the GNU Lesser General Public
  // License along with this program; if not, write to the Free Software
  // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  //////////////////////////////////////////////////////////////////////////////
  package opennlp.ccg.test;
  
  import java.io.*;
  import java.net.URL;
  import java.text.*;
  import java.util.*;
  import java.util.prefs.Preferences;
  
  import opennlp.ccg.TextCCG;
  import opennlp.ccg.grammar.Grammar;
  import opennlp.ccg.hylo.*;
  import opennlp.ccg.lexicon.Tokenizer;
  import opennlp.ccg.lexicon.Word;
  import opennlp.ccg.ngrams.*;
  import opennlp.ccg.parse.ParseException;
  import opennlp.ccg.parse.Parser;
  import opennlp.ccg.parse.Supertagger;
  import opennlp.ccg.parse.supertagger.WordAndPOSDictionaryLabellingStrategy;
  import opennlp.ccg.realize.*;
  import opennlp.ccg.realize.hypertagger.ZLMaxentHypertagger;
  import opennlp.ccg.synsem.*;
  import opennlp.ccg.util.Pair;
  import opennlp.ccg.util.SingletonList;
  import opennlp.ccg.perceptron.*;
  
  import org.jdom2.*;
  import org.jdom2.output.Format;
  import org.jdom2.output.XMLOutputter;
  
  /**
   * Automates the testing of CCG grammars.
   *
   * @author  Jason Baldridge
   * @author  Michael White
   * @version $Revision: 1.151 $, $Date: 2011/12/11 16:51:58 $
   */
  public class Regression {
  
      /** Flag for whether to do parsing. */
      public boolean doParsing = true;
  
      /** Flag for whether to do realization. */
      public boolean doRealization = true;
  
      /** Flag for whether to just do even items. */
      public boolean evenOnly = false;
  
      /** Flag for whether to just do odd items. */
      public boolean oddOnly = false;
  
      /** Flag for whether to do garbage collection before each iteration. */
      public boolean doGC = false;
  
      /** File to write events to (if any). */
      public String eventfile = null;
  
      /** Flag for whether to include the gold sign when generating events. */
      public boolean includeGoldInEvents = false;
  
      /** Directory for writing APML files (if any). */
      public String apmldir = null;
  
      /** Flag for whether to show realization stats. */
      public boolean showStats = true;
  
      /** Flag for whether to show parsing stats. */
      public boolean showParseStats = false;
  
      /** File to dump realizer stats to (if any). */
      public String statsfile = null;
  
      /** File prefix to write bleu test files to (if any). */
      public String bleufileprefix = null;
  
      /** File to write n-best realizations to (if any). */
      public String nbestrealfile = null;
  
      /** Flag for whether to normalize strings as for BLEU scoring in n-best output. */
      public boolean nbestnormbleu = false;
  
      /** Directory to save best realization serializations to (if any). */
      public String realserdir = null;
  
      /** File to write rescored sign scores to (if any). */
      public String rescorefile = null;
  
      /** Map from info keys to best realization signs for serialization (if any). */
      public Map<String,Sign> bestRealMap = null;
  
      /** Flag for whether to include LFs in n-best output. */
      public boolean nbestincludelfs = false;
  
      /** File to write n-best parses to (if any). */
      public String nbestparsefile = null;
  
      /** The grammar to use for testing. */
      public Grammar grammar = null;
  
      /** The parser to use for testing. */
      public Parser parser = null;
  
      /** The realizer to use for testing. */
      public Realizer realizer = null;
  
      /** The scorer to use for realizer testing (or null, for default). */
      public SignScorer scorer = null;
  
      /** The scorer to use for parser testing. */
      public SignScorer parseScorer = null;
  
      /** Flag for whether to only allow exact matches with the default scorer. */
      public boolean exactMatches = false;
  
      /** The n-gram order to use with the default scorer (or 0, for default). */
      public int ngramOrder = 0;
  
      /** The feature extractor to use in extracting events. */
      public FeatureExtractor featureExtractor = null;
  
      //
      // the various totals
      //
  
      public int pCount = 0;
      public int pBadCount = 0;
      public int pFailedCount = 0;
      public int pExactCount = 0;
      public double totalF = 0.0;
      public double totalRecall = 0.0;
      public double totalPrecision = 0.0;
      public double totalDepsF = 0.0;
      public double totalDepsRecall = 0.0;
      public double totalDepsPrecision = 0.0;
      public double totalUnlabeledDepsF = 0.0;
      public double totalUnlabeledDepsRecall = 0.0;
      public double totalUnlabeledDepsPrecision = 0.0;
      public double totalFComplete = 0.0;
      public double totalRecallComplete = 0.0;
      public double totalPrecisionComplete = 0.0;
      public double totalDepsFComplete = 0.0;
      public double totalDepsRecallComplete = 0.0;
      public double totalDepsPrecisionComplete = 0.0;
      public double totalUnlabeledDepsFComplete = 0.0;
      public double totalUnlabeledDepsRecallComplete = 0.0;
      public double totalUnlabeledDepsPrecisionComplete = 0.0;
      public int pTotalEdges = 0;
      public int pTotalEdgesGood = 0;
      public int pMaxEdges = 0;
      public int pMaxEdgesGood = 0;
      public int pTotalUnpackingEdges = 0;
      public int pMaxUnpackingEdges = 0;
      public int pTotalCellMax = 0;
      public int pTotalCellMaxGood = 0;
      public int pMaxCellMax = 0;
      public int pMaxCellMaxGood = 0;
      public int pTotalLexTime = 0;
      public int pTotalParseTime = 0;
      public int pTotalChartTime = 0;
      public int pTotalUnpackingTime = 0;
      public int pMaxLexTime = 0;
      public int pMaxParseTime = 0;
      public int pMaxChartTime = 0;
      public int pMaxUnpackingTime = 0;
      public Map<Double,Integer> pBetaTallies = null;
  
      public int rCount = 0;
      public int rDoneCount = 0;
      public int rBadCount = 0;
      public int rExactCount = 0;
      public double totalScore = 0.0;
      public double totalScoreComplete = 0.0;
      public double totalReciprocalRank = 0.0;
      public int totalNominals = 0;
      public int totalTokens = 0;
      public int minTokens = 0;
      public int maxTokens = 0;
      public int totalRuleApps = 0;
      public int totalEdges = 0;
      public int totalEdgesCreated = 0;
      public int totalUnprunedEdges = 0;
      public int totalPrunedRemoved = 0;
      public int totalPrunedNeverAdded = 0;
      public int totalCellMax = 0;
      public int totalNewBest = 0;
      public int totalLex = 0;
      public int totalFirst = 0;
      public int totalBest = 0;
      public int totalPacked = 0;
      public int totalStoppedOrDone = 0;
      public int maxLex = 0;
      public int maxFirst = 0;
      public int maxBest = 0;
      public int maxNewBest = 0;
      public int maxPacked = 0;
      public int maxStoppedOrDone = 0;
      public int oracleBetter = 0;
      public int goldMissing = 0;
      public String maxLexStr = null;
      public String maxFirstStr = null;
      public String maxBestStr = null;
      public String maxNewBestStr = null;
      public String maxPackedStr = null;
      public String maxStoppedOrDoneStr = null;
      public List<Double> bestEstimatedScores = null;
      public List<Double> bestActualScores = null;
      public List<Integer> itemRanks = null;
      public TimingMap lexMap = null;
      public TimingMap firstMap = null;
      public TimingMap bestMap = null;
      public TimingMap allMap = null;
  
      private PrintWriter events = null;
      private PrintWriter bleuGen = null;
      private PrintWriter bleuRef = null;
      private PrintWriter bleuSrc = null;
      private PrintWriter nbestrealPW = null;
      private PrintWriter rescorePW = null;
      private PrintWriter nbestparsePW = null;
  
      private XMLOutputter xmlOutputter = new XMLOutputter(); // for xml-escaping strings
  
      /** Constructor. */
      public Regression() {
          // init
          resetTotals();
      }
  
      /** Resets the various totals. */
      public void resetTotals() {
      	// parser
          pCount = 0; pBadCount = 0; pFailedCount = 0; pExactCount = 0;
          totalF = 0.0; totalRecall = 0.0; totalPrecision = 0.0;
          totalDepsF = 0.0; totalDepsRecall = 0.0; totalDepsPrecision = 0.0;
          totalUnlabeledDepsF = 0.0; totalUnlabeledDepsRecall = 0.0; totalUnlabeledDepsPrecision = 0.0;
          totalFComplete = 0.0; totalRecallComplete = 0.0; totalPrecisionComplete = 0.0;
          totalDepsFComplete = 0.0; totalDepsRecallComplete = 0.0; totalDepsPrecisionComplete = 0.0;
          totalUnlabeledDepsFComplete = 0.0; totalUnlabeledDepsRecallComplete = 0.0; totalUnlabeledDepsPrecisionComplete = 0.0;
          pTotalEdges = 0; pTotalEdgesGood = 0; pMaxEdges = 0; pMaxEdgesGood = 0;
          pTotalUnpackingEdges = 0; pMaxUnpackingEdges = 0;
          pTotalCellMax = 0; pTotalCellMaxGood = 0; pMaxCellMax = 0; pMaxCellMaxGood = 0;
          pTotalLexTime = 0; pTotalParseTime = 0; pTotalChartTime = 0; pTotalUnpackingTime = 0;
          pMaxLexTime = 0; pMaxParseTime = 0; pMaxChartTime = 0; pMaxUnpackingTime = 0;
          if (doParsing) {
          	pBetaTallies = new TreeMap<Double,Integer>();
          }
          // realizer
          rCount = 0; rDoneCount = 0; rBadCount = 0; rExactCount = 0;
          totalScore = 0.0; totalScoreComplete = 0.0;
          totalReciprocalRank = 0.0;
          totalNominals = 0;
          totalTokens = 0; minTokens = 0; maxTokens = 0;
          totalRuleApps = 0;
          totalEdges = 0; totalEdgesCreated = 0; totalUnprunedEdges = 0; totalPrunedRemoved = 0; totalPrunedNeverAdded = 0; totalCellMax = 0;
          totalNewBest = 0; totalLex = 0; totalFirst = 0; totalBest = 0; totalPacked = 0; totalStoppedOrDone = 0;
          maxLex = 0; maxFirst = 0; maxBest = 0; maxNewBest = 0; maxPacked = 0; maxStoppedOrDone = 0;
          oracleBetter = 0; goldMissing = 0;
          maxLexStr = null; maxFirstStr = null; maxBestStr = null; maxNewBestStr = null; maxPackedStr = null; maxStoppedOrDoneStr = null;
          if (doRealization) {
              bestActualScores = new ArrayList<Double>(); bestEstimatedScores = new ArrayList<Double>(); itemRanks = new ArrayList<Integer>();
              lexMap = new TimingMap("lex"); firstMap = new TimingMap("first"); bestMap = new TimingMap("best"); allMap = new TimingMap("all");
          }
      }
  
  
      // sets up bleu output
      private void bleuSetup() throws IOException {
      	// setup bleu files, if apropos
      	if (bleufileprefix != null && doRealization) {
      		bleuGen = new PrintWriter(new BufferedWriter(new FileWriter(bleufileprefix + "-gen.sgm")));
      		bleuRef = new PrintWriter(new BufferedWriter(new FileWriter(bleufileprefix + "-ref.sgm")));
      		bleuSrc = new PrintWriter(new BufferedWriter(new FileWriter(bleufileprefix + "-src.sgm")));
      		bleuGen.println("<tstset setid=\"ccg-test\" srclang=\"en\" trglang=\"en\">");
      		bleuRef.println("<refset setid=\"ccg-test\" srclang=\"en\" trglang=\"en\">");
      		bleuSrc.println("<srcset setid=\"ccg-test\" srclang=\"en\">");
      	}
      }
  
      // sets up n-best realization output
      private void nbestrealSetup() throws IOException {
          // set up file to write ref & n-best realizations (if any)
          if (nbestrealfile != null && doRealization) {
              nbestrealPW = new PrintWriter(new BufferedWriter(new FileWriter(nbestrealfile)));
              nbestrealPW.println("<nbest>");
          }
      }
  
      // sets up rescored sign score output
      private void rescoreSetup() throws IOException {
          // set up rescoring file
          if (rescorefile != null) {
              rescorePW = new PrintWriter(new BufferedWriter(new FileWriter(rescorefile)));
              rescorePW.println("<rescored>");
          }
      }
  
      // sets up n-best parsing output
      private void nbestparseSetup() throws IOException {
          // set up file to write sentence & n-best parses (if any)
          if (nbestparsefile != null && doParsing) {
              nbestparsePW = new PrintWriter(new BufferedWriter(new FileWriter(nbestparsefile)));
              nbestparsePW.println("<nbest>");
          }
      }
  
      // starts a doc
      private void bleuStartDoc(String id) {
      	if (bleufileprefix != null && doRealization) {
  			bleuGen.println("<doc docid=\"" + id + "\" sysid=\"openccg\">");
  			bleuRef.println("<doc docid=\"" + id + "\" sysid=\"ref\">");
  			bleuSrc.println("<doc docid=\"" + id + "\">");
      	}
      }
  
      // ends a doc
      private void bleuEndDoc() {
      	if (bleufileprefix != null && doRealization) {
      		bleuGen.println("</doc>");
      		bleuRef.println("</doc>");
      		bleuSrc.println("</doc>");
      	}
      }
  
      // finishes bleu output
      private void bleuFinish() throws IOException {
      	// finish bleu files, if apropos
      	if (bleufileprefix != null && doRealization) {
      		bleuGen.println("</tstset>");
      		bleuRef.println("</refset>");
      		bleuSrc.println("</srcset>");
      		bleuGen.flush(); bleuGen.close();
      		bleuRef.flush(); bleuRef.close();
      		bleuSrc.flush(); bleuSrc.close();
      	}
      }
  
      // finishes n-best realization output
      private void nbestrealFinish() throws IOException {
      	// finish n-best real file, if apropos
      	if (nbestrealfile != null && doRealization) {
      		nbestrealPW.println("</nbest>");
      		nbestrealPW.flush(); nbestrealPW.close();
      	}
      }
  
      // finishes rescored sign output
      private void rescoreFinish() throws IOException {
      	// finish rescoring file
      	if (rescorefile != null) {
      		rescorePW.println("</rescored>");
      		rescorePW.flush(); rescorePW.close();
      	}
      }
  
      // finishes n-best parsing output
      private void nbestparseFinish() throws IOException {
      	// finish n-best real file, if apropos
      	if (nbestparsefile != null && doParsing) {
      		nbestparsePW.println("</nbest>");
      		nbestparsePW.flush(); nbestparsePW.close();
      	}
      }
  
      // resets bestRealMap
      private void realserStartDoc() {
      	if (realserdir != null && doRealization) {
      		bestRealMap = new HashMap<String,Sign>();
      	}
      }
  
      // serializes bestRealMap
      private void realserEndDoc(String testName) throws IOException {
      	if (realserdir != null && doRealization) {
  	    	File serFile = new File(new File(realserdir), testName + ".ser");
  	    	ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(serFile));
  	    	oos.writeObject(bestRealMap);
  	    	oos.close();
      	}
      }
  
      // escapes string for xml output
      private String xmlEscape(String s) { return xmlOutputter.outputString(new Text(s)); }
  
      // normalizes realizations for BLEU scoring
      // at present, this means replacing underscores with spaces and escaping for xml output
      private String norm_bleu(String s) { return xmlEscape(s.replace('_', ' ')); }
  
  
      /** Runs the test on the items in the given file or directory of files. */
      public void runTest(File regressionFile) throws IOException {
      	// set up event file (if any)
      	if (eventfile != null) events = EventFile.openWriter(new File(eventfile));
          // set up bleu output, n-best realizations, rescoring, n-best parses (if apropos)
      	bleuSetup(); nbestrealSetup(); rescoreSetup(); nbestparseSetup();
      	// do each file or files
      	for (File f : getXMLFiles(regressionFile))
      		runSingleTest(f);
      	// finish bleu, n-best realization output, rescoring, n-best parses (if apropos)
      	bleuFinish(); nbestrealFinish(); rescoreFinish(); nbestparseFinish();
      	// close event file (if any)
      	if (events != null) { events.flush(); events.close(); }
      	// show stats (if apropos)
      	if (rescorefile != null) return;
      	if (doParsing && showParseStats) showParseStats();
          if (doRealization && showStats) showStats();
      }
  
      /** Returns a list of xml files from the given file or directory. */
      public static List<File> getXMLFiles(File file) {
      	if (!file.isDirectory()) return new SingletonList<File>(file);
      	List<File> retval = new ArrayList<File>();
  		File[] files = file.listFiles();
  		Arrays.sort(files);
  		for (int i = 0; i < files.length; i++) {
  			if (!files[i].isDirectory() && files[i].getName().endsWith(".xml"))
  				retval.add(files[i]);
  		}
      	return retval;
      }
  
      /** Runs the test on the items in the given file. */
      private void runSingleTest(File regressionFile) throws IOException {
  
      	String testName = regressionFile.getName();
      	int lastDot = testName.lastIndexOf('.');
      	if (lastDot > 0) testName = testName.substring(0, lastDot);
  
          // load testfile
      	System.out.println("Loading: " + testName);
      	System.out.println();
          RegressionInfo rinfo = new RegressionInfo(grammar, regressionFile);
  
          // start bleu doc (if apropos)
          bleuStartDoc(testName);
          // start storing best realizations for serialization (if apropos)
          realserStartDoc();
  
          // do each test
          int numItems = rinfo.numberOfItems();
          System.out.println("Parse\tRealize\tString");
          System.out.println("-----\t-------\t------");
  
          for (int i=0; i < numItems; i++) {
  
              // check even/odd only
              if (i % 2 == 1 && evenOnly) continue;
              if (i % 2 == 0 && oddOnly) continue;
  
              RegressionInfo.TestItem testItem = rinfo.getItem(i);
              if (doGC) System.gc();
  
              // short circuit for sign rescoring; realization only at the moment
              if (rescorefile != null) {
                  String id = testItem.info;
                  double score = scorer.score(testItem.sign, true);
              	rescorePW.println("<seg id=\"" + id + "\" score=\"" + score + "\"/>");
                  showOutcome("-", nfE.format(score), "", testItem.sign.getOrthography());
              	continue;
              }
  
              List<Sign> parses = null;
              List<Double> parseScores = null;
              LF parsedLF = null;
              LF compactedLF = null;
              LF transformedParsedLF = null;
              boolean parsed = false;
              boolean parsedComplete = false;
              if (doParsing) {
                  try {
              		// use full-words or words from stored sign if possible
              		List<Word> words = null;
              		if (testItem.fullWords != null) {
                  		words = grammar.lexicon.tokenizer.tokenize(testItem.fullWords, true);
                  		// strip <s> and </s>
                  		if (words.get(0).getForm() == "<s>") words.remove(0);
                  		if (words.get(words.size()-1).getForm() == "</s>") words.remove(words.size()-1);
              		}
              		else if (testItem.sign != null) {
              			words = testItem.sign.getWords();
              		}
              		if (words != null) {
  	            		// parse 'em
  	            		parser.parse(words);
              		}
                  	else {
                  		parser.parse(testItem.sentence);
                  	}
                  	// retrieve results
                      parses = parser.getResult();
                      parseScores = parser.getScores();
                      parsed = true;
                      parsedComplete = !parses.get(0).getCategory().isFragment();
                      // get LF of best parse, if needed
                      if (showParseStats || (doRealization && testItem.lfElt == null && testItem.sign == null)) {
  	                    Sign sign = parses.get(0);
  	                    Category cat = sign.getCategory().copy();
  	                    Nominal index = cat.getIndexNominal();
  	                    parsedLF = cat.getLF();
  	                    index = HyloHelper.convertNominals(parsedLF, sign, index);
  	                    compactedLF = HyloHelper.compact(parsedLF, index);
  	                    // get transformed version if needed
  	                    if (testItem.sign == null) {
  		                    transformedParsedLF = grammar.transformLF(compactedLF);
  	                    }
                      }
                  } catch (ParseException e) {
                  	parses = Collections.emptyList();
                      parsed = false;
                  } catch (Exception e) {
                  	parses = Collections.emptyList();
                      parsed = false;
                      System.err.println("Uncaught exception in parsing: " + testItem.sentence);
                      e.printStackTrace(System.err);
                  }
  
                  // update parse stats
                  int count = parser.edgeCount();
                  pTotalEdges += count;
                  if (count > pMaxEdges) pMaxEdges = count;
                  if (parsedComplete) {
                  	pTotalEdgesGood += count;
                  	if (count > pMaxEdgesGood) pMaxEdgesGood = count;
                  }
                  count = parser.unpackingEdgeCount();
                  pTotalUnpackingEdges += count;
                  if (count > pMaxUnpackingEdges) pMaxUnpackingEdges = count;
                  int cellMax = parser.maxCellSize();
                  pTotalCellMax += cellMax;
                  if (cellMax > pMaxCellMax) pMaxCellMax = cellMax;
                  if (parsedComplete) {
                  	pTotalCellMaxGood += cellMax;
                      if (cellMax > pMaxCellMaxGood) pMaxCellMaxGood = cellMax;
                  }
                  int time = parser.getLexTime();
                  pTotalLexTime += time;
                  if (time > pMaxLexTime) pMaxLexTime = time;
                  time = parser.getParseTime();
                  pTotalParseTime += time;
                  if (time > pMaxParseTime) pMaxParseTime = time;
                  time = parser.getChartTime();
                  pTotalChartTime += time;
                  if (time > pMaxChartTime) pMaxChartTime = time;
                  time = parser.getUnpackingTime();
                  pTotalUnpackingTime += time;
                  if (time > pMaxUnpackingTime) pMaxUnpackingTime = time;
                  double beta = parser.getSupertaggerBeta();
                  Integer betaTally = pBetaTallies.get(beta);
                  pBetaTallies.put(beta, (betaTally != null) ? ++betaTally : 1);
              }
  
              // get test item LF, if needed
              LF testItemLF = null;
              if (testItem.lfElt != null && (doRealization || (showParseStats && parsed && testItem.sign == null))) {
                  Element lfElt = testItem.lfElt;
                  Document doc = new Document();
                  lfElt.detach();
                  doc.setRootElement(lfElt);
                  testItemLF = grammar.loadLF(doc);
              }
  
              // compare EPs
              EPsScorer.Results parseScore = null;
              LF goldLF = null;
              if (showParseStats && parsedLF != null) {
                  // get LF to score, gold LF
              	LF lfToScore = null;
                  if (testItem.sign != null) {
                  	// use LF from stored sign if available
                  	lfToScore = parsedLF;
                      Category cat = testItem.sign.getCategory().copy();
                      Nominal index = cat.getIndexNominal();
                      goldLF = cat.getLF();
                      index = HyloHelper.convertNominals(goldLF, testItem.sign, index);
                  }
                  else {
                  	// otherwise use test item LF
                  	lfToScore = transformedParsedLF;
                  	goldLF = testItemLF;
                  }
          		if (goldLF == null) {
          			throw new RuntimeException(
          				"Can't score parse: " + testItem.sentence + "!\n" +
          				"No gold LF."
      				);
          		}
                  // score parse
              	parseScore = EPsScorer.score(lfToScore, goldLF);
              }
  
              // update parsing results
              if (showParseStats && testItem.numOfParses > 0) {
              	pCount++;
              	if (parses.size() == 0 || !parsedComplete) pBadCount++;
              	if (parses.size() == 0) pFailedCount++;
              	if (parseScore != null) {
              		if (parseScore.fscore == 1.0) pExactCount++;
              		totalF += parseScore.fscore;
              		totalRecall += parseScore.recall;
              		totalPrecision += parseScore.precision;
              		totalDepsF += parseScore.depsFscore;
              		totalDepsRecall += parseScore.depsRecall;
              		totalDepsPrecision += parseScore.depsPrecision;
              		totalUnlabeledDepsF += parseScore.unlabeledDepsFscore;
              		totalUnlabeledDepsRecall += parseScore.unlabeledDepsRecall;
              		totalUnlabeledDepsPrecision += parseScore.unlabeledDepsPrecision;
              		if (parsedComplete) {
                  		totalFComplete += parseScore.fscore;
                  		totalRecallComplete += parseScore.recall;
                  		totalPrecisionComplete += parseScore.precision;
                  		totalDepsFComplete += parseScore.depsFscore;
                  		totalDepsRecallComplete += parseScore.depsRecall;
                  		totalDepsPrecisionComplete += parseScore.depsPrecision;
                  		totalUnlabeledDepsFComplete += parseScore.unlabeledDepsFscore;
                  		totalUnlabeledDepsRecallComplete += parseScore.unlabeledDepsRecall;
                  		totalUnlabeledDepsPrecisionComplete += parseScore.unlabeledDepsPrecision;
              		}
              	}
              }
  
              // events output
              if (events != null && doParsing) {
              	// nb: only dealing with complete parses at the moment
              	// nb: gold LF must come from saved sign
              	if (parses.size() > 0 && testItem.sign != null) {
              		List<Sign> bestSigns = new ArrayList<Sign>(parses);
              		Sign best = parses.get(0);
              		// update best if not exact match
              		if (parseScore.fscore != 1.0) {
              			// check oracle best
              			Pair<Sign,Boolean> bestPair = parser.oracleBest(goldLF);
              			if (bestPair.a != null) oracleBetter++;
              			if (bestPair.b) {
              				best = bestPair.a;
              				if (!bestSigns.contains(best)) bestSigns.add(best);
              			}
              			// add gold if missing, if apropos
              			else {
              				goldMissing++;
              				if (includeGoldInEvents) {
  	                			best = testItem.sign;
  	                    		//parser.addSupertaggerLogProbs(best);
  	                    		bestSigns.add(best);
              				}
              			}
              		}
          			EventFile.writeEvents(events, bestSigns, best, featureExtractor);
              	}
              }
  
              // n-best parses output
              if (nbestparsePW != null) {
                  XMLOutputter outputter = new XMLOutputter();
                  outputter.setFormat(Format.getPrettyFormat());
              	// header for item
                  String extras = "";
                  if (parsedComplete) extras += " complete=\"true\"";
                  String id = testItem.info;
                  if (id == null) id = "" + i;
                  nbestparsePW.println("<seg id=\"" + id + "\" str=\"" + xmlEscape(testItem.sentence) + "\"" + extras + ">");
              	String tagend = (nbestincludelfs) ? ">" : "/>";
              	// add best parse
              	if (parseScore != null) {
              		double edgeScore = parseScores.get(0);
                  	String scores = "score=\"" + nf.format(parseScore.fscore) + "\" edge-score=\"" + nfE.format(edgeScore) + "\"";
                  	nbestparsePW.println("<best " + scores + tagend);
                  	if (nbestincludelfs) {
                      	Element lfElt = grammar.makeLfElt(compactedLF);
                      	nbestparsePW.println(outputter.outputString(lfElt));
                      	nbestparsePW.println("</best>");
                  	}
              	}
              	// add remaining n-best
              	for (int k=1; k < parses.size(); k++) {
                      Sign sign = parses.get(k);
              		double edgeScore = parseScores.get(k);
                      Category cat = sign.getCategory().copy();
                      Nominal index = cat.getIndexNominal();
                      LF parsedLFk = cat.getLF();
                      index = HyloHelper.convertNominals(parsedLFk, sign, index);
                      LF compactedLFk = HyloHelper.compact(parsedLFk, index);
                      LF lfToScore = parsedLFk;
                      if (testItem.sign != null) {
                          lfToScore = grammar.transformLF(compactedLFk);
                      }
                      EPsScorer.Results parseScoreK = EPsScorer.score(lfToScore, goldLF);
                  	String scores = "score=\"" + nf.format(parseScoreK.fscore) + "\" edge-score=\"" + nfE.format(edgeScore) + "\"";
                  	nbestparsePW.println("<next " + scores + tagend);
                  	if (nbestincludelfs) {
                      	Element lfElt = grammar.makeLfElt(compactedLFk);
                      	nbestparsePW.println(outputter.outputString(lfElt));
                      	nbestparsePW.println("</next>");
                  	}
              	}
                  // close item
                  nbestparsePW.println("</seg>");
              }
  
              // determine string to show for parse result
              String starForBadSentence = "";
              if (testItem.numOfParses == 0) starForBadSentence = "*";
  
              String parseResult;
              if (!doParsing) {
                  parseResult = "-";
              } else if (parseScore != null) {
                  parseResult = nf.format(parseScore.fscore);
                  if (!parsedComplete) parseResult = "[" + parseResult + "]";
              } else if (testItem.numOfParses == parses.size()) {
                  parseResult = "ok";
              } else if (testItem.numOfParses > 0 && parses.size() > 0) {
                  // show num parses, if not the number expected
                  parseResult = "(" + parses.size() + ")";
              } else if (testItem.knownFailure) {
                  parseResult = "(known)";
              } else {
                  parseResult = "FAILED";
              }
  
              if (!doRealization || (doParsing && !parsed) || testItem.numOfParses == 0) {
                  showOutcome(parseResult, "-", starForBadSentence, testItem.sentence);
                  continue;
              }
  
              LF inputLF = null;
              // use given LF
              if (testItemLF != null) inputLF = testItemLF;
              // or LF from stored sign
              else if (testItem.sign != null) {
                  Sign sign = testItem.sign;
                  Category cat = sign.getCategory().copy();
                  Nominal index = cat.getIndexNominal();
                  LF convertedLF = HyloHelper.compactAndConvertNominals(cat.getLF(), index, sign);
                  inputLF = grammar.transformLF(convertedLF);
              }
              // otherwise use first parse
              else if (transformedParsedLF != null) inputLF = transformedParsedLF;
              // otherwise give up
              else {
                  String suggestion = (!doParsing) ? "Try leaving off -noparsing option." : "";
                  throw new RuntimeException("No LF to realize! " + suggestion);
              }
  
              // set up n-gram precision scorer for default scoring and/or scoring results
              String[] targets = (testItem.alt == null)
              	? new String[] { testItem.sentence }
              	: new String[] { testItem.sentence, testItem.alt };
              NgramPrecisionModel defaultNgramScorer = new NgramPrecisionModel(targets);
              SignScorer scorerToUse = scorer;
              if (scorerToUse == null) {
                  if (ngramOrder > 0 || exactMatches) {
                  	if (ngramOrder > 0) scorerToUse = new NgramPrecisionModel(targets, ngramOrder);
                  	else scorerToUse = new NgramPrecisionModel(targets);
                  	((NgramPrecisionModel)scorerToUse).setExactMatches(exactMatches);
                  }
                  else scorerToUse = defaultNgramScorer;
              }
              // set targets for self-paraphrase biasing
              else if (scorerToUse instanceof SelfParaphraseBiaser) {
              	((SelfParaphraseBiaser)scorerToUse).setTargets(targets);
              }
  
              if (doGC) System.gc();
              try {
              	//Add gold std pred supertag mapping to the hypertagger class
              	if (events != null && realizer.hypertagger != null && testItem.predInfo != null) {
                  	realizer.hypertagger.storeGoldStdPredInfo(testItem.predInfo);
                  }
              	realizer.realize(inputLF, scorerToUse);
              }
              catch (Throwable thrwbl) {
                  System.out.println("Unable to process: " + testItem.sentence);
                  thrwbl.printStackTrace(System.out);
                  continue;
              }
              opennlp.ccg.realize.Chart chart = realizer.getChart();
              String realizeResult = "ok"; boolean gramcomplete = true; boolean joined = false;
              Edge bestEdge = chart.bestEdge;
              if (!bestEdge.complete() || bestEdge.getSign().getCategory().isFragment()) {
                  realizeResult = "[ok]"; gramcomplete = false;
                  rBadCount++;
                  if (chart.joinFragments) {
                  	bestEdge = chart.bestJoinedEdge;
                  	joined = true;
                  }
              }
              String bestRealization = bestEdge.getSign().getOrthography();
              double score = defaultNgramScorer.score(bestEdge.getSign(), false); // nb: use default n-gram precision score for reporting
  
              // events output
              if (events != null) {
              	List<Edge> bestEdges = chart.bestEdges();
              	// nb: only dealing with complete realizations at the moment
              	if (bestEdges.size() > 0) {
              		Pair<Edge,Boolean> bestPair = chart.oracleBest(testItem.sentence);
              		Edge oracleBest = bestPair.a;
              		if (oracleBest != null) {
                      	Sign best = oracleBest.getSign();
                      	List<Sign> bestSigns = new ArrayList<Sign>(bestEdges.size()+1);
                      	for (Edge e : bestEdges) bestSigns.add(e.getSign());
              			if (bestEdge != oracleBest) oracleBetter++;
              			if (!bestPair.b) {
              				goldMissing++;
              				if (includeGoldInEvents) {
              					best = testItem.sign;
              				}
              			}
              			if (!bestSigns.contains(best)) bestSigns.add(best);
              			EventFile.writeEvents(events, bestSigns, best, featureExtractor);
              		}
              	}
              }
  
              // bleu output
              if (bleufileprefix != null) {
                  String extras = " time=\"";
                  if (chart.done) extras += chart.timeTilDone;
                  else extras += chart.timeTilStopped;
                  extras += "\"";
                  extras += " score=\"" + nf.format(score) + "\"";
                  if (gramcomplete) extras += " complete=\"true\"";
                  if (joined) extras += " joined=\"true\"";
                  String id = testItem.info;
                  if (id == null) id = "" + i;
              	bleuGen.println("<seg id=\"" + id + "\"" + extras + ">" + norm_bleu(bestRealization) + "</seg>");
              	String sent = norm_bleu(testItem.sentence);
              	bleuRef.println("<seg id=\"" + id + "\">" + sent + "</seg>");
              	bleuSrc.println("<seg id=\"" + id + "\">" + sent + "</seg>");
              }
  
              // n-best realization output
              if (nbestrealPW != null) {
                  XMLOutputter outputter = new XMLOutputter();
                  outputter.setFormat(Format.getPrettyFormat());
              	// header for item
                  String extras = "";
                  if (gramcomplete) extras += " complete=\"true\"";
                  if (joined) extras += " joined=\"true\"";
                  String id = testItem.info;
                  if (id == null) id = "" + i;
              	nbestrealPW.println("<seg id=\"" + id + "\"" + extras + ">");
              	// add ref sentence
              	String ref = (nbestnormbleu) ? norm_bleu(testItem.sentence) : xmlEscape(testItem.sentence);
              	nbestrealPW.println("<ref>" + ref + "</ref>");
              	// add best realization
              	String scores = "score=\"" + nf.format(score) + "\" edge-score=\"" + nfE.format(bestEdge.score) + "\"";
              	String best = (nbestnormbleu) ? norm_bleu(bestRealization) : xmlEscape(bestRealization);
              	if (!nbestincludelfs)
              		nbestrealPW.println("<best " + scores + ">" + best + "</best>");
              	else {
              		nbestrealPW.println("<best " + scores + ">");
              		nbestrealPW.println("<str>" + best + "</str>");
                  	Sign sign = bestEdge.getSign();
                      Category cat = sign.getCategory().copy();
                      Nominal index = cat.getIndexNominal();
                      LF lf = cat.getLF();
                  	index = HyloHelper.convertNominalsToVars(lf, index);
                      index = HyloHelper.convertNominals(lf, sign, index);
                      LF lfc = HyloHelper.compact(lf, index);
                  	Element lfElt = grammar.makeLfElt(lfc);
                  	nbestrealPW.println(outputter.outputString(lfElt));
              		nbestrealPW.println("</best>");
  
              	}
                  // if complete, add remaining n-best
                  if (bestEdge.complete()) {
                      List<Edge> bestEdges = chart.bestEdges();
                      for (int j=1; j < bestEdges.size(); j++) {
                      	Edge e = bestEdges.get(j);
                          String eSent = e.getSign().getOrthography();
                          double eScore = defaultNgramScorer.score(e.getSign(), false); // nb: use default n-gram precision score for reporting
                      	String eScores = " score=\"" + nf.format(eScore) + "\" edge-score=\"" + nfE.format(e.score) + "\"";
                      	// add next realization
                      	String next = (nbestnormbleu) ? norm_bleu(eSent) : xmlEscape(eSent);
                      	if (!nbestincludelfs)
                      		nbestrealPW.println("<next" + eScores + ">" + next + "</next>");
                      	else {
                      		nbestrealPW.println("<next" + eScores + ">");
                      		nbestrealPW.println("<str>" + next + "</str>");
                          	Sign sign = e.getSign();
                              Category cat = sign.getCategory().copy();
                              Nominal index = cat.getIndexNominal();
                              LF lf = cat.getLF();
                          	index = HyloHelper.convertNominalsToVars(lf, index);
                              index = HyloHelper.convertNominals(lf, sign, index);
                              LF lfc = HyloHelper.compact(lf, index);
                          	Element lfElt = grammar.makeLfElt(lfc);
                          	nbestrealPW.println(outputter.outputString(lfElt));
                      		nbestrealPW.println("</next>");
                      	}
                      }
                  }
                  // close item
              	nbestrealPW.println("</seg>");
              }
  
          	// if apmldir non-null, output APML as apmldir/ex(i+1).apml
              if (apmldir != null) {
                  String apmlfn = apmldir + "/ex" + (i+1) + ".apml";
                  grammar.saveToApml(bestEdge.getSign(), apmlfn);
              }
  
              // store best realization, if apropos and grammatically complete, keyed by info string or item position
              if (realserdir != null && gramcomplete) {
              	String id = testItem.info;
              	if (id == null) id = "i" + i;
              	bestRealMap.put(testItem.info, bestEdge.getSign());
              }
  
              // compute stats, show outcome
              rCount++;
              totalScore += score;
              if (gramcomplete) totalScoreComplete += score;
              int itemRank = 1;
              Tokenizer tokenizer = grammar.lexicon.tokenizer;
              String itemOrth = tokenizer.getOrthography(tokenizer.tokenize(testItem.sentence));
              if (!bestRealization.equals(itemOrth)) {
                  itemRank = 0;
                  List<Edge> bestEdges = chart.bestEdges();
                  for (int j = 0; j < bestEdges.size(); j++) {
                      Edge edge = bestEdges.get(j);
                      String str = edge.getSign().getOrthography();
                      if (str.equals(itemOrth)) {
                          itemRank = j+1; break;
                      }
                  }
                  if (itemRank > 0) totalReciprocalRank += (1.0 / itemRank);
              	if (gramcomplete) {
  	                realizeResult = nf.format(score);
  	                if (itemRank > 0 && itemRank < 10) realizeResult += " ";
  	                if (itemRank > 0 && itemRank < 100) realizeResult += "#" + itemRank;
              	}
              	else {
              		realizeResult = "[" + nf.format(score) + "]";
              		if (joined) realizeResult += "j";
              	}
                  showOutcome(parseResult, realizeResult, starForBadSentence, testItem.sentence, bestRealization);
              }
              else {
                  rExactCount++;
                  totalReciprocalRank += 1.0;
                  showOutcome(parseResult, realizeResult, starForBadSentence, testItem.sentence);
              }
  
              totalNominals += chart.numNominals;
              int tokens = testItem.sentence.split("\\s+").length;
              totalTokens += tokens;
              if (tokens < minTokens || minTokens == 0) minTokens = tokens;
              if (tokens > maxTokens) maxTokens = tokens;
              totalRuleApps += chart.edgeFactory.ruleApps();
              totalEdges += chart.numEdgesInChart();
              totalEdgesCreated += chart.numEdges;
              totalUnprunedEdges += chart.numUnprunedEdges();
              totalPrunedRemoved += chart.numPrunedRemoved;
              totalPrunedNeverAdded += chart.numPrunedNeverAdded;
              totalCellMax += chart.cellMax;
              totalNewBest += chart.newBest;
  
              bestActualScores.add(new Double(score));
              bestEstimatedScores.add(new Double(bestEdge.score));
              itemRanks.add(new Integer(itemRank));
  
              totalLex += chart.timeTilLex;
              if (chart.timeTilLex > maxLex) {
                  maxLex = chart.timeTilLex;
                  maxLexStr = testItem.sentence;
              }
              lexMap.add(chart.numNominals, chart.timeTilLex);
  
              totalFirst += chart.timeTilFirst;
              if (chart.timeTilFirst > maxFirst) {
                  maxFirst = chart.timeTilFirst;
                  maxFirstStr = testItem.sentence;
              }
              firstMap.add(chart.numNominals, chart.timeTilFirst);
  
              totalBest += chart.timeTilBest;
              if (chart.timeTilBest > maxBest) {
                  maxBest = chart.timeTilBest;
                  maxBestStr = testItem.sentence;
              }
              bestMap.add(chart.numNominals, chart.timeTilBest);
  
              if (chart.newBest > 0 && (chart.timeTilBest - chart.timeTilFirst) >= maxNewBest) {
                  maxNewBest = chart.timeTilBest - chart.timeTilFirst;
                  maxNewBestStr = testItem.sentence;
              }
  
              totalPacked += chart.timeTilPacked;
              if (chart.timeTilPacked > maxPacked) {
                  maxPacked = chart.timeTilPacked;
                  maxPackedStr = testItem.sentence;
              }
  
              if (chart.done) {
                  rDoneCount++;
                  totalStoppedOrDone += chart.timeTilDone;
                  if (chart.timeTilDone > maxStoppedOrDone) {
                      maxStoppedOrDone = chart.timeTilDone;
                      maxStoppedOrDoneStr = testItem.sentence;
                  }
                  allMap.add(chart.numNominals, chart.timeTilDone);
              }
              else {
                  totalStoppedOrDone += chart.timeTilStopped;
                  if (chart.timeTilStopped > maxStoppedOrDone) {
                      maxStoppedOrDone = chart.timeTilStopped;
                      maxStoppedOrDoneStr = testItem.sentence;
                  }
                  allMap.add(chart.numNominals, chart.timeTilStopped);
              }
          }
  
          // end bleu doc (if apropos)
          bleuEndDoc();
          System.out.println();
  
          // serialize best realizations (if apropos)
          realserEndDoc(testName);
      }
  
      /** Shows the various parsing totals. */
      public void showParseStats() {
      	int pCompleteCount = pCount - pBadCount;
      	int pFragCount = pBadCount - pFailedCount;
          String pComplete = "" + pCompleteCount;
          String pCompletePct = "" + nf.format(100.0 * pCompleteCount / pCount) + "%";
          System.out.println("Strings parsed completely (in fragments, failed): " + pComplete + " (" + pFragCount + ", " + pFailedCount + ") " + pCompletePct);
          String pInexact = "" + (pCount - pExactCount);
          String pExactPct = "" + nf.format(100.0 * pExactCount / pCount) + "%";
          System.out.println("Strings parsed exactly (inexactly): " + pExactCount + " (" + pInexact + ") " + pExactPct);
          String avgEdges = nf.format(1.0 * pTotalEdges / pCount);
          System.out.println("Average edge count (before unpacking): " + avgEdges);
          String avgEdgesGood = nf.format(1.0 * pTotalEdgesGood / pCompleteCount);
          System.out.println("Average edge count (before unpacking) for complete parses: " + avgEdgesGood);
          System.out.println("Max edge count: " + pMaxEdges);
          System.out.println("Max edge count for complete parses: " + pMaxEdgesGood);
          String avgUnpacked = nf.format(1.0 * pTotalUnpackingEdges / pCount);
          System.out.println("Average edges unpacked (created while unpacking): " + avgUnpacked);
          System.out.println("Max unpacked edges: " + pMaxUnpackingEdges);
          String avgLexTime = nf.format(1.0 * pTotalLexTime / pCount);
          String avgCellMax = nf.format(1.0 * pTotalCellMax / pCount);
          System.out.println("Average max cell size (before unpacking): " + avgCellMax);
          String avgCellMaxGood = nf.format(1.0 * pTotalCellMaxGood / pCompleteCount);
          System.out.println("Average max cell size (before unpacking) for complete parses: " + avgCellMaxGood);
          System.out.println("Max max cell size: " + pMaxCellMax);
          System.out.println("Max max cell size for complete parses: " + pMaxCellMaxGood);
          System.out.println("Average lex lookup time: " + avgLexTime);
          String avgParseTime = nf.format(1.0 * pTotalParseTime / pCount);
          System.out.println("Max lex lookup time: " + pMaxLexTime);
          System.out.println("Average parse time: " + avgParseTime);
          System.out.println("Max parse time: " + pMaxParseTime);
          String avgChartTime = nf.format(1.0 * pTotalChartTime / pCount);
          System.out.println("Average chart construction time: " + avgChartTime);
          System.out.println("Max chart construction time: " + pMaxChartTime);
          String avgUnpackingTime = nf.format(1.0 * pTotalUnpackingTime / pCount);
          System.out.println("Average unpacking time: " + avgUnpackingTime);
          System.out.println("Max unpacking time: " + pMaxUnpackingTime);
          System.out.println("Supertagger beta tallies:");
          Set<Double> betas = pBetaTallies.keySet();
          for (double beta : betas) {
          	int tally = pBetaTallies.get(beta);
              System.out.println(beta + "\t" + tally);
          }
          String avgF = nf.format(100.0 * totalF / pCount);
          System.out.println("Labeled f-score: " + avgF);
          String avgRecall = nf.format(100.0 * totalRecall / pCount);
          System.out.println("Labeled recall: " + avgRecall);
          String avgPrecision = nf.format(100.0 * totalPrecision / pCount);
          System.out.println("Labeled precision: " + avgPrecision);
          String avgDepsF = nf.format(100.0 * totalDepsF / pCount);
          System.out.println("Labeled f-score deps only: " + avgDepsF);
          String avgDepsRecall = nf.format(100.0 * totalDepsRecall / pCount);
          System.out.println("Labeled recall deps only: " + avgDepsRecall);
          String avgDepsPrecision = nf.format(100.0 * totalDepsPrecision / pCount);
          System.out.println("Labeled precision deps only: " + avgDepsPrecision);
          String avgUnlabeledDepsF = nf.format(100.0 * totalUnlabeledDepsF / pCount);
          System.out.println("Unlabeled deps f-score: " + avgUnlabeledDepsF);
          String avgUnlabeledDepsRecall = nf.format(100.0 * totalUnlabeledDepsRecall / pCount);
          System.out.println("Unlabeled deps recall: " + avgUnlabeledDepsRecall);
          String avgUnlabeledDepsPrecision = nf.format(100.0 * totalUnlabeledDepsPrecision / pCount);
          System.out.println("Unlabeled deps precision: " + avgUnlabeledDepsPrecision);
          String avgFComplete = nf.format(100.0 * totalFComplete / pCompleteCount);
          System.out.println("Labeled f-score for complete parses: " + avgFComplete);
          String avgRecallComplete = nf.format(100.0 * totalRecallComplete / pCompleteCount);
          System.out.println("Labeled recall for complete parses: " + avgRecallComplete);
          String avgPrecisionComplete = nf.format(100.0 * totalPrecisionComplete / pCompleteCount);
          System.out.println("Labeled precision for complete parses: " + avgPrecisionComplete);
          String avgDepsFComplete = nf.format(100.0 * totalDepsFComplete / pCompleteCount);
          System.out.println("Labeled f-score deps only for complete parses: " + avgDepsFComplete);
          String avgDepsRecallComplete = nf.format(100.0 * totalDepsRecallComplete / pCompleteCount);
          System.out.println("Labeled recall deps only for complete parses: " + avgDepsRecallComplete);
          String avgDepsPrecisionComplete = nf.format(100.0 * totalDepsPrecisionComplete / pCompleteCount);
          System.out.println("Labeled precision deps only for complete parses: " + avgDepsPrecisionComplete);
          String avgUnlabeledDepsFComplete = nf.format(100.0 * totalUnlabeledDepsFComplete / pCompleteCount);
          System.out.println("Unlabeled deps f-score for complete parses: " + avgUnlabeledDepsFComplete);
          String avgUnlabeledDepsRecallComplete = nf.format(100.0 * totalUnlabeledDepsRecallComplete / pCompleteCount);
          System.out.println("Unlabeled deps recall for complete parses: " + avgUnlabeledDepsRecallComplete);
          String avgUnlabeledDepsPrecisionComplete = nf.format(100.0 * totalUnlabeledDepsPrecisionComplete / pCompleteCount);
          System.out.println("Unlabeled deps precision for complete parses: " + avgUnlabeledDepsPrecisionComplete);
          if (oracleBetter > 0) System.out.println("Oracle better: " + oracleBetter);
          if (goldMissing > 0) System.out.println("Gold missing: " + goldMissing);
          System.out.println();
      }
  
      /** Shows the various realization totals. */
      public void showStats() {
      	int rCompleteCount = rCount - rBadCount;
          String rComplete = "" + rCompleteCount;
          String rCompletePct = "" + nf.format(100.0 * rCompleteCount / rCount) + "%";
          System.out.println("Strings realized completely (in fragments): " + rComplete + " (" + rBadCount + ") " + rCompletePct);
          String rInexact = "" + (rCount - rExactCount);
          String rExactPct = "" + nf.format(100.0 * rExactCount / rCount) + "%";
          System.out.println("Strings realized exactly (inexactly): " + rExactCount + " (" + rInexact + ") " + rExactPct);
          System.out.println("Strings where realization finished: " + rDoneCount);
          String avgScore = nf.format(totalScore / rCount);
          System.out.println("Avg score: " + avgScore);
          String avgScoreComplete = nf.format(totalScoreComplete / rCompleteCount);
          System.out.println("Avg score for complete realizations: " + avgScoreComplete);
          String meanReciprocalRank = nf.format(totalReciprocalRank / rCount);
          System.out.println("Mean reciprocal rank: " + meanReciprocalRank);
          String residualMRR = (rCount == rExactCount)
              ? "n/a"
              : nf.format((totalReciprocalRank - rExactCount) / (rCount - rExactCount));
          System.out.println("Residual mean reciprocal rank: " + residualMRR);
          String avgNodes = nf.format(totalNominals * 1.0 / rCount);
          String avgTokens = nf.format(totalTokens * 1.0 / rCount);
          System.out.println("Avg num nodes, words: " + avgNodes + ", " + avgTokens);
          System.out.println("Num words (min-max): " + minTokens + "-" + maxTokens);
          String avgRuleApps = nf.format(totalRuleApps * 1.0 / rCount);
          System.out.println("Avg num rule apps: " + avgRuleApps);
          String avgEdges = nf.format(totalEdges * 1.0 / rCount);
          String avgEdgesCreated = nf.format(totalEdgesCreated * 1.0 / rCount);
          String avgUnprunedEdges = nf.format(totalUnprunedEdges * 1.0 / rCount);
          System.out.println("Avg num edges in chart: " + avgEdges);
          System.out.println("Avg num edges created: " + avgEdgesCreated);
          System.out.println("Avg num unpruned edges: " + avgUnprunedEdges);
          String avgRemoved = nf.format(totalPrunedRemoved * 1.0 / rCount);
          String avgNeverAdded = nf.format(totalPrunedNeverAdded * 1.0 / rCount);
          System.out.println("Avg num pruned edges removed, never added: "  + avgRemoved + ", " + avgNeverAdded);
          String avgCellMax = nf.format(totalCellMax * 1.0 / rCount);
          System.out.println("Avg cell max: " + avgCellMax);
          String avgNewBest = nf.format(totalNewBest * 1.0 / rCount);
          System.out.println("Total, avg num new best realizations: " + totalNewBest + ", " + avgNewBest);
          String avgLex = nf.format(lexMap.mean());
          String stdLex = nf.format(lexMap.sigma());
          System.out.println("Avg (std) time 'til lex lookup finished: " + avgLex + " (" + stdLex + ")");
          System.out.println("Max time 'til lex lookup finished: " + maxLex + " (" + maxLexStr + ")");
          String avgFirst = nf.format(firstMap.mean());
          String stdFirst = nf.format(firstMap.sigma());
          System.out.println("Avg (std) time 'til first realization: " + avgFirst + " (" + stdFirst + ")");
          System.out.println("Max time 'til first realization: " + maxFirst + " (" + maxFirstStr + ")");
          String avgBest = nf.format(bestMap.mean());
          String stdBest = nf.format(bestMap.sigma());
          System.out.println("Avg (std) time 'til best realization: " + avgBest + " (" + stdBest + ")");
          System.out.println("Max time 'til best realization: " + maxBest + " (" + maxBestStr +")");
          System.out.println("Max time 'til new best realization: " + maxNewBest + " (" + maxNewBestStr +")");
          String avgPacked = nf.format(totalPacked / rCount);
          System.out.println("Avg time 'til done packing: " + avgPacked);
          System.out.println("Max time 'til done packing: " + maxPacked + " (" + maxPackedStr +")");
          String avgStoppedOrDone = nf.format(allMap.mean());
          String stdStoppedOrDone = nf.format(allMap.sigma());
          System.out.println("Avg (std) time 'til stopped/done with realizations: " + avgStoppedOrDone + " (" + stdStoppedOrDone + ")");
          System.out.println("Max time 'til stopped/done with realizations: " + maxStoppedOrDone + " (" + maxStoppedOrDoneStr +")");
          if (oracleBetter > 0) System.out.println("Oracle better: " + oracleBetter);
          if (goldMissing > 0) System.out.println("Gold missing: " + goldMissing);
          if (statsfile != null) {
              Document doc = new Document();
              Element root = new Element("rstats");
              doc.setRootElement(root);
              Element counts = new Element("counts");
              root.addContent(counts);
              counts.setAttribute("realized", "" + rCount);
              counts.setAttribute("complete", "" + rComplete);
              counts.setAttribute("fragments", "" + rBadCount);
              counts.setAttribute("exact", "" + rExactCount);
              counts.setAttribute("inexact", rInexact);
              counts.setAttribute("finished", "" + rDoneCount);
              Element overall = new Element("overall");
              root.addContent(overall);
              overall.setAttribute("avg-score", avgScore);
              overall.setAttribute("mean-reciprocal-rank", meanReciprocalRank);
              overall.setAttribute("residual-mrr", residualMRR);
              overall.setAttribute("avg-nodes", avgNodes);
              overall.setAttribute("avg-words", avgTokens);
              overall.setAttribute("min-words", "" + minTokens);
              overall.setAttribute("max-words", "" + maxTokens);
              Element rules = new Element("rules");
              root.addContent(rules);
              rules.setAttribute("avg-apps", avgRuleApps);
              Element edges = new Element("edges");
              root.addContent(edges);
              edges.setAttribute("avg", avgEdges);
              edges.setAttribute("avg-unpruned", avgUnprunedEdges);
              edges.setAttribute("avg-removed", avgRemoved);
              edges.setAttribute("avg-never-added", avgNeverAdded);
              edges.setAttribute("avg-cell-max", avgCellMax);
              if (oracleBetter > 0) edges.setAttribute("oracle-better", "" + oracleBetter);
              if (goldMissing > 0) edges.setAttribute("gold-missing", "" + goldMissing);
              Element newBest = new Element("new-best");
              root.addContent(newBest);
              newBest.setAttribute("total", "" + totalNewBest);
              newBest.setAttribute("avg", avgNewBest);
              Element times = new Element("times-summary");
              root.addContent(times);
              times.setAttribute("avg-lex", avgLex);
              times.setAttribute("std-lex", stdLex);
              times.setAttribute("avg-first", avgFirst);
              times.setAttribute("std-first", stdFirst);
              times.setAttribute("max-first", "" + maxFirst);
              times.setAttribute("avg-best", avgBest);
              times.setAttribute("std-best", stdBest);
              times.setAttribute("max-best", "" + maxBest);
              times.setAttribute("max-new-best", "" + maxNewBest);
              times.setAttribute("avg-packed", avgPacked);
              times.setAttribute("max-packed", "" + maxPacked);
              times.setAttribute("avg-stopped-or-done", avgStoppedOrDone);
              times.setAttribute("std-stopped-or-done", stdStoppedOrDone);
              times.setAttribute("max-stopped-or-done", "" + maxStoppedOrDone);
              Element strings = new Element("max-strings");
              root.addContent(strings);
              Element lex = new Element("lex");
              strings.addContent(lex);
              lex.addContent(maxLexStr);
              Element first = new Element("first");
              strings.addContent(first);
              first.addContent(maxFirstStr);
              Element best = new Element("best");
              strings.addContent(best);
              best.addContent(maxBestStr);
              Element newBest2 = new Element("new-best");
              strings.addContent(newBest2);
              newBest2.addContent(maxNewBestStr);
              Element packed = new Element("packed");
              strings.addContent(packed);
              packed.addContent(maxPackedStr);
              Element stoppedOrDone = new Element("stopped-or-done");
              strings.addContent(stoppedOrDone);
              stoppedOrDone.addContent(maxStoppedOrDoneStr);
              Element scores = new Element("scores");
              root.addContent(scores);
              for (int i = 0; i < bestActualScores.size(); i++) {
                  Element score = new Element("score");
                  scores.addContent(score);
                  score.setAttribute("val", bestActualScores.get(i).toString());
                  score.setAttribute("est", bestEstimatedScores.get(i).toString());
                  score.setAttribute("rank", itemRanks.get(i).toString());
              }
              firstMap.saveTimes(root);
              bestMap.saveTimes(root);
              allMap.saveTimes(root);
              try {
                  FileOutputStream out = new FileOutputStream(statsfile);
                  grammar.serializeXml(doc, out);
                  out.flush();
              }
              catch (IOException exc) {
                  System.out.println("Unable to write stats to: " + statsfile + " (" + exc + ")");
              }
          }
      }
  
      // show outcome, with wrapping
      private static void showOutcome(String parseResult, String realizeResult, String starForBadSentence, String str) {
          showOutcome(parseResult, realizeResult, starForBadSentence, str, null);
      }
  
      // show outcome including best realization
      private static void showOutcome(String parseResult, String realizeResult, String starForBadSentence,
                                      String str, String bestRealization)
      {
          System.out.print(parseResult + "\t" + realizeResult + "\t");
          simpleWrap(starForBadSentence + str);
          if (bestRealization != null) {
              System.out.print("\t\t");
              simpleWrap("(best: " + bestRealization + ")");
          }
      }
  
      // does simple wrapping at TEXTWIDTH
      private static void simpleWrap(String str) {
          int TEXTWIDTH = 60;
          for (int i = 0; i <= (str.length()-1)/TEXTWIDTH; i++) {
              if (i != 0) {
                  System.out.print("\t\t");
              }
              System.out.println(str.substring(i*TEXTWIDTH, Math.min(i*TEXTWIDTH + TEXTWIDTH, str.length())));
          }
      }
  
      // formats to three decimal places
      private static final NumberFormat nf = initNF();
      private static NumberFormat initNF() {
          NumberFormat f = NumberFormat.getInstance();
          f.setMinimumIntegerDigits(1);
          f.setMinimumFractionDigits(1);
          f.setMaximumFractionDigits(2);
          return f;
      }
  
      // formats to three decimal places in scientific notation
      private static final NumberFormat nfE = initNFE();
      private static NumberFormat initNFE() {
      	DecimalFormat f = new DecimalFormat("0.###E0");
          return f;
      }
  
      /** Shows realizer settings for current test. */
      static void showRealizerSettings() {
          // get, show prefs
          Preferences prefs = Preferences.userNodeForPackage(TextCCG.class);
          boolean useIndexing = prefs.getBoolean(EdgeFactory.USE_INDEXING, true);
          boolean useChunks = prefs.getBoolean(EdgeFactory.USE_CHUNKS, true);
          boolean useLicensing = prefs.getBoolean(EdgeFactory.USE_FEATURE_LICENSING, true);
          boolean useCombos = prefs.getBoolean(opennlp.ccg.realize.Chart.USE_COMBOS, true);
          boolean usePacking = prefs.getBoolean(opennlp.ccg.realize.Chart.USE_PACKING, false);
          int timeLimit = prefs.getInt(opennlp.ccg.realize.Chart.TIME_LIMIT, opennlp.ccg.realize.Chart.NO_TIME_LIMIT);
          double nbTimeLimit = prefs.getDouble(opennlp.ccg.realize.Chart.NEW_BEST_TIME_LIMIT, opennlp.ccg.realize.Chart.NO_TIME_LIMIT);
          int pruningVal = prefs.getInt(opennlp.ccg.realize.Chart.PRUNING_VALUE, opennlp.ccg.realize.Chart.NO_PRUNING);
          int cellPruningVal = prefs.getInt(opennlp.ccg.realize.Chart.CELL_PRUNING_VALUE, opennlp.ccg.realize.Chart.NO_PRUNING);
          String msg = "Timing realization with index filtering " + ((useIndexing) ? "on" : "off") + ", ";
          msg += "chunks " + ((useChunks) ? "on" : "off") + ", ";
          msg += "licensing " + ((useLicensing) ? "on" : "off") + ", ";
          if (usePacking) msg += "packing on, ";
          else {
              msg += "combos " + ((useCombos) ? "on" : "off") + ", ";
              if (timeLimit == opennlp.ccg.realize.Chart.NO_TIME_LIMIT) msg += "no time limit, ";
              else msg += "a time limit of " + timeLimit + " ms, ";
              if (nbTimeLimit == opennlp.ccg.realize.Chart.NO_TIME_LIMIT) msg += "no new best time limit, ";
              else {
                  msg += "a new best time limit of ";
                  if (nbTimeLimit >= 1) msg += ((int)nbTimeLimit) + " ms, ";
                  else msg += nbTimeLimit + " of first, ";
              }
          }
          if (pruningVal == opennlp.ccg.realize.Chart.NO_PRUNING) msg += "no pruning, ";
          else msg += "a pruning value of " + pruningVal + ", ";
          msg += "and ";
          if (cellPruningVal == opennlp.ccg.realize.Chart.NO_PRUNING) msg += "no cell pruning";
          else msg += "a cell pruning value of " + cellPruningVal;
          System.out.println(msg);
          System.out.println();
      }
  
  
      /**
       * Writes the target strings from the given testbed to the given textfile.
       */
      public void writeTargets(File tbFile, String textfile) throws IOException {
          writeTargets(tbFile, textfile, false, false, false);
      }
  
      /**
       * Writes the target strings with semantic class replacement
       * from the given testbed to the given textfile.
       */
      public void writeTargetsSC(File tbFile, String textfile) throws IOException {
          writeTargets(tbFile, textfile, true, false, false);
      }
  
      /**
       * Writes the target strings with all associated factors
       * from the given testbed to the given textfile.
       */
      public void writeTargetsF(File tbFile, String textfile) throws IOException {
          writeTargets(tbFile, textfile, false, true, false);
      }
  
      /**
       * Writes the target strings with all associated factors with semantic class replacement
       * from the given testbed to the given textfile.
       */
      public void writeTargetsFSC(File tbFile, String textfile) throws IOException {
          writeTargets(tbFile, textfile, true, true, false);
      }
  
      // writes targets, optionally with sem class replacement or factors,
      // and optionally reversing the words; ungrammatical options are filtered out
      private void writeTargets(
      		File tbFile, String filename,
      		boolean semClassReplacement, boolean withFactors,
      		boolean reverse
      ) throws IOException {
          // open text file
          String option = "";
          if (withFactors) option = " with factors";
          if (semClassReplacement) option += " with semantic class replacement";
          if (reverse) option += ", reversed";
          System.out.println("Writing text file" + option + ": " + filename);
          System.out.println();
          PrintWriter tOut = new PrintWriter(new BufferedWriter(new FileWriter(filename)));
          HashSet<String> unique = new HashSet<String>();
          Tokenizer tokenizer = grammar.lexicon.tokenizer;
          // loop through files
          for (File f : getXMLFiles(tbFile)) {
  	        // load testbed
  	        System.out.println("Loading testbed from: " + f);
  	        RegressionInfo tbInfo = new RegressionInfo(grammar, f);
  	        int numItems = tbInfo.numberOfItems();
  	        // do each test item
  	        for (int i = 0; i < numItems; i++) {
  	            // check even/odd only
  	            if (i % 2 == 1 && evenOnly) continue;
  	            if (i % 2 == 0 && oddOnly) continue;
  	            RegressionInfo.TestItem testItem = tbInfo.getItem(i);
  	        	// check grammatical
  	        	if (testItem.numOfParses == 0) continue;
  	            String s = testItem.sentence;
  	            // get parsed words if doing more than just text
  	            List<Word> words = null;
  	            if (semClassReplacement || withFactors) {
  	                // use words from sign or pre-parsed full words if available
  	            	if (testItem.sign != null)
  	            		words = testItem.sign.getWords();
  	            	else if (testItem.fullWords != null)
  	                    words = tokenizer.tokenize(testItem.fullWords, true);
  	                // otherwise parse
  	                else words = grammar.getParsedWords(s);
  	            }
  	            else words = tokenizer.tokenize(s);
  	            // reverse, if apropos
  	            if (reverse) {
  	                List<Word> tmp = words;
  	                words = new ArrayList<Word>(words.size());
  	                words.add(Word.createWord("<s>"));
  	                for (int j = tmp.size()-1; j >= 0; j--) {
  	                    Word w = tmp.get(j);
  	                    if (w.getForm() == "<s>" || w.getForm() == "</s>") continue; // skip <s> or </s>
  	                    words.add(w);
  	                }
  	                words.add(Word.createWord("</s>"));
  	            }
  	            // write str, add to unique set
  	            String str = (!withFactors)
  	                ? tokenizer.getOrthography(words, semClassReplacement)
  	                : tokenizer.format(words, semClassReplacement);
  	            tOut.println(str);
  	            unique.add(str);
  	            System.out.print("."); // indicate progress
  	        }
  	        System.out.println();
          }
          tOut.flush(); tOut.close();
          System.out.println();
          System.out.println("Unique strings: " + unique.size());
          System.out.println();
      }
  
      private void writeDerivationFactors(File tbFile, String filename) throws IOException {
          // open text file
          System.out.println("Writing derivation factors file: " + filename);
          System.out.println();
          PrintWriter tOut = new PrintWriter(new BufferedWriter(new FileWriter(filename)));
          Tokenizer tokenizer = grammar.lexicon.tokenizer;
          // loop through files
          for (File f : getXMLFiles(tbFile)) {
  	        // load testbed
  	        System.out.println("Loading testbed from: " + f);
  	        RegressionInfo tbInfo = new RegressionInfo(grammar, f);
  	        int numItems = tbInfo.numberOfItems();
  	        // do each test item, using the saved sign
  	        for (int i = 0; i < numItems; i++) {
  	            RegressionInfo.TestItem testItem = tbInfo.getItem(i);
  	        	if (testItem.numOfParses == 0) continue; // check grammatical
  	        	Sign sign = testItem.sign;
  	        	List<Word> factors = GenerativeSyntacticModel.getFactors(sign);
  	        	for (Word w : factors) {
  	        		tOut.print(tokenizer.format(w));
  	        		tOut.print(" ");
  	        	}
  	        	tOut.println();
  	            System.out.print("."); // indicate progress
  	        }
  	        System.out.println();
          }
          tOut.flush(); tOut.close();
          System.out.println();
      }
  
  
      /** Command-line routine for regression testing. */
      public static void main(String[] args) throws IOException {
  
          String usage = "java opennlp.ccg.test.Regression \n" +
                         "  (-noparsing) (-norealization) (-even|-odd) (-gc) \n" +
                         "  (-nullscorer) (-randomscorer) \n" +
                         "  (-depthfirst) (-exactmatches) (-aanfilter (<excfile>)) \n" +
                         "  (-scorer <scorerclass>) \n" +
                         "  (-parsescorer <scorerclass>) \n" +
                         "  (-extractor <extractorclass>) \n" +
                         "  (-ngrampruningstrategy) (-pruningstrategy <pruningstrategyclass>) \n" +
                         "  (-hypertagger <hypertaggerclass> | -htconfig <configfile>) (-htgold) \n" +
                         "  (-supertagger <supertaggerclass> | -stconfig <configfile>) \n" +
                         "  (-ngramorder N) (-lm|-lmsc <lmfile>) \n" +
                         "  (-srilm " + Arrays.toString(SRILMNgramModelType.values()) + ")\n"+
                         "  (-flm|-flmsc <flmfile>) \n" +
                         "  (-text|-textsc|-textf|-textfsc <textfile>) (-reverse) \n" +
                         "  (-derivf <derivfactorsfile>) \n" +
                         "  (-2events <eventfile>) (-includegoldinevents) \n" +
                         "  (-2apml <apmldir>) (-bleu <bleufileprefix>) \n" +
                         "  (-nbestrealfile <nbestrealfile>) (-nbestnormbleu) (-realserdir <realserdir>) \n" +
                         "  (-nbestincludelfs) \n" +
                         "  (-rescorefile <rescorefile>) \n" +
                         "  (-nbestparsefile <nbestparsefile>) \n" +
                         "  (-g <grammarfile>) (-s <statsfile>) (<regressionfile>|<regressiondir>)";
  
          if (args.length > 0 && args[0].equals("-h")) {
              System.out.println("Usage: \n\n" + usage);
              System.exit(0);
          }
  
          // setup Regression tester
          Regression tester = new Regression();
  
          // args
          String grammarfile = "grammar.xml";
          String regressionfile = "testbed.xml";
          boolean depthFirst = false;
          boolean aanfilter = false;
          String excfile = null;
          String scorerClass = null;
          String parseScorerClass = null;
          String extractorClass = null;
          boolean ngrampruningstrategy = false;
          String pruningStrategyClass = null;
          String hypertaggerClass = null, htconfig = null;
          boolean htgold = false;
          String supertaggerClass = null, stconfig = null;
          String lmfile = null;
          String flmfile = null;
          boolean useSemClasses = false;
          boolean withFactors = false;
          boolean reverse = false;
          String textfile = null;
          String derivfactorsfile = null;
          boolean srilm = false;
          SRILMNgramModelType srilmModelType = SRILMNgramModelType.STANDARD;
  
          for (int i = 0; i < args.length; i++) {
          	if (args[i].startsWith("-D")) {
          		String prop = args[i].substring(2); int equalpos = prop.indexOf("=");
          		String key = prop.substring(0, equalpos); String val = prop.substring(equalpos+1);
          		System.setProperty(key, val); continue;
          	}
              if (args[i].equals("-noparsing")) { tester.doParsing = false; continue; }
              if (args[i].equals("-norealization")) { tester.doRealization = false; continue; }
              if (args[i].equals("-even")) { tester.evenOnly = true; continue; }
              if (args[i].equals("-odd")) { tester.oddOnly = true; continue; }
              if (args[i].equals("-gc")) { tester.doGC = true; continue; }
              if (args[i].equals("-nullscorer")) {
              	tester.scorer = SignScorer.nullScorer; tester.parseScorer = SignScorer.nullScorer; continue;
              }
              if (args[i].equals("-randomscorer")) {
              	tester.scorer = SignScorer.randomScorer; tester.parseScorer = SignScorer.randomScorer; continue;
              }
              if (args[i].equals("-depthfirst")) { depthFirst = true; continue; }
              if (args[i].equals("-exactmatches")) { tester.exactMatches = true; continue; }
  