gt:     for (IN wi : doc) {
context: package edu.stanford.nlp.sequences;
  
  import edu.stanford.nlp.ling.CoreAnnotations;
  import edu.stanford.nlp.ling.CoreLabel;
  import edu.stanford.nlp.ling.CoreAnnotations.AnswerAnnotation;
  import edu.stanford.nlp.ling.CoreAnnotations.BeforeAnnotation;
  import edu.stanford.nlp.ling.CoreAnnotations.AfterAnnotation;
  import edu.stanford.nlp.ling.CoreAnnotations.PositionAnnotation;
  import edu.stanford.nlp.ling.CoreAnnotations.TextAnnotation;
  import edu.stanford.nlp.objectbank.TokenizerFactory;
  import edu.stanford.nlp.process.CoreLabelTokenFactory;
  import edu.stanford.nlp.process.CoreTokenFactory;
  import edu.stanford.nlp.process.PTBTokenizer;
  import edu.stanford.nlp.process.Tokenizer;
  import edu.stanford.nlp.process.WordToSentenceProcessor;
  import edu.stanford.nlp.util.CoreMap;
  import edu.stanford.nlp.util.StringUtils;
  import edu.stanford.nlp.util.XMLUtils;
  
  import java.io.*;
  import java.util.*;
  import java.util.regex.*;
  
  /**
   * This class provides methods for reading plain text documents and writing out
   * those documents once classified in several different formats.
   * <p>
   * <i>Implementation note:</i> see
   * itest/src/edu/stanford/nlp/ie/crf/CRFClassifierITest.java for examples and
   * test cases for the output options.
   *
   * It can be over anything that extends {@link CoreMap}, and the default is
   * {@link CoreLabel}
   *
   * @author Jenny Finkel
   * @author Christopher Manning (new output options organization)
   * @author Sonal Gupta (made the class generic)
   */
  public class PlainTextDocumentReaderAndWriter<IN extends CoreMap> implements DocumentReaderAndWriter<IN> {
  
    // todo: This is hardwired for PTBTokenizer, and hence languages roughly
    // like English. There should be flags which would allow it to be used with
    // other languages/tokenizers, perhaps instead involving changes in
    // CRFClassifier, etc.
  
    private static final long serialVersionUID = -2420535144980273136L;
  
    public enum OutputStyle {
      SLASH_TAGS    ("slashTags"),
      XML           ("xml"),
      INLINE_XML    ("inlineXML"),
      TSV           ("tsv");
  
      private final String shortName;
      OutputStyle(String shortName) {
        this.shortName = shortName;
      }
  
      private static final Map<String, OutputStyle> shortNames =
        new HashMap<String, OutputStyle>();
      static {
        for (OutputStyle style : OutputStyle.values())
          shortNames.put(style.shortName, style);
      }
  
      /*
       * Convert a String expressing an output format to its internal
       * coding as an OutputStyle.
       *
       * @param outputFormat
       *        The String
       * @return OutputStyle The internal constant
       */
      public static OutputStyle fromShortName(String name) {
        OutputStyle result = shortNames.get(name);
        if (result == null)
          throw new IllegalArgumentException(name + " is not an OutputStyle");
        return result;
      }
    }
  
    private static final Pattern sgml = Pattern.compile("<[^>]*>");
    private final WordToSentenceProcessor<IN> wts = new WordToSentenceProcessor<IN>();
  
    private SeqClassifierFlags flags; // = null;
    private TokenizerFactory<IN> tokenizerFactory;
    private CoreTokenFactory<IN> tokenFactory;
  
    /**
     * Construct a PlainTextDocumentReaderAndWriter. You should call init() after
     * using the constructor.
     */
    public PlainTextDocumentReaderAndWriter() {
    }
  
    public void init(SeqClassifierFlags flags) {
      String options = "tokenizeNLs=false,invertible=true";
      if (flags.tokenizerOptions != null) {
        options = options + "," + flags.tokenizerOptions;
      }
      init(flags, (TokenizerFactory<IN>) PTBTokenizer.PTBTokenizerFactory.newCoreLabelTokenizerFactory(options));
    }
  
    public void init(SeqClassifierFlags flags, TokenizerFactory<IN> tokenizerFactory) {
      if (flags.tokenFactory == null)
        tokenFactory = (CoreTokenFactory<IN>) new CoreLabelTokenFactory();
      else {
        try {
          this.tokenFactory = (CoreTokenFactory<IN>) Class.forName(flags.tokenFactory).newInstance();
        } catch (Exception e) {
          throw new RuntimeException(e);
        }
      }
      init(flags, tokenizerFactory, tokenFactory);
    }
  
    public void init(SeqClassifierFlags flags, TokenizerFactory<IN> tokenizerFactory, CoreTokenFactory<IN> tokenFactory) {
      this.flags = flags;
      this.tokenizerFactory = tokenizerFactory;
      this.tokenFactory = tokenFactory;
    }
  
    // todo: give options for document splitting. A line or the whole file or
    // sentence splitting as now
    public Iterator<List<IN>> getIterator(Reader r) {
      Tokenizer<IN> tokenizer = tokenizerFactory.getTokenizer(r);
      // PTBTokenizer.newPTBTokenizer(r, false, true);
      List<IN> words = new ArrayList<IN>();
      IN previous = tokenFactory.makeToken();
      StringBuilder prepend = new StringBuilder();
  
      /*
       * This changes SGML tags into whitespace -- it should maybe be moved
       * elsewhere
       */
      while (tokenizer.hasNext()) {
        IN w = tokenizer.next();
        String word = w.get(CoreAnnotations.TextAnnotation.class);
        Matcher m = sgml.matcher(word);
        if (m.matches()) {
  
          String before = StringUtils.getNotNullString(w.get(CoreAnnotations.BeforeAnnotation.class));
          String after = StringUtils.getNotNullString(w.get(CoreAnnotations.AfterAnnotation.class));
          prepend.append(before).append(word);
          String previousTokenAfter = StringUtils.getNotNullString(previous.get(CoreAnnotations.AfterAnnotation.class));
          previous.set(AfterAnnotation.class, previousTokenAfter + word + after);
          // previous.appendAfter(w.word() + w.after());
        } else {
  
          String before = StringUtils.getNotNullString(w.get(CoreAnnotations.BeforeAnnotation.class));
          if (prepend.length() > 0) {
            w.set(BeforeAnnotation.class, prepend.toString() + before);
            // w.prependBefore(prepend.toString());
            prepend = new StringBuilder();
          }
          words.add(w);
          previous = w;
        }
      }
  
      List<List<IN>> sentences = wts.process(words);
      String after = "";
      IN last = null;
      for (List<IN> sentence : sentences) {
        int pos = 0;
        for (IN w : sentence) {
          w.set(PositionAnnotation.class, Integer.toString(pos));
          after = StringUtils.getNotNullString(w.get(CoreAnnotations.AfterAnnotation.class));
          w.remove(AfterAnnotation.class);
          last = w;
        }
      }
      if (last != null) {
        last.set(AfterAnnotation.class, after);
      }
  
      return sentences.iterator();
    }
  
  
    /**
     * Print the classifications for the document to the given Writer. This method
     * now checks the <code>outputFormat</code> property, and can print in
     * slashTags, inlineXML, or xml (stand-Off XML). For both the XML output
     * formats, it preserves spacing, while for the slashTags format, it prints
     * tokenized (since preserveSpacing output is somewhat dysfunctional with the
     * slashTags format).
     *
     * @param list
     *          List of tokens with classifier answers
     * @param out
     *          Where to print the output to
     */
    public void printAnswers(List<IN> list, PrintWriter out) {
      String style = null;
      if (flags != null) {
        style = flags.outputFormat;
      }
      if (style == null || "".equals(style)) {
        style = "slashTags";
      }
      OutputStyle outputStyle = OutputStyle.fromShortName(style);
      printAnswers(list, out, outputStyle, !"slashTags".equals(style));
    }
  
    public String getAnswers(List<IN> l,
                             OutputStyle outputStyle, boolean preserveSpacing) {
      StringWriter sw = new StringWriter();
      PrintWriter pw = new PrintWriter(sw);
      printAnswers(l, pw, outputStyle, preserveSpacing);
      pw.flush();
      return sw.toString();
    }
  
    public void printAnswers(List<IN> l, PrintWriter out,
                             OutputStyle outputStyle, boolean preserveSpacing) {
      switch (outputStyle) {
      case SLASH_TAGS:
        if (preserveSpacing) {
          printAnswersAsIsText(l, out);
        } else {
          printAnswersTokenizedText(l, out);
        }
        break;
      case XML:
        if (preserveSpacing) {
          printAnswersXML(l, out);
        } else {
          printAnswersTokenizedXML(l, out);
        }
        break;
      case INLINE_XML:
        if (preserveSpacing) {
          printAnswersInlineXML(l, out);
        } else {
          printAnswersTokenizedInlineXML(l, out);
        }
        break;
      default:
        throw new IllegalArgumentException(outputStyle +
                                           " is an unsupported OutputStyle");
      }
    }
  
    private static <IN extends CoreMap> void printAnswersTokenizedText(List<IN> l, PrintWriter out) {
      for (IN wi : l) {
        out.print(StringUtils.getNotNullString(wi.get(TextAnnotation.class)));
        out.print('/');
        out.print(StringUtils.getNotNullString(wi.get(AnswerAnnotation.class)));
        out.print(' ');
      }
      out.println(); // put a single newline at the end [added 20091024].
    }
  
    private static <IN extends CoreMap> void printAnswersAsIsText(List<IN> l, PrintWriter out) {
      for (IN wi : l) {
        out.print(StringUtils.getNotNullString(wi.get(BeforeAnnotation.class)));
        out.print(StringUtils.getNotNullString(wi.get(TextAnnotation.class)));
        out.print('/');
        out.print(StringUtils.getNotNullString(wi.get(AnswerAnnotation.class)));
        out.print(StringUtils.getNotNullString(wi.get(AfterAnnotation.class)));
      }
    }
  
    private static <IN extends CoreMap> void printAnswersXML(List<IN> doc, PrintWriter out) {
      int num = 0;
  