gt: 				chunk.append ((char) c); // don't eat escape character
context: /* 
   * Copyright (C) 1997 Roger Whitney <whitney@cs.sdsu.edu>
   *
   * This file is part of the San Diego State University Java Library.
   *
   * This program is free software; you can redistribute it and/or modify
   * it under the terms of the GNU General Public License as published by
   * the Free Software Foundation; either version 2 of the License, or
   * (at your option) any later version.
   *
   * This program is distributed in the hope that it will be useful,
   * but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   * GNU General Public License for more details.
   *
   * You should have received a copy of the GNU General Public License
   * along with this program; if not, write to the Free Software
   * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
   */
   
  package sdsu.util;
  
  import java.io.StringBufferInputStream;
  import java.io.IOException;
  import java.io.InputStream;
  import java.io.OutputStream;
  import java.io.PrintStream;
  import java.io.PushbackInputStream;
  
  /**
   * This class performs some simple parsing of strings or streams. The input is a sequence of 
   * ascii characters. The sequence is divided into tokens, whitespace, and comments.
   * Comments start with the comment character and continue to the next newline (\n ) character.
   * Comments are removed from the input characters and not returned as part of a token.
   * A token is string from the current location to the next separator or whitespace character.
   * Characters defined as whitespace (tab, newline, and space default values ) help delineate tokens
   * but are not part of tokens. That is whitespace characters are removed after finding a token.
   * If a token must contain whitespace character, a possible separator, or comment character,  
   * the token must be placed between two quote characters.  A quoted token can 
   * contain a quote character.
   *
   * @see		Stringizer
   * @see		TokenCharacters
   * @version 1.0 2 February 1997 
   * @author Roger Whitney (<a href=mailto:whitney@cs.sdsu.edu>whitney@cs.sdsu.edu</a>)
   */
  
  public class SimpleTokenizer 
  	{
  	/*
  	 * Class invariants: 
  	 *		a call to nextToken always consumes at least one character
  	 *		tokenizer is always left at the next non-whitespace or non-comment
  	 */
  
  	private TokenCharacters charTable;
  	private char lastFoundSeparator;
  
  	/**
  	 * Inputstream that is the internal source for tokens
  	 */
  	private PushbackInputStream input;
  	
  	private static final int EOF = -1;
  
  	private boolean eatEscapeChar = true;
  	    
  	/**
  	 * Create a SimpleTokenizer on string with default settings
  	 */
  	public SimpleTokenizer( String parsable )
  		{
  		this( new StringBufferInputStream( parsable ) ) ;
  		}
  
  	/**
  	 * Create a SimpleTokenizer on string
  	 * @param	commentChar	character used to indicate start of a comment
  	 * @param	quoteChar	character used to quote a string containing special characters
  	 * @param	whitespace	characters used for whitespace. Use null or empty string for no whitespace characters
  	 */
  	public SimpleTokenizer( String parsable, TokenCharacters charTable )
  		{
  		this( new StringBufferInputStream( parsable ), charTable );
  		}
  
  	/**
  	 * Create a SimpleTokenizer on tokenSource with default settings
  	 */
  	public SimpleTokenizer( InputStream tokenSource )
  		{
  		this( tokenSource, new TokenCharacters()  );
  		}
  
  
  	/**
  	 * Create a SimpleTokenizer on tokenSource 
  	 * @param	commentChar	character used to indicate start of a comment
  	 * @param	beginQuoteChar	character used to start a quote of a string containing special characters
  	 * @param	endQuoteChar	character used to end a quote of a string containing special characters
  	 * @param	whitespace	characters used for whitespace. Use null or empty string 
  	 *         for no whitespace characters
  	 */
  	public SimpleTokenizer( InputStream tokenSource, TokenCharacters charTable )
  		{
  		input = new PushbackInputStream( tokenSource );
  		this.charTable = charTable;
  		
  		try
  			{
  			removeCommentAndWhitespace();
  			}
  		catch (IOException readError ) 
  			//abstraction does not require IO here, hide IO to allow changes
  			{}							
  		}
  
  	
  	/**
  	 * Returns true if not at end of source stream or source string
  	 */
  	 public boolean hasMoreTokens()
  	 	{
  	 	// Since whitespace has been removed, EOF is equivalent to no more tokens
  	 	try
  			{
  		 	int c = input.read();
  		 	if ( c == EOF )
  		 		return false;
  		 	else 
  		 		{
  		 		input.unread( c );
  		 		return true;
  		 		}
  		 	}
  		catch (IOException readError )
  			{
  			return false;
  			}
  
  	 	}
  
  	/**
  	 * Returns true if not at end of source stream or source string
  	 */
  	 public boolean hasMoreElements()
  	 	{
  	 	return hasMoreTokens();
  	 	}
  
  	/**
  	 * Returns the separator found by the last call to nextToken
  	 */
  	public char separator()
  		{
  		return lastFoundSeparator;
  		}
  
  	/**
  	 * Enable/Disable the eating of the escape character.
  	 */
  	public void setEatEscapeChar (boolean value)
  		{
  		eatEscapeChar = value;
  		}
  	    
  	/**
  	 * Returns string containing all characters up to the given separator, unquoted
  	 * whitespace, or EOF
  	 * if the separator is not found. The separator is removed from the stream, but not returned
  	 * as part of token.
  	 * @param separator can be any character except the current comment or quote character
  	 * @exception	IOException	If separator or EOF does not follow this token 
  	 */
  	public String nextToken( String  newSeparators ) throws IOException
  		{
  		charTable.setSeparatorChars( newSeparators );
  		return nextToken( );
  		}
  	 	
  	/**
  	 * Returns string containing all characters up to the given separator, unquoted
  	 * whitespace, or EOF
  	 * if the separator is not found. The separator is removed from the stream, but not returned
  	 * as part of token.
  	 * @param separator set of characters to be used as separator after token. Can be any non-null or 
  	 *  nonempty string of characters
  	 *  except the current comment or quote character 
  	 * @exception	IOException	If separator or EOF does not follow this token 
  	 */
  	public String nextToken( ) throws IOException
  	 	{
  	 	String token;
  	 	int c = input.read();
  	 	
  	 	//System.out.println( (char) c );
  	 	if ( charTable.isBeginQuote( (char) c ) )
  	 		token = getQuotedChunk( (char) c );
  	 	else
  	 		{
  	 		input.unread( c );
  	 		token = getNextChunk();
  	 		}
  		//System.out.println( "token " + token );
  		if ( !isSeparatorNext() )
  			throw new IOException( "Missing separator ");
  		
  		removeCommentAndWhitespace();	
  		return token;
  	 	}
  
  	/**
  	 * Returns next token. String terminateChars indicates end of token
  	 */
  	private String getNextChunk( ) throws IOException
  		{
  		
  		int c;
  		StringBuffer chunk = new StringBuffer();
  		
  		while ( ( c = input.read()) != EOF )
  			{
  			//System.out.println( "chunk " + (char) c );
  			
  			if ( charTable.isTokenTerminator( (char) c ) )
  				{
  				//System.out.println( "T Termin " + (char) c );
  				break;
  				}
  			chunk.append( (char) c );
  			}
  		
  		if ( c != EOF )
  			input.unread( c );
  		
  		return chunk.toString();
  		}
  
  	/**
  	 * Returns quoted token. Assumes leading comment char has been removed from stream
  	 */
  	private String getQuotedChunk( char beginQuoteChar ) throws IOException
  		{
  		int c;
  		StringBuffer chunk = new StringBuffer();
  		
  		while ( ( c = input.read()) != EOF )
  			{
  			if ( charTable.isQuotePair( beginQuoteChar, (char) c ) )
  				break;
  				
  			if ( charTable.isEscape( (char) c ) )
  			{
  			    if (! eatEscapeChar)
  