Bug 13907 Scanner does not report whitespace tokens at end of input
The Scanner does not report the whitespace at the end of its input. The whitespace is added to the EOF token. If I explicitely ask for whitespace this behavior seems to be wrong (comments are reported correctly). Run the following testcase: import org.eclipse.jdt.core.compiler.InvalidInputException; import org.eclipse.jdt.internal.compiler.parser.Scanner; import org.eclipse.jdt.core.compiler.ITerminalSymbols; public class ScannerTest { public static void main(String[] args) { String input= "package com.ibm.itp.compare.ui; //foo \r\n"; int l= input.length(); Scanner scanner= new Scanner(true, true); // returns comments & whitespace char[] chars= new char[l]; input.getChars(0, l, chars, 0); scanner.setSource(chars); try { for (;;) { int t= 0; switch (t= scanner.getNextToken()) { case ITerminalSymbols.TokenNameEOF: System.out.println("EOF"); return; case ITerminalSymbols.TokenNameWHITESPACE: System.out.println("WS"); break; case ITerminalSymbols.TokenNameCOMMENT_LINE: System.out.println("COMMENT"); break; case ITerminalSymbols.TokenNameSEMICOLON: System.out.println("SEMICOLON"); break; default: System.out.println("Token: " + t); break; } } } catch (InvalidInputException ex) { } } }
org.eclipse.jdt.ui/ui/org/eclipse/jdt/internal/ui/compare/JavaTokenComparator.java
*/
	public JavaTokenComparator(String text, boolean shouldEscape) {
		
		Assert.isNotNull(text);
		
		fText= text;
		fShouldEscape= shouldEscape;
		
		int length= fText.length();
		fStarts= new int[length];
		fLengths= new int[length];
		fCount= 0;
		
		Scanner scanner= new Scanner(true, true);	
		scanner.setSource(fText.toCharArray());
		try {
			while (scanner.getNextToken() != ITerminalSymbols.TokenNameEOF) {
				fStarts[fCount]= scanner.startPosition;
				fLengths[fCount]= scanner.currentPosition - fStarts[fCount];
				fCount++;
			}
		} catch (InvalidInputException ex) {
		}
	}	
	/**
	 * Returns the number of token in the string.
	 *
	 * @return number of token in the string
	 */
	public int getRangeCount() {