Bug 13907 Scanner does not report whitespace tokens at end of input
The Scanner does not report the whitespace at the end of its input. The whitespace is added to the EOF token. If I explicitely ask for whitespace this behavior seems to be wrong (comments are reported correctly). Run the following testcase: import org.eclipse.jdt.core.compiler.InvalidInputException; import org.eclipse.jdt.internal.compiler.parser.Scanner; import org.eclipse.jdt.core.compiler.ITerminalSymbols; public class ScannerTest { public static void main(String[] args) { String input= "package com.ibm.itp.compare.ui; //foo \r\n"; int l= input.length(); Scanner scanner= new Scanner(true, true); // returns comments & whitespace char[] chars= new char[l]; input.getChars(0, l, chars, 0); scanner.setSource(chars); try { for (;;) { int t= 0; switch (t= scanner.getNextToken()) { case ITerminalSymbols.TokenNameEOF: System.out.println("EOF"); return; case ITerminalSymbols.TokenNameWHITESPACE: System.out.println("WS"); break; case ITerminalSymbols.TokenNameCOMMENT_LINE: System.out.println("COMMENT"); break; case ITerminalSymbols.TokenNameSEMICOLON: System.out.println("SEMICOLON"); break; default: System.out.println("Token: " + t); break; } } } catch (InvalidInputException ex) { } } }
org.eclipse.jdt.ui/ui/org/eclipse/jdt/internal/ui/compare/JavaTokenComparator.java
JavaTokenComparator tc= (JavaTokenComparator) other;	
			int thisLen= getTokenLength(thisIndex);
			int otherLen= tc.getTokenLength(otherIndex);
			if (thisLen == otherLen)
				return fText.regionMatches(false, getTokenStart(thisIndex), tc.fText, tc.getTokenStart(otherIndex), thisLen);
		}
		return false;
	}
	/**
	 * Aborts the comparison if the number of tokens is too large.
	 *
	 * @return <code>true</code> to abort a token comparison
	 */
	public boolean skipRangeComparison(int length, int max, IRangeComparator other) {
		if (!fShouldEscape)
			return false;
		if (getRangeCount() < 50 || other.getRangeCount() < 50)
			return false;
		if (max < 100)
			return false;
		if (length < 100)
			return false;
		if (max > 800)
			return true;
		if (length < max / 4)
			return false;
		return true;
	}
}