Bug 13907 Scanner does not report whitespace tokens at end of input
The Scanner does not report the whitespace at the end of its input. The whitespace is added to the EOF token. If I explicitely ask for whitespace this behavior seems to be wrong (comments are reported correctly). Run the following testcase: import org.eclipse.jdt.core.compiler.InvalidInputException; import org.eclipse.jdt.internal.compiler.parser.Scanner; import org.eclipse.jdt.core.compiler.ITerminalSymbols; public class ScannerTest { public static void main(String[] args) { String input= "package com.ibm.itp.compare.ui; //foo \r\n"; int l= input.length(); Scanner scanner= new Scanner(true, true); // returns comments & whitespace char[] chars= new char[l]; input.getChars(0, l, chars, 0); scanner.setSource(chars); try { for (;;) { int t= 0; switch (t= scanner.getNextToken()) { case ITerminalSymbols.TokenNameEOF: System.out.println("EOF"); return; case ITerminalSymbols.TokenNameWHITESPACE: System.out.println("WS"); break; case ITerminalSymbols.TokenNameCOMMENT_LINE: System.out.println("COMMENT"); break; case ITerminalSymbols.TokenNameSEMICOLON: System.out.println("SEMICOLON"); break; default: System.out.println("Token: " + t); break; } } } catch (InvalidInputException ex) { } } }
org.eclipse.jdt.ui/ui/org/eclipse/jdt/internal/ui/compare/JavaTokenComparator.java
return fCount;
	}
	/* (non Javadoc)
	 * see ITokenComparator.getTokenStart
	 */
	public int getTokenStart(int index) {
		if (index < fCount)
			return fStarts[index];
		return fText.length();
	}
	/* (non Javadoc)
	 * see ITokenComparator.getTokenLength
	 */
	public int getTokenLength(int index) {
		if (index < fCount)
			return fLengths[index];
		return 0;
	}
	
	/**
	 * Returns <code>true</code> if a token given by the first index
	 * matches a token specified by the other <code>IRangeComparator</code> and index.
	 *
	 * @param thisIndex	the number of the token within this range comparator
	 * @param other the range comparator to compare this with
	 * @param otherIndex the number of the token within the other comparator
	 * @return <code>true</code> if the token are equal
	 */
	public boolean rangesEqual(int thisIndex, IRangeComparator other, int otherIndex) {
		if (other != null && getClass() == other.getClass()) {