Bug 13907 Scanner does not report whitespace tokens at end of input
The Scanner does not report the whitespace at the end of its input. The whitespace is added to the EOF token. If I explicitely ask for whitespace this behavior seems to be wrong (comments are reported correctly). Run the following testcase: import org.eclipse.jdt.core.compiler.InvalidInputException; import org.eclipse.jdt.internal.compiler.parser.Scanner; import org.eclipse.jdt.core.compiler.ITerminalSymbols; public class ScannerTest { public static void main(String[] args) { String input= "package com.ibm.itp.compare.ui; //foo \r\n"; int l= input.length(); Scanner scanner= new Scanner(true, true); // returns comments & whitespace char[] chars= new char[l]; input.getChars(0, l, chars, 0); scanner.setSource(chars); try { for (;;) { int t= 0; switch (t= scanner.getNextToken()) { case ITerminalSymbols.TokenNameEOF: System.out.println("EOF"); return; case ITerminalSymbols.TokenNameWHITESPACE: System.out.println("WS"); break; case ITerminalSymbols.TokenNameCOMMENT_LINE: System.out.println("COMMENT"); break; case ITerminalSymbols.TokenNameSEMICOLON: System.out.println("SEMICOLON"); break; default: System.out.println("Token: " + t); break; } } } catch (InvalidInputException ex) { } } }
org.eclipse.jdt.ui/ui/org/eclipse/jdt/internal/ui/compare/JavaTokenComparator.java
/*
 * (c) Copyright IBM Corp. 2000, 2001.
 * All Rights Reserved.
 */
package org.eclipse.jdt.internal.ui.compare;
import org.eclipse.jface.util.Assert;
import org.eclipse.jdt.core.compiler.InvalidInputException;
import org.eclipse.jdt.internal.compiler.parser.Scanner;
import org.eclipse.jdt.core.compiler.ITerminalSymbols;
import org.eclipse.compare.contentmergeviewer.ITokenComparator;
import org.eclipse.compare.rangedifferencer.IRangeComparator;
/**
 * A comparator for Java tokens.
 */
class JavaTokenComparator implements ITokenComparator {
		
	private String fText;
	private boolean fShouldEscape= true;
	private int fCount;
	private int[] fStarts;
	private int[] fLengths;
	/**
	 * Creates a TokenComparator for the given string.