gt: 							if (!charBuffer.hasRemaining()) {
context: /*
   * jETeL/CloverETL - Java based ETL application framework.
   * Copyright (c) Javlin, a.s. (info@cloveretl.com)
   *  
   * This library is free software; you can redistribute it and/or
   * modify it under the terms of the GNU Lesser General Public
   * License as published by the Free Software Foundation; either
   * version 2.1 of the License, or (at your option) any later version.
   *
   * This library is distributed in the hope that it will be useful,
   * but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   * Lesser General Public License for more details.
   *
   * You should have received a copy of the GNU Lesser General Public
   * License along with this library; if not, write to the Free Software
   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
   */
  package org.jetel.data.parser;
  
  import java.io.IOException;
  import java.nio.ByteBuffer;
  import java.nio.CharBuffer;
  import java.nio.InvalidMarkException;
  import java.nio.channels.FileChannel;
  import java.nio.channels.ReadableByteChannel;
  import java.nio.charset.Charset;
  import java.nio.charset.CharsetDecoder;
  import java.nio.charset.CoderResult;
  
  import javax.naming.OperationNotSupportedException;
  
  import org.jetel.data.Defaults;
  import org.jetel.data.parser.AbstractTextParser.CharsetDecoderException;
  import org.jetel.exception.JetelRuntimeException;
  import org.jetel.exception.PolicyType;
  import org.jetel.metadata.DataFieldMetadata;
  import org.jetel.metadata.DataRecordMetadata;
  import org.jetel.util.bytes.ByteBufferUtils;
  import org.jetel.util.bytes.CloverBuffer;
  
  /**
   * An abstract class for input readers able to provide mixed char/byte data
   * 
   * @author jhadrava (info@cloveretl.com) (c) Javlin, a.s. (www.cloveretl.com)
   * 
   * @created Dec 7, 2010
   */
  public abstract class CharByteInputReader implements ICharByteInputReader {
  
  	/** minimal size of buffer operation (reading, decoding, ..) */
  	protected static final int MIN_BUFFER_OPERATION_SIZE = 512;
  
  	protected ReadableByteChannel channel;
  	protected int inputBytesConsumed;
  
  	public CharByteInputReader() {
  		channel = null;
  		inputBytesConsumed = 0;
  	}
  
  	@Override
  	public int readChar() throws IOException, OperationNotSupportedException {
  		throw new OperationNotSupportedException("Input reader doesn't support readChar() operation. Choose another implementation");
  	}
  
  	@Override
  	public int readByte() throws IOException, OperationNotSupportedException {
  		throw new OperationNotSupportedException("Input reader doesn't support readByte() operation. Choose another implementation");
  	}
  
  	@Override
  	public int skipBytes(int num) {
  		throw new UnsupportedOperationException("Input reader doesn't support skipBytes() operation. Choose another implementation");
  	}
  
  	@Override
  	public int skipChars(int num) {
  		throw new UnsupportedOperationException("Input reader doesn't support skipChars() operation. Choose another implementation");
  	}
  
  	@Override
  	public void mark() throws OperationNotSupportedException {
  		throw new OperationNotSupportedException("Input reader doesn't support mark() operation. Choose another implementation");
  	}
  	@Override
  	public void revert() throws OperationNotSupportedException, InvalidMarkException {
  		throw new OperationNotSupportedException("Input reader doesn't support revert() operation. Choose another implementation");
  	}
  
  	@Override
  	public CharSequence getCharSequence(int relativeEnd) throws OperationNotSupportedException, InvalidMarkException {
  		throw new OperationNotSupportedException("Input reader doesn't support getCharSequence() operation. Choose another implementation");
  	}
  
  	@Override
  	public CloverBuffer getByteSequence(int relativeEnd) throws OperationNotSupportedException, InvalidMarkException {
  		throw new OperationNotSupportedException("Input reader doesn't support getByteSequence() operation. Choose another implementation");
  	}
  
  	@Override
  	public void setInputSource(ReadableByteChannel channel) {
  		this.channel = channel;
  	}
  	
  	/** used by DoubleMarkCharByteInputReader */
  	protected abstract void setMark(int mark);
  
  	/** used by DoubleMarkCharByteInputReader */
  	protected abstract int getMark();
  
  	@Override
  	public int getPosition() {
  		return inputBytesConsumed;
  	}
  
  	@Override
  	public void setPosition(int position) throws OperationNotSupportedException, IOException {
  		if (inputBytesConsumed != 0) {
  			throw new OperationNotSupportedException("setPosition() must be called before reading starts");
  		}
  		if (position < 0) {
  			throw new OperationNotSupportedException("Illegal position: " + position);
  		}
  		if (channel instanceof FileChannel) {
  			((FileChannel)channel).position(position);
  		} else {
  			ByteBuffer buf = ByteBuffer.allocateDirect(Defaults.Record.RECORD_INITIAL_SIZE);
  			int bytesRemaining = position;
  			while (bytesRemaining > 0) {
  				buf.clear();
  				if (buf.remaining() > bytesRemaining) {
  					buf.limit(bytesRemaining);
  				}
  				int n = channel.read(buf);
  				if (n == -1) {
  					bytesRemaining = 0;
  					break;  // setting position greater than input size is legal but next reading operation will produce end-of-input result
  				}
  				if (n == 0) {
  					throw new OperationNotSupportedException("Asynchronous input source not support");
  				}
  				bytesRemaining -= n;
  			}
  			assert bytesRemaining == 0 : "Unexpected internal state occured during code execution";
  		}
  		inputBytesConsumed = position;				
  	}
  	
  	/**
  	 * Creates input reader suitable for parsing specified metadata
  	 * @param metadata
  	 * @param charset
  	 * @param needByteInput
  	 * @param needCharInput
  	 * @return
  	 */
  	public static CharByteInputReader createInputReader(DataRecordMetadata metadata, Charset charset, boolean needByteInput, boolean needCharInput, PolicyType policyType) {
  		return createInputReader(new DataRecordMetadata[]{metadata}, charset, needByteInput, needCharInput, policyType);
  	}
  	
  	/**
  	 * Creates input reader suitable for parsing specified metadata
  	 * @param metadataArray
  	 * @param charset
  	 * @param needByteInput
  	 * @param needCharInput
  	 * @return
  	 */
  	public static CharByteInputReader createInputReader(DataRecordMetadata[] metadataArray, Charset charset, boolean needByteInput, boolean needCharInput, PolicyType policyType) {
  		int maxBackShift = 0;
  
  		for (DataRecordMetadata metadata : metadataArray) {
  			if(metadata == null) {
  				continue;
  			}
  			
  			int lastNonAutoFilledField;
  			for (lastNonAutoFilledField = metadata.getNumFields() - 1; lastNonAutoFilledField >= 0; lastNonAutoFilledField--) {
  				if (!metadata.getField(lastNonAutoFilledField).isAutoFilled()) {
  					break;
  				}
  			}
  
  			for (DataFieldMetadata field : metadata.getFields()) {
  				if (field.isAutoFilled()) {
  					continue;
  				}
  				if (field.isByteBased()) {
  					needByteInput = true;
  				} else {
  					needCharInput = true;
  				}
  				maxBackShift = Math.max(maxBackShift, -field.getShift());				
  			}
  
  			if (metadata.isSpecifiedRecordDelimiter() && !metadata.getField(lastNonAutoFilledField).isDelimited()) {
  				needCharInput = true;
  			}
  		}
  
  		CharByteInputReader reader;
  
  		if (!needCharInput) {
  			reader = new CharByteInputReader.ByteInputReader(maxBackShift);
  		} else if (!needByteInput) {
  			reader = new CharByteInputReader.CharInputReader(charset, maxBackShift, policyType);
  		} else if (TextParserConfiguration.isSingleByteCharset(charset)) {
  			reader = new CharByteInputReader.SingleByteCharsetInputReader(charset, maxBackShift, policyType);
  		} else {
  			reader = new CharByteInputReader.RobustInputReader(charset, maxBackShift, policyType);
  		}
  		
  		return reader;
  	}
  
  	/**
  	 * This input reader can be used only for data inputs which don't contain any delimiters or char-based fields
  	 * 
  	 * @author jhadrava (info@cloveretl.com) (c) Javlin, a.s. (www.cloveretl.com)
  	 * 
  	 * @created Dec 7, 2010
  	 */
  	public static class ByteInputReader extends CharByteInputReader {
  		private CloverBuffer byteBuffer;
  		private int currentMark;
  		private boolean endOfInput;
  		private int maxBackMark;
  
  		/**
  		 * Sole constructor
  		 * @param maxBackMark Max span for backward skip.
  		 */
  		public ByteInputReader(int maxBackMark) {
  			super();
  			byteBuffer = CloverBuffer.allocate(Defaults.Record.RECORD_INITIAL_SIZE, Defaults.Record.RECORD_LIMIT_SIZE);
  			channel = null;
  			currentMark = INVALID_MARK;
  			endOfInput = false;
  			this.maxBackMark = maxBackMark;
  		}
  
  		@Override
  		public int readByte() throws IOException, OperationNotSupportedException {
  			if (!byteBuffer.hasRemaining()) { // read data from input
  				if (endOfInput) {
  					return END_OF_INPUT;
  				}
  				// byteBuffer gets ready to receive data
  				int numBytesToPreserve;
  				int markSpan;
  				if (currentMark == INVALID_MARK) {
  					markSpan = 0;
  				} else {
  					markSpan = byteBuffer.position() - currentMark;
  				}
  				numBytesToPreserve = Math.min(byteBuffer.position(), Math.max(markSpan, maxBackMark));
  				if (byteBuffer.capacity() - numBytesToPreserve < MIN_BUFFER_OPERATION_SIZE) {
  					byteBuffer.limit(numBytesToPreserve + MIN_BUFFER_OPERATION_SIZE);
  					if (byteBuffer.capacity() - numBytesToPreserve < MIN_BUFFER_OPERATION_SIZE) {
  						return BLOCKED_BY_MARK; // there's not enough space for buffer operations due to the span of the
  												// mark
  					}
  				}
  				// preserve data between mark and current position
  				byteBuffer.position(byteBuffer.position() - numBytesToPreserve);
  				byteBuffer.compact();
  				currentMark = numBytesToPreserve - markSpan;
  
  				byteBuffer.limit(byteBuffer.capacity()).position(numBytesToPreserve);
  				int bytesConsumed = channel.read(byteBuffer.buf());
  				byteBuffer.flip().position(numBytesToPreserve); // get ready to provide data
  				switch (bytesConsumed) {
  				case 0:
  					throw new OperationNotSupportedException("Non-blocking input source not supported by the selected input reader. Choose another implementation");
  				case -1: // end of input
  					endOfInput = true;
  					return END_OF_INPUT;
  				}
  			}
  			return byteBuffer.get();
  		}
  
  		@Override
  		public int skipBytes(int numBytes) {
  			if (numBytes == 0) {
  				return 0;
  			}
  			if (numBytes > 0) {
  				int idx;
  				for (idx = 0; idx < numBytes; idx++) {
  					try {
  						readByte();
  					} catch (IOException e) {
  						break;
  					} catch (OperationNotSupportedException e) {
  						assert false : "Unexpected execution flow";						
  					}
  				}
  				return idx;
  			}			
  			int pos = byteBuffer.position();
  			if (-numBytes > pos) {
  				byteBuffer.position(0);
  				return -pos;
  			}
  			byteBuffer.position(pos + numBytes);
  			return numBytes;
  		}
  		
  		@Override
  		public boolean isEndOfInput() {
  			// CLO-5610: return false if there are remaining bytes in the buffer
  			return endOfInput && !byteBuffer.hasRemaining();
  		}
  		
  		@Override
  		public void mark() throws OperationNotSupportedException {
  			currentMark = byteBuffer.position();
  		}
  
  		@Override
  		public void revert() throws OperationNotSupportedException {
  			if (currentMark == INVALID_MARK) {
  				throw new InvalidMarkException();
  			}
  			byteBuffer.position(currentMark);
  			currentMark = INVALID_MARK;
  		}
  
  		@Override
  		public CloverBuffer getByteSequence(int relativeEnd) throws OperationNotSupportedException, InvalidMarkException {
  			if (currentMark == INVALID_MARK) {
  				throw new InvalidMarkException();
  			}
  			int pos = byteBuffer.position();
  			byteBuffer.position(currentMark);
  			CloverBuffer seq = byteBuffer.slice();
  			seq.limit(pos + relativeEnd - currentMark); // set the end of the sequence
  			byteBuffer.position(pos); // restore original position
  			currentMark = INVALID_MARK;
  			return seq;
  		}
  
  		@Override
  		public void setInputSource(ReadableByteChannel channel) {
  			super.setInputSource(channel);
  			byteBuffer.clear().flip(); // make it appear empty, so that it is clear we need to fill it
  			currentMark = INVALID_MARK;
  			endOfInput = false;
  		}
  
  		@Override
  		protected void setMark(int mark) {
  			currentMark = mark;
  		}
  
  		@Override
  		protected int getMark() {
  			return currentMark;
  		}
  
  		@Override
  		public Charset getCharset() {
  			return null;
  		}
  
  	}
  
  	/**
  	 * This input reader can be used only for data inputs which don't contain any byte-based fields
  	 * 
  	 * @author jhadrava (info@cloveretl.com) (c) Javlin, a.s. (www.cloveretl.com)
  	 * 
  	 * @created Dec 7, 2010
  	 */
  	public static class CharInputReader extends CharByteInputReader {
  		private Charset charset;
  		private ByteBuffer byteBuffer;
  		private CharBuffer charBuffer;
  		private CharsetDecoder decoder;
  		private int currentMark;
  		private boolean endOfInput;
  		private int maxBackMark;
  
  		/**
  		 * Sole constructor
  		 * @param charset Input charset
  		 * @param maxBackMark Max span for backward skip.
  		 */
  		public CharInputReader(Charset charset, int maxBackMark, PolicyType policyType) {
  			super();
  			byteBuffer = ByteBuffer.allocateDirect(Defaults.Record.RECORD_INITIAL_SIZE);
  			charBuffer = CharBuffer.allocate(Defaults.Record.RECORD_INITIAL_SIZE + MIN_BUFFER_OPERATION_SIZE);
  			channel = null;
  			this.charset = charset;
  			if (charset == null) {
  				charset = Charset.forName(Defaults.DataParser.DEFAULT_CHARSET_DECODER);
  			}
  			decoder = charset.newDecoder();
  			AbstractTextParser.applyDecoderPolicy(decoder, policyType);
  			currentMark = INVALID_MARK;
  			endOfInput = false;
  			this.maxBackMark = maxBackMark;
  		}
  
  		@Override
  		public int readChar() throws IOException, OperationNotSupportedException {
  			// simple cases first
  			if (charBuffer.hasRemaining()) {
  				return charBuffer.get();
  			}
  			if (endOfInput) {
  				return END_OF_INPUT;
  			}
  
  			// need to get more data in charBuffer
  
  			// charBuffer gets ready to receive data
  			int numCharsToPreserve;
  			int markSpan;
  			if (currentMark == INVALID_MARK) {
  				markSpan = 0;
  			} else {
  				markSpan = charBuffer.position() - currentMark;
  			}
  			numCharsToPreserve = Math.min(charBuffer.position(), Math.max(markSpan, maxBackMark));
  			if (charBuffer.capacity() - numCharsToPreserve < MIN_BUFFER_OPERATION_SIZE) {
  				charBuffer = ByteBufferUtils.expandCharBuffer(charBuffer, numCharsToPreserve + MIN_BUFFER_OPERATION_SIZE, Defaults.Record.RECORD_LIMIT_SIZE);
  				if (charBuffer.capacity() - numCharsToPreserve < MIN_BUFFER_OPERATION_SIZE) {
  					return BLOCKED_BY_MARK; // there's not enough space for buffer operations due to the span of the
  										    // mark
  				}
  			}
  			// preserve data between mark and current position
  			charBuffer.position(charBuffer.position() - numCharsToPreserve);
  			charBuffer.compact();
  			currentMark = numCharsToPreserve - markSpan;
  			do {
  				charBuffer.limit(charBuffer.capacity()).position(numCharsToPreserve); // get ready to receive data
  				checkDecoderResult(decoder.decode(byteBuffer, charBuffer, false));
  				charBuffer.flip().position(numCharsToPreserve); // get ready to provide data
  				if (!charBuffer.hasRemaining()) { // need to read more data from input
  					byteBuffer.compact(); // get ready to receive data
  					int bytesConsumed = channel.read(byteBuffer);
  					byteBuffer.flip(); // get ready to provide data
  					switch (bytesConsumed) {
  					case 0:
  						throw new OperationNotSupportedException("Non-blocking input source not supported by the selected input reader. Choose another implementation");
  					case -1: // end of input
  						charBuffer.limit(charBuffer.capacity()).position(numCharsToPreserve); // get ready to receive
  																								// data
  						checkDecoderResult(decoder.decode(byteBuffer, charBuffer, true));
  						charBuffer.flip().position(numCharsToPreserve); // get ready to provide data
  						if (!charBuffer.hasRemaining()) {
  							charBuffer.limit(charBuffer.capacity()).position(numCharsToPreserve); // get ready to
  																									// receive data
  							checkDecoderResult(decoder.flush(charBuffer));
  							charBuffer.flip().position(numCharsToPreserve);
  