class AnyCharTokenizer extends Tokenizer {

  private static final int MAX_WORD_LEN = Integer.MAX_VALUE; // extend the word length!

  private final CharacterUtils.CharacterBuffer ioBuffer = CharacterUtils.newCharacterBuffer(4096);
  private final CharacterUtils charUtils = CharacterUtils.getInstance();
  private final CharTermAttribute termAtt = this.addAttribute(CharTermAttribute.class);
  private final OffsetAttribute offsetAtt = this.addAttribute(OffsetAttribute.class);

  private int bufferIndex = 0;
  private int dataLen = 0;
  private int offset = 0;
  private int finalOffset = 0;
  
  /**
   * Construct a new AnyCharTokenizer.
   */
  public AnyCharTokenizer() {
    super();
  }

  /**
   * Construct a new AnyCharTokenizer using a given
   * {@link org.apache.lucene.util.AttributeFactory}.
   * @param factory the attribute factory to use for this {@link org.apache.lucene.analysis.Tokenizer}
   */
  public AnyCharTokenizer(AttributeFactory factory) {
    super(factory);
  }

  /**
   * Collects any characters.
   */
  protected boolean isTokenChar(int c) {
    return true;
  }

  protected int normalize(int c) {
    return c;
  }

  @Override
  public boolean incrementToken() throws IOException {<FILL_FUNCTION_BODY>}

  @Override
  public void end() throws IOException {
    super.end();
    this.offsetAtt.setOffset(this.finalOffset, this.finalOffset);
  }

  @Override
  public void reset() throws IOException {
    super.reset();
    this.bufferIndex = 0;
    this.offset = 0;
    this.dataLen = 0;
    this.finalOffset = 0;
    this.ioBuffer.reset();
  }

}

    this.clearAttributes();
    int length = 0;
    int start = -1;
    int end = -1;
    char[] buffer = this.termAtt.buffer();

    while(true) {
      if(this.bufferIndex >= this.dataLen) {
        this.offset += this.dataLen;
        this.charUtils.fill(this.ioBuffer, this.input);
        if(this.ioBuffer.getLength() == 0) {
          this.dataLen = 0;
          if(length <= 0) {
            this.finalOffset = this.correctOffset(this.offset);
            return false;
          }
          break;
        }

        this.dataLen = this.ioBuffer.getLength();
        this.bufferIndex = 0;
      }

      int c = this.charUtils.codePointAt(this.ioBuffer.getBuffer(), this.bufferIndex, this.ioBuffer.getLength());
      int charCount = Character.charCount(c);
      this.bufferIndex += charCount;
      if(this.isTokenChar(c)) {
        if(length == 0) {
          assert start == -1;

          start = this.offset + this.bufferIndex - charCount;
          end = start;
        } else if(length >= buffer.length - 1) {
          buffer = this.termAtt.resizeBuffer(2 + length);
        }

        end += charCount;
        length += Character.toChars(this.normalize(c), buffer, length);
        if(length >= MAX_WORD_LEN) {
          break;
        }
      } else if(length > 0) {
        break;
      }
    }

    this.termAtt.setLength(length);

    assert start != -1;

    this.offsetAtt.setOffset(this.correctOffset(start), this.finalOffset = this.correctOffset(end));
    return true;
  
<no_super_class>