test_class.declarator: tokenizedToken = new HashSet<TestToken>()
test_class.modifier: private
test_class.original_string: private Set<TestToken> tokenizedToken = new HashSet<TestToken>();
test_class.type: Set<TestToken>
test_class.var_name: tokenizedToken
test_class.declarator: content = new StringReader("삼성전자absc1234엠피3mp3버전1.2  띄어쓰기");
test_class.modifier: private
test_class.original_string: private StringReader content = new StringReader("삼성전자absc1234엠피3mp3버전1.2  띄어쓰기");
test_class.type: StringReader
test_class.var_name: content
test_class.declarator: tokenizer = new KoreanCharacterTokenizer(content)
test_class.modifier: private
test_class.original_string: private KoreanCharacterTokenizer tokenizer = new KoreanCharacterTokenizer(content);
test_class.type: KoreanCharacterTokenizer
test_class.var_name: tokenizer
test_class.file: korean-analyzer-4.x/src/test/java/com/tistory/devyongsik/analyzer/KoreanCharacterTokenizerTest.java
test_class.identifier: KoreanCharacterTokenizerTest
test_class.superclass: extends AnalyzerTestUtil
test_case.body: @Test
  	public void testIncrementToken() throws IOException {
  		CharTermAttribute charTermAtt = tokenizer.getAttribute(CharTermAttribute.class);
  		OffsetAttribute offSetAtt = tokenizer.getAttribute(OffsetAttribute.class);
  		
  		while(tokenizer.incrementToken()) {
  			TestToken t = getToken(charTermAtt.toString(), offSetAtt.startOffset(), offSetAtt.endOffset());
  			System.out.println("termAtt.term() : " + charTermAtt.toString());
  			System.out.println("offSetAtt : " + offSetAtt.startOffset());
  			System.out.println("offSetAtt : " + offSetAtt.endOffset());
  
  			Assert.assertTrue(tokenizedToken.contains(t));
  		}
  	}
test_case.class_method_signature: KoreanCharacterTokenizerTest.testIncrementToken()
test_case.full_signature: @Test public void testIncrementToken()
test_case.identifier: testIncrementToken
test_case.invocations: getAttribute getAttribute incrementToken getToken toString startOffset endOffset println toString println startOffset println endOffset assertTrue contains
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testIncrementToken()
focal_class.declarator: logger = LoggerFactory.getLogger(KoreanCharacterTokenizer.class)
focal_class.modifier: private
focal_class.original_string: private Logger logger = LoggerFactory.getLogger(KoreanCharacterTokenizer.class);
focal_class.type: Logger
focal_class.var_name: logger
focal_class.declarator: offset = 0
focal_class.modifier: private
focal_class.original_string: private int offset = 0, bufferIndex = -1, dataLen = 0, finalOffset = 0;
focal_class.type: int
focal_class.var_name: offset
focal_class.declarator: MAX_WORD_LEN = 255
focal_class.modifier: private static final
focal_class.original_string: private static final int MAX_WORD_LEN = 255;
focal_class.type: int
focal_class.var_name: MAX_WORD_LEN
focal_class.declarator: IO_BUFFER_SIZE = 4096
focal_class.modifier: private static final
focal_class.original_string: private static final int IO_BUFFER_SIZE = 4096;
focal_class.type: int
focal_class.var_name: IO_BUFFER_SIZE
focal_class.declarator: termAtt = addAttribute(CharTermAttribute.class)
focal_class.modifier: private final
focal_class.original_string: private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
focal_class.type: CharTermAttribute
focal_class.var_name: termAtt
focal_class.declarator: offsetAtt = addAttribute(OffsetAttribute.class)
focal_class.modifier: private final
focal_class.original_string: private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
focal_class.type: OffsetAttribute
focal_class.var_name: offsetAtt
focal_class.declarator: typeAtt = addAttribute(TypeAttribute.class)
focal_class.modifier: private final
focal_class.original_string: private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
focal_class.type: TypeAttribute
focal_class.var_name: typeAtt
focal_class.declarator: positionAtt = addAttribute(PositionIncrementAttribute.class)
focal_class.modifier: private final
focal_class.original_string: private final PositionIncrementAttribute positionAtt = addAttribute(PositionIncrementAttribute.class);
focal_class.type: PositionIncrementAttribute
focal_class.var_name: positionAtt
focal_class.declarator: charUtils
focal_class.modifier: private final
focal_class.original_string: private final CharacterUtils charUtils;
focal_class.type: CharacterUtils
focal_class.var_name: charUtils
focal_class.declarator: ioBuffer = CharacterUtils.newCharacterBuffer(IO_BUFFER_SIZE)
focal_class.modifier: private final
focal_class.original_string: private final CharacterBuffer ioBuffer = CharacterUtils.newCharacterBuffer(IO_BUFFER_SIZE);
focal_class.type: CharacterBuffer
focal_class.var_name: ioBuffer
focal_class.declarator: preChar = 0
focal_class.modifier: private
focal_class.original_string: private int preChar = 0;
focal_class.type: int
focal_class.var_name: preChar
focal_class.declarator: preCharType = 99
focal_class.modifier: private
focal_class.original_string: private int preCharType = 99;
focal_class.type: int
focal_class.var_name: preCharType
focal_class.declarator: nowCharType = 99
focal_class.modifier: private
focal_class.original_string: private int nowCharType = 99;
focal_class.type: int
focal_class.var_name: nowCharType
focal_class.declarator: DIGIT = 0
focal_class.modifier: private final
focal_class.original_string: private final int DIGIT = 0;
focal_class.type: int
focal_class.var_name: DIGIT
focal_class.declarator: KOREAN = 1
focal_class.modifier: private final
focal_class.original_string: private final int KOREAN = 1;
focal_class.type: int
focal_class.var_name: KOREAN
focal_class.declarator: ALPHA = 2
focal_class.modifier: private final
focal_class.original_string: private final int ALPHA = 2;
focal_class.type: int
focal_class.var_name: ALPHA
focal_class.file: korean-analyzer-4.x/src/main/java/com/tistory/devyongsik/analyzer/KoreanCharacterTokenizer.java
focal_class.identifier: KoreanCharacterTokenizer
focal_class.class_method_signature: KoreanCharacterTokenizer.KoreanCharacterTokenizer(Reader input)
focal_class.full_signature: public  KoreanCharacterTokenizer(Reader input)
focal_class.identifier: KoreanCharacterTokenizer
focal_class.modifiers: public
focal_class.parameters: (Reader input)
focal_class.signature:  KoreanCharacterTokenizer(Reader input)
focal_class.class_method_signature: KoreanCharacterTokenizer.KoreanCharacterTokenizer(Version matchVersion, Reader input)
focal_class.full_signature: public  KoreanCharacterTokenizer(Version matchVersion, Reader input)
focal_class.identifier: KoreanCharacterTokenizer
focal_class.modifiers: public
focal_class.parameters: (Version matchVersion, Reader input)
focal_class.signature:  KoreanCharacterTokenizer(Version matchVersion, Reader input)
focal_class.class_method_signature: KoreanCharacterTokenizer.KoreanCharacterTokenizer(Version matchVersion, AttributeSource source,
  			Reader input)
focal_class.full_signature: public  KoreanCharacterTokenizer(Version matchVersion, AttributeSource source,
  			Reader input)
focal_class.identifier: KoreanCharacterTokenizer
focal_class.modifiers: public
focal_class.parameters: (Version matchVersion, AttributeSource source,
  			Reader input)
focal_class.signature:  KoreanCharacterTokenizer(Version matchVersion, AttributeSource source,
  			Reader input)
focal_class.class_method_signature: KoreanCharacterTokenizer.KoreanCharacterTokenizer(Version matchVersion, AttributeFactory factory,
  			Reader input)
focal_class.full_signature: public  KoreanCharacterTokenizer(Version matchVersion, AttributeFactory factory,
  			Reader input)
focal_class.identifier: KoreanCharacterTokenizer
focal_class.modifiers: public
focal_class.parameters: (Version matchVersion, AttributeFactory factory,
  			Reader input)
focal_class.signature:  KoreanCharacterTokenizer(Version matchVersion, AttributeFactory factory,
  			Reader input)
focal_class.class_method_signature: KoreanCharacterTokenizer.isTokenChar(int c)
focal_class.full_signature: protected boolean isTokenChar(int c)
focal_class.identifier: isTokenChar
focal_class.modifiers: protected
focal_class.parameters: (int c)
focal_class.return: boolean
focal_class.signature: boolean isTokenChar(int c)
focal_class.class_method_signature: KoreanCharacterTokenizer.normalize(int c)
focal_class.full_signature: protected int normalize(int c)
focal_class.identifier: normalize
focal_class.modifiers: protected
focal_class.parameters: (int c)
focal_class.return: int
focal_class.signature: int normalize(int c)
focal_class.class_method_signature: KoreanCharacterTokenizer.incrementToken()
focal_class.full_signature: @Override public final boolean incrementToken()
focal_class.identifier: incrementToken
focal_class.modifiers: @Override public final
focal_class.parameters: ()
focal_class.return: boolean
focal_class.signature: boolean incrementToken()
focal_class.class_method_signature: KoreanCharacterTokenizer.end()
focal_class.full_signature: @Override public final void end()
focal_class.identifier: end
focal_class.modifiers: @Override public final
focal_class.parameters: ()
focal_class.return: void
focal_class.signature: void end()
focal_class.class_method_signature: KoreanCharacterTokenizer.reset()
focal_class.full_signature: @Override public void reset()
focal_class.identifier: reset
focal_class.modifiers: @Override public
focal_class.parameters: ()
focal_class.return: void
focal_class.signature: void reset()
focal_class.superclass: extends Tokenizer
focal_method.body: @Override
  	public final boolean incrementToken() throws IOException {
  		clearAttributes();
  		
  		logger.debug("incrementToken");
  		
  		int length = 0;
  		int start = -1; // this variable is always initialized
  		int end = -1;
  		char[] buffer = termAtt.buffer();
  		while (true) {
  			
  			if (bufferIndex >= dataLen) {
  				offset += dataLen;
  				charUtils.fill(ioBuffer, input);
  				
  				if(ioBuffer.getLength() == 0) { // read supplementary char aware with CharacterUtils
  					dataLen = 0; // so next offset += dataLen won't decrement offset
  					if (length > 0) {
  						break;
  					} else {
  						finalOffset = correctOffset(offset);
  						return false;
  					}
  				}
  				dataLen = ioBuffer.getLength();
  				bufferIndex = 0;
  			}
  			
  			// use CharacterUtils here to support < 3.1 UTF-16 code unit behavior if the char based methods are gone
  			final int c = charUtils.codePointAt(ioBuffer.getBuffer(), bufferIndex, ioBuffer.getBuffer().length);
  			final int charCount = Character.charCount(c);
  			bufferIndex += charCount;
  
  			if (isTokenChar(c)) {               // if it's a token char
  				
  				//전 문자와 현재 문자를 비교해서 속성이 다르면  분리해낸다.
  				if (length > 0) {
  					//이전문자의 속성 set
  					if(Character.isDigit(preChar)) preCharType = this.DIGIT;
  					else if(preChar < 127) preCharType = this.ALPHA;
  					else preCharType = this.KOREAN;
  
  					//현재문자의 속성set
  					if(Character.isDigit(c)) nowCharType = this.DIGIT;
  					else if(c < 127) nowCharType = this.ALPHA;
  					else nowCharType = this.KOREAN;
  
  					if(preCharType != nowCharType) { //앞뒤 Character가 서로 다른 형식
  						bufferIndex--;
  
  						//여기서 토큰을 하나 끊어야 함
  						termAtt.setLength(length);
  					    offsetAtt.setOffset(correctOffset(start), correctOffset(start+length));
  					    typeAtt.setType("word");
  					    positionAtt.setPositionIncrement(1);
  
  						return true;
  					}
  				}
  				
  				preChar = c;
  				
  				if (length == 0) {                // start of token
  					assert start == -1;
  					start = offset + bufferIndex - charCount;
  					end = start;
  				} else if (length >= buffer.length-1) { // check if a supplementary could run out of bounds
  					
  					buffer = termAtt.resizeBuffer(2+length); // make sure a supplementary fits in the buffer
  				
  				}
  				
  				end += charCount;
  				length += Character.toChars(normalize(c), buffer, length); // buffer it, normalized
  				
  				if (length >= MAX_WORD_LEN) // buffer overflow! make sure to check for >= surrogate pair could break == test
  					break;
  			
  			} else if (length > 0) {            // at non-Letter w/ chars
  				break;                           // return 'em
  			}
  		}
  
  		termAtt.setLength(length);
  		assert start != -1;
  		offsetAtt.setOffset(correctOffset(start), finalOffset = correctOffset(end));
  		typeAtt.setType("word");
  	    positionAtt.setPositionIncrement(1);
  	    
  		return true;
  	}
focal_method.class_method_signature: KoreanCharacterTokenizer.incrementToken()
focal_method.full_signature: @Override public final boolean incrementToken()
focal_method.identifier: incrementToken
focal_method.invocations: clearAttributes debug buffer fill getLength correctOffset getLength codePointAt getBuffer getBuffer charCount isTokenChar isDigit isDigit setLength setOffset correctOffset correctOffset setType setPositionIncrement resizeBuffer toChars normalize setLength setOffset correctOffset correctOffset setType setPositionIncrement
focal_method.modifiers: @Override public final
focal_method.parameters: ()
focal_method.return: boolean
focal_method.signature: boolean incrementToken()