in lib/twitter_cldr/segmentation/state_machine.rb [72:124]
def handle_next(cursor)
result = initial_position = cursor.position
state = START_STATE
row = state * (metadata.category_count + NEXT_STATES)
category = 3
mode = :run
if ftable.bof_required?
category = 2
mode = :start
end
until state == STOP_STATE
if cursor.eos?
break if mode == :stop
mode = :stop
category = 1
elsif mode == :run
category = category_table.get(cursor.codepoint)
cursor.advance
else
mode = :run
end
state = ftable[row + NEXT_STATES + category]
row = state * (metadata.category_count + NEXT_STATES)
accepting = ftable[row + ACCEPTING]
if accepting == ACCEPTING_UNCONDITIONAL
result = cursor.position
elsif accepting > ACCEPTING_UNCONDITIONAL
if (lookahead_result = @lookahead_matches[accepting]) >= 0
cursor.position = lookahead_result
return lookahead_result
end
end
if (rule = ftable[row + LOOKAHEAD]) != 0
@lookahead_matches[rule] = cursor.position
end
end
cursor.position = result
if cursor.position == initial_position
cursor.advance
end
result
end