in lib/twitter_cldr/segmentation/word_iterator.rb [38:133]
def each_boundary_helper(str, &block)
dict_cursor = create_cursor(str)
rule_cursor = dict_cursor.dup
yield 0
until dict_cursor.eos? || rule_cursor.eos?
m = dictionary_re.match(dict_cursor.text, dict_cursor.position)
break unless m
dict_cursor.position = m.begin(0)
dict_break_engine = dictionary_break_engine_for(dict_cursor.codepoint)
dict_enum = dict_break_engine.each_boundary(dict_cursor)
dict_boundary = begin
dict_enum.peek
rescue StopIteration
nil
end
if dict_boundary
if rule_cursor.position < m.begin(0)
rule_set.each_boundary(rule_cursor, m.begin(0), &block)
end
loop do
yield dict_enum.next
end
yield dict_cursor.position
rule_cursor.position = dict_cursor.position
else
dict_cursor.advance
end
end
unless rule_cursor.eos?
rule_set.each_boundary(rule_cursor, &block)
end
yield rule_cursor.length
end