handle_next

in lib/twitter_cldr/segmentation/state_machine.rb [72:124]


      def handle_next(cursor)
        result = initial_position = cursor.position
        state = START_STATE
        row = state * (metadata.category_count + NEXT_STATES)
        category = 3
        mode = :run

        if ftable.bof_required?
          category = 2
          mode = :start
        end

        until state == STOP_STATE
          if cursor.eos?
            break if mode == :stop
            mode = :stop
            category = 1
          elsif mode == :run
            category = category_table.get(cursor.codepoint)
            cursor.advance
          else
            mode = :run
          end

          state = ftable[row + NEXT_STATES + category]
          row = state * (metadata.category_count + NEXT_STATES)
          accepting = ftable[row + ACCEPTING]

          if accepting == ACCEPTING_UNCONDITIONAL
            
            result = cursor.position
          elsif accepting > ACCEPTING_UNCONDITIONAL
            if (lookahead_result = @lookahead_matches[accepting]) >= 0
              cursor.position = lookahead_result
              return lookahead_result
            end
          end

          if (rule = ftable[row + LOOKAHEAD]) != 0
            @lookahead_matches[rule] = cursor.position
          end
        end

        cursor.position = result

        
        if cursor.position == initial_position
          cursor.advance
        end

        result
      end