buildTokenizedDataset()

in src/models/SentenTreeBuilder.js [34:47]

• 13 lines of code
• 2 McCabe index (conditional complexity)

  buildTokenizedDataset(entries) {
    const tokenizedEntries = entries
      .map(entry => ({
        id: entry.id,
        count: entry.count || 1,
        tokens: this._tokenize(entry.text)
          .map(this._transformToken)
          .filter(this._filterToken),
        rawText: entry.text,
      }))
      .filter(entry => entry.tokens.length > 0);

    return new TokenizedDataset(tokenizedEntries);
  }