tokenize function

List<String> tokenize(
  1. String input
)

Tokenize a string for fuzzy matching.

Splits on whitespace, punctuation, and word boundaries inside camelCase identifiers. Returns lower-cased, diacritic-folded tokens.

Diacritics are folded BEFORE the alpha check so accented letters (Spanish "botón", "canción") count as alphabetic and remain inside the same token.

Implementation

List<String> tokenize(String input) {
  if (input.isEmpty) return const [];
  final out = <String>[];
  final buffer = StringBuffer();

  void flush() {
    if (buffer.isEmpty) return;
    out.add(buffer.toString().toLowerCase());
    buffer.clear();
  }

  for (var i = 0; i < input.length; i++) {
    final raw = input.codeUnitAt(i);
    final folded = _foldDiacritic(raw);
    final isAlpha =
        (folded >= 0x41 && folded <= 0x5A) ||
        (folded >= 0x61 && folded <= 0x7A);
    final isDigit = folded >= 0x30 && folded <= 0x39;
    final isAlphaNum = isAlpha || isDigit;
    final isUpper = folded >= 0x41 && folded <= 0x5A;

    if (!isAlphaNum) {
      flush();
      continue;
    }

    // camelCase split: uppercase letter inside a non-empty buffer that ended
    // in a lowercase letter starts a new token.
    if (isUpper && buffer.isNotEmpty) {
      final last = buffer.toString().codeUnitAt(buffer.length - 1);
      final lastIsLower = last >= 0x61 && last <= 0x7A;
      if (lastIsLower) flush();
    }

    buffer.writeCharCode(folded);
  }
  flush();
  return out;
}