charTokenIndices property

List<int> get charTokenIndices

Maps every character in text to its source token index in tokens.

Implementation

List<int> get charTokenIndices {
  final List<int> result = [];
  for (int i = 0; i < tokens.length; i++) {
    final int len = tokens[i].length;
    for (int c = 0; c < len; c++) {
      result.add(i);
    }
  }
  return result;
}