shannonEntropy function

double shannonEntropy(
  1. String token
)

Shannon entropy of token in bits per character.

Implementation

double shannonEntropy(String token) {
  final counts = <int, int>{};
  for (final c in token.codeUnits) {
    counts[c] = (counts[c] ?? 0) + 1;
  }
  var entropy = 0.0;
  final n = token.length;
  for (final count in counts.values) {
    final p = count / n;
    entropy -= p * (math.log(p) / math.ln2);
  }
  return entropy;
}