genUiCoverage function

GenUiCoverage genUiCoverage({
  1. required Catalog catalog,
  2. required Iterable<GenUiTrace> traces,
})

Reads traces and reports what the agent did with catalog.

Every component the agent composed is counted, along with the properties it filled and the enum values it chose. What is left over is the part of the catalog that travels in every request and has never been used.

final coverage = genUiCoverage(catalog: genUiCatalog, traces: recorded);
print(coverage.describe());
expect(coverage.unusedComponents, isEmpty,
    reason: 'the prompt is paying for components nobody composes');

A component being unused is not automatically wrong: a catalog is written before the conversations that use it, and an error state should be rare. It is a number to look at, not a rule to enforce, which is why this hands back the counts rather than a verdict.

Implementation

GenUiCoverage genUiCoverage({
  required Catalog catalog,
  required Iterable<GenUiTrace> traces,
}) {
  final componentUses = <String, int>{};
  final propertyUses = <String, Set<String>>{};
  final enumValueUses = <String, Set<String>>{};
  final surfaces = <String>{};

  var sessions = 0;
  for (final GenUiTrace trace in traces) {
    sessions++;
    for (final GenUiTraceStep step in trace.steps) {
      if (step is! GenUiMessageStep) continue;
      final Object? update = step.message['updateComponents'];
      if (update is! Map) continue;
      final Object? surfaceId = update['surfaceId'];
      if (surfaceId is String) surfaces.add('$sessions/$surfaceId');
      final Object? components = update['components'];
      if (components is! List) continue;
      for (final Object? component in components) {
        if (component is! Map) continue;
        final Object? name = component['component'];
        if (name is! String) continue;
        componentUses[name] = (componentUses[name] ?? 0) + 1;
        final props = propertyUses.putIfAbsent(name, () => <String>{});
        for (final Object? key in component.keys) {
          if (key is! String || key == 'id' || key == 'component') continue;
          props.add(key);
          final Object? value = component[key];
          if (value is String) {
            enumValueUses
                .putIfAbsent('$name.$key', () => <String>{})
                .add(value);
          }
        }
      }
    }
  }

  final sorted = componentUses.entries.toList()
    ..sort((a, b) => b.value.compareTo(a.value));

  final catalogProperties = <String, Set<String>>{};
  final catalogEnumValues = <String, Set<String>>{};
  for (final CatalogItem item in catalog.items) {
    final Object schema = item.dataSchema.value;
    if (schema is! Map) continue;
    final Object? properties = schema['properties'];
    if (properties is! Map) continue;
    final names = <String>{
      for (final Object? key in properties.keys)
        if (key is String && key != 'component') key,
    };
    catalogProperties[item.name] = names;
    for (final String name in names) {
      final Set<String> values = _enumValuesIn(properties[name]);
      if (values.isNotEmpty) {
        catalogEnumValues['${item.name}.$name'] = values;
      }
    }
  }

  GenUiCatalogWeight? weight;
  if (catalog.catalogId != null) {
    weight = genUiCatalogWeight(genUiCatalogJson(catalog));
  }

  return GenUiCoverage(
    sessions: sessions,
    surfaces: surfaces.length,
    componentUses: <String, int>{for (final e in sorted) e.key: e.value},
    propertyUses: propertyUses,
    enumValueUses: enumValueUses,
    catalogComponents: catalog.items.map((i) => i.name).toSet(),
    catalogProperties: catalogProperties,
    catalogEnumValues: catalogEnumValues,
    weight: weight,
  );
}