loadWebPage function

Implementation

Future<WebPage> loadWebPage(
  http.Client client,
  Uri uri, {
  required Duration timeout,
  required int maxBytes,
  Map<String, String>? headers,
}) async {
  final request = http.Request('GET', uri)
    ..followRedirects = true
    ..maxRedirects = 5
    ..headers.addAll({
      'User-Agent': webSearchUserAgent,
      'Accept':
          'text/html,application/xhtml+xml,application/xml;q=0.9,'
          'text/plain;q=0.8,*/*;q=0.5',
      'Accept-Language': 'en-US,en;q=0.5',
    });
  if (headers != null) request.headers.addAll(headers);

  final response = await client.send(request).timeout(timeout);
  final rawContentType = response.headers['content-type'] ?? '';
  final contentType = rawContentType.split(';').first.trim().toLowerCase();

  final chunks = <int>[];
  var truncated = false;
  await for (final chunk in response.stream.timeout(timeout)) {
    final remaining = maxBytes - chunks.length;
    if (chunk.length > remaining) {
      chunks.addAll(chunk.sublist(0, remaining));
      truncated = true;
      break;
    }
    chunks.addAll(chunk);
  }

  return WebPage(
    url: uri,
    finalUrl: response.request?.url ?? uri,
    statusCode: response.statusCode,
    contentType: contentType,
    body: _decodeBody(chunks, rawContentType),
    truncated: truncated,
  );
}