fetchHtmlWithRateLimiting method

Future<String> fetchHtmlWithRateLimiting({
  1. required String url,
  2. Map<String, String>? headers,
  3. int? timeout,
  4. int? retries,
  5. RequestPriority priority = RequestPriority.normal,
  6. bool ignoreRobotsTxt = false,
})

Fetches HTML content with rate limiting

url is the URL to fetch headers are additional headers to send with the request timeout is the timeout for the request in milliseconds retries is the number of retry attempts priority is the priority of the request ignoreRobotsTxt whether to ignore robots.txt rules (default: false)

Implementation

Future<String> fetchHtmlWithRateLimiting({
  required String url,
  Map<String, String>? headers,
  int? timeout,
  int? retries,
  RequestPriority priority = RequestPriority.normal,
  bool ignoreRobotsTxt = false,
}) async {
  final uri = Uri.parse(url);
  final domain = uri.host;

  return _requestQueue.enqueue(
    domain: domain,
    priority: priority,
    execute:
        () => fetchHtml(
          url: url,
          headers: headers,
          timeout: timeout,
          retries: retries,
          ignoreRobotsTxt: ignoreRobotsTxt,
        ),
  );
}