fetchHtml method

Future<String> fetchHtml({
  1. required String url,
  2. int priority = 0,
  3. Map<String, String>? headers,
  4. int? timeout,
  5. int? retries,
  6. bool ignoreRobotsTxt = false,
})

Fetches HTML content from a URL with priority

url is the URL to fetch priority is the priority of the task (higher values = higher priority) headers are additional headers to send with the request timeout is the timeout for the request in milliseconds retries is the number of retry attempts ignoreRobotsTxt whether to ignore robots.txt rules (default: false)

Implementation

Future<String> fetchHtml({
  required String url,
  int priority = 0,
  Map<String, String>? headers,
  int? timeout,
  int? retries,
  bool ignoreRobotsTxt = false,
}) {
  return _taskQueue.addTask<String>(
    task:
        () => _webScraper.fetchHtml(
          url: url,
          headers: headers,
          timeout: timeout,
          retries: retries,
          ignoreRobotsTxt: ignoreRobotsTxt,
        ),
    priority: priority,
    taskName: 'FetchHTML-$url',
  );
}