scrapeWithLazyLoadingAndPagination<T> method

Future<PaginationResult<T>> scrapeWithLazyLoadingAndPagination<T>({
  1. required String url,
  2. required PaginationConfig paginationConfig,
  3. required LazyLoadConfig lazyLoadConfig,
  4. required Future<T> extractor(
    1. String html,
    2. String pageUrl
    ),
  5. Map<String, String>? headers,
  6. int? timeout,
  7. int? retries,
})

Fetches HTML content with both lazy loading and pagination support

url is the starting URL paginationConfig is the pagination configuration lazyLoadConfig is the lazy loading configuration extractor is a function that extracts data from each page headers are additional headers to send with the request timeout is the timeout for the request in milliseconds retries is the number of retry attempts

Implementation

Future<PaginationResult<T>> scrapeWithLazyLoadingAndPagination<T>({
  required String url,
  required PaginationConfig paginationConfig,
  required LazyLoadConfig lazyLoadConfig,
  required Future<T> Function(String html, String pageUrl) extractor,
  Map<String, String>? headers,
  int? timeout,
  int? retries,
}) async {
  // Create a new extractor that applies lazy loading before extraction
  Future<T> lazyLoadingExtractor(String html, String pageUrl) async {
    // Apply lazy loading to the HTML
    final lazyLoadResult = await lazyLoadHandler.handleLazyLoading(
      url: pageUrl,
      config: lazyLoadConfig,
      headers: headers,
    );

    // Extract data from the lazy-loaded HTML
    return extractor(lazyLoadResult.html, pageUrl);
  }

  // Use the pagination handler with the lazy loading extractor
  return paginationHandler.scrapeWithPagination(
    url: url,
    config: paginationConfig,
    extractor: lazyLoadingExtractor,
    headers: headers,
    timeout: timeout,
    retries: retries,
  );
}