WebScraper constructor

WebScraper({
  1. required ProxyManager proxyManager,
  2. ProxyHttpClient? httpClient,
  3. String? defaultUserAgent,
  4. Map<String, String>? defaultHeaders,
  5. int defaultTimeout = 30000,
  6. int maxRetries = 3,
  7. AdaptiveScrapingStrategy? adaptiveStrategy,
  8. SiteReputationTracker? reputationTracker,
  9. ScrapingLogger? logger,
})

Creates a new WebScraper with the given parameters

Implementation

WebScraper({
  required this.proxyManager,
  ProxyHttpClient? httpClient,
  String? defaultUserAgent,
  Map<String, String>? defaultHeaders,
  int defaultTimeout = 30000,
  int maxRetries = 3,
  AdaptiveScrapingStrategy? adaptiveStrategy,
  SiteReputationTracker? reputationTracker,
  ScrapingLogger? logger,
}) : _httpClient =
         httpClient ??
         ProxyHttpClient(
           proxyManager: proxyManager,
           useValidatedProxies: true,
           rotateProxies: true,
         ),
     _defaultUserAgent =
         defaultUserAgent ??
         'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
     _defaultHeaders = defaultHeaders ?? {},
     _defaultTimeout = defaultTimeout,
     _maxRetries = maxRetries,
     _reputationTracker = reputationTracker ?? SiteReputationTracker(),
     _logger = logger ?? ScrapingLogger(),
     _adaptiveStrategy =
         adaptiveStrategy ??
         AdaptiveScrapingStrategy(reputationTracker: reputationTracker);