Skip to main content

BasicCrawlerOptions

crawlee.basic_crawler.basic_crawler.BasicCrawlerOptions

Copy of the parameter types of BasicCrawler.__init__ meant for typing forwarded init args in subclasses.

Index

Properties

concurrency_settings

concurrency_settings: NotRequired[ConcurrencySettings]

configuration

configuration: NotRequired[Configuration]

configure_logging

configure_logging: NotRequired[bool]

http_client

http_client: NotRequired[BaseHttpClient]

max_request_retries

max_request_retries: NotRequired[int]

max_requests_per_crawl

max_requests_per_crawl: NotRequired[int | None]

max_session_rotations

max_session_rotations: NotRequired[int]

proxy_configuration

proxy_configuration: NotRequired[ProxyConfiguration]

request_handler

request_handler: NotRequired[Callable[[TCrawlingContext], Awaitable[None]]]

request_handler_timeout

request_handler_timeout: NotRequired[timedelta]

request_provider

request_provider: NotRequired[RequestProvider]

retry_on_blocked

retry_on_blocked: NotRequired[bool]

session_pool

session_pool: NotRequired[SessionPool]

statistics

statistics: NotRequired[Statistics[StatisticsState]]

use_session_pool

use_session_pool: NotRequired[bool]