Skip to main content
Version: Next

Error handling

This page demonstrates how Crawlee deals with errors during crawling and the techniques available for handling them. For how retries are counted and how to turn them off for a single request, see Requests.

Handling proxy errors​

Low-quality proxies can cause problems even with high settings for max_request_retries and max_session_rotations in BasicCrawlerOptions. If you can't get data because of proxy errors, you might want to try again. You can do this using failed_request_handler:

Run on
import asyncio

from crawlee import Request
from crawlee.crawlers import BasicCrawlingContext, HttpCrawler, HttpCrawlingContext
from crawlee.errors import ProxyError


async def main() -> None:
    # Set how many session rotations will happen before calling the error handler
    # when ProxyError occurs
    crawler = HttpCrawler(max_session_rotations=5, max_request_retries=6)

    # For this example, we'll create a proxy error in our handler
    @crawler.router.default_handler
    async def default_handler(context: HttpCrawlingContext) -> None:
        context.log.info(f'Processing {context.request.url} ...')
        raise ProxyError('Simulated proxy error')

    # This handler runs after all retry attempts are exhausted
    @crawler.failed_request_handler
    async def failed_handler(context: BasicCrawlingContext, error: Exception) -> None:
        context.log.error(f'Failed request {context.request.url}, after 5 rotations')
        request = context.request
        # For proxy errors, we can add a new `Request` to try again
        if isinstance(error, ProxyError) and not request.unique_key.startswith('retry'):
            context.log.info(f'Retrying {request.url} ...')
            # Create a new `Request` with a modified key to avoid deduplication
            new_request = Request.from_url(
                request.url, unique_key=f'retry{request.unique_key}'
            )

            # Add the new `Request` to the `Queue`
            rq = await crawler.get_request_manager()
            await rq.add_request(new_request)

    await crawler.run(['https://crawlee.dev/'])


if __name__ == '__main__':
    asyncio.run(main())

You can use this same approach when testing different proxy providers. To better manage this process, you can count proxy errors and stop the crawler if you get too many.

Changing how error status codes are handled​

By default, when Sessions get status codes like 401, 403, or 429, Crawlee marks the Session as retire and switches to a new one. This might not be what you want, especially when working with authentication. You can learn more on the Session management page.

Here's an example of how to change this behavior:

Run on
import asyncio
import json

from crawlee import HttpHeaders
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
from crawlee.errors import HttpStatusCodeError
from crawlee.sessions import SessionPool

# Using a placeholder refresh token for this example
REFRESH_TOKEN = 'PLACEHOLDER'
UNAUTHORIZED_CODE = 401


async def main() -> None:
    crawler = HttpCrawler(
        max_request_retries=2,
        # Only treat 403 as a blocking status code, not 401
        session_pool=SessionPool(create_session_settings={'blocked_status_codes': [403]}),
        # Don't treat 401 responses as errors
        ignore_http_error_status_codes=[UNAUTHORIZED_CODE],
    )

    @crawler.router.default_handler
    async def default_handler(context: HttpCrawlingContext) -> None:
        context.log.info(f'Processing {context.request.url} ...')
        # Now we can handle 401 responses ourselves
        if context.http_response.status_code == UNAUTHORIZED_CODE:
            # Get a fresh access token
            headers = {'authorization': f'Bearer {REFRESH_TOKEN}'}
            response = await context.send_request(
                'https://placeholder.org/refresh', headers=headers
            )
            data = json.loads(await response.read())
            # Add the new token to our `Request` headers
            context.request.headers |= HttpHeaders(
                {'authorization': f'Bearer {data["access_token"]}'},
            )
            # Trigger a retry with our updated headers
            raise HttpStatusCodeError('Unauthorized', status_code=UNAUTHORIZED_CODE)

    await crawler.run(['http://httpbingo.org/status/401'])


if __name__ == '__main__':
    asyncio.run(main())

Turning off retries for non-network errors​

Sometimes you might get unexpected errors when parsing data, like when a website has an unusual structure. Crawlee normally tries again based on your max_request_retries setting, but sometimes you don't want that.

Here's how to turn off retries for non-network errors using error_handler, which runs before Crawlee tries again:

Run on
import asyncio

from crawlee.crawlers import BasicCrawlingContext, HttpCrawler, HttpCrawlingContext
from crawlee.errors import HttpStatusCodeError, SessionError


async def main() -> None:
    crawler = HttpCrawler(max_request_retries=5)

    # Create a parsing error for demonstration
    @crawler.router.default_handler
    async def default_handler(context: HttpCrawlingContext) -> None:
        context.log.info(f'Processing {context.request.url} ...')
        raise ValueError('Simulated parsing error')

    # This handler runs before any retry attempts
    @crawler.error_handler
    async def retry_handler(context: BasicCrawlingContext, error: Exception) -> None:
        context.log.error(f'Failed request {context.request.url}')
        # Only allow retries for network-related errors
        if not isinstance(error, (SessionError, HttpStatusCodeError)):
            context.log.error('Non-network error detected')
            # Stop further retry attempts for this `Request`
            context.request.no_retry = True

    await crawler.run(['https://crawlee.dev/'])


if __name__ == '__main__':
    asyncio.run(main())

Capturing page snapshots on errors​

Debugging a failing request is much easier when you can see what the page looked like at the moment of failure. Set save_error_snapshots=True in the crawler's Statistics and Crawlee automatically captures a page snapshot on the first occurrence of each unique error. The snapshot can contain an HTML file and a JPEG screenshot of the page where the unhandled exception was raised, and it's saved to the default key-value store, or to the one named by the persist_state_kvs_name option of Statistics. Both PlaywrightCrawler and HTTP crawlers capture the HTML file, but only PlaywrightCrawler can capture a screenshot as well.

Run on
import asyncio
from random import choice

from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.statistics import Statistics


async def main() -> None:
    crawler = ParselCrawler(
        statistics=Statistics.with_default_state(save_error_snapshots=True)
    )

    @crawler.router.default_handler
    async def request_handler(context: ParselCrawlingContext) -> None:
        context.log.info(f'Processing {context.request.url} ...')
        # Simulate various errors to demonstrate `ErrorSnapshotter`
        # saving only the first occurrence of unique error.
        await context.enqueue_links()
        random_number = choice(range(10))
        if random_number == 1:
            raise KeyError('Some KeyError')
        if random_number == 2:
            raise ValueError('Some ValueError')
        if random_number == 3:
            raise RuntimeError('Some RuntimeError')

    await crawler.run(['https://crawlee.dev'])


if __name__ == '__main__':
    asyncio.run(main())