Playwright with Stagehand
Stagehand is a framework that combines Playwright with AI-driven natural language understanding and decision-making capabilities. With Stagehand, you can use natural language instructions to interact with web pages instead of writing complex selectors and automation logic.
Stagehand supports multiple AI models through LiteLLM. This guide demonstrates how to integrate Stagehand with PlaywrightCrawler using Gemini as the AI model provider.
This guide is based on stagehand-python v0.4.0 with local configuration settings and may not be compatible with newer versions.
Get Gemini API key
You need to register with Google AI Studio and navigate to Get API key to obtain your API key.
Create support classes for Stagehand
To integrate Stagehand with Crawlee, you need to create wrapper classes that allow PlaywrightBrowserPlugin to manage the Playwright lifecycle.
Create CrawleeStagehand - a custom Stagehand subclass that overrides the init method to prevent Stagehand from launching its own Playwright instance.
Create CrawleeStagehandPage - a wrapper class for StagehandPage that implements the Playwright Page behavior expected by PlaywrightCrawler.
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from stagehand import Stagehand, StagehandPage
if TYPE_CHECKING:
from types import TracebackType
class CrawleeStagehandPage:
"""StagehandPage wrapper for Crawlee."""
def __init__(self, page: StagehandPage) -> None:
self._page = page
async def goto(
self,
url: str,
*,
referer: str | None = None,
timeout: int | None = None,
wait_until: str | None = None,
) -> Any:
"""Navigate to the specified URL."""
# Override goto to return navigation result that `PlaywrightCrawler` expects
return await self._page._page.goto( # noqa: SLF001
url,
referer=referer,
timeout=timeout,
wait_until=wait_until,
)
def __getattr__(self, name: str) -> Any:
"""Delegate all other methods to the underlying StagehandPage."""
return getattr(self._page, name)
async def __aenter__(self) -> CrawleeStagehandPage:
"""Enter the context manager."""
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
exc_traceback: TracebackType | None,
) -> None:
await self._page.close()
class CrawleeStagehand(Stagehand):
"""Stagehand wrapper for Crawlee to disable the launch of Playwright."""
async def init(self) -> None:
# Skip Stagehand's own Playwright initialization
# Let Crawlee's PlaywrightBrowserPlugin manage the browser lifecycle
self._initialized = True
Create browser integration classes
You need to create a custom browser plugin and controller that properly initialize Stagehand and obtain browser pages from StagehandContext.
Create StagehandPlugin - a subclass of PlaywrightBrowserPlugin that holds the Stagehand instance and creates PlaywrightPersistentBrowser instances.
Create StagehandBrowserController - a subclass of PlaywrightBrowserController that lazily initializes StagehandContext and creates new pages with AI capabilities on demand.
from __future__ import annotations
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any, cast
from stagehand.context import StagehandContext
from typing_extensions import override
from crawlee.browsers import (
PlaywrightBrowserController,
PlaywrightBrowserPlugin,
PlaywrightPersistentBrowser,
)
from .support_classes import CrawleeStagehandPage
if TYPE_CHECKING:
from collections.abc import Mapping
from playwright.async_api import Page
from stagehand import Stagehand
from crawlee.proxy_configuration import ProxyInfo
class StagehandBrowserController(PlaywrightBrowserController):
@override
def __init__(
self, browser: PlaywrightPersistentBrowser, stagehand: Stagehand, **kwargs: Any
) -> None:
# Initialize with browser context instead of browser instance
super().__init__(browser, **kwargs)
self._stagehand = stagehand
self._stagehand_context: StagehandContext | None = None
@override
async def new_page(
self,
browser_new_context_options: Mapping[str, Any] | None = None,
proxy_info: ProxyInfo | None = None,
) -> Page:
# Initialize browser context if not already done
if not self._browser_context:
self._browser_context = await self._create_browser_context(
browser_new_context_options=browser_new_context_options,
proxy_info=proxy_info,
)
# Initialize Stagehand context if not already done
if not self._stagehand_context:
self._stagehand_context = await StagehandContext.init(
self._browser_context, self._stagehand
)
# Create a new page using Stagehand context
page = await self._stagehand_context.new_page()
pw_page = page._page # noqa: SLF001
# Handle page close event
pw_page.on(event='close', f=self._on_page_close)
# Update internal state
self._pages.append(pw_page)
self._last_page_opened_at = datetime.now(timezone.utc)
self._total_opened_pages += 1
# Wrap StagehandPage to provide Playwright Page interface
return cast('Page', CrawleeStagehandPage(page))
class StagehandPlugin(PlaywrightBrowserPlugin):
"""Browser plugin that integrates Stagehand with Crawlee's browser management."""
@override
def __init__(self, stagehand: Stagehand, **kwargs: Any) -> None:
super().__init__(**kwargs)
self._stagehand = stagehand
@override
async def new_browser(self) -> StagehandBrowserController:
if not self._playwright:
raise RuntimeError('Playwright browser plugin is not initialized.')
browser = PlaywrightPersistentBrowser(
# Stagehand can run only on a Chromium-based browser.
self._playwright.chromium,
self._user_data_dir,
self._browser_launch_options,
)
# Return custom controller with Stagehand
return StagehandBrowserController(
browser=browser,
stagehand=self._stagehand,
header_generator=None,
fingerprint_generator=None,
)
Create a crawler
Now you can create a PlaywrightCrawler that uses Stagehand's AI capabilities to interact with web pages using natural language commands:
from __future__ import annotations
import asyncio
import os
from typing import cast
from stagehand import StagehandConfig, StagehandPage
from crawlee import ConcurrencySettings
from crawlee.browsers import BrowserPool
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
from .browser_classes import StagehandPlugin
from .support_classes import CrawleeStagehand
async def main() -> None:
# Configure local Stagehand with Gemini model
config = StagehandConfig(
env='LOCAL',
model_name='google/gemini-2.5-flash-preview-05-20',
model_api_key=os.getenv('GEMINI_API_KEY'),
)
# Create Stagehand instance
stagehand = CrawleeStagehand(config)
# Create crawler with custom browser pool using Stagehand
crawler = PlaywrightCrawler(
# Limit the crawl to max requests. Remove or increase it for crawling all links.
max_requests_per_crawl=10,
# Custom browser pool. Gives users full control over browsers used by the crawler.
concurrency_settings=ConcurrencySettings(max_tasks_per_minute=10),
browser_pool=BrowserPool(
plugins=[
StagehandPlugin(stagehand, browser_launch_options={'headless': True})
],
),
)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Cast to StagehandPage for proper type hints in IDE
page = cast('StagehandPage', context.page)
# Use regular Playwright method
playwright_title = await page.title()
context.log.info(f'Playwright page title: {playwright_title}')
# Use AI-powered extraction with natural language
gemini_title = await page.extract('Extract page title')
context.log.info(f'Gemini page title: {gemini_title}')
await context.enqueue_links()
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
The integration works through several key components:
CrawleeStagehandprevents Stagehand from launching its own Playwright instance, allowing Crawlee to manage the browser lifecycleStagehandPluginextends the Playwright browser plugin to create Stagehand-enabled browser instancesStagehandBrowserControllerusesStagehandContextto create pages with AI capabilitiesCrawleeStagehandPageprovides interface compatibility between Stagehand pages and Crawlee's expectations
In the request handler, you can use natural language commands like page.extract('Extract title page') to perform intelligent data extraction without writing complex selectors.