Browser Automation

Headless browser control for agents: navigation, extraction, interaction, and testing.

Browser Automation Tools

Playwright (Recommended)

Setup

pip install playwright
playwright install  # Installs Chromium, Firefox, WebKit

Basic Usage

from playwright.async_api import async_playwright

async def scrape_example():
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        page = await browser.new_page()

        await page.goto("https://example.com")
        title = await page.title()
        print(title)

        # Screenshot
        await page.screenshot(path="screenshot.png")

        await browser.close()

asyncio.run(scrape_example())

Navigation & Interaction

# Navigation
await page.goto("https://example.com", wait_until="networkidle")
await page.wait_for_load_state("domcontentloaded")

# Click
await page.click("button#submit")
await page.click("text=Login")

# Type
await page.fill("input[name='email']", "user@example.com")
await page.press("input[name='password'", "Enter")

# Select
await page.select_option("select#country", "US")

# Hover
await page.hover(".dropdown-menu")

# Drag & Drop
await page.drag_and_drop("#source", "#target")

Waiting Strategies

# Explicit waits (preferred)
await page.wait_for_selector(".results", state="visible")
await page.wait_for_load_state("networkidle")
await page.wait_for_function("() => document.querySelector('.ready')")

# Implicit (avoid)
page.set_default_timeout(30000)

Data Extraction

# Text
text = await page.inner_text(".price")
all_text = await page.locator(".item").all_inner_texts()

# HTML
html = await page.inner_html(".container")

# Attributes
href = await page.get_attribute("a.link", "href")

# Tables
rows = await page.locator("table tr").all()
for row in rows:
    cells = await row.locator("td").all_inner_texts()
    print(cells)

# JSON from script tag
data = await page.evaluate("() => window.__INITIAL_DATA__")

Screenshots & PDFs

# Screenshot
await page.screenshot(path="full.png", full_page=True)
await page.screenshot(path="element.png", clip={"x": 0, "y": 0, "width": 800, "height": 600})

# PDF
await page.pdf(path="page.pdf", format="A4", print_background=True)

Authentication & Cookies

# Set cookies
await page.context.add_cookies([
    {"name": "session", "value": "abc123", "domain": ".example.com", "path": "/"}
])

# Storage state (persist login)
storage = await page.context.storage_state(path="auth.json")

# Reuse
context = await browser.new_context(storage_state="auth.json")

Browserbase (Cloud)

Setup

pip install browserbase
export BROWSERBASE_API_KEY="your-key"

Usage

from browserbase import Browserbase

bb = Browserbase(api_key="your-key")

# Create session
session = bb.sessions.create(
    project_id="your-project-id",
    browser_settings={
        "viewport": {"width": 1280, "height": 720},
        "block_ads": True,
        "block_tracking": True
    }
)

# Connect via CDP
from playwright.async_api import async_playwright

async with async_playwright() as p:
    browser = await p.chromium.connect_over_cdp(session.connect_url)
    page = browser.contexts[0].pages[0] if browser.contexts else await browser.new_page()

    await page.goto("https://example.com")
    # ... interact

    await browser.close()

# Cleanup
bb.sessions.delete(session.id)

Scaling

# Run many sessions in parallel
async def scrape_many(urls: list[str]):
    sessions = []
    for _ in urls:
        session = bb.sessions.create(project_id="proj")
        sessions.append(session)

    async def scrape_one(session, url):
        async with async_playwright() as p:
            browser = await p.chromium.connect_over_cdp(session.connect_url)
            page = await browser.new_page()
            await page.goto(url)
            content = await page.content()
            await browser.close()
            return content

    results = await asyncio.gather(*[
        scrape_one(s, url) for s, url in zip(sessions, urls)
    ])

    # Cleanup
    for s in sessions:
        bb.sessions.delete(s.id)

    return results

Stagehand (AI-Driven)

Setup

pip install stagehand
export OPENAI_API_KEY="your-key"

Usage

from stagehand import Stagehand

stagehand = Stagehand(
    env="LOCAL",  # or "BROWSERBASE"
    model_name="gpt-4o",
    headless=True
)

await stagehand.init()

# Natural language actions
await stagehand.page.act("Click the login button")
await stagehand.page.act("Fill in the email field with user@example.com")
await stagehand.page.act("Click submit and wait for dashboard")

# Extraction
data = await stagehand.page.extract({
    "instruction": "Extract all product names and prices",
    "schema": {
        "type": "array",
        "items": {
            "type": "object",
            "properties": {
                "name": {"type": "string"},
                "price": {"type": "string"}
            }
        }
    }
})

await stagehand.close()

Crawl4AI (AI + Crawler)

Setup

pip install crawl4ai

Usage

from crawl4ai import AsyncWebCrawler

async with AsyncWebCrawler() as crawler:
    # Basic crawl
    result = await crawler.arun(url="https://example.com")
    print(result.markdown)

    # With extraction
    result = await crawler.arun(
        url="https://example.com",
        extraction_strategy="llm",
        extraction_prompt="Extract all product info as JSON"
    )
    print(result.extracted_content)

    # Deep crawl
    results = await crawler.arun(
        url="https://example.com",
        deep_crawl=True,
        max_depth=2,
        max_pages=50
    )

Scrapy (Large-Scale Scraping)

Setup

pip install scrapy
scrapy startproject myproject

Spider

import scrapy

class ProductSpider(scrapy.Spider):
    name = "products"
    start_urls = ["https://example.com/products"]

    def parse(self, response):
        for product in response.css(".product"):
            yield {
                "name": product.css(".name::text").get(),
                "price": product.css(".price::text").get(),
                "url": response.urljoin(product.css("a::attr(href)").get())
            }

        # Pagination
        next_page = response.css(".next a::attr(href)").get()
        if next_page:
            yield response.follow(next_page, self.parse)

Run

scrapy crawl products -o products.json

Middleware (Playwright Integration)

# settings.py
DOWNLOADER_MIDDLEWARES = {
    'scrapy_playwright.middleware.ScrapyPlaywrightMiddleware': 800,
}

# Spider
def parse(self, response):
    page = response.meta["playwright_page"]
    await page.click("button.load-more")
    content = await page.content()
    yield scrapy.Request(response.url, meta={"playwright": True, "playwright_include_page": True})

Common Patterns

1. Login Flow

async def login(page, username, password):
    await page.goto("https://app.example.com/login")
    await page.fill("input[name='email']", username)
    await page.fill("input[name='password']", password)
    await page.click("button[type='submit']")
    await page.wait_for_url("**/dashboard")

    # Save storage state
    await page.context.storage_state(path="auth.json")

2. Infinite Scroll

async def scroll_to_bottom(page, max_scrolls=50):
    last_height = 0
    for _ in range(max_scrolls):
        await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
        await page.wait_for_timeout(1000)
        new_height = await page.evaluate("document.body.scrollHeight")
        if new_height == last_height:
            break
        last_height = new_height

3. Handle Dialogs

page.on("dialog", lambda dialog: dialog.accept())
# or
page.on("dialog", lambda dialog: dialog.dismiss())

4. File Download

async with page.expect_download() as download_info:
    await page.click("a.download-link")
download = await download_info.value
await download.save_as("downloaded_file.pdf")

5. Network Interception

# Block resources
await page.route("**/*.{png,jpg,jpeg,css,woff2}", lambda route: route.abort())

# Mock API
await page.route("**/api/user", lambda route: route.fulfill(
    status=200,
    content_type="application/json",
    body='{"name": "Test User", "role": "admin"}'
))

5. Stealth / Anti-Detection

# Playwright stealth
await page.add_init_script("""
    Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
    window.chrome = {runtime: {}};
""")

# Or use playwright-stealth
# pip install playwright-stealth
from playwright_stealth import stealth_async
await stealth_async(page)

Testing Patterns

E2E Testing

import pytest
from playwright.async_api import async_playwright

@pytest.fixture
async def page():
    async with async_playwright() as p:
        browser = await p.chromium.launch()
        page = await browser.new_page()
        yield page
        await browser.close()

@pytest.mark.asyncio
async def test_login(page):
    await page.goto("https://app.example.com/login")
    await page.fill("#email", "test@example.com")
    await page.fill("#password", "password")
    await page.click("button[type=submit]")
    await page.wait_for_url("**/dashboard")
    assert await page.locator(".welcome").is_visible()

Visual Regression

await page.screenshot(path="baseline.png")
# Later...
await page.screenshot(path="current.png")
# Compare with pixelmatch or similar

Cloud Scaling

Related Notes

• Function Calling — Browser as tool
• MCP Protocol — Browser via MCP
• Code Execution — Code execution in browser
• API Integration — API vs browser
• Custom Tools — Building browser tools
• OpenAI Agents SDK — Computer use tool
• LangGraph — Browser in LangGraph
• AgentKit — Browser in AgentKit
• Research Agents — Web research agents
• Customer Facing Agents — Browser automation for support
• Secure Coding — Browser security
• Observability — Browser observability
• Security — Browser security

Sources: Playwright docs, Puppeteer docs, Selenium docs, Browserbase docs, Stagehand docs, Crawl4AI docs, Scrapy docs, Playwright stealth, browser automation best practices.