//go:build playwright

// camoufox_playwright.go — real Camoufox browser fetcher using playwright-go.
//
// Compiled only when the "playwright" build tag is present:
//
//	go build -tags playwright ./...
//	go test  -tags playwright ./fetch/...
//
// Prerequisites:
//  1. Add playwright-go to the module:
//     go get github.com/playwright-community/playwright-go@latest
//  2. Install the Firefox (Camoufox) binary once per environment:
//     go run github.com/playwright-community/playwright-go/cmd/playwright install firefox
//
// Design notes:
//   - One playwright.Playwright + one playwright.Browser is shared across all
//     Fetch calls; a fresh playwright.BrowserContext is created per Fetch to
//     provide per-request cookie/session isolation.
//   - Camoufox CAMOU_CONFIG environment vars come from identity.Profile.CamoufoxEnv
//     and are injected via playwright BrowserTypeLaunchOptions.Env.
//   - Images, media, and fonts are intercepted and aborted when blockImages=true
//     to reduce bandwidth for content-only scraping.
//   - Navigation uses WaitUntilStateDomcontentloaded by default for fast page
//     loads. When a solver extension is active, WaitUntilStateLoad is used
//     to ensure content scripts are injected.

package fetch

import (
	"context"
	"encoding/json"
	"fmt"
	"log/slog"
	"math/rand/v2"
	"net/http"
	neturl "net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/playwright-community/playwright-go"
	foxhound "github.com/sadewadee/foxhound"
	"github.com/sadewadee/foxhound/behavior"
	"github.com/sadewadee/foxhound/captcha"
	"github.com/sadewadee/foxhound/identity"
)

// defaultBrowserTimeout is the per-navigation ceiling when no explicit timeout
// is set on the CamoufoxFetcher.
const defaultBrowserTimeout = 60 * time.Second

// resourceBlockPatterns is the route glob list aborted when blockImages=true.
// Blocking binary resources cuts typical page-load time by 30–70 % for
// content-only scraping.
var resourceBlockPatterns = []string{
	"**/*.{png,jpg,jpeg,gif,svg,webp,ico,avif,bmp,tiff}",
	"**/*.{mp4,webm,ogg,mp3,wav,flac}",
	"**/*.{woff,woff2,ttf,otf,eot}",
}

// BrowserCookie represents a cookie to inject into the browser context.
type BrowserCookie struct {
	Name     string
	Value    string
	Domain   string
	Path     string
	Secure   bool
	HttpOnly bool
}

// StorageState represents the full browser session state (cookies + localStorage + sessionStorage).
type StorageState struct {
	Cookies    []BrowserCookie `json:"cookies"`
	Origins    []OriginStorage `json:"origins"`
	ExportedAt time.Time       `json:"exported_at"`
}

// OriginStorage represents localStorage/sessionStorage for a single origin.
type OriginStorage struct {
	Origin       string            `json:"origin"`
	LocalStorage map[string]string `json:"local_storage"`
}

// CamoufoxOption is a functional option for configuring a CamoufoxFetcher.
type CamoufoxOption func(*CamoufoxFetcher)

// WithBrowserCookies sets cookies to inject into the browser context before
// page navigation. Useful for pre-authenticated sessions.
func WithBrowserCookies(cookies []BrowserCookie) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.cookies = cookies
	}
}

// WithBehaviorProfile sets the behavior profile used for scroll and keyboard
// simulation in browser steps. When nil, defaults are used.
func WithBehaviorProfile(p *behavior.BehaviorProfile) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.behaviorProfile = p
	}
}

// WithBrowserIdentity sets the identity profile used to configure the Camoufox
// browser context and CAMOU_CONFIG environment variables.
func WithBrowserIdentity(p *identity.Profile) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.identity = p
	}
}

// WithBlockImages controls whether image/media/font requests are intercepted
// and aborted to reduce bandwidth for content-only scraping.
func WithBlockImages(block bool) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.blockImages = block
	}
}

// WithHeadless sets the display mode for the Camoufox browser:
//   - "virtual":  Xvfb virtual display (recommended on servers without a GPU)
//   - "true":     native headless mode
//   - "false":    full visible browser (useful for local debugging)
func WithHeadless(mode string) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.headless = mode
	}
}

// WithBrowserTimeout overrides the default per-navigation timeout.
func WithBrowserTimeout(d time.Duration) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.timeout = d
	}
}

// WithMaxBrowserRequests configures the browser instance to be restarted
// after serving n requests. This clears accumulated state (cookies, cache,
// memory leaks) and effectively rotates the browser fingerprint over long runs.
//
// Set n=0 to disable automatic restarts. The default is 300.
func WithMaxBrowserRequests(n int) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.maxRequests = n
	}
}

// WithPersistSession controls whether the same BrowserContext is reused
// across requests for the same walker session. When true, cookies and
// localStorage are preserved between fetches — necessary for sites that
// require login state to be maintained across page visits. When false
// (default) a fresh isolated context is created per fetch.
func WithPersistSession(persist bool) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.persistSession = persist
	}
}

// CamoufoxFetcher drives a Camoufox (Firefox fork) browser instance via the
// Juggler protocol using playwright-go.
//
// A single browser instance is shared; each Fetch call creates an isolated
// BrowserContext so cookies and sessions never bleed across requests.
// The browser is automatically restarted every maxRequests fetches to clear
// accumulated state and rotate the fingerprint.
//
// When persistSession=true a single BrowserContext is reused across all Fetch
// calls so cookies, localStorage, and session state are preserved between
// requests — useful for sites that require login state.
type CamoufoxFetcher struct {
	pw                     *playwright.Playwright
	browser                playwright.Browser
	identity               *identity.Profile
	blockImages            bool
	headless               string
	timeout                time.Duration
	proxyURL               string       // SOCKS5 or HTTP proxy URL
	extensionPath          string       // path to Firefox extension dir (e.g. NopeCHA)
	maxRequests            int          // restart after this many fetches (0 = disabled)
	requestCount           atomic.Int64 // total fetches served by the current browser instance
	mu                     sync.Mutex   // serialises browser restart and Close()
	closing                atomic.Bool  // set by Close() to prevent restart() after shutdown begins
	persistSession         bool
	sessionCtx             playwright.BrowserContext // reused when persistSession=true
	sessionMu              sync.Mutex                // guards sessionCtx lifecycle
	hasExtension           bool                      // true when solver extension is loaded
	initScript             string                    // JS injected into every page via AddInitScript
	userDataDir            string                    // persistent profile dir; uses LaunchPersistentContext
	persistCtx             playwright.BrowserContext // context from LaunchPersistentContext
	cdpURL                 string                    // connect to existing browser via CDP
	useRealChrome          bool                      // use pw.Chromium with channel=chrome
	capturePatterns        []*regexp.Regexp          // URL patterns for XHR/fetch response capture
	poolSize               int                       // max pooled pages (0 = disabled)
	pageReuseLimit         int                       // max reuses per pooled page (0 = unlimited)
	pool                   *PagePool                 // page pool (non-nil when poolSize > 0 and !persistSession)
	cookies                []BrowserCookie           // cookies to inject into browser context before navigation
	behaviorProfile        *behavior.BehaviorProfile // optional profile for scroll/keyboard configs
	tempDirs               []string                  // temp directories to clean up on Close/restart
	storageStatePath       string                    // path to load/save storage state JSON
	socksBridge            *socks5Bridge             // local SOCKS5 bridge for authenticated proxies (nil when not needed)
	skipExtension          bool                      // skip auto-loading NopeCHA addon (API solver active)
	displayMgr             *DisplayManager           // Xvfb manager (non-nil in "virtual" mode on Linux)
	hasCamoufox            bool                      // true when Camoufox binary is in use (not plain Firefox)
	cloudflareSolveTimeout time.Duration             // when >0, verify Cloudflare solve via cookie+DOM polling
	interceptConfig        *InterceptConfig          // route-level resource/domain blocking config
}

// WithBrowserProxy sets the proxy URL for all browser requests.
// Supports SOCKS5 (socks5://user:pass@host:port) and HTTP (http://user:pass@host:port).
func WithBrowserProxy(proxyURL string) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.proxyURL = proxyURL
	}
}

// WithExtensionPath sets the path to a Firefox extension directory to load.
// The extension is installed as a temporary addon when the browser launches.
// This is used to load NopeCHA or similar solver extensions that auto-solve
// CAPTCHA challenges the browser encounters.
func WithExtensionPath(path string) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.extensionPath = path
	}
}

// WithSkipExtension prevents the NopeCHA addon from being auto-loaded.
// Use when the NopeCHA Token API solver is active — the API and addon
// should not run simultaneously.
func WithSkipExtension() CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.skipExtension = true
	}
}

// WithInitScript sets a JavaScript snippet that is injected into every new
// page before the page's own scripts execute, via page.AddInitScript. Useful
// for overriding navigator properties (e.g. navigator.webdriver=false) or
// installing global stubs before any site JS runs.
func WithInitScript(script string) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.initScript = script
	}
}

// WithUserDataDir sets a persistent profile directory. When non-empty,
// NewCamoufox uses pw.Firefox.LaunchPersistentContext (or
// pw.Chromium.LaunchPersistentContext when WithRealChrome is true) so cookies,
// localStorage, and cached resources survive across browser restarts.
func WithUserDataDir(dir string) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.userDataDir = dir
	}
}

// WithCDPURL sets a Chrome DevTools Protocol endpoint URL (e.g.
// "http://localhost:9222"). When non-empty, NewCamoufox connects to an
// existing browser process via pw.Chromium.ConnectOverCDP instead of launching
// a new browser. Useful for remote debugging setups or distributed scraping
// where the browser lifecycle is managed externally.
func WithCDPURL(url string) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.cdpURL = url
	}
}

// WithRealChrome switches from Firefox/Camoufox to pw.Chromium.Launch with
// channel="chrome". Use when Chrome rendering is required or when a Chrome
// installation is available but Camoufox is not. Falls back silently to
// Chromium if the Chrome channel binary is not installed.
func WithRealChrome(use bool) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.useRealChrome = use
	}
}

// WithPoolSize enables page pooling with the given max size.
// When enabled, browser contexts and pages are reused instead of created
// fresh for each request, significantly reducing per-request overhead (~3s
// context creation cost is eliminated after warmup).
// Only applies when PersistSession is false (non-persistent mode).
// Set to 0 (default) to disable pooling and use per-request contexts.
func WithPoolSize(n int) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.poolSize = n
	}
}

// WithPageReuseLimit sets the maximum number of requests a pooled page handles
// before being destroyed and replaced. This prevents long-lived pages from
// accumulating state that increases detection risk. Default 0 means unlimited.
// Recommended: 50-200 for anti-detection scraping.
func WithPageReuseLimit(n int) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.pageReuseLimit = n
	}
}

// WithStorageState sets a file path for session state persistence.
// On startup, existing state is loaded into the cookie list. On Close(), the
// current browser session state is saved to disk.
func WithStorageState(path string) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.storageStatePath = path
	}
}

// WithSolveCloudflare enables verified Cloudflare challenge resolution. After
// the existing handler runs (extension addon or manual click), the fetcher
// polls for up to timeout for three success signals: the cf_clearance cookie
// is present, Turnstile DOM markers are gone, and the cf-turnstile-response
// token is set. When all three pass within the budget the resulting Response
// has CloudflareSolved=true. On timeout the fetch is NOT failed; the response
// is still returned with CloudflareSolved=false so callers can decide whether
// to retry, escalate, or accept a partial page. Pass 0 to disable.
func WithSolveCloudflare(timeout time.Duration) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.cloudflareSolveTimeout = timeout
	}
}

// WithInterceptConfig wires a route-level blocking configuration into the
// browser. Resources matching BlockedResourceTypes are aborted before they
// reach the network, and requests to BlockedDomains (or their subdomains) are
// also aborted. Use NewInterceptConfig to construct the value.
func WithInterceptConfig(ic *InterceptConfig) CamoufoxOption {
	return func(f *CamoufoxFetcher) {
		f.interceptConfig = ic
	}
}

// NewCamoufox initialises playwright, applies the supplied options, launches a
// Firefox (Camoufox) browser, and returns a ready-to-use CamoufoxFetcher.
//
// The browser is kept alive until Close is called. If the playwright runtime or
// the Camoufox binary is not installed, NewCamoufox auto-downloads it via
// `python3 -m camoufox fetch`. No manual setup is required.
func NewCamoufox(opts ...CamoufoxOption) (*CamoufoxFetcher, error) {
	f := &CamoufoxFetcher{
		headless:    "virtual",
		timeout:     defaultBrowserTimeout,
		maxRequests: 300,
	}
	for _, opt := range opts {
		opt(f)
	}

	// When headless mode is "virtual", start an Xvfb virtual display server
	// so the browser has an X11 display to render into. On non-Linux platforms
	// or when DISPLAY is already set, this is a no-op (NewDisplayManager returns nil).
	if f.headless == "virtual" && needsXvfb() {
		dm, dmErr := NewDisplayManager()
		if dmErr != nil {
			slog.Warn("fetch/camoufox: Xvfb failed, falling back to native headless",
				"err", dmErr)
			f.headless = "true"
		} else if dm != nil {
			f.displayMgr = dm
			slog.Info("fetch/camoufox: using Xvfb virtual display", "display", dm.Display())
		}
	}

	// Start the playwright runtime.
	pw, err := playwright.Run()
	if err != nil {
		// If we started Xvfb, clean it up on failure.
		if f.displayMgr != nil {
			f.displayMgr.Close()
		}
		return nil, fmt.Errorf("fetch/camoufox: starting playwright runtime: %w", err)
	}

	// If the proxy is SOCKS5 with auth credentials, start a local bridge
	// because Firefox/Playwright does not support SOCKS5 authentication.
	// Detection is automatic from the URL — no flags needed.
	if upstream, user, pass, ok := needsSocks5Bridge(f.proxyURL); ok {
		bridge, err := newSocks5Bridge(upstream, user, pass)
		if err != nil {
			pw.Stop()
			return nil, fmt.Errorf("fetch/camoufox: starting SOCKS5 auth bridge: %w", err)
		}
		f.socksBridge = bridge
		f.proxyURL = "socks5://" + bridge.localAddr
		slog.Info("fetch/camoufox: SOCKS5 auth bridge started",
			"local", bridge.localAddr, "upstream", upstream)
	}

	// Locate or auto-install the Camoufox binary.
	// Camoufox is a C++ patched Firefox with built-in anti-fingerprinting:
	// canvas, WebGL, audio, font, navigator — all spoofed at engine level.
	execPath, err := findOrInstallCamoufox()
	if err != nil {
		slog.Warn("fetch/camoufox: Camoufox binary not available, falling back to plain Firefox",
			"err", err)
		// Fall back to plain Firefox (playwright's bundled version)
		execPath = ""
	}

	headlessBool := f.headless != "false"

	launchOpts := playwright.BrowserTypeLaunchOptions{
		Headless: playwright.Bool(headlessBool),
	}

	// Use Camoufox binary if found — this enables ALL C++ level anti-detection:
	// canvas spoofing, WebGL spoofing, font spoofing, navigator.webdriver=false,
	// WebRTC IP protection, audio context spoofing, BrowserForge fingerprints.
	if execPath != "" {
		launchOpts.ExecutablePath = playwright.String(execPath)
		f.hasCamoufox = true
		slog.Info("fetch/camoufox: using Camoufox binary", "path", execPath)
	} else {
		slog.Warn("fetch/camoufox: using plain Firefox — anti-fingerprint features disabled")
	}

	// Set proxy at browser launch level.
	if f.proxyURL != "" {
		proxyOpt := parsePlaywrightProxy(f.proxyURL)
		launchOpts.Proxy = proxyOpt
		slog.Info("fetch/camoufox: proxy configured", "server", proxyOpt.Server)
	}

	// When skipExtension is true (NopeCHA API solver active), do not
	// auto-download or load the addon — API and addon must not run together.
	if f.skipExtension && f.extensionPath == "" {
		f.extensionPath = "none"
		slog.Info("fetch/camoufox: NopeCHA addon skipped — API solver active")
	}

	// Auto-load NopeCHA extension by default unless explicitly disabled.
	// If extensionPath is empty, auto-download NopeCHA to cache.
	// If extensionPath is "none", skip extension loading entirely.
	if f.extensionPath == "" {
		nopechaPath, nopechaErr := ensureNopeCHA()
		if nopechaErr != nil {
			slog.Warn("fetch/camoufox: NopeCHA auto-download failed, continuing without extension",
				"err", nopechaErr)
		} else {
			f.extensionPath = nopechaPath
			slog.Info("fetch/camoufox: NopeCHA extension auto-loaded", "path", nopechaPath)
		}
	}

	// Build CAMOU_CONFIG env vars: fingerprint config + addon config merged
	// into a single JSON blob split across CAMOU_CONFIG_1, CAMOU_CONFIG_2, ...
	//
	// Camoufox reads these at C++ level to spoof screen, navigator, WebGL,
	// fonts, canvas, geolocation, timezone, and locale — all invisible to JS.
	if launchOpts.Env == nil {
		launchOpts.Env = map[string]string{}
	}

	// Resolve addon path (if any) for merging into CAMOU_CONFIG.
	var addonExtra map[string]any
	if f.extensionPath != "" && f.extensionPath != "none" && execPath != "" {
		absExt, _ := filepath.Abs(f.extensionPath)
		// If .xpi, it must be extracted first — Camoufox expects directories.
		if strings.HasSuffix(absExt, ".xpi") {
			extractDir, tmpErr := os.MkdirTemp("", "foxhound-addon-*")
			if tmpErr == nil {
				unzipCmd := exec.Command("unzip", "-o", absExt, "-d", extractDir)
				if uzErr := unzipCmd.Run(); uzErr != nil {
					slog.Warn("fetch/camoufox: failed to extract .xpi", "err", uzErr)
				} else {
					absExt = extractDir
					f.tempDirs = append(f.tempDirs, extractDir)
					slog.Info("fetch/camoufox: extracted .xpi", "dir", extractDir)
				}
			}
		}
		addonExtra = map[string]any{
			"addons": []string{absExt},
		}
		f.hasExtension = true
		slog.Info("fetch/camoufox: addon will be injected via CAMOU_CONFIG",
			"path", absExt)
	}

	// Merge fingerprint config + addon config into CAMOU_CONFIG_N env vars.
	if f.identity != nil && execPath != "" {
		if addonExtra != nil {
			// Merge: fingerprint + addons in one JSON blob
			for k, v := range f.identity.MergeCamoufoxConfig(addonExtra) {
				launchOpts.Env[k] = v
			}
		} else {
			// Fingerprint only
			for k, v := range f.identity.BuildCamoufoxEnv() {
				launchOpts.Env[k] = v
			}
		}
		slog.Info("fetch/camoufox: fingerprint config injected via CAMOU_CONFIG")
	} else if addonExtra != nil {
		// No identity but addon needed — addon-only config
		configJSON, _ := json.Marshal(addonExtra)
		launchOpts.Env["CAMOU_CONFIG_1"] = string(configJSON)
	}

	// --- Browser acquisition: CDP connect > persistent context > normal launch ---

	// Mode 1: connect to an existing browser over CDP (e.g. remote Chrome).
	// cdpURL takes precedence over all other launch options.
	if f.cdpURL != "" {
		browser, cdpErr := pw.Chromium.ConnectOverCDP(f.cdpURL)
		if cdpErr != nil {
			_ = pw.Stop()
			return nil, fmt.Errorf("fetch/camoufox: ConnectOverCDP(%s): %w", f.cdpURL, cdpErr)
		}
		f.pw = pw
		f.browser = browser
		slog.Info("fetch/camoufox: connected to existing browser via CDP", "url", f.cdpURL)
		return f, nil
	}

	// Mode 2: real Chrome channel (pw.Chromium with channel="chrome").
	if f.useRealChrome {
		chromeLaunchOpts := playwright.BrowserTypeLaunchOptions{
			Headless: playwright.Bool(f.headless != "false"),
			Channel:  playwright.String("chrome"),
		}
		if f.proxyURL != "" {
			chromeLaunchOpts.Proxy = parsePlaywrightProxy(f.proxyURL)
		}

		// Persistent context takes priority when userDataDir is also set.
		if f.userDataDir != "" {
			persistOpts := playwright.BrowserTypeLaunchPersistentContextOptions{
				Headless: playwright.Bool(f.headless != "false"),
				Channel:  playwright.String("chrome"),
			}
			if f.proxyURL != "" {
				persistOpts.Proxy = parsePlaywrightProxy(f.proxyURL)
			}
			bctx, persistErr := pw.Chromium.LaunchPersistentContext(f.userDataDir, persistOpts)
			if persistErr != nil {
				// Fall back to regular Chrome launch.
				slog.Warn("fetch/camoufox: LaunchPersistentContext (Chrome) failed, falling back",
					"err", persistErr)
			} else {
				f.pw = pw
				f.persistCtx = bctx
				// Obtain the browser handle from the context for restart logic.
				f.browser = bctx.Browser()
				slog.Info("fetch/camoufox: Chrome with persistent context ready",
					"dir", f.userDataDir)
				return f, nil
			}
		}

		browser, chromeErr := pw.Chromium.Launch(chromeLaunchOpts)
		if chromeErr != nil {
			// Chrome not installed — fall back to plain Chromium.
			slog.Warn("fetch/camoufox: Chrome channel not found, falling back to Chromium", "err", chromeErr)
			chromeLaunchOpts.Channel = nil
			browser, chromeErr = pw.Chromium.Launch(chromeLaunchOpts)
			if chromeErr != nil {
				_ = pw.Stop()
				return nil, fmt.Errorf("fetch/camoufox: Chromium launch failed: %w", chromeErr)
			}
		}
		f.pw = pw
		f.browser = browser
		slog.Info("fetch/camoufox: Chrome/Chromium browser ready",
			"headless", f.headless,
			"timeout", f.timeout,
		)
		return f, nil
	}

	// Mode 3: Firefox/Camoufox with persistent profile directory.
	if f.userDataDir != "" {
		persistOpts := playwright.BrowserTypeLaunchPersistentContextOptions{
			Headless: playwright.Bool(f.headless != "false"),
		}
		if execPath != "" {
			persistOpts.ExecutablePath = playwright.String(execPath)
		}
		if f.proxyURL != "" {
			persistOpts.Proxy = parsePlaywrightProxy(f.proxyURL)
		}
		bctx, persistErr := pw.Firefox.LaunchPersistentContext(f.userDataDir, persistOpts)
		if persistErr != nil {
			slog.Warn("fetch/camoufox: LaunchPersistentContext failed, falling back to regular launch",
				"err", persistErr)
			// Fall through to Mode 4 (normal Firefox launch).
		} else {
			f.pw = pw
			f.persistCtx = bctx
			f.browser = bctx.Browser()
			slog.Info("fetch/camoufox: Firefox/Camoufox with persistent context ready",
				"dir", f.userDataDir)
			return f, nil
		}
	}

	// Mode 4: standard Firefox/Camoufox launch (default path).
	browser, err := pw.Firefox.Launch(launchOpts)
	if err != nil {
		// If Camoufox binary failed, try plain Firefox as last resort.
		if execPath != "" {
			slog.Warn("fetch/camoufox: Camoufox binary launch failed, trying plain Firefox", "err", err)
			launchOpts.ExecutablePath = nil
			browser, err = pw.Firefox.Launch(launchOpts)
		}
		if err != nil {
			// Auto-install playwright Firefox if nothing works.
			slog.Info("fetch/camoufox: installing playwright Firefox...")
			if installErr := playwright.Install(&playwright.RunOptions{
				Browsers: []string{"firefox"},
			}); installErr != nil {
				_ = pw.Stop()
				return nil, fmt.Errorf("fetch/camoufox: all browser launch attempts failed: %w", err)
			}
			browser, err = pw.Firefox.Launch(launchOpts)
			if err != nil {
				_ = pw.Stop()
				return nil, fmt.Errorf("fetch/camoufox: launch failed after install: %w", err)
			}
		}
	}

	f.pw = pw
	f.browser = browser

	// Pre-warm: create and immediately close one context to initialize
	// browser internals. Subsequent context creation is ~50% faster.
	if warmCtx, warmErr := f.browser.NewContext(); warmErr == nil {
		warmCtx.Close()
	}

	browserType := "plain Firefox"
	if execPath != "" {
		browserType = "Camoufox"
	}
	slog.Info("fetch/camoufox: browser ready",
		"browser", browserType,
		"headless", f.headless,
		"block_images", f.blockImages,
		"timeout", f.timeout,
	)

	// Load previously saved storage state (cookies) into the fetcher.
	if f.storageStatePath != "" {
		state, loadErr := f.LoadStorageState()
		if loadErr != nil {
			slog.Warn("fetch/camoufox: error loading storage state", "err", loadErr)
		} else if state != nil {
			f.cookies = state.Cookies
			slog.Debug("fetch/camoufox: loaded storage state", "cookies", len(state.Cookies))
		}
	}

	// Initialise page pool when poolSize > 0 and not in persistent-session mode.
	// Each pooled entry owns a dedicated BrowserContext + Page so contexts are
	// fully isolated between pool slots and sessions never bleed.
	if f.poolSize > 0 && !f.persistSession {
		f.pool = NewPagePool(
			f.poolSize,
			func() (any, error) {
				// IMPORTANT: snapshot f.browser under f.mu at invocation time.
				// The create closure is called lazily by pool.Acquire outside any
				// lock; Close() or restart() may nil f.browser between Acquire
				// calls, so capturing a live snapshot here prevents nil-pointer
				// dereference (v0.0.22 fix for B3 — see .dev-squad/v0.0.22-rca.md).
				f.mu.Lock()
				b := f.browser
				f.mu.Unlock()
				if b == nil {
					return nil, fmt.Errorf("fetch/camoufox: browser unavailable for page pool")
				}
				bctx, err := b.NewContext(f.buildContextOptions())
				if err != nil {
					return nil, fmt.Errorf("pagepool: new context: %w", err)
				}
				pg, err := bctx.NewPage()
				if err != nil {
					_ = bctx.Close()
					return nil, fmt.Errorf("pagepool: new page: %w", err)
				}
				return pg, nil
			},
			func(v any) error {
				pg := v.(playwright.Page)
				return pg.Context().Close()
			},
			WithPageReset(func(v any) error {
				pg := v.(playwright.Page)
				// Clear cookies + storage to prevent session bleed across requests.
				if err := pg.Context().ClearCookies(); err != nil {
					return fmt.Errorf("pagepool: clear cookies: %w", err)
				}
				// Navigate to about:blank so no prior page content is visible.
				if _, err := pg.Goto("about:blank", playwright.PageGotoOptions{
					WaitUntil: playwright.WaitUntilStateDomcontentloaded,
					Timeout:   playwright.Float(5000),
				}); err != nil {
					return fmt.Errorf("pagepool: reset navigate: %w", err)
				}
				return nil
			}),
			WithPoolReuseLimit(f.pageReuseLimit),
		)
		slog.Info("fetch/camoufox: page pool initialised",
			"pool_size", f.poolSize,
			"page_reuse_limit", f.pageReuseLimit)
	}

	return f, nil
}

// findOrInstallCamoufox locates the Camoufox binary on the system.
// If not found, it attempts to auto-install via `python3 -m camoufox fetch`.
// Returns the executable path or an error.
func findOrInstallCamoufox() (string, error) {
	// Check known paths per OS.
	path := findCamoufoxBinary()
	if path != "" {
		slog.Debug("fetch/camoufox: binary found", "path", path)
		return path, nil
	}

	// Not found — try auto-install.
	slog.Info("fetch/camoufox: binary not found, auto-downloading...")

	// Try python3 first, then python.
	for _, py := range []string{"python3", "python"} {
		// First ensure the camoufox package is installed.
		installCmd := exec.Command(py, "-m", "pip", "install", "-q", "camoufox")
		installCmd.Stdout = os.Stderr
		installCmd.Stderr = os.Stderr
		if err := installCmd.Run(); err != nil {
			continue
		}

		// Download the browser binary.
		fetchCmd := exec.Command(py, "-m", "camoufox", "fetch")
		fetchCmd.Stdout = os.Stderr
		fetchCmd.Stderr = os.Stderr
		if err := fetchCmd.Run(); err != nil {
			slog.Warn("fetch/camoufox: download failed", "python", py, "err", err)
			continue
		}

		// Re-check after install.
		path = findCamoufoxBinary()
		if path != "" {
			slog.Info("fetch/camoufox: auto-install successful", "path", path)
			return path, nil
		}
	}

	return "", fmt.Errorf("camoufox binary not found and auto-install failed (is Python installed?)")
}

// findCamoufoxBinary checks known locations for the Camoufox executable.
func findCamoufoxBinary() string {
	home, _ := os.UserHomeDir()

	var candidates []string
	switch runtime.GOOS {
	case "darwin":
		// macOS: installed via pip
		cacheDir := filepath.Join(home, "Library", "Caches", "camoufox")
		candidates = []string{
			filepath.Join(cacheDir, "Camoufox.app", "Contents", "MacOS", "camoufox"),
		}
	case "linux":
		candidates = []string{
			filepath.Join(home, ".cache", "camoufox", "camoufox"),
			filepath.Join(home, ".local", "share", "camoufox", "camoufox"),
			"/usr/local/bin/camoufox",
		}
	case "windows":
		appData := os.Getenv("LOCALAPPDATA")
		if appData == "" {
			appData = filepath.Join(home, "AppData", "Local")
		}
		candidates = []string{
			filepath.Join(appData, "camoufox", "camoufox.exe"),
		}
	}

	// PATH check is LAST — the `camoufox` in PATH is usually the Python CLI
	// wrapper, not the actual browser binary. Prefer the cache directory.
	if path, err := exec.LookPath("camoufox"); err == nil {
		candidates = append(candidates, path)
	}

	for _, p := range candidates {
		if info, err := os.Stat(p); err == nil && !info.IsDir() {
			return p
		}
	}
	return ""
}

// nopechaCacheDir returns the directory where NopeCHA extension is cached.
func nopechaCacheDir() string {
	home, _ := os.UserHomeDir()
	return filepath.Join(home, ".cache", "foxhound", "extensions", "nopecha")
}

// nopeCHAGitHubReleasesAPI is the GitHub API endpoint for NopeCHA releases.
const nopeCHAGitHubReleasesAPI = "https://api.github.com/repos/NopeCHALLC/nopecha-extension/releases/latest"

// ensureNopeCHA returns the path to a cached NopeCHA extension directory,
// downloading it from GitHub releases if not already present.
func ensureNopeCHA() (string, error) {
	cacheDir := nopechaCacheDir()
	manifestPath := filepath.Join(cacheDir, "manifest.json")

	// Already cached — return immediately.
	if _, err := os.Stat(manifestPath); err == nil {
		slog.Debug("fetch/nopecha: using cached extension", "dir", cacheDir)
		return cacheDir, nil
	}

	slog.Info("fetch/nopecha: downloading NopeCHA extension from GitHub...")

	// Query GitHub API for latest release to find firefox_automation.zip asset.
	downloadURL, err := findNopeCHADownloadURL()
	if err != nil {
		return "", fmt.Errorf("fetch/nopecha: find download URL: %w", err)
	}

	// Download the zip to a temp file.
	tmpFile, err := os.CreateTemp("", "nopecha-*.zip")
	if err != nil {
		return "", fmt.Errorf("fetch/nopecha: create temp file: %w", err)
	}
	tmpPath := tmpFile.Name()
	tmpFile.Close()
	defer os.Remove(tmpPath)

	// Use curl for the download (follows redirects, handles GitHub).
	dlCmd := exec.Command("curl", "-fsSL", "-o", tmpPath, downloadURL)
	dlCmd.Stderr = os.Stderr
	if err := dlCmd.Run(); err != nil {
		return "", fmt.Errorf("fetch/nopecha: download failed: %w", err)
	}

	// Create cache directory and extract.
	if err := os.MkdirAll(cacheDir, 0o755); err != nil {
		return "", fmt.Errorf("fetch/nopecha: create cache dir: %w", err)
	}

	unzipCmd := exec.Command("unzip", "-o", tmpPath, "-d", cacheDir)
	unzipCmd.Stderr = os.Stderr
	if err := unzipCmd.Run(); err != nil {
		os.RemoveAll(cacheDir)
		return "", fmt.Errorf("fetch/nopecha: extract failed: %w", err)
	}

	// Verify manifest.json exists after extraction.
	if _, err := os.Stat(manifestPath); err != nil {
		// Some zips have a nested directory — check one level deep.
		entries, _ := os.ReadDir(cacheDir)
		for _, e := range entries {
			if e.IsDir() {
				nested := filepath.Join(cacheDir, e.Name(), "manifest.json")
				if _, nerr := os.Stat(nested); nerr == nil {
					// Move contents up one level.
					nestedDir := filepath.Join(cacheDir, e.Name())
					innerEntries, _ := os.ReadDir(nestedDir)
					for _, ie := range innerEntries {
						os.Rename(
							filepath.Join(nestedDir, ie.Name()),
							filepath.Join(cacheDir, ie.Name()),
						)
					}
					os.Remove(nestedDir)
					break
				}
			}
		}
	}

	// Final check.
	if _, err := os.Stat(manifestPath); err != nil {
		return "", fmt.Errorf("fetch/nopecha: manifest.json not found after extraction")
	}

	slog.Info("fetch/nopecha: extension downloaded and cached", "dir", cacheDir)
	return cacheDir, nil
}

// findNopeCHADownloadURL queries GitHub releases API and returns the
// firefox.zip asset download URL. We use the regular extension build (not
// firefox_automation.zip) because the automation build requires an API key
// for cloud-based solving. The regular build works like a normal browser
// extension — no API key needed.
func findNopeCHADownloadURL() (string, error) {
	// Use curl to fetch the releases API (avoids importing net/http for this one call).
	out, err := exec.Command("curl", "-fsSL", nopeCHAGitHubReleasesAPI).Output()
	if err != nil {
		return "", fmt.Errorf("GitHub API request failed: %w", err)
	}

	var release struct {
		Assets []struct {
			Name               string `json:"name"`
			BrowserDownloadURL string `json:"browser_download_url"`
		} `json:"assets"`
	}
	if err := json.Unmarshal(out, &release); err != nil {
		return "", fmt.Errorf("parse GitHub API response: %w", err)
	}

	// Use firefox.zip (regular extension, no API key needed).
	// Do NOT use firefox_automation.zip — that build requires a paid API key.
	for _, a := range release.Assets {
		if a.Name == "firefox.zip" {
			return a.BrowserDownloadURL, nil
		}
	}

	return "", fmt.Errorf("no firefox extension asset found in latest NopeCHA release")
}

// Fetch navigates to job.URL inside a fresh, isolated BrowserContext, waits
// for the network to become idle, extracts the fully-rendered HTML, and returns
// a foxhound.Response with FetchMode=FetchBrowser.
//
// Each call creates a new BrowserContext (and therefore a new cookie jar,
// localStorage, and cache partition) so requests cannot share state unless the
// caller deliberately reuses a context — which this implementation intentionally
// avoids to prevent identity leakage across scraping targets.
//
// When the number of completed fetches reaches maxRequests, the browser is
// automatically restarted before continuing. The restart is serialised under a
// mutex so concurrent Fetch calls never race on the browser handle.
func (f *CamoufoxFetcher) Fetch(ctx context.Context, job *foxhound.Job) (*foxhound.Response, error) {
	if job == nil {
		return nil, fmt.Errorf("fetch/camoufox: job must not be nil")
	}
	if job.URL == "" {
		return nil, fmt.Errorf("fetch/camoufox: job URL must not be empty")
	}

	// Lifecycle: restart the browser after maxRequests fetches.
	if f.maxRequests > 0 {
		count := f.requestCount.Add(1)
		if count > int64(f.maxRequests) {
			if err := f.restart(); err != nil {
				// Non-fatal: log and continue with the existing browser.
				slog.Warn("fetch/camoufox: browser restart failed, continuing with existing instance",
					"err", err,
					"request_count", count,
				)
			}
		}
	}

	// Propagate context cancellation into playwright by running navigation in a
	// goroutine and aborting via page.Close on context done.
	type result struct {
		resp *foxhound.Response
		err  error
	}
	ch := make(chan result, 1)

	go func() {
		resp, err := f.navigate(ctx, job)
		ch <- result{resp, err}
	}()

	select {
	case <-ctx.Done():
		return nil, fmt.Errorf("fetch/camoufox: context cancelled for %s: %w", job.URL, ctx.Err())
	case res := <-ch:
		return res.resp, res.err
	}
}

// getOrCreateContext returns the BrowserContext to use for this request.
//
// Priority order:
//  1. persistCtx — set when LaunchPersistentContext succeeded (userDataDir).
//     The persistent context IS the only context; never closed by callers.
//  2. persistSession=true — a single BrowserContext shared across requests.
//     Initialised on first call, never closed by callers.
//  3. Default — a fresh isolated BrowserContext per request; caller must close.
//
// Returns (ctx, shouldClose, err).
func (f *CamoufoxFetcher) getOrCreateContext() (playwright.BrowserContext, bool, error) {

	// IMPORTANT: snapshot f.persistCtx and f.browser under f.mu before use.
	// Close() and restart() both mutate these fields while holding f.mu, so
	// reading them once under the lock and using local copies eliminates the
	// TOCTOU race that would crash with a nil-pointer dereference (v0.0.22 fix
	// for B2 — see .dev-squad/v0.0.22-rca.md).
	f.mu.Lock()
	persistCtx := f.persistCtx
	browser := f.browser
	f.mu.Unlock()

	// When a persistent context was obtained from LaunchPersistentContext,
	// use it directly — it already embeds the profile directory state.
	if persistCtx != nil {
		return persistCtx, false, nil // caller must NOT close
	}

	if !f.persistSession {
		if browser == nil {
			return nil, false, fmt.Errorf("fetch/camoufox: browser is not running")
		}
		bctx, err := browser.NewContext(f.buildContextOptions())
		return bctx, true, err // caller should close
	}

	f.sessionMu.Lock()
	defer f.sessionMu.Unlock()

	if f.sessionCtx == nil {
		if browser == nil {
			return nil, false, fmt.Errorf("fetch/camoufox: browser is not running")
		}
		bctx, err := browser.NewContext(f.buildContextOptions())
		if err != nil {
			return nil, false, err
		}
		f.sessionCtx = bctx
	}
	return f.sessionCtx, false, nil // caller must NOT close
}

// simulateHumanBehavior performs lightweight mouse and scroll actions after a
// page has finished loading. These interactions prove to behavioural anti-bot
// systems that a real user is present and interacting with the page.
//
// Actions performed (all are best-effort; errors are silently ignored so the
// underlying content extraction is not blocked by a failed gesture):
//  1. Random reading pause after page load (1-3 s)
//  2. Traverse bezier path to first random viewport position
//  3. Idle micro-drift while "reading"
//  4. Scroll down 200-600 px with idle drift during scroll pause
//  5. Optionally scroll back up (30 % probability, natural reading pattern)
//  6. Traverse bezier path to second random viewport position
func (f *CamoufoxFetcher) simulateHumanBehavior(page playwright.Page) {
	mouse := behavior.NewMouse(f.mouseConfig())

	// 1. Reading pause — simulates time-to-first-interaction after load.
	time.Sleep(time.Duration(1000+rand.IntN(2000)) * time.Millisecond)

	viewportSize := page.ViewportSize()
	if viewportSize == nil || viewportSize.Width <= 100 || viewportSize.Height <= 100 {
		return
	}

	// 2. Traverse bezier path to first random position
	startX := float64(viewportSize.Width / 2)
	startY := float64(viewportSize.Height / 2)
	endX := float64(rand.IntN(viewportSize.Width-100) + 50)
	endY := float64(rand.IntN(viewportSize.Height-100) + 50)

	path := mouse.MoveTo(
		behavior.Point{X: startX, Y: startY},
		behavior.Point{X: endX, Y: endY},
	)
	for _, pt := range path {
		if err := page.Mouse().Move(pt.X, pt.Y); err != nil {
			break
		}
		// Inter-point delay: 5-15ms per waypoint (mimics 60-120Hz mouse polling)
		time.Sleep(time.Duration(5+rand.IntN(11)) * time.Millisecond)
	}

	// Idle micro-drift while "reading" (0.5-3px tremor)
	idleDuration := time.Duration(500+rand.IntN(1500)) * time.Millisecond
	idleEnd := time.Now().Add(idleDuration)
	lastX, lastY := endX, endY
	for time.Now().Before(idleEnd) {
		drift := mouse.IdleDrift()
		lastX += drift.X
		lastY += drift.Y
		page.Mouse().Move(lastX, lastY)
		time.Sleep(time.Duration(80+rand.IntN(120)) * time.Millisecond)
	}

	// 3. Scroll down — shows the user is reading past the fold.
	scroller := behavior.NewScroll(f.scrollConfig())
	dist, pause, _ := scroller.ScrollGesture(behavior.ScrollReading)
	if err := page.Mouse().Wheel(0, float64(dist)); err != nil {
		slog.Debug("fetch/camoufox: simulateHuman: scroll down failed", "err", err)
	}

	// Idle micro-drift during scroll pause
	scrollPauseEnd := time.Now().Add(pause)
	for time.Now().Before(scrollPauseEnd) {
		drift := mouse.IdleDrift()
		lastX += drift.X
		lastY += drift.Y
		page.Mouse().Move(lastX, lastY)
		time.Sleep(time.Duration(80+rand.IntN(120)) * time.Millisecond)
	}

	// 4. Occasionally scroll back up — natural reading / re-reading behaviour.
	if rand.Float64() < 0.3 {
		upDist, upPause, _ := scroller.ScrollGesture(behavior.ScrollReading)
		if err := page.Mouse().Wheel(0, -float64(upDist/2)); err != nil {
			slog.Debug("fetch/camoufox: simulateHuman: scroll up failed", "err", err)
		}
		time.Sleep(upPause)
	}

	// 5. Traverse bezier path to second position
	endX2 := float64(rand.IntN(viewportSize.Width-100) + 50)
	endY2 := float64(rand.IntN(viewportSize.Height-100) + 50)
	path2 := mouse.MoveTo(
		behavior.Point{X: lastX, Y: lastY},
		behavior.Point{X: endX2, Y: endY2},
	)
	for _, pt := range path2 {
		if err := page.Mouse().Move(pt.X, pt.Y); err != nil {
			break
		}
		time.Sleep(time.Duration(5+rand.IntN(11)) * time.Millisecond)
	}
}

// executeStep performs a single Trail step on the loaded page. It handles
// Click, Wait, and Scroll actions. Extract steps are skipped (handled later
// by the Walker's Processor).
// executeStep runs a single JobStep on the page. For Evaluate steps, the
// return value of the JS expression is returned as the first value.
// For all other steps the first return value is nil.
func (f *CamoufoxFetcher) executeStep(page playwright.Page, step foxhound.JobStep) (any, error) {
	switch step.Action {
	case foxhound.JobStepClick:
		el, err := page.QuerySelector(step.Selector)
		if err != nil {
			return nil, fmt.Errorf("query selector %q: %w", step.Selector, err)
		}
		if el == nil {
			return nil, fmt.Errorf("selector %q not found", step.Selector)
		}
		if err := el.Click(); err != nil {
			return nil, fmt.Errorf("click %q: %w", step.Selector, err)
		}
		// Brief pause after click to let any JS handlers fire.
		time.Sleep(time.Duration(200+rand.IntN(300)) * time.Millisecond)

	case foxhound.JobStepWait:
		timeout := step.Duration
		if timeout <= 0 {
			timeout = 10 * time.Second
		}
		_, err := page.WaitForSelector(step.Selector, playwright.PageWaitForSelectorOptions{
			Timeout: playwright.Float(float64(timeout.Milliseconds())),
		})
		if err != nil {
			return nil, fmt.Errorf("wait for %q (timeout %v): %w", step.Selector, timeout, err)
		}

	case foxhound.JobStepScroll:
		scroller := behavior.NewScroll(f.scrollConfig())
		axis := behavior.ScrollAxis(step.ScrollAxis)
		extent := step.ScrollExtent
		if extent <= 0 {
			extent = 3000
		}
		mode := behavior.ScrollMode(step.ScrollMode)
		actions := scroller.ScrollSequenceAxis(extent, mode, axis)
		for _, a := range actions {
			var dx, dy float64
			if a.Axis == behavior.ScrollHorizontal {
				dx = float64(a.Distance)
				if a.Up {
					dx = -dx
				}
			} else {
				dy = float64(a.Distance)
				if a.Up {
					dy = -dy
				}
			}
			if err := page.Mouse().Wheel(dx, dy); err != nil {
				slog.Debug("fetch/camoufox: scroll step failed", "err", err)
			}
			time.Sleep(a.Pause)
		}

	case foxhound.JobStepInfiniteScroll:
		maxScrolls := step.MaxScrolls
		if maxScrolls <= 0 {
			maxScrolls = 50
		}

		// Custom scroll container: if Selector is set, scroll inside that
		// element (e.g. Google Maps panel) instead of document.body.
		container := step.Selector // "" = whole page (default)

		slog.Info("fetch/camoufox: infinite scroll started",
			"max", maxScrolls, "container", container,
			"stop_selector", step.StopSelector, "stop_count", step.StopCount)

		scroller := behavior.NewScroll(f.scrollConfig())

		// Build JS snippets for scroll height depending on container.
		getHeightJS := "() => document.body.scrollHeight"
		scrollToBottomJS := "() => window.scrollTo(0, document.body.scrollHeight)"
		if container != "" {
			getHeightJS = fmt.Sprintf("() => { const el = document.querySelector(%q); return el ? el.scrollHeight : 0 }", container)
			scrollToBottomJS = fmt.Sprintf("() => { const el = document.querySelector(%q); if (el) el.scrollTop = el.scrollHeight }", container)
		}

		stopCount := step.StopCount
		if stopCount <= 0 {
			stopCount = 1
		}

		scrollWait := step.ScrollWait
		if scrollWait <= 0 {
			scrollWait = 2 * time.Second
		}

		// Track element count across iterations for StopSelector progress.
		prevCount := 0
		if step.StopSelector != "" {
			initResult, _ := page.Evaluate(fmt.Sprintf(
				"() => document.querySelectorAll(%q).length", step.StopSelector))
			if cnt, ok := initResult.(float64); ok {
				prevCount = int(cnt)
			}
		}

		for i := 0; i < maxScrolls; i++ {
			// Check StopSelector target count before scrolling.
			if step.StopSelector != "" {
				countResult, _ := page.Evaluate(fmt.Sprintf(
					"() => document.querySelectorAll(%q).length", step.StopSelector))
				if cnt, ok := countResult.(float64); ok && int(cnt) >= stopCount {
					slog.Info("fetch/camoufox: infinite scroll stopped — target reached",
						"selector", step.StopSelector, "count", int(cnt), "target", stopCount)
					break
				}
			}

			// Get current scroll height.
			prevHeight, _ := page.Evaluate(getHeightJS)
			prevH, _ := prevHeight.(float64)

			// Scroll with human-like gesture.
			dist, pause, _ := scroller.ScrollGesture(behavior.ScrollScan)
			if container != "" {
				// Scroll inside the container element.
				page.Evaluate(fmt.Sprintf(
					"(d) => { const el = document.querySelector(%q); if (el) el.scrollTop += d }", container),
					dist)
			} else {
				_ = page.Mouse().Wheel(0, float64(dist))
			}
			time.Sleep(pause)

			// Also jump to bottom for pages that need it.
			page.Evaluate(scrollToBottomJS)
			// Wait for new content to load.
			time.Sleep(scrollWait + time.Duration(rand.IntN(500))*time.Millisecond)

			// When StopSelector is set, use element count as progress indicator
			// instead of scrollHeight — more reliable for lazy-loaded content.
			if step.StopSelector != "" {
				countResult, _ := page.Evaluate(fmt.Sprintf(
					"() => document.querySelectorAll(%q).length", step.StopSelector))
				if cnt, ok := countResult.(float64); ok && int(cnt) >= stopCount {
					slog.Info("fetch/camoufox: infinite scroll stopped — target reached",
						"selector", step.StopSelector, "count", int(cnt), "target", stopCount)
					break
				}
				// If count changed from last iteration, content is still loading.
				newCount, _ := countResult.(float64)
				if int(newCount) > prevCount {
					prevCount = int(newCount)
					continue // skip scrollHeight check — content IS loading
				}
			}

			// Check if new content appeared.
			newHeight, _ := page.Evaluate(getHeightJS)
			newH, _ := newHeight.(float64)

			if newH <= prevH {
				slog.Info("fetch/camoufox: infinite scroll complete — no new content",
					"iterations", i+1)
				break
			}
		}

	case foxhound.JobStepLoadMore:
		maxClicks := step.MaxClicks
		if maxClicks <= 0 {
			maxClicks = 20
		}
		sel := step.Selector
		if sel == "" {
			// Common "load more" / "show more" selectors.
			sel = "button:has-text('Load more'), button:has-text('Show more'), " +
				"button:has-text('Muat lagi'), a:has-text('Load more'), " +
				"[class*='load-more'], [class*='show-more'], [class*='loadMore']"
		}
		slog.Info("fetch/camoufox: load more started", "selector", sel, "max", maxClicks)

		for i := 0; i < maxClicks; i++ {
			el, err := page.QuerySelector(sel)
			if err != nil || el == nil {
				slog.Info("fetch/camoufox: load more button gone", "clicks", i)
				break
			}
			visible, _ := el.IsVisible()
			if !visible {
				slog.Info("fetch/camoufox: load more button hidden", "clicks", i)
				break
			}
			// Human-like pause before click.
			time.Sleep(time.Duration(500+rand.IntN(1000)) * time.Millisecond)
			if err := el.Click(); err != nil {
				slog.Debug("fetch/camoufox: load more click failed", "err", err)
				break
			}
			// Wait for new content to load.
			time.Sleep(time.Duration(1500+rand.IntN(2000)) * time.Millisecond)
		}

	case foxhound.JobStepPaginate:
		maxPages := step.MaxPages
		if maxPages <= 0 {
			maxPages = 10
		}
		sel := step.Selector
		if sel == "" {
			// Common pagination "next" selectors.
			sel = "a[rel='next'], a:has-text('Next'), a:has-text('Selanjutnya'), " +
				"li.next a, a.next, [aria-label='Next'], [aria-label='Next page'], " +
				"a:has-text('›'), a:has-text('»')"
		}
		slog.Info("fetch/camoufox: pagination started", "selector", sel, "max", maxPages)

		// Capture initial page content before navigating away.
		var pages []string
		if content, contentErr := page.Content(); contentErr == nil {
			pages = append(pages, content)
		}

		for i := 0; i < maxPages; i++ {
			el, err := page.QuerySelector(sel)
			if err != nil || el == nil {
				slog.Info("fetch/camoufox: no more pagination links", "pages", i)
				break
			}
			visible, _ := el.IsVisible()
			if !visible {
				slog.Info("fetch/camoufox: pagination link hidden", "pages", i)
				break
			}
			// Human-like pause before navigating.
			time.Sleep(time.Duration(1000+rand.IntN(2000)) * time.Millisecond)
			if err := el.Click(); err != nil {
				slog.Debug("fetch/camoufox: pagination click failed", "err", err)
				break
			}
			// Wait for next page to load.
			page.WaitForLoadState(playwright.PageWaitForLoadStateOptions{
				State: playwright.LoadStateNetworkidle,
			})
			time.Sleep(time.Duration(1000+rand.IntN(1500)) * time.Millisecond)

			// Capture this page's content.
			if content, contentErr := page.Content(); contentErr == nil {
				pages = append(pages, content)
			}
		}

		// Return accumulated pages — caller stores in Response.StepResults.
		if len(pages) > 0 {
			return pages, nil
		}

	case foxhound.JobStepFill:
		if step.Selector == "" {
			return nil, fmt.Errorf("fill step has empty selector")
		}
		el, err := page.QuerySelector(step.Selector)
		if err != nil {
			return nil, fmt.Errorf("fill: query %q: %w", step.Selector, err)
		}
		if el == nil {
			return nil, fmt.Errorf("fill: selector %q not found", step.Selector)
		}
		// Clear existing value.
		if err := el.Fill(""); err != nil {
			slog.Debug("fetch/camoufox: fill clear failed", "err", err)
		}
		// Type with human-like keystrokes using behavior.Keyboard.
		kb := behavior.NewKeyboard(f.keyboardConfig())
		actions := kb.TypeString(step.Value)
		for _, a := range actions {
			if a.IsBackspace {
				if err := el.Press("Backspace"); err != nil {
					slog.Debug("fetch/camoufox: fill backspace failed", "err", err)
				}
			} else {
				if err := el.Type(string(a.Char)); err != nil {
					slog.Debug("fetch/camoufox: fill type failed", "err", err)
				}
			}
			time.Sleep(a.Delay)
		}
		time.Sleep(time.Duration(200+rand.IntN(300)) * time.Millisecond)

	case foxhound.JobStepEvaluate:
		if step.Script == "" {
			return nil, fmt.Errorf("evaluate step has empty script")
		}
		result, err := page.Evaluate(step.Script)
		if err != nil {
			return nil, fmt.Errorf("evaluate script: %w", err)
		}
		return result, nil

	case foxhound.JobStepExtract:
		// Handled by Walker after fetch — no-op here.

	default:
		slog.Debug("fetch/camoufox: unknown step action", "action", step.Action)
	}
	return nil, nil
}

// handleCookieConsent attempts to click common cookie-consent accept buttons.
// It tries each selector in order, clicking the first visible match and
// returning immediately. If no consent button is found the call is a no-op.
//
// Accepting cookie banners prevents the banner from obscuring page content
// and matches the behaviour of a real user who dismisses these prompts.
func (f *CamoufoxFetcher) handleCookieConsent(page playwright.Page) {
	consentSelectors := []string{
		"button[id*='accept']",
		"button[class*='accept']",
		"button[id*='consent']",
		"a[id*='accept']",
		"[data-testid*='accept']",
		"button:has-text('Accept')",
		"button:has-text('I agree')",
		"button:has-text('Accept all')",
		"button:has-text('Accept All')",
		"button:has-text('Terima')", // Indonesian
		"button:has-text('Setuju')", // Indonesian
		"#onetrust-accept-btn-handler",
		".cookie-accept",
		"#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll",
	}

	for _, sel := range consentSelectors {
		el, err := page.QuerySelector(sel)
		if err != nil || el == nil {
			continue
		}
		visible, visErr := el.IsVisible()
		if visErr != nil || !visible {
			continue
		}
		if clickErr := el.Click(); clickErr != nil {
			slog.Debug("fetch/camoufox: cookie consent click failed",
				"selector", sel, "err", clickErr)
			continue
		}
		slog.Debug("fetch/camoufox: clicked cookie consent", "selector", sel)
		time.Sleep(500 * time.Millisecond)
		return
	}
}

// handlePerimeterX detects a PerimeterX "Robot or human?" press-and-hold
// challenge and attempts to solve it via behavior.PressAndHold before any
// NopeCHA or token-based solver fires. If the challenge is not present, or the
// context is already done, this is a no-op. NopeCHA path is not touched.
func (f *CamoufoxFetcher) handlePerimeterX(ctx context.Context, page playwright.Page) {
	if ctx.Err() != nil {
		return
	}
	// Quick DOM check before calling the full solver: look for the PX wrapper.
	hasPX := false
	for _, sel := range []string{"#px-captcha-wrapper", "#px-captcha", "div[id^='px-captcha']"} {
		count, err := page.Locator(sel).Count()
		if err == nil && count > 0 {
			hasPX = true
			break
		}
	}
	if !hasPX {
		return
	}
	if err := captcha.SolvePerimeterX(ctx, page); err != nil {
		slog.Warn("fetch/camoufox: PerimeterX press-hold failed (continuing)", "err", err)
	}
}

// handleRecaptcha detects and attempts to solve reCAPTCHA v2 checkbox challenges.
// reCAPTCHA renders inside an iframe — we locate the iframe, find the checkbox,
// move the mouse naturally toward it, and click. If Google's behavioral score
// (based on prior mouse movements + timing) is good enough, the checkbox
// resolves immediately without an image challenge.
func (f *CamoufoxFetcher) handleRecaptcha(ctx context.Context, page playwright.Page) {
	if ctx.Err() != nil {
		return
	}
	// Check if page contains reCAPTCHA indicators
	hasRecaptcha := false
	indicators := []string{
		"iframe[src*='recaptcha']",
		"iframe[src*='google.com/recaptcha']",
		"#captcha-form",
		".g-recaptcha",
		"div[class*='recaptcha']",
	}
	for _, sel := range indicators {
		el, err := page.QuerySelector(sel)
		if err == nil && el != nil {
			hasRecaptcha = true
			break
		}
	}
	if !hasRecaptcha {
		return
	}

	slog.Info("fetch/camoufox: reCAPTCHA detected, attempting to solve...")

	// Small random delay before interacting — a real user pauses to read
	if !sleepCtx(ctx, time.Duration(1500+rand.IntN(2000))*time.Millisecond) {
		return
	}

	// Strategy 1: Find the reCAPTCHA iframe and click the checkbox inside it
	iframeSelectors := []string{
		"iframe[src*='recaptcha/api2/anchor']",
		"iframe[src*='recaptcha/enterprise/anchor']",
		"iframe[title*='reCAPTCHA']",
		"iframe[src*='google.com/recaptcha']",
	}

	for _, iframeSel := range iframeSelectors {
		iframeEl, err := page.QuerySelector(iframeSel)
		if err != nil || iframeEl == nil {
			continue
		}

		frame, err := iframeEl.ContentFrame()
		if err != nil || frame == nil {
			continue
		}

		// The checkbox inside the reCAPTCHA iframe
		checkboxSelectors := []string{
			"#recaptcha-anchor",
			".recaptcha-checkbox",
			"span[role='checkbox']",
			"div.recaptcha-checkbox-border",
		}

		for _, cbSel := range checkboxSelectors {
			cb, err := frame.QuerySelector(cbSel)
			if err != nil || cb == nil {
				continue
			}

			visible, _ := cb.IsVisible()
			if !visible {
				continue
			}

			// Move mouse toward the checkbox with human-like trajectory
			box, err := cb.BoundingBox()
			if err == nil && box != nil {
				// Move to a random point near the checkbox (not dead center)
				targetX := box.X + box.Width*0.3 + float64(rand.IntN(int(box.Width*0.4)))
				targetY := box.Y + box.Height*0.3 + float64(rand.IntN(int(box.Height*0.4)))

				// Slow, multi-step mouse movement (human-like)
				currentX := float64(200 + rand.IntN(400))
				currentY := float64(200 + rand.IntN(200))
				steps := 8 + rand.IntN(8)
				for step := 1; step <= steps; step++ {
					t := float64(step) / float64(steps)
					// Ease-in-out curve
					t = t * t * (3 - 2*t)
					mx := currentX + (targetX-currentX)*t + float64(rand.IntN(3)-1)
					my := currentY + (targetY-currentY)*t + float64(rand.IntN(3)-1)
					page.Mouse().Move(mx, my)
					if !sleepCtx(ctx, time.Duration(20+rand.IntN(40))*time.Millisecond) {
						return
					}
				}

				// Small pause before click (human reaction time)
				if !sleepCtx(ctx, time.Duration(200+rand.IntN(300))*time.Millisecond) {
					return
				}
			}

			// Click the checkbox
			if err := cb.Click(); err != nil {
				slog.Debug("fetch/camoufox: reCAPTCHA checkbox click failed", "err", err)
				continue
			}

			slog.Info("fetch/camoufox: reCAPTCHA checkbox clicked, waiting for verification...")

			// Wait for Google to verify — this takes 2-5 seconds
			if !sleepCtx(ctx, time.Duration(3000+rand.IntN(3000))*time.Millisecond) {
				return
			}

			// Check if solved (checkbox gets aria-checked="true")
			checked, _ := cb.GetAttribute("aria-checked")
			if checked == "true" {
				slog.Info("fetch/camoufox: reCAPTCHA SOLVED via checkbox click!")

				// If there's a submit button after solving, click it
				f.submitAfterCaptcha(ctx, page)

				// Wait for page to load after form submission
				if !sleepCtx(ctx, time.Duration(2000+rand.IntN(2000))*time.Millisecond) {
					return
				}
				page.WaitForLoadState(playwright.PageWaitForLoadStateOptions{
					State: playwright.LoadStateNetworkidle,
				})
				return
			}

			slog.Warn("fetch/camoufox: reCAPTCHA checkbox clicked but image challenge appeared — no extension loaded to solve it")
			return
		}
	}

	// Strategy 2: Some pages embed reCAPTCHA inline (not in iframe)
	inlineSelectors := []string{
		"#recaptcha-anchor",
		".recaptcha-checkbox",
		"span[role='checkbox'][aria-labelledby*='recaptcha']",
	}
	for _, sel := range inlineSelectors {
		el, err := page.QuerySelector(sel)
		if err != nil || el == nil {
			continue
		}
		visible, _ := el.IsVisible()
		if !visible {
			continue
		}
		slog.Info("fetch/camoufox: clicking inline reCAPTCHA checkbox")
		el.Click()
		sleepCtx(ctx, time.Duration(3000+rand.IntN(3000))*time.Millisecond)
		return
	}

	slog.Warn("fetch/camoufox: reCAPTCHA found but could not locate clickable checkbox")
}

// handleHCaptcha detects and attempts to solve hCaptcha checkbox challenges.
// hCaptcha renders inside an iframe similar to reCAPTCHA — we locate the
// iframe, find the checkbox, move the mouse naturally, and click.
func (f *CamoufoxFetcher) handleHCaptcha(ctx context.Context, page playwright.Page) {
	if ctx.Err() != nil {
		return
	}
	// Check if page contains hCaptcha indicators.
	hasHCaptcha := false
	indicators := []string{
		"iframe[src*='hcaptcha.com']",
		"iframe[data-hcaptcha-widget-id]",
		".h-captcha",
		"div[class*='h-captcha']",
	}
	for _, sel := range indicators {
		el, err := page.QuerySelector(sel)
		if err == nil && el != nil {
			hasHCaptcha = true
			break
		}
	}
	if !hasHCaptcha {
		return
	}

	slog.Info("fetch/camoufox: hCaptcha detected, attempting to solve...")
	if !sleepCtx(ctx, time.Duration(1500+rand.IntN(2000))*time.Millisecond) {
		return
	}

	// hCaptcha checkbox lives inside an iframe.
	iframeSelectors := []string{
		"iframe[src*='hcaptcha.com/captcha/checkbox']",
		"iframe[src*='assets.hcaptcha.com']",
		"iframe[src*='hcaptcha.com']",
		"iframe[data-hcaptcha-widget-id]",
	}

	for _, iframeSel := range iframeSelectors {
		iframeEl, err := page.QuerySelector(iframeSel)
		if err != nil || iframeEl == nil {
			continue
		}

		frame, err := iframeEl.ContentFrame()
		if err != nil || frame == nil {
			continue
		}

		checkboxSelectors := []string{
			"#checkbox",
			"div[id='checkbox']",
			"div[class*='check']",
		}

		for _, cbSel := range checkboxSelectors {
			cb, err := frame.QuerySelector(cbSel)
			if err != nil || cb == nil {
				continue
			}
			visible, _ := cb.IsVisible()
			if !visible {
				continue
			}

			// Human-like mouse movement toward the checkbox.
			box, err := cb.BoundingBox()
			if err == nil && box != nil {
				targetX := box.X + box.Width*0.3 + float64(rand.IntN(int(box.Width*0.4)))
				targetY := box.Y + box.Height*0.3 + float64(rand.IntN(int(box.Height*0.4)))
				currentX := float64(200 + rand.IntN(400))
				currentY := float64(200 + rand.IntN(200))
				steps := 8 + rand.IntN(8)
				for step := 1; step <= steps; step++ {
					t := float64(step) / float64(steps)
					t = t * t * (3 - 2*t)
					mx := currentX + (targetX-currentX)*t + float64(rand.IntN(3)-1)
					my := currentY + (targetY-currentY)*t + float64(rand.IntN(3)-1)
					page.Mouse().Move(mx, my)
					if !sleepCtx(ctx, time.Duration(20+rand.IntN(40))*time.Millisecond) {
						return
					}
				}
				if !sleepCtx(ctx, time.Duration(200+rand.IntN(300))*time.Millisecond) {
					return
				}
			}

			if err := cb.Click(); err != nil {
				slog.Debug("fetch/camoufox: hCaptcha checkbox click failed", "err", err)
				continue
			}

			slog.Info("fetch/camoufox: hCaptcha checkbox clicked, waiting for verification...")
			if !sleepCtx(ctx, time.Duration(3000+rand.IntN(3000))*time.Millisecond) {
				return
			}

			// Check if solved.
			checked, _ := cb.GetAttribute("aria-checked")
			if checked == "true" {
				slog.Info("fetch/camoufox: hCaptcha SOLVED via checkbox click!")
				f.submitAfterCaptcha(ctx, page)
				if !sleepCtx(ctx, time.Duration(2000+rand.IntN(2000))*time.Millisecond) {
					return
				}
				page.WaitForLoadState(playwright.PageWaitForLoadStateOptions{
					State: playwright.LoadStateNetworkidle,
				})
				return
			}

			slog.Warn("fetch/camoufox: hCaptcha checkbox clicked but image challenge appeared — no extension loaded to solve it")
			return
		}
	}

	slog.Warn("fetch/camoufox: hCaptcha found but could not locate clickable checkbox")
}

// waitForExtensionSolve detects captcha type on the page and waits for the
// solver extension (NopeCHA) to solve it. Used when extension is loaded —
// no manual clicking needed, the extension handles everything.
//
// Per-type timeout budgets reflect realistic solve times:
//   - turnstile:  20s (usually <5s, allow buffer)
//   - hcaptcha:   45s (image challenges take ~20-30s)
//   - recaptcha:  90s (Enterprise/v3 can take 30-60s)
//   - geetest:    45s (slider+image, ~20-30s)
func (f *CamoufoxFetcher) waitForExtensionSolve(ctx context.Context, page playwright.Page) {
	if ctx.Err() != nil {
		return
	}
	// Early exit if the extension was explicitly disabled — nothing can solve.
	if !f.hasExtension || f.extensionPath == "none" {
		slog.Debug("fetch/camoufox: captcha extension disabled, skipping solve wait")
		return
	}

	// Detect captcha FIRST — skip immediately when no captcha is present.
	captchaType := f.detectCaptchaType(page)
	if captchaType == "" {
		return
	}

	// Per-type budget — reCAPTCHA Enterprise can take 30-60s.
	budget := 30 * time.Second
	switch captchaType {
	case "turnstile":
		budget = 20 * time.Second
	case "hcaptcha", "geetest":
		budget = 45 * time.Second
	case "recaptcha":
		budget = 90 * time.Second
	}

	slog.Info("fetch/camoufox: captcha detected, waiting for extension to solve",
		"type", captchaType, "budget", budget)

	// Give extension time to init and start solving now that we know there's work.
	initWait := 3 * time.Second
	if !sleepCtx(ctx, initWait) {
		slog.Debug("fetch/camoufox: captcha solve wait cancelled during init",
			"type", captchaType)
		return
	}

	// Poll for solve completion — check-then-sleep to avoid trailing delay.
	deadline := time.Now().Add(budget)
	pollInterval := 1 * time.Second
	for time.Now().Before(deadline) {
		if ctx.Err() != nil {
			slog.Debug("fetch/camoufox: captcha solve wait cancelled", "type", captchaType)
			return
		}
		if f.isCaptchaSolved(page, captchaType) {
			elapsed := int(time.Since(deadline.Add(-budget)).Seconds())
			slog.Info("fetch/camoufox: extension solved captcha!",
				"type", captchaType, "elapsed_seconds", elapsed)
			f.submitAfterCaptcha(ctx, page)
			sleepCtx(ctx, 1*time.Second)
			return
		}
		if !sleepCtx(ctx, pollInterval) {
			slog.Debug("fetch/camoufox: captcha solve wait cancelled during poll",
				"type", captchaType)
			return
		}
	}
	slog.Warn("fetch/camoufox: extension did not solve captcha in time",
		"type", captchaType, "budget", budget)
}

// detectCaptchaType checks the page content for known captcha indicators.
// Returns the captcha type string or "" if none detected.
func (f *CamoufoxFetcher) detectCaptchaType(page playwright.Page) string {
	content, err := page.Content()
	if err != nil {
		slog.Debug("fetch/camoufox: failed to get page content for captcha detection", "err", err)
		return ""
	}
	lower := strings.ToLower(content)

	switch {
	case strings.Contains(lower, "challenges.cloudflare.com/turnstile") || strings.Contains(lower, "cf-turnstile"):
		return "turnstile"
	case strings.Contains(lower, "hcaptcha.com") || strings.Contains(lower, "h-captcha"):
		return "hcaptcha"
	case strings.Contains(lower, "google.com/recaptcha") || strings.Contains(lower, "g-recaptcha"):
		return "recaptcha"
	case strings.Contains(lower, "geetest.com") || strings.Contains(lower, "gt_captcha"):
		return "geetest"
	default:
		return ""
	}
}

// isCaptchaSolved checks whether the captcha of the given type has been solved
// by verifying that a response token exists in the DOM.
func (f *CamoufoxFetcher) isCaptchaSolved(page playwright.Page, captchaType string) bool {
	var js string
	switch captchaType {
	case "turnstile":
		js = `() => {
			const inputs = document.querySelectorAll('input[name="cf-turnstile-response"]');
			for (const input of inputs) {
				if (input.value && input.value.length > 10) return true;
			}
			return false;
		}`
	case "recaptcha":
		// reCAPTCHA v2/v3/Enterprise: token can land in multiple places.
		// Check all textareas (some Enterprise widgets create multiple),
		// then fall back to the grecaptcha API if available.
		js = `() => {
			// 1. Standard textarea (v2 + Enterprise checkbox flow)
			const tas = document.querySelectorAll('textarea[name^="g-recaptcha-response"]');
			for (const ta of tas) {
				if (ta.value && ta.value.length > 10) return true;
			}
			// 2. grecaptcha API (v2 + Enterprise programmatic)
			try {
				if (typeof grecaptcha !== 'undefined' && grecaptcha.getResponse) {
					const r = grecaptcha.getResponse();
					if (r && r.length > 10) return true;
				}
				// Enterprise namespace
				if (typeof grecaptcha !== 'undefined' && grecaptcha.enterprise && grecaptcha.enterprise.getResponse) {
					const r = grecaptcha.enterprise.getResponse();
					if (r && r.length > 10) return true;
				}
			} catch (e) {}
			// 3. Hidden inputs sometimes used by Enterprise
			const hidden = document.querySelectorAll('input[name="g-recaptcha-response"], input[name^="g-recaptcha-response-"]');
			for (const inp of hidden) {
				if (inp.value && inp.value.length > 10) return true;
			}
			return false;
		}`
	case "hcaptcha":
		js = `() => {
			const ta = document.querySelector('textarea[name="h-captcha-response"]');
			return ta && ta.value && ta.value.length > 10;
		}`
	case "geetest":
		js = `() => {
			const inp = document.querySelector('input[name="geetest_validate"]');
			return inp && inp.value && inp.value.length > 10;
		}`
	default:
		return false
	}
	val, err := page.Evaluate(js)
	if err != nil {
		slog.Debug("fetch/camoufox: captcha solve check failed", "type", captchaType, "err", err)
		return false
	}
	b, ok := val.(bool)
	return ok && b
}

// detectCloudflare inspects the fully-loaded page content and returns the type
// of Cloudflare challenge present, or "" if none is detected.
//
// Recognised challenge types:
//   - "js_challenge"  — "Checking your browser" / "Just a moment" interstitial
//   - "turnstile"     — Cloudflare Turnstile widget (visible or managed/invisible)
//   - "under_attack"  — Under Attack Mode (extended JS challenge)
func (f *CamoufoxFetcher) detectCloudflare(page playwright.Page) string {
	content, err := page.Content()
	if err != nil {
		slog.Debug("fetch/camoufox: detectCloudflare page.Content failed", "err", err)
		return ""
	}
	lower := strings.ToLower(content)

	// Turnstile widget — check FIRST because Turnstile pages also contain
	// "challenge-platform" which would falsely match js_challenge.
	if strings.Contains(lower, "challenges.cloudflare.com/turnstile") ||
		strings.Contains(lower, "cf-turnstile") {
		solved, _ := page.Evaluate(`() => {
			const inputs = document.querySelectorAll('input[name="cf-turnstile-response"]');
			for (const input of inputs) {
				if (input.value && input.value.length > 10) return true;
			}
			return false;
		}`)
		if val, ok := solved.(bool); ok && val {
			return "" // already solved
		}
		return "turnstile"
	}

	// JS Challenge — "Checking your browser" / "Just a moment" interstitial.
	// NOTE: "challenge-platform" removed — it appears in Turnstile pages too.
	if strings.Contains(lower, "checking your browser") ||
		strings.Contains(lower, "just a moment") ||
		strings.Contains(lower, "cf-browser-verification") {
		return "js_challenge"
	}

	// Under Attack Mode — require cf-chl-bypass specifically, not just "ray id" + "cloudflare"
	// which matches normal Cloudflare error page footers.
	if strings.Contains(lower, "cf-chl-bypass") {
		return "under_attack"
	}

	return ""
}

// handleCloudflare attempts to resolve a Cloudflare challenge on the given page.
// It returns true when the challenge was resolved and the page content has
// changed, and false when no challenge was detected or the challenge timed out.
//
// Three challenge types are handled:
//   - js_challenge: wait up to 12 s for the automatic JS solve + redirect.
//   - turnstile: locate the Turnstile iframe and click the checkbox; if not
//     found, wait 5 s for managed auto-resolution.
//   - under_attack: wait up to 15 s for the extended JS challenge to complete.
func (f *CamoufoxFetcher) handleCloudflare(ctx context.Context, page playwright.Page, cfType string) bool {
	if cfType == "" {
		return false
	}
	if ctx.Err() != nil {
		return false
	}

	switch cfType {
	case "js_challenge":
		slog.Info("fetch/camoufox: waiting for Cloudflare JS challenge to resolve...")
		for i := 0; i < 12; i++ {
			if !sleepCtx(ctx, 1*time.Second) {
				slog.Debug("fetch/camoufox: JS challenge wait cancelled")
				return false
			}
			// Lightweight check: JS challenge pages have specific markers in DOM.
			gone, _ := page.Evaluate(`() => {
				const body = document.body ? document.body.innerText.toLowerCase() : '';
				return !body.includes('checking your browser') && !body.includes('just a moment');
			}`)
			if val, ok := gone.(bool); ok && val {
				slog.Info("fetch/camoufox: Cloudflare JS challenge resolved!")
				page.WaitForLoadState(playwright.PageWaitForLoadStateOptions{
					State: playwright.LoadStateNetworkidle,
				})
				return true
			}
		}
		slog.Warn("fetch/camoufox: Cloudflare JS challenge did not resolve in time")

	case "turnstile":
		slog.Info("fetch/camoufox: attempting Turnstile challenge...")

		// Turnstile can render as:
		//  (a) an iframe with src*='challenges.cloudflare.com' (Cloudflare-hosted interstitial)
		//  (b) a div container with JS-injected widget (embedded via api.js — no iframe)
		// We try both strategies: iframe first, then container div.

		// Strategy 1: Turnstile iframe (Cloudflare interstitial pages).
		iframeSelectors := []string{
			"iframe[src*='challenges.cloudflare.com/turnstile']",
			"iframe[src*='challenges.cloudflare.com/cdn-cgi']",
			"iframe[src*='challenges.cloudflare.com']",
			"div.cf-turnstile iframe",
		}

		var widget playwright.ElementHandle
		for _, sel := range iframeSelectors {
			el, err := page.QuerySelector(sel)
			if err == nil && el != nil {
				if box, _ := el.BoundingBox(); box != nil && box.Width >= 30 && box.Height >= 30 {
					widget = el
					slog.Info("fetch/camoufox: Turnstile iframe found", "selector", sel)
					break
				}
			}
		}

		// Strategy 2: find widget by hidden input name="cf-turnstile-response".
		// The hidden input is always present — its parent div IS the widget container.
		if widget == nil {
			inputs, err := page.QuerySelectorAll("input[name='cf-turnstile-response']")
			if err == nil {
				for _, input := range inputs {
					// The widget container is the closest ancestor with visible dimensions.
					parent, _ := page.EvaluateHandle(
						"el => { let p = el.parentElement; while (p && (p.offsetWidth < 200 || p.offsetHeight < 30)) p = p.parentElement; return p; }",
						input)
					if parent == nil {
						continue
					}
					el, ok := parent.(playwright.ElementHandle)
					if !ok {
						continue
					}
					box, _ := el.BoundingBox()
					// Turnstile widget is ~300x65. Skip anything too wide (parent wrapper).
					if box != nil && box.Width >= 200 && box.Width <= 500 && box.Height >= 30 {
						widget = el
						slog.Info("fetch/camoufox: Turnstile widget found via hidden input",
							"w", box.Width, "h", box.Height)
						break
					}
				}
			}
		}

		// Strategy 3: Turnstile container div by class (fallback).
		if widget == nil {
			containerSelectors := []string{
				"div.cf-turnstile",
				"div[class*='cf-turnstile']",
			}
			for _, sel := range containerSelectors {
				els, err := page.QuerySelectorAll(sel)
				if err != nil || len(els) == 0 {
					continue
				}
				for _, el := range els {
					box, _ := el.BoundingBox()
					if box != nil && box.Width >= 200 && box.Width <= 500 && box.Height >= 30 {
						widget = el
						slog.Info("fetch/camoufox: Turnstile container found",
							"selector", sel, "w", box.Width, "h", box.Height)
						break
					}
				}
				if widget != nil {
					break
				}
			}
		}

		clicked := false
		if widget != nil {
			// Wait for widget to fully render.
			if !sleepCtx(ctx, time.Duration(1500+rand.IntN(1500))*time.Millisecond) {
				return false
			}

			box, err := widget.BoundingBox()
			if err == nil && box != nil {
				slog.Info("fetch/camoufox: clicking Turnstile checkbox",
					"x", box.X, "y", box.Y, "w", box.Width, "h", box.Height)

				// Checkbox is in the left-center area of the widget (~28px from left).
				targetX := box.X + 26 + float64(rand.IntN(10))
				targetY := box.Y + box.Height/2 + float64(rand.IntN(6)-3)

				// Human-like mouse movement.
				startX := float64(200 + rand.IntN(300))
				startY := float64(150 + rand.IntN(200))
				steps := 6 + rand.IntN(6)
				for step := 1; step <= steps; step++ {
					t := float64(step) / float64(steps)
					t = t * t * (3 - 2*t) // ease-in-out
					mx := startX + (targetX-startX)*t + float64(rand.IntN(3)-1)
					my := startY + (targetY-startY)*t + float64(rand.IntN(3)-1)
					_ = page.Mouse().Move(mx, my)
					if !sleepCtx(ctx, time.Duration(15+rand.IntN(30))*time.Millisecond) {
						return false
					}
				}

				if !sleepCtx(ctx, time.Duration(200+rand.IntN(400))*time.Millisecond) {
					return false
				}
				_ = page.Mouse().Click(targetX, targetY)
				clicked = true
				slog.Info("fetch/camoufox: clicked Turnstile", "x", targetX, "y", targetY)
			}
		} else {
			slog.Warn("fetch/camoufox: no Turnstile widget found in DOM")
		}

		// Poll for resolution.
		waitSecs := 20
		if !clicked {
			slog.Info("fetch/camoufox: waiting for Turnstile auto-resolution...")
			waitSecs = 10
		}
		for i := 0; i < waitSecs; i++ {
			if !sleepCtx(ctx, 1*time.Second) {
				slog.Debug("fetch/camoufox: Turnstile poll cancelled")
				return false
			}
			// Lightweight: check if Turnstile markup is gone (page redirected) OR token is set.
			gone, _ := page.Evaluate(`() => {
				const html = document.documentElement.innerHTML.toLowerCase();
				if (!html.includes('cf-turnstile') && !html.includes('challenges.cloudflare.com/turnstile')) return true;
				const inputs = document.querySelectorAll('input[name="cf-turnstile-response"]');
				for (const inp of inputs) { if (inp.value && inp.value.length > 10) return true; }
				return false;
			}`)
			if val, ok := gone.(bool); ok && val {
				slog.Info("fetch/camoufox: Turnstile challenge resolved!")
				page.WaitForLoadState(playwright.PageWaitForLoadStateOptions{
					State: playwright.LoadStateNetworkidle,
				})
				return true
			}
			// Retry click every 5s if first click didn't resolve.
			if clicked && i > 0 && i%5 == 0 && widget != nil {
				if box, err := widget.BoundingBox(); err == nil && box != nil {
					tx := box.X + 26 + float64(rand.IntN(10))
					ty := box.Y + box.Height/2 + float64(rand.IntN(6)-3)
					_ = page.Mouse().Click(tx, ty)
					slog.Info("fetch/camoufox: retried Turnstile click")
				}
			}
		}
		slog.Warn("fetch/camoufox: Turnstile challenge did not resolve in time")

	case "under_attack":
		slog.Info("fetch/camoufox: Cloudflare Under Attack mode, waiting...")
		for i := 0; i < 15; i++ {
			if !sleepCtx(ctx, 1*time.Second) {
				slog.Debug("fetch/camoufox: Under Attack poll cancelled")
				return false
			}
			gone, _ := page.Evaluate(`() => {
				const html = document.documentElement.innerHTML.toLowerCase();
				return !html.includes('cf-chl-bypass');
			}`)
			if val, ok := gone.(bool); ok && val {
				slog.Info("fetch/camoufox: Under Attack mode resolved!")
				page.WaitForLoadState(playwright.PageWaitForLoadStateOptions{
					State: playwright.LoadStateNetworkidle,
				})
				return true
			}
		}
		slog.Warn("fetch/camoufox: Cloudflare Under Attack mode did not resolve in time")
	}

	return false
}

// verifyCloudflareSolve polls up to timeout for three success signals after a
// Cloudflare challenge handler has run:
//
//  1. The cf_clearance cookie is present in the browser context for the page's host.
//  2. The Turnstile DOM markers (cf-turnstile element / challenges.cloudflare.com
//     iframe) are gone OR the cf-turnstile-response token has length > 10.
//  3. isCaptchaSolved("turnstile") returns true.
//
// All three must pass within the budget for the function to return true.
// On timeout it logs a warning and returns false (the caller decides what
// to do — typically attach the result to Response.CloudflareSolved and
// return the page anyway).
func (f *CamoufoxFetcher) verifyCloudflareSolve(ctx context.Context, page playwright.Page, bctx playwright.BrowserContext, timeout time.Duration) bool {
	if timeout <= 0 {
		return false
	}
	if ctx.Err() != nil {
		return false
	}
	deadline := time.Now().Add(timeout)
	pollInterval := 500 * time.Millisecond

	pageHost := ""
	if u, err := neturl.Parse(page.URL()); err == nil {
		pageHost = u.Hostname()
	}

	for time.Now().Before(deadline) {
		if ctx.Err() != nil {
			slog.Debug("fetch/camoufox: Cloudflare verify cancelled")
			return false
		}
		// Signal 1: cf_clearance cookie present.
		hasClearance := false
		if cookies, err := bctx.Cookies(); err == nil {
			for _, c := range cookies {
				if c.Name != "cf_clearance" || c.Value == "" {
					continue
				}
				if pageHost == "" || c.Domain == pageHost ||
					strings.HasSuffix(pageHost, strings.TrimPrefix(c.Domain, ".")) ||
					strings.HasSuffix(strings.TrimPrefix(c.Domain, "."), pageHost) {
					hasClearance = true
					break
				}
			}
		}

		// Signal 2: Turnstile DOM markers gone OR token set.
		domClean := false
		if val, err := page.Evaluate(`() => {
			const html = document.documentElement.innerHTML.toLowerCase();
			const markersGone = !html.includes('cf-turnstile') && !html.includes('challenges.cloudflare.com/turnstile');
			if (markersGone) return true;
			const inputs = document.querySelectorAll('input[name="cf-turnstile-response"]');
			for (const inp of inputs) { if (inp.value && inp.value.length > 10) return true; }
			return false;
		}`); err == nil {
			if b, ok := val.(bool); ok && b {
				domClean = true
			}
		}

		// Signal 3: token verification.
		tokenSolved := f.isCaptchaSolved(page, "turnstile")

		if hasClearance && domClean && tokenSolved {
			slog.Info("fetch/camoufox: Cloudflare solve verified",
				"clearance", hasClearance, "dom_clean", domClean, "token", tokenSolved)
			return true
		}

		if !sleepCtx(ctx, pollInterval) {
			slog.Debug("fetch/camoufox: Cloudflare verify cancelled during poll")
			return false
		}
	}

	slog.Warn("fetch/camoufox: Cloudflare solve verification timed out",
		"timeout", timeout)
	return false
}

// submitAfterCaptcha looks for and clicks a submit button after CAPTCHA is solved.
func (f *CamoufoxFetcher) submitAfterCaptcha(ctx context.Context, page playwright.Page) {
	if ctx.Err() != nil {
		return
	}
	submitSelectors := []string{
		"#captcha-form input[type='submit']",
		"#captcha-form button[type='submit']",
		"#challenge-form input[type='submit']",
		"#challenge-form button[type='submit']",
		"#captcha-form button:has-text('Submit')",
		"#captcha-form button:has-text('Verify')",
		"#challenge-form button:has-text('Continue')",
		"#challenge-form button:has-text('Verify')",
		"form[action*='captcha'] button[type='submit']",
		"form[action*='challenge'] button[type='submit']",
	}
	for _, sel := range submitSelectors {
		el, err := page.QuerySelector(sel)
		if err != nil || el == nil {
			continue
		}
		visible, _ := el.IsVisible()
		if !visible {
			continue
		}
		if !sleepCtx(ctx, time.Duration(500+rand.IntN(500))*time.Millisecond) {
			return
		}
		if err := el.Click(); err != nil {
			slog.Debug("fetch/camoufox: submitAfterCaptcha click failed", "selector", sel, "err", err)
			continue
		}
		slog.Info("fetch/camoufox: submitted form after CAPTCHA solve", "selector", sel)
		return
	}
}

// hasWaitStep returns true if the job has at least one Wait step. When a job
// includes explicit Wait steps the caller already handles element readiness, so
// the initial page.Goto can use the faster DOMContentLoaded event instead of
// waiting for full network idle — saving 1-3 seconds per navigation.
func hasWaitStep(job *foxhound.Job) bool {
	for _, s := range job.Steps {
		if s.Action == foxhound.JobStepWait {
			return true
		}
	}
	return false
}

// isPlaywrightTimeout returns true when the error is a playwright navigation
// timeout (the error message contains "Timeout" and "exceeded").
func isPlaywrightTimeout(err error) bool {
	if err == nil {
		return false
	}
	msg := err.Error()
	return strings.Contains(msg, "Timeout") && strings.Contains(msg, "exceeded")
}

// navigate performs the actual playwright navigation. It is called from a
// goroutine so that context cancellation can abort it cleanly.
func (f *CamoufoxFetcher) navigate(ctx context.Context, job *foxhound.Job) (*foxhound.Response, error) {
	// When a page pool is active, acquire a pre-warmed page+context instead of
	// creating a fresh BrowserContext for every request. The pool resets cookies
	// and navigates to about:blank between uses to prevent session bleed.
	//
	// IMPORTANT: snapshot f.pool under f.mu before use. restart() nil-s f.pool
	// while holding f.mu, so reading f.pool once under the lock and using the
	// local copy eliminates the TOCTOU race that caused a nil-pointer SIGSEGV
	// (v0.0.21 fix — see .dev-squad/v0.0.21-rca.md).
	f.mu.Lock()
	pool := f.pool
	f.mu.Unlock()

	if pool != nil {
		pooledAny, err := pool.Acquire(ctx)
		if err != nil {
			return nil, fmt.Errorf("fetch/camoufox: acquiring pooled page for %s: %w", job.URL, err)
		}
		page := pooledAny.(playwright.Page)
		bctx := page.Context()
		resp, err := f.navigateWithPage(ctx, job, bctx, page)
		// Always release back to pool; the pool's reset func will clean state.
		// If restart() closed and nil-ed f.pool between Acquire and here, pool
		// is still a valid pointer to the now-closed pool. Release on a closed
		// pool is safe — it destroys the page and returns immediately.
		pool.Release(page)
		return resp, err
	}

	// Obtain a BrowserContext — either a fresh one or the shared session
	// context, depending on the persistSession configuration.
	bctx, shouldClose, err := f.getOrCreateContext()
	if err != nil {
		return nil, fmt.Errorf("fetch/camoufox: creating browser context for %s: %w", job.URL, err)
	}
	if shouldClose {
		defer func() {
			if closeErr := bctx.Close(); closeErr != nil {
				slog.Warn("fetch/camoufox: error closing browser context",
					"url", job.URL,
					"err", closeErr,
				)
			}
		}()
	}

	page, err := bctx.NewPage()
	if err != nil {
		return nil, fmt.Errorf("fetch/camoufox: opening page for %s: %w", job.URL, err)
	}
	defer func() {
		if closeErr := page.Close(); closeErr != nil {
			slog.Warn("fetch/camoufox: error closing page",
				"url", job.URL,
				"err", closeErr,
			)
		}
	}()

	return f.navigateWithPage(ctx, job, bctx, page)
}

// navigateWithPage performs navigation using the given page and context.
// It is called by navigate for both the pooled and non-pooled paths.
// The caller is responsible for closing or releasing the page after return.
// ctx threads through the captcha / Cloudflare solve helpers so that a cancelled
// hunt does not have to wait out the full solve budget.
func (f *CamoufoxFetcher) navigateWithPage(ctx context.Context, job *foxhound.Job, bctx playwright.BrowserContext, page playwright.Page) (*foxhound.Response, error) {
	// Inject Accept-Language from the identity profile as an extra header so
	// the browser reports the correct locale to the server.
	if f.identity != nil && len(f.identity.Languages) > 0 {
		if err := bctx.SetExtraHTTPHeaders(map[string]string{
			"Accept-Language": buildAcceptLanguage(f.identity.Languages),
		}); err != nil {
			slog.Warn("fetch/camoufox: could not set Accept-Language header",
				"url", job.URL,
				"err", err,
			)
		}
	}

	// Inject cookies into browser context before navigation.
	if len(f.cookies) > 0 {
		var pwCookies []playwright.OptionalCookie
		for _, c := range f.cookies {
			domain := c.Domain
			path := c.Path
			pwCookies = append(pwCookies, playwright.OptionalCookie{
				Name:     c.Name,
				Value:    c.Value,
				Domain:   &domain,
				Path:     &path,
				Secure:   playwright.Bool(c.Secure),
				HttpOnly: playwright.Bool(c.HttpOnly),
			})
		}
		if err := bctx.AddCookies(pwCookies); err != nil {
			slog.Warn("fetch/camoufox: failed to inject cookies", "err", err)
		}
	}

	// Inject init script before any page JS executes. This runs on every
	// page navigation, making it the right place for navigator overrides or
	// global stubs that must survive across client-side route changes.
	if f.initScript != "" {
		if scriptErr := page.AddInitScript(playwright.Script{
			Content: playwright.String(f.initScript),
		}); scriptErr != nil {
			slog.Warn("fetch/camoufox: AddInitScript failed (non-fatal)",
				"url", job.URL,
				"err", scriptErr,
			)
		}
	}

	// Block binary resources to reduce bandwidth for content-only scraping.
	if f.blockImages {
		for _, pattern := range resourceBlockPatterns {
			p := pattern // capture for closure
			if routeErr := page.Route(p, func(route playwright.Route) {
				if abortErr := route.Abort(); abortErr != nil {
					slog.Debug("fetch/camoufox: route abort error",
						"pattern", p,
						"err", abortErr,
					)
				}
			}); routeErr != nil {
				slog.Warn("fetch/camoufox: could not install route handler",
					"pattern", p,
					"err", routeErr,
				)
			}
		}
	}

	// Apply route-level intercept config: resource type and domain blocking.
	// This is in addition to the legacy blockImages glob handler above so both
	// configurations can coexist.
	if f.interceptConfig != nil && f.interceptConfig.IsActive() {
		ic := f.interceptConfig
		if routeErr := page.Route("**/*", func(route playwright.Route) {
			req := route.Request()
			if ic.ShouldBlock(req.ResourceType(), req.URL()) {
				if abortErr := route.Abort(); abortErr != nil {
					slog.Debug("fetch/camoufox: intercept abort error", "err", abortErr)
				}
				return
			}
			if contErr := route.Continue(); contErr != nil {
				slog.Debug("fetch/camoufox: intercept continue error", "err", contErr)
			}
		}); routeErr != nil {
			slog.Warn("fetch/camoufox: could not install intercept route handler", "err", routeErr)
		}
	}

	// Set up XHR/fetch response capture if patterns are configured.
	// Patterns may come from the fetcher (global, via WithCaptureXHR) and/or
	// from the job's meta (per-trail, via Trail.CaptureXHR). The two sets are
	// merged for this fetch.
	var capturedXHR []map[string]any
	activePatterns := f.capturePatterns
	if jobPatterns := capturePatternsFromJob(job); len(jobPatterns) > 0 {
		merged := make([]*regexp.Regexp, 0, len(f.capturePatterns)+len(jobPatterns))
		merged = append(merged, f.capturePatterns...)
		merged = append(merged, jobPatterns...)
		activePatterns = merged
	}
	if len(activePatterns) > 0 {
		page.On("response", func(response playwright.Response) {
			url := response.URL()
			for _, p := range activePatterns {
				if p.MatchString(url) {
					body, _ := response.Body()
					headers, _ := response.AllHeaders()
					capturedXHR = append(capturedXHR, map[string]any{
						"request_url":    url,
						"request_method": response.Request().Method(),
						"status":         response.Status(),
						"headers":        headers,
						"body":           string(body),
					})
					break
				}
			}
		})
	}

	// Per-job timeout override takes precedence over the fetcher default.
	navTimeout := f.timeout
	if job.NavigationTimeout > 0 {
		navTimeout = job.NavigationTimeout
	}
	timeoutMs := float64(navTimeout.Milliseconds())

	slog.Debug("fetch/camoufox: navigating",
		"url", job.URL,
		"job_id", job.ID,
		"timeout_ms", timeoutMs,
	)

	// Default to domcontentloaded — sufficient for most content scraping and
	// significantly faster than networkidle (saves 2-10s on pages with many
	// third-party resources like Google SERP). Use load only when the
	// extension needs content scripts injected.
	waitUntil := playwright.WaitUntilStateDomcontentloaded
	// Extension needs at least "load" to ensure content scripts are injected.
	if f.hasExtension {
		waitUntil = playwright.WaitUntilStateLoad
	}

	start := time.Now()
	navResp, err := page.Goto(job.URL, playwright.PageGotoOptions{
		WaitUntil: waitUntil,
		Timeout:   playwright.Float(timeoutMs),
	})
	duration := time.Since(start)

	// On navigation timeout, retry once with 2x timeout and the faster
	// domcontentloaded event. Many pages (especially Google SERP later
	// pages) are content-ready well before networkidle/load.
	if err != nil && isPlaywrightTimeout(err) {
		retryTimeout := navTimeout * 2
		// Cap at 120s to avoid absurdly long waits.
		if retryTimeout > 120*time.Second {
			retryTimeout = 120 * time.Second
		}
		retryTimeoutMs := float64(retryTimeout.Milliseconds())
		slog.Warn("fetch/camoufox: navigation timeout, retrying with extended timeout",
			"url", job.URL,
			"original_timeout", navTimeout,
			"retry_timeout", retryTimeout,
			"wait_until", "domcontentloaded",
		)
		start = time.Now()
		navResp, err = page.Goto(job.URL, playwright.PageGotoOptions{
			WaitUntil: playwright.WaitUntilStateDomcontentloaded,
			Timeout:   playwright.Float(retryTimeoutMs),
		})
		duration = time.Since(start)
	}

	if err != nil {
		return nil, fmt.Errorf("fetch/camoufox: navigating to %s: %w", job.URL, err)
	}

	// Handle Cloudflare challenges (JS check, Turnstile, Under Attack Mode).
	// Some sites (e.g. Bing) present sequential challenges — retry up to 3 times.
	cfChallengeSeen := false
	cloudflareSolved := false
	for cfAttempt := 0; cfAttempt < 3; cfAttempt++ {
		cfType := f.detectCloudflare(page)
		if cfType == "" {
			break
		}
		cfChallengeSeen = true
		// When extension is present, let it handle Turnstile —
		// manual clicking in handleCloudflare conflicts with the extension's solver.
		if cfType == "turnstile" && f.hasExtension {
			slog.Debug("fetch/camoufox: Turnstile detected, deferring to extension solver",
				"url", job.URL)
			break
		}
		slog.Info("fetch/camoufox: Cloudflare challenge detected",
			"type", cfType, "attempt", cfAttempt+1, "url", job.URL)
		if !f.handleCloudflare(ctx, page, cfType) {
			slog.Warn("fetch/camoufox: Cloudflare handling did not resolve challenge",
				"attempt", cfAttempt+1)
			break
		}
		if !sleepCtx(ctx, 2*time.Second) {
			return nil, fmt.Errorf("fetch/camoufox: context cancelled after Cloudflare solve for %s: %w", job.URL, ctx.Err())
		}
	}

	// Human behaviour simulation — only when the page is real content,
	// not a Cloudflare challenge page where random interactions are harmful.
	if f.detectCloudflare(page) == "" {
		f.simulateHumanBehavior(page)
	}

	// Dismiss cookie consent banners — only on real content pages.
	if f.detectCloudflare(page) == "" {
		f.handleCookieConsent(page)
	}

	// PerimeterX press-and-hold: attempt before NopeCHA so the gesture fires
	// while the challenge page is still live. NopeCHA path is not affected —
	// if PX clears, NopeCHA never fires; if PX fails, NopeCHA fires as normal.
	f.handlePerimeterX(ctx, page)

	if f.hasExtension {
		f.waitForExtensionSolve(ctx, page)
		// Fallback: only run manual handling if a captcha is present AND unsolved.
		// detectCaptchaType returning non-empty just means captcha markup exists —
		// after a successful solve the markup is still there, only the token is set.
		// We must check isCaptchaSolved to avoid running manual fallback after success.
		if ct := f.detectCaptchaType(page); ct != "" && !f.isCaptchaSolved(page, ct) {
			slog.Warn("fetch/camoufox: extension did not solve, falling back to manual",
				"type", ct)
			f.handleRecaptcha(ctx, page)
			f.handleHCaptcha(ctx, page)
		}
	} else {
		f.handleRecaptcha(ctx, page)
		f.handleHCaptcha(ctx, page)
	}

	// Verified Cloudflare solve mode: when WithSolveCloudflare(timeout) was set
	// AND a Cloudflare challenge was actually seen, poll for cookie + DOM +
	// token signals before continuing. The result is attached to the Response
	// so callers can branch on it. Verification timeout is non-fatal — the
	// page is still returned with CloudflareSolved=false on timeout.
	if f.cloudflareSolveTimeout > 0 && cfChallengeSeen {
		cloudflareSolved = f.verifyCloudflareSolve(ctx, page, bctx, f.cloudflareSolveTimeout)
	}

	// Execute Trail-attached steps. Click and Wait are "hard" steps — their
	// failure means the page content will be wrong, so we abort the fetch.
	// Scroll and Evaluate are best-effort (warn and continue).
	var stepResults map[string]any
	for i, step := range job.Steps {
		stepResult, stepErr := f.executeStep(page, step)
		if stepErr != nil {
			// Hard steps: Click, Wait — abort on failure.
			if !step.Optional && (step.Action == foxhound.JobStepClick || step.Action == foxhound.JobStepWait) {
				return nil, fmt.Errorf("fetch/camoufox: step %d (%d) failed for %s: %w",
					i, step.Action, job.URL, stepErr)
			}
			slog.Warn("fetch/camoufox: step execution failed (non-fatal)",
				"step_index", i,
				"action", step.Action,
				"selector", step.Selector,
				"err", stepErr,
			)
		}
		// Capture Evaluate step return values into Response.StepResults.
		if stepResult != nil {
			if stepResults == nil {
				stepResults = make(map[string]any)
			}
			stepResults[fmt.Sprintf("step_%d", i)] = stepResult
		}
	}

	// Extract the fully-rendered HTML. page.Content() returns the live DOM
	// after all JS has executed, which is the primary reason to use a browser.
	content, err := page.Content()
	if err != nil {
		return nil, fmt.Errorf("fetch/camoufox: extracting content from %s: %w", job.URL, err)
	}

	// If paginate steps accumulated pages, join them all for Response.Body so
	// processors can extract data from every page at once. The delimiter
	// <!-- foxhound:page-break --> allows processors to split pages if needed.
	for _, v := range stepResults {
		if pages, ok := v.([]string); ok && len(pages) > 1 {
			content = strings.Join(pages, "\n<!-- foxhound:page-break -->\n")
			break
		}
	}

	// Collect status code and final URL from the navigation response.
	statusCode := 200
	finalURL := job.URL
	if navResp != nil {
		statusCode = navResp.Status()
		if u := navResp.URL(); u != "" {
			finalURL = u
		}
	}
	// page.URL() gives the current address after all client-side redirects,
	// which is more accurate than the navigation response URL for SPAs.
	if pageURL := page.URL(); pageURL != "" && pageURL != "about:blank" {
		finalURL = pageURL
	}

	slog.Debug("fetch/camoufox: navigation complete",
		"status", statusCode,
		"url", finalURL,
		"duration_ms", duration.Milliseconds(),
		"body_bytes", len(content),
		"job_id", job.ID,
	)

	// Export browser cookies into Response.Cookies.
	var respCookies []*http.Cookie
	if pwCookies, cookieErr := bctx.Cookies(); cookieErr == nil {
		for _, c := range pwCookies {
			respCookies = append(respCookies, &http.Cookie{
				Name:     c.Name,
				Value:    c.Value,
				Domain:   c.Domain,
				Path:     c.Path,
				Secure:   c.Secure,
				HttpOnly: c.HttpOnly,
			})
		}
	}

	return &foxhound.Response{
		StatusCode:       statusCode,
		Headers:          make(map[string][]string),
		Body:             []byte(content),
		URL:              finalURL,
		FetchMode:        foxhound.FetchBrowser,
		Duration:         duration,
		Job:              job,
		StepResults:      stepResults,
		CapturedXHR:      capturedXHR,
		Cookies:          respCookies,
		CloudflareSolved: cloudflareSolved,
	}, nil
}

// buildContextOptions constructs playwright.BrowserNewContextOptions from the
// identity profile.
//
// When the Camoufox binary is active (hasCamoufox=true), UserAgent, Locale,
// and TimezoneId are NOT set here because Camoufox handles them at C++ level
// via CAMOU_CONFIG. Setting them via Playwright would create a detectable
// mismatch between the JS-injected values and the C++ values.
//
// Viewport is always set (it controls window size, separate from screen size).
// Geolocation permissions may still be needed for navigator.geolocation API.
//
// When no identity is set and we're on plain Firefox, sensible defaults are
// applied so the browser still presents a plausible fingerprint.
func (f *CamoufoxFetcher) buildContextOptions() playwright.BrowserNewContextOptions {
	opts := playwright.BrowserNewContextOptions{
		ColorScheme: playwright.ColorSchemeLight,
	}

	if f.identity == nil {
		// No identity — apply defaults. When using Camoufox, skip UA/locale/tz
		// since Camoufox has its own defaults from BrowserForge.
		if !f.hasCamoufox {
			opts.UserAgent = playwright.String(
				"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:136.0) Gecko/20100101 Firefox/136.0",
			)
			opts.Locale = playwright.String("en-US")
			opts.TimezoneId = playwright.String("America/New_York")
		}
		opts.Viewport = &playwright.Size{Width: 1920, Height: 1080}
		return opts
	}

	p := f.identity

	// Viewport controls the browser window inner size — always set this.
	opts.Viewport = &playwright.Size{
		Width:  p.ScreenW,
		Height: p.ScreenH,
	}

	if p.PixelRatio != 0 {
		opts.DeviceScaleFactor = playwright.Float(p.PixelRatio)
	}

	// When Camoufox is active, it handles UA, locale, and timezone at C++
	// level via CAMOU_CONFIG. Do NOT override them via Playwright context
	// because that creates a detectable mismatch.
	if !f.hasCamoufox {
		opts.UserAgent = playwright.String(p.UA)
		opts.Locale = playwright.String(p.Locale)
		opts.TimezoneId = playwright.String(p.Timezone)
	}

	// Only attach geolocation when real coordinates are present; a zero-zero
	// coordinate (null island) is worse than no geolocation because it is an
	// obvious detection signal.
	if p.Lat != 0 || p.Lng != 0 {
		opts.Geolocation = &playwright.Geolocation{
			Latitude:  p.Lat,
			Longitude: p.Lng,
		}
		opts.Permissions = []string{"geolocation"}
	}

	return opts
}

// parsePlaywrightProxy converts a proxy URL like
// "socks5://user:pass@host:port" into a playwright.Proxy struct
// with Server, Username, and Password separated (required by Playwright).
func parsePlaywrightProxy(rawURL string) *playwright.Proxy {
	proxy := &playwright.Proxy{}

	// Parse the URL to extract components
	u, err := neturl.Parse(rawURL)
	if err != nil {
		// Fallback: use as-is
		proxy.Server = rawURL
		return proxy
	}

	// Server = scheme://host:port (no auth)
	proxy.Server = fmt.Sprintf("%s://%s", u.Scheme, u.Host)

	// Extract username and password separately
	if u.User != nil {
		username := u.User.Username()
		proxy.Username = &username
		if pass, ok := u.User.Password(); ok {
			proxy.Password = &pass
		}
	}

	return proxy
}

// cleanTempDirs removes all tracked temporary directories.
func (f *CamoufoxFetcher) cleanTempDirs() {
	for _, dir := range f.tempDirs {
		if err := os.RemoveAll(dir); err != nil {
			slog.Debug("fetch/camoufox: error removing temp dir", "dir", dir, "err", err)
		}
	}
	f.tempDirs = nil
}

// restart closes the current browser instance and launches a new one, resetting
// the request counter. It is serialised under mu so that concurrent Fetch calls
// do not race on the browser handle. If the new browser fails to launch the
// old instance (if still alive) is used and an error is returned.
func (f *CamoufoxFetcher) restart() error {
	f.mu.Lock()
	defer f.mu.Unlock()

	// Double-check: another goroutine may have already restarted the browser
	// between the counter check and acquiring the lock.
	if f.requestCount.Load() <= int64(f.maxRequests) {
		return nil
	}

	// If Close() has been called, the fetcher is shutting down. Restarting the
	// browser would be pointless and would race with the teardown sequence
	// (v0.0.22 fix for B5 — see .dev-squad/v0.0.22-rca.md).
	if f.closing.Load() {
		return nil
	}

	slog.Info("fetch/camoufox: restarting browser instance",
		"request_count", f.requestCount.Load(),
		"max_requests", f.maxRequests,
	)

	// Close the persistent session context first — it references the old
	// browser and will be invalid after the browser is torn down.
	f.sessionMu.Lock()
	if f.sessionCtx != nil {
		if closeErr := f.sessionCtx.Close(); closeErr != nil {
			if !strings.Contains(closeErr.Error(), "Target closed") &&
				!strings.Contains(closeErr.Error(), "Browser has been closed") {
				slog.Warn("fetch/camoufox: error closing session context during restart", "err", closeErr)
			}
		}
		f.sessionCtx = nil
	}
	f.sessionMu.Unlock()

	// Drain the page pool — old pages reference the dead browser.
	if f.pool != nil {
		f.pool.Close()
		f.pool = nil
	}

	// Clean up temporary directories from the previous browser session.
	f.cleanTempDirs()

	// Close the existing browser (best-effort; errors are logged only).
	if f.browser != nil {
		if err := f.browser.Close(); err != nil {
			if !strings.Contains(err.Error(), "Target closed") &&
				!strings.Contains(err.Error(), "Browser has been closed") {
				slog.Warn("fetch/camoufox: error closing browser during restart", "err", err)
			}
		}
		f.browser = nil
	}

	// Launch a fresh browser with the same configuration — replicate the
	// full launch options from NewCamoufox so Camoufox binary, proxy, and
	// extension settings are preserved across restarts.
	headlessBool := f.headless != "false"
	launchOpts := playwright.BrowserTypeLaunchOptions{
		Headless: playwright.Bool(headlessBool),
	}

	// Restore Camoufox binary path.
	execPath, _ := findOrInstallCamoufox()
	if execPath != "" {
		launchOpts.ExecutablePath = playwright.String(execPath)
	}

	// Restore proxy settings.
	if f.proxyURL != "" {
		launchOpts.Proxy = parsePlaywrightProxy(f.proxyURL)
	}

	// Restore CAMOU_CONFIG (fingerprint + extension) on restart.
	if launchOpts.Env == nil {
		launchOpts.Env = map[string]string{}
	}
	var restartAddonExtra map[string]any
	if f.hasExtension && f.extensionPath != "" && execPath != "" {
		absExt, _ := filepath.Abs(f.extensionPath)
		restartAddonExtra = map[string]any{"addons": []string{absExt}}
	}
	if f.identity != nil && execPath != "" {
		if restartAddonExtra != nil {
			for k, v := range f.identity.MergeCamoufoxConfig(restartAddonExtra) {
				launchOpts.Env[k] = v
			}
		} else {
			for k, v := range f.identity.BuildCamoufoxEnv() {
				launchOpts.Env[k] = v
			}
		}
	} else if restartAddonExtra != nil {
		configJSON, _ := json.Marshal(restartAddonExtra)
		launchOpts.Env["CAMOU_CONFIG_1"] = string(configJSON)
	}

	browser, err := f.pw.Firefox.Launch(launchOpts)
	if err != nil {
		return fmt.Errorf("fetch/camoufox: launching replacement browser: %w", err)
	}

	f.browser = browser
	f.requestCount.Store(0)

	// Recreate page pool for the new browser.
	if f.poolSize > 0 && !f.persistSession {
		f.pool = NewPagePool(
			f.poolSize,
			func() (any, error) {
				// IMPORTANT: snapshot f.browser under f.mu at invocation time.
				// The create closure is called lazily outside f.mu; a concurrent
				// Close() or second restart() may nil f.browser before this runs
				// (v0.0.22 fix for B3 — see .dev-squad/v0.0.22-rca.md).
				f.mu.Lock()
				b := f.browser
				f.mu.Unlock()
				if b == nil {
					return nil, fmt.Errorf("fetch/camoufox: browser unavailable for page pool")
				}
				bctx, err := b.NewContext(f.buildContextOptions())
				if err != nil {
					return nil, fmt.Errorf("pool create: %w", err)
				}
				pg, err := bctx.NewPage()
				if err != nil {
					bctx.Close()
					return nil, fmt.Errorf("pool create page: %w", err)
				}
				return pg, nil
			},
			func(page any) error {
				pg := page.(playwright.Page)
				return pg.Context().Close()
			},
			WithPageReset(func(page any) error {
				pg := page.(playwright.Page)
				if err := pg.Context().ClearCookies(); err != nil {
					return err
				}
				_, err := pg.Goto("about:blank", playwright.PageGotoOptions{
					WaitUntil: playwright.WaitUntilStateDomcontentloaded,
					Timeout:   playwright.Float(5000),
				})
				return err
			}),
			WithPoolReuseLimit(f.pageReuseLimit),
		)
		slog.Info("fetch/camoufox: page pool recreated after restart",
			"pool_size", f.poolSize,
			"page_reuse_limit", f.pageReuseLimit)
	}

	slog.Info("fetch/camoufox: browser restarted successfully")
	return nil
}

// scrollConfig returns the scroll configuration from the behavior profile,
// or defaults when no profile is set.
func (f *CamoufoxFetcher) scrollConfig() behavior.ScrollConfig {
	if f.behaviorProfile != nil {
		return f.behaviorProfile.Scroll
	}
	return behavior.DefaultScrollConfig()
}

// keyboardConfig returns the keyboard configuration from the behavior profile,
// or defaults when no profile is set.
func (f *CamoufoxFetcher) keyboardConfig() behavior.KeyboardConfig {
	if f.behaviorProfile != nil {
		return f.behaviorProfile.Keyboard
	}
	return behavior.DefaultKeyboardConfig()
}

// mouseConfig returns the mouse configuration from the behavior profile,
// or defaults when no profile is set.
func (f *CamoufoxFetcher) mouseConfig() behavior.MouseConfig {
	if f.behaviorProfile != nil {
		return f.behaviorProfile.Mouse
	}
	return behavior.DefaultMouseConfig()
}

// Close gracefully shuts down the browser and stops the playwright runtime.
//
// Resources are released in order: persistent session context (if any),
// LaunchPersistentContext context (if any), browser, then the playwright
// process. Errors from each step are logged but do not prevent the subsequent
// steps from running.
//
// Close() acquires f.mu for the entire browser teardown sequence so it cannot
// race with restart() (v0.0.22 fix for B1/B4/B5 — see .dev-squad/v0.0.22-rca.md).
// Lock order: f.mu → f.sessionMu (must not be inverted).
func (f *CamoufoxFetcher) Close() error {
	var firstErr error

	// Signal that shutdown is in progress. restart() checks this flag under
	// f.mu and returns immediately if set, preventing a concurrent restart
	// from racing with teardown.
	f.closing.Store(true)

	// Save storage state before acquiring f.mu — SaveStorageState only reads
	// browser state and does not mutate the fields guarded by f.mu.
	if err := f.SaveStorageState(); err != nil {
		slog.Warn("fetch/camoufox: error saving storage state", "err", err)
	}

	// Hold f.mu for the entire browser/pool teardown so that restart() cannot
	// concurrently nil or replace f.pool, f.browser, or f.persistCtx while we
	// are closing them.
	f.mu.Lock()
	defer f.mu.Unlock()

	// Close the page pool first so all pooled contexts are released before
	// the browser is torn down.
	if f.pool != nil {
		if closeErr := f.pool.Close(); closeErr != nil {
			slog.Warn("fetch/camoufox: error closing page pool", "err", closeErr)
			if firstErr == nil {
				firstErr = fmt.Errorf("fetch/camoufox: closing page pool: %w", closeErr)
			}
		}
		f.pool = nil
	}

	// Close the persistent session context so its resources are freed
	// before the browser itself is torn down.
	// Lock order: f.mu (already held) → f.sessionMu.
	f.sessionMu.Lock()
	if f.sessionCtx != nil {
		if closeErr := f.sessionCtx.Close(); closeErr != nil {
			if !strings.Contains(closeErr.Error(), "Target closed") &&
				!strings.Contains(closeErr.Error(), "Browser has been closed") {
				slog.Warn("fetch/camoufox: error closing persistent session context", "err", closeErr)
				if firstErr == nil {
					firstErr = fmt.Errorf("fetch/camoufox: closing session context: %w", closeErr)
				}
			}
		}
		f.sessionCtx = nil
	}
	f.sessionMu.Unlock()

	// Close the LaunchPersistentContext context (userDataDir mode).
	// This also implicitly closes the underlying browser process.
	if f.persistCtx != nil {
		if closeErr := f.persistCtx.Close(); closeErr != nil {
			if !strings.Contains(closeErr.Error(), "Target closed") &&
				!strings.Contains(closeErr.Error(), "Browser has been closed") {
				slog.Warn("fetch/camoufox: error closing persistent profile context", "err", closeErr)
				if firstErr == nil {
					firstErr = fmt.Errorf("fetch/camoufox: closing persistent profile context: %w", closeErr)
				}
			}
		}
		f.persistCtx = nil
		f.browser = nil // browser was bound to the persistent context
	}

	if f.browser != nil {
		if err := f.browser.Close(); err != nil {
			// Filter out "Target closed" noise which is benign on shutdown.
			if !strings.Contains(err.Error(), "Target closed") &&
				!strings.Contains(err.Error(), "Browser has been closed") {
				slog.Error("fetch/camoufox: error closing browser", "err", err)
				firstErr = fmt.Errorf("fetch/camoufox: closing browser: %w", err)
			}
		}
		f.browser = nil
	}

	if f.pw != nil {
		if err := f.pw.Stop(); err != nil {
			slog.Error("fetch/camoufox: error stopping playwright runtime", "err", err)
			if firstErr == nil {
				firstErr = fmt.Errorf("fetch/camoufox: stopping playwright: %w", err)
			}
		}
		f.pw = nil
	}

	// Clean up temporary directories while f.mu is held, mirroring restart()'s
	// call pattern to prevent a race between concurrent Close() and restart().
	f.cleanTempDirs()

	// Shut down the SOCKS5 auth bridge if it was started.
	if f.socksBridge != nil {
		if err := f.socksBridge.Close(); err != nil {
			slog.Warn("fetch/camoufox: error closing SOCKS5 bridge", "err", err)
			if firstErr == nil {
				firstErr = fmt.Errorf("fetch/camoufox: closing SOCKS5 bridge: %w", err)
			}
		}
		f.socksBridge = nil
	}

	// Shut down Xvfb display manager (last, after all browser resources are freed).
	if f.displayMgr != nil {
		if err := f.displayMgr.Close(); err != nil {
			slog.Warn("fetch/camoufox: error closing display manager", "err", err)
			if firstErr == nil {
				firstErr = fmt.Errorf("fetch/camoufox: closing display manager: %w", err)
			}
		}
		f.displayMgr = nil
	}

	return firstErr
}

// getActiveContext returns the current browser context, preferring the
// persistent context (LaunchPersistentContext) over the session context.
func (f *CamoufoxFetcher) getActiveContext() playwright.BrowserContext {
	if f.persistCtx != nil {
		return f.persistCtx
	}
	f.sessionMu.Lock()
	defer f.sessionMu.Unlock()
	return f.sessionCtx
}

// ExportStorageState captures the current browser session state (cookies).
func (f *CamoufoxFetcher) ExportStorageState() (*StorageState, error) {
	ctx := f.getActiveContext()
	if ctx == nil {
		return nil, fmt.Errorf("fetch/camoufox: no active browser context for storage state export")
	}

	playwrightCookies, err := ctx.Cookies()
	if err != nil {
		return nil, fmt.Errorf("fetch/camoufox: export cookies: %w", err)
	}

	var cookies []BrowserCookie
	for _, c := range playwrightCookies {
		cookies = append(cookies, BrowserCookie{
			Name:     c.Name,
			Value:    c.Value,
			Domain:   c.Domain,
			Path:     c.Path,
			Secure:   c.Secure,
			HttpOnly: c.HttpOnly,
		})
	}

	return &StorageState{
		Cookies:    cookies,
		ExportedAt: time.Now(),
	}, nil
}

// SaveStorageState exports and saves the session state to the configured file path.
func (f *CamoufoxFetcher) SaveStorageState() error {
	if f.storageStatePath == "" {
		return nil
	}
	state, err := f.ExportStorageState()
	if err != nil {
		return err
	}
	data, err := json.MarshalIndent(state, "", "  ")
	if err != nil {
		return fmt.Errorf("fetch/camoufox: marshal storage state: %w", err)
	}
	dir := filepath.Dir(f.storageStatePath)
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return fmt.Errorf("fetch/camoufox: create storage state dir: %w", err)
	}
	return os.WriteFile(f.storageStatePath, data, 0o644)
}

// LoadStorageState reads saved state from the configured file path and returns
// it for injection. Returns (nil, nil) when no path is set or the file does not
// exist yet.
func (f *CamoufoxFetcher) LoadStorageState() (*StorageState, error) {
	if f.storageStatePath == "" {
		return nil, nil
	}
	data, err := os.ReadFile(f.storageStatePath)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, fmt.Errorf("fetch/camoufox: read storage state: %w", err)
	}
	var state StorageState
	if err := json.Unmarshal(data, &state); err != nil {
		return nil, fmt.Errorf("fetch/camoufox: parse storage state: %w", err)
	}
	return &state, nil
}