`,
captcha.CaptchaHCaptcha,
},
{
"no captcha",
`Normal page`,
captcha.CaptchaNone,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
resp := &foxhound.Response{
StatusCode: 200,
Body: []byte(tc.body),
URL: "https://example.com",
}
result := captcha.Detect(resp)
if result.Type != tc.expected {
t.Errorf("expected %q, got %q", tc.expected, result.Type)
}
t.Logf("Detected: %s (sitekey=%q)", result.Type, result.SiteKey)
})
}
}
// ---------------------------------------------------------------------------
// Test 11: Parse — all modes (CSS, JSON, regex, structured)
// ---------------------------------------------------------------------------
func TestParseAllModes(t *testing.T) {
htmlBody := `
Test
Widget
$19.99
Gadget
$39.99
`
resp := &foxhound.Response{
StatusCode: 200,
Body: []byte(htmlBody),
URL: "https://example.com/products",
}
// CSS selectors
doc, err := parse.NewDocument(resp)
if err != nil {
t.Fatal(err)
}
titles := doc.Texts("div.product h2")
if len(titles) != 2 {
t.Errorf("expected 2 product titles, got %d", len(titles))
}
t.Logf("CSS: found %d products: %v", len(titles), titles)
// Regex
prices, err := parse.RegexExtractAll(resp, `\$\d+\.\d+`)
if err != nil {
t.Fatal(err)
}
if len(prices) != 2 {
t.Errorf("expected 2 prices, got %d", len(prices))
}
t.Logf("Regex: %v", prices)
// Structured schema
schema := &parse.Schema{
Fields: []parse.FieldDef{
{Name: "title", Selector: "title"},
{Name: "first_product", Selector: "div.product h2"},
},
}
item, err := schema.Extract(resp)
if err != nil {
t.Fatal(err)
}
if v, _ := item.Get("title"); v != "Test" {
t.Errorf("expected title 'Test', got %q", v)
}
t.Logf("Structured: %v", item.Fields)
// JSON parse
jsonResp := &foxhound.Response{
StatusCode: 200,
Body: []byte(`{"data":{"items":[{"name":"A"},{"name":"B"}]}}`),
URL: "https://api.example.com",
}
val, err := parse.JSONPath(jsonResp, "data.items")
if err != nil {
t.Fatal(err)
}
items, ok := val.([]any)
if !ok || len(items) != 2 {
t.Errorf("expected 2 JSON items, got %v", val)
}
t.Logf("JSONPath: %v", val)
}
// ---------------------------------------------------------------------------
// Test 12: SQLite queue persistence
// ---------------------------------------------------------------------------
func TestSQLiteQueue_Persistence(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "test.db")
// Write jobs
q1, err := queue.NewSQLite(dbPath)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
q1.Push(ctx, &foxhound.Job{ID: "j1", URL: "http://1", Priority: foxhound.PriorityHigh, CreatedAt: time.Now()})
q1.Push(ctx, &foxhound.Job{ID: "j2", URL: "http://2", Priority: foxhound.PriorityNormal, CreatedAt: time.Now()})
q1.Close()
// Reopen and verify
q2, err := queue.NewSQLite(dbPath)
if err != nil {
t.Fatal(err)
}
defer q2.Close()
if q2.Len() != 2 {
t.Errorf("expected 2 pending jobs, got %d", q2.Len())
}
job, err := q2.Pop(ctx)
if err != nil {
t.Fatal(err)
}
if job.ID != "j1" {
t.Errorf("expected j1 (high priority), got %s", job.ID)
}
t.Logf("SQLite persistence: 2 jobs survived close/reopen, popped %s first", job.ID)
}
// ---------------------------------------------------------------------------
// Test 13: GeoIP country lookup
// ---------------------------------------------------------------------------
func TestGeoIP_CountryLookup(t *testing.T) {
countries := []struct {
code string
timezone string
locale string
}{
{"US", "America/New_York", "en-US"},
{"GB", "Europe/London", "en-GB"},
{"JP", "Asia/Tokyo", "ja-JP"},
{"DE", "Europe/Berlin", "de-DE"},
}
for _, tc := range countries {
t.Run(tc.code, func(t *testing.T) {
geo, ok := identity.LookupCountry(tc.code)
if !ok {
t.Fatalf("country %q not found", tc.code)
}
if geo.Timezone != tc.timezone {
t.Errorf("expected tz %q, got %q", tc.timezone, geo.Timezone)
}
if geo.Locale != tc.locale {
t.Errorf("expected locale %q, got %q", tc.locale, geo.Locale)
}
if geo.Lat == 0 && geo.Lng == 0 {
t.Error("coordinates are zero")
}
t.Logf("%s: tz=%s, locale=%s, lat=%.2f, lng=%.2f", tc.code, geo.Timezone, geo.Locale, geo.Lat, geo.Lng)
})
}
// WithCountry should apply geo to identity
prof := identity.Generate(
identity.WithBrowser(identity.BrowserFirefox),
identity.WithOS(identity.OSWindows),
identity.WithCountry("JP"),
)
if prof.Timezone != "Asia/Tokyo" {
t.Errorf("expected Asia/Tokyo, got %s", prof.Timezone)
}
if prof.Locale != "ja-JP" {
t.Errorf("expected ja-JP, got %s", prof.Locale)
}
t.Logf("WithCountry(JP): tz=%s, locale=%s", prof.Timezone, prof.Locale)
}
// ---------------------------------------------------------------------------
// Test 14: Config loading
// ---------------------------------------------------------------------------
func TestConfigLoading(t *testing.T) {
cfgContent := `
hunt:
domain: "test.example.com"
walkers: 5
identity:
browser: "chrome"
os: ["windows"]
fetch:
static:
timeout: 15s
browser:
instances: 0
middleware:
ratelimit:
enabled: true
requests_per_sec: 3.0
burst_size: 5
depth_limit:
max: 4
queue:
backend: memory
logging:
level: debug
format: text
`
cfgPath := filepath.Join(t.TempDir(), "test-config.yaml")
os.WriteFile(cfgPath, []byte(cfgContent), 0644)
cfg, err := foxhound.LoadConfig(cfgPath)
if err != nil {
t.Fatal(err)
}
if cfg.Hunt.Domain != "test.example.com" {
t.Errorf("domain: %q", cfg.Hunt.Domain)
}
if cfg.Hunt.Walkers != 5 {
t.Errorf("walkers: %d", cfg.Hunt.Walkers)
}
if cfg.Identity.Browser != "chrome" {
t.Errorf("browser: %q", cfg.Identity.Browser)
}
if cfg.Fetch.Static.Timeout.Duration != 15*time.Second {
t.Errorf("timeout: %s", cfg.Fetch.Static.Timeout.Duration)
}
if cfg.Middleware.RateLimit.RequestsPerSec != 3.0 {
t.Errorf("rps: %f", cfg.Middleware.RateLimit.RequestsPerSec)
}
if cfg.Logging.Level != "debug" {
t.Errorf("log level: %q", cfg.Logging.Level)
}
t.Logf("Config: domain=%s, walkers=%d, browser=%s, rps=%.1f, depth=%d",
cfg.Hunt.Domain, cfg.Hunt.Walkers, cfg.Identity.Browser,
cfg.Middleware.RateLimit.RequestsPerSec, cfg.Middleware.DepthLimit.Max)
}