package parse_test
import (
"strings"
"testing"
"time"
"github.com/sadewadee/foxhound/parse"
)
// ---------------------------------------------------------------------------
// Test fixtures
// ---------------------------------------------------------------------------
const articleHTML = `
The Future of Sustainable Energy | GreenTech Blog
The Future of Sustainable Energy
By Jane Smith
Sustainable energy is rapidly transforming how we power our homes, businesses, and cities around the world. The transition from fossil fuels to renewable sources has accelerated dramatically in recent years, driven by declining costs and growing environmental awareness among consumers and policymakers alike.
Solar panel installations have doubled in the past three years, with residential rooftop systems becoming increasingly affordable for middle-class families. Wind farms, both onshore and offshore, now generate a significant share of electricity in many countries across Europe and North America.
Battery storage technology has made remarkable advances, solving the intermittency problem that once plagued renewable energy. Modern lithium-ion and solid-state batteries can store enough energy to power entire neighborhoods through cloudy days and calm nights without any fossil fuel backup.
Government policies continue to play a crucial role in this transition. Tax incentives, feed-in tariffs, and carbon pricing mechanisms have created favorable conditions for clean energy investment. The private sector has responded enthusiastically, with billions of dollars flowing into green technology startups and infrastructure projects worldwide.
Looking ahead, hydrogen fuel cells and next-generation nuclear reactors promise to fill gaps that solar and wind cannot easily address, particularly in heavy industry and long-distance transportation sectors.
`
const listingHTML = `
Search Results
Item 1
Description one
Item 2
Description two
Item 3
Description three
Item 4
Description four
Item 5
Description five
Item 6
Description six
`
const productHTML = `
Premium Headphones
Premium Wireless Headphones
$29.99
High quality sound with noise cancellation and 30-hour battery life.
`
const searchHTML = `
Search: coffee shops
Result 1
Result 2
Result 3
Result 4
Result 5
`
const feedHTML = `
Activity Feed
First update
Second update
Third update
Fourth update
`
const minimalHTML = `
Hi
Hello
`
const listingJSONLDHTML = `
Local Restaurants
Local Restaurants
Find the best dining near you.
`
// ---------------------------------------------------------------------------
// DetectContentType tests
// ---------------------------------------------------------------------------
func TestDetectContentType_Article(t *testing.T) {
resp := newHTMLResponse(articleHTML)
ct := parse.DetectContentType(resp)
if ct != parse.ContentArticle {
t.Errorf("expected ContentArticle, got %v (%s)", ct, ct)
}
}
func TestDetectContentType_Listing(t *testing.T) {
resp := newHTMLResponse(listingHTML)
ct := parse.DetectContentType(resp)
if ct != parse.ContentListing {
t.Errorf("expected ContentListing, got %v (%s)", ct, ct)
}
}
func TestDetectContentType_Product(t *testing.T) {
resp := newHTMLResponse(productHTML)
ct := parse.DetectContentType(resp)
if ct != parse.ContentProduct {
t.Errorf("expected ContentProduct, got %v (%s)", ct, ct)
}
}
func TestDetectContentType_Search(t *testing.T) {
resp := newHTMLResponse(searchHTML)
ct := parse.DetectContentType(resp)
if ct != parse.ContentSearch {
t.Errorf("expected ContentSearch, got %v (%s)", ct, ct)
}
}
func TestDetectContentType_Feed(t *testing.T) {
resp := newHTMLResponse(feedHTML)
ct := parse.DetectContentType(resp)
if ct != parse.ContentFeed {
t.Errorf("expected ContentFeed, got %v (%s)", ct, ct)
}
}
func TestDetectContentType_Unknown(t *testing.T) {
resp := newHTMLResponse(minimalHTML)
ct := parse.DetectContentType(resp)
if ct != parse.ContentUnknown {
t.Errorf("expected ContentUnknown, got %v (%s)", ct, ct)
}
}
// ---------------------------------------------------------------------------
// AutoExtract tests
// ---------------------------------------------------------------------------
func TestAutoExtract_Article(t *testing.T) {
resp := newHTMLResponse(articleHTML)
result, err := parse.AutoExtract(resp)
if err != nil {
t.Fatalf("AutoExtract error: %v", err)
}
if result.Type != parse.ContentArticle {
t.Fatalf("expected ContentArticle, got %s", result.Type)
}
if result.Article == nil {
t.Fatal("Article is nil")
}
if result.Article.Title == "" {
t.Error("Article.Title is empty")
}
}
func TestAutoExtract_Listing(t *testing.T) {
resp := newHTMLResponse(listingJSONLDHTML)
result, err := parse.AutoExtract(resp)
if err != nil {
t.Fatalf("AutoExtract error: %v", err)
}
if result.Type != parse.ContentListing {
t.Fatalf("expected ContentListing, got %s", result.Type)
}
if len(result.Listings) == 0 {
t.Error("expected non-empty Listings")
}
}
// ---------------------------------------------------------------------------
// ExtractArticle tests
// ---------------------------------------------------------------------------
func TestExtractArticle_FullExtraction(t *testing.T) {
resp := newHTMLResponse(articleHTML)
article, err := parse.ExtractArticle(resp)
if err != nil {
t.Fatalf("ExtractArticle error: %v", err)
}
if article == nil {
t.Fatal("article is nil")
}
// Title
if article.Title != "The Future of Sustainable Energy" {
t.Errorf("Title = %q, want %q", article.Title, "The Future of Sustainable Energy")
}
// Author
if article.Author != "Jane Smith" {
t.Errorf("Author = %q, want %q", article.Author, "Jane Smith")
}
// Published date
if article.PublishedDate != "2025-06-15T08:00:00Z" {
t.Errorf("PublishedDate = %q, want %q", article.PublishedDate, "2025-06-15T08:00:00Z")
}
// ContentText should contain substantive text.
if len(article.ContentText) < 100 {
t.Errorf("ContentText too short: %d chars", len(article.ContentText))
}
// WordCount
if article.WordCount < 50 {
t.Errorf("WordCount = %d, want >= 50", article.WordCount)
}
// ReadingTime should be positive.
if article.ReadingTime <= 0 {
t.Errorf("ReadingTime = %v, want > 0", article.ReadingTime)
}
// Sanity check: reading time matches word count at 200 wpm.
expectedRT := time.Duration(float64(article.WordCount)/200.0*60.0) * time.Second
if article.ReadingTime != expectedRT {
t.Errorf("ReadingTime = %v, want %v", article.ReadingTime, expectedRT)
}
// Summary should be populated and not too long.
if article.Summary == "" {
t.Error("Summary is empty")
}
if len(article.Summary) > 210 {
t.Errorf("Summary too long: %d chars", len(article.Summary))
}
// Images
if len(article.Images) < 1 {
t.Errorf("expected at least 1 image, got %d", len(article.Images))
}
// Tags — from article:tag + keywords.
if len(article.Tags) < 3 {
t.Errorf("expected at least 3 tags, got %d: %v", len(article.Tags), article.Tags)
}
}
func TestExtractArticle_TitleFromOGTag(t *testing.T) {
html := `
Article Title | My Website Name
Heading Title
` + strings.Repeat("This is a paragraph of content with enough text to be scored. ", 10) + `
`
resp := newHTMLResponse(html)
article, err := parse.ExtractArticle(resp)
if err != nil {
t.Fatalf("ExtractArticle error: %v", err)
}
if article == nil {
t.Fatal("article is nil")
}
if article.Title != "Clean OG Title" {
t.Errorf("Title = %q, want %q", article.Title, "Clean OG Title")
}
}
func TestExtractArticle_ReadabilityScoring(t *testing.T) {
// A page with rich content in a scored container should have Score > 0.
html := `
Scored Article
Main Article
` + strings.Repeat("This is a well-written paragraph with commas, clauses, and detail. ", 8) + `
` + strings.Repeat("Another substantial paragraph discussing important topics in depth. ", 8) + `
` + strings.Repeat("A third paragraph providing additional context, examples, and analysis. ", 8) + `
