153 lines
3.5 KiB
Go
153 lines
3.5 KiB
Go
package search
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"golang.org/x/net/html"
|
|
)
|
|
|
|
func init() {
|
|
RegisterProvider(&DuckDuckGo{})
|
|
// Default active provider — no API key needed
|
|
_ = SetActive("duckduckgo")
|
|
}
|
|
|
|
// DuckDuckGo implements Provider using DDG's HTML lite endpoint.
|
|
// No API key required. Returns organic web results.
|
|
type DuckDuckGo struct{}
|
|
|
|
func (d *DuckDuckGo) Name() string { return "duckduckgo" }
|
|
|
|
func (d *DuckDuckGo) Search(ctx context.Context, query string, maxResults int) ([]SearchResult, error) {
|
|
if maxResults <= 0 {
|
|
maxResults = 5
|
|
}
|
|
if maxResults > 10 {
|
|
maxResults = 10
|
|
}
|
|
|
|
endpoint := "https://html.duckduckgo.com/html/?q=" + url.QueryEscape(query)
|
|
|
|
req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("build request: %w", err)
|
|
}
|
|
req.Header.Set("User-Agent", "ChatSwitchboard/1.0 (Web Search Tool)")
|
|
|
|
client := &http.Client{Timeout: 10 * time.Second}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("search request failed: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("search returned HTTP %d", resp.StatusCode)
|
|
}
|
|
|
|
// Limit body read to 512KB
|
|
body, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read response: %w", err)
|
|
}
|
|
|
|
return parseDDGHTML(string(body), maxResults), nil
|
|
}
|
|
|
|
// parseDDGHTML extracts search results from DuckDuckGo's HTML lite page.
|
|
// Results are in <a class="result__a"> tags with snippets in <a class="result__snippet">.
|
|
func parseDDGHTML(htmlContent string, maxResults int) []SearchResult {
|
|
doc, err := html.Parse(strings.NewReader(htmlContent))
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
var results []SearchResult
|
|
|
|
// Walk the DOM looking for result blocks
|
|
var walk func(*html.Node)
|
|
walk = func(n *html.Node) {
|
|
if len(results) >= maxResults {
|
|
return
|
|
}
|
|
|
|
if n.Type == html.ElementNode && n.Data == "a" {
|
|
class := getAttr(n, "class")
|
|
|
|
if strings.Contains(class, "result__a") {
|
|
href := getAttr(n, "href")
|
|
title := textContent(n)
|
|
// DDG wraps URLs through a redirect; extract the actual URL
|
|
actualURL := extractDDGURL(href)
|
|
if actualURL != "" && title != "" {
|
|
results = append(results, SearchResult{
|
|
Title: strings.TrimSpace(title),
|
|
URL: actualURL,
|
|
})
|
|
}
|
|
}
|
|
|
|
if strings.Contains(class, "result__snippet") {
|
|
snippet := strings.TrimSpace(textContent(n))
|
|
// Attach snippet to the most recent result
|
|
if len(results) > 0 && results[len(results)-1].Snippet == "" {
|
|
results[len(results)-1].Snippet = snippet
|
|
}
|
|
}
|
|
}
|
|
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
|
walk(c)
|
|
}
|
|
}
|
|
walk(doc)
|
|
|
|
return results
|
|
}
|
|
|
|
// extractDDGURL extracts the actual URL from DDG's redirect wrapper.
|
|
// DDG links look like: //duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com&rut=...
|
|
func extractDDGURL(href string) string {
|
|
if strings.Contains(href, "uddg=") {
|
|
u, err := url.Parse(href)
|
|
if err != nil {
|
|
return href
|
|
}
|
|
uddg := u.Query().Get("uddg")
|
|
if uddg != "" {
|
|
return uddg
|
|
}
|
|
}
|
|
// Direct URL
|
|
if strings.HasPrefix(href, "http") {
|
|
return href
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func getAttr(n *html.Node, key string) string {
|
|
for _, a := range n.Attr {
|
|
if a.Key == key {
|
|
return a.Val
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func textContent(n *html.Node) string {
|
|
if n.Type == html.TextNode {
|
|
return n.Data
|
|
}
|
|
var sb strings.Builder
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
|
sb.WriteString(textContent(c))
|
|
}
|
|
return sb.String()
|
|
}
|