tell-me/tools/search.go

package tools

import (
	"context"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

const (
	maxRetries = 3
	timeout    = 60 * time.Second
)

// SearchResult represents a single search result
type SearchResult struct {
	Title   string `json:"title"`
	URL     string `json:"url"`
	Content string `json:"content"`
}

// WebSearch performs a web search using DuckDuckGo
func WebSearch(query string) (string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()

	// Create HTTP client with proxy support
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
	}

	client := &http.Client{
		Timeout:   timeout,
		Transport: transport,
	}

	// Perform search with retry logic
	results, err := searchDuckDuckGo(ctx, client, query)
	if err != nil {
		return "", fmt.Errorf("DuckDuckGo search failed: %w", err)
	}

	// Format results as text
	if len(results) == 0 {
		return "No results found.", nil
	}

	var output strings.Builder
	fmt.Fprintf(&output, "Found %d results:\n\n", len(results))

	for i, result := range results {
		if i >= 10 { // Limit to top 10 results
			break
		}
		fmt.Fprintf(&output, "%d. %s\n", i+1, result.Title)
		fmt.Fprintf(&output, "   URL: %s\n", result.URL)
		if result.Content != "" {
			fmt.Fprintf(&output, "   %s\n", result.Content)
		}
		fmt.Fprintf(&output, "\n")
	}

	return output.String(), nil
}

// searchDuckDuckGo performs the actual DuckDuckGo search
func searchDuckDuckGo(ctx context.Context, client *http.Client, query string) ([]SearchResult, error) {
	// Build DuckDuckGo search URL
	searchURL := fmt.Sprintf("https://html.duckduckgo.com/html/?q=%s", url.QueryEscape(query))

	// Create request with proper headers
	req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to create search request: %w", err)
	}

	// Set browser-like user agent
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	req.Header.Set("Accept-Language", "en-US,en;q=0.5")

	// Execute request with retry logic
	resp, err := executeWithRetry(ctx, client, req, "search DuckDuckGo")
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	// Check response status
	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
		return nil, fmt.Errorf("search request failed with status: %d %s", resp.StatusCode, resp.Status)
	}

	// Parse HTML response
	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to parse search results: %w", err)
	}

	// Extract search results
	var results []SearchResult
	doc.Find(".web-result").Each(func(i int, s *goquery.Selection) {
		titleNode := s.Find(".result__a")
		title := strings.TrimSpace(titleNode.Text())
		info := strings.TrimSpace(s.Find(".result__snippet").Text())

		// Extract URL
		var resultURL string
		if titleNode.Length() > 0 {
			if href, exists := titleNode.Attr("href"); exists {
				// Validate URL format
				if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
					resultURL = href
				}
			}
		}

		if title != "" && resultURL != "" {
			results = append(results, SearchResult{
				Title:   title,
				URL:     resultURL,
				Content: info,
			})
		}
	})

	return results, nil
}

// executeWithRetry executes an HTTP request with exponential backoff retry logic
func executeWithRetry(ctx context.Context, client *http.Client, req *http.Request, operation string) (*http.Response, error) {
	var resp *http.Response
	var err error

	// Retry logic with exponential backoff
	for attempt := 0; attempt <= maxRetries; attempt++ {
		resp, err = client.Do(req)
		if err == nil {
			return resp, nil
		}

		// Don't retry on the last attempt
		if attempt == maxRetries {
			break
		}

		// Exponential backoff: 1s, 2s, 4s
		backoffDuration := time.Duration(1<<uint(attempt)) * time.Second

		select {
		case <-ctx.Done():
			return nil, fmt.Errorf("%s cancelled: %v", operation, ctx.Err())
		case <-time.After(backoffDuration):
			// Continue to next retry
		}
	}

	return nil, fmt.Errorf("failed to %s after %d retries: %v", operation, maxRetries, err)
}

// logProxyConfiguration logs the proxy configuration for debugging
func init() {
	if httpsProxy := os.Getenv("HTTPS_PROXY"); httpsProxy != "" {
		fmt.Printf("Using HTTPS_PROXY: %s\n", httpsProxy)
	} else if httpProxy := os.Getenv("HTTP_PROXY"); httpProxy != "" {
		fmt.Printf("Using HTTP_PROXY: %s\n", httpProxy)
	} else if allProxy := os.Getenv("ALL_PROXY"); allProxy != "" {
		fmt.Printf("Using ALL_PROXY: %s\n", allProxy)
	}
}