package tools import ( "context" "fmt" "net/http" "net/url" "os" "strings" "time" "github.com/PuerkitoBio/goquery" ) const ( maxRetries = 3 timeout = 60 * time.Second ) // SearchResult represents a single search result type SearchResult struct { Title string `json:"title"` URL string `json:"url"` Content string `json:"content"` } // WebSearch performs a web search using DuckDuckGo func WebSearch(query string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() // Create HTTP client with proxy support transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, } client := &http.Client{ Timeout: timeout, Transport: transport, } // Perform search with retry logic results, err := searchDuckDuckGo(ctx, client, query) if err != nil { return "", fmt.Errorf("DuckDuckGo search failed: %w", err) } // Format results as text if len(results) == 0 { return "No results found.", nil } var output strings.Builder fmt.Fprintf(&output, "Found %d results:\n\n", len(results)) for i, result := range results { if i >= 10 { // Limit to top 10 results break } fmt.Fprintf(&output, "%d. %s\n", i+1, result.Title) fmt.Fprintf(&output, " URL: %s\n", result.URL) if result.Content != "" { fmt.Fprintf(&output, " %s\n", result.Content) } fmt.Fprintf(&output, "\n") } return output.String(), nil } // searchDuckDuckGo performs the actual DuckDuckGo search func searchDuckDuckGo(ctx context.Context, client *http.Client, query string) ([]SearchResult, error) { // Build DuckDuckGo search URL searchURL := fmt.Sprintf("https://html.duckduckgo.com/html/?q=%s", url.QueryEscape(query)) // Create request with proper headers req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil) if err != nil { return nil, fmt.Errorf("failed to create search request: %w", err) } // Set browser-like user agent req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") req.Header.Set("Accept-Language", "en-US,en;q=0.5") // Execute request with retry logic resp, err := executeWithRetry(ctx, client, req, "search DuckDuckGo") if err != nil { return nil, err } defer resp.Body.Close() // Check response status if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { return nil, fmt.Errorf("search request failed with status: %d %s", resp.StatusCode, resp.Status) } // Parse HTML response doc, err := goquery.NewDocumentFromReader(resp.Body) if err != nil { return nil, fmt.Errorf("failed to parse search results: %w", err) } // Extract search results var results []SearchResult doc.Find(".web-result").Each(func(i int, s *goquery.Selection) { titleNode := s.Find(".result__a") title := strings.TrimSpace(titleNode.Text()) info := strings.TrimSpace(s.Find(".result__snippet").Text()) // Extract URL var resultURL string if titleNode.Length() > 0 { if href, exists := titleNode.Attr("href"); exists { // Validate URL format if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") { resultURL = href } } } if title != "" && resultURL != "" { results = append(results, SearchResult{ Title: title, URL: resultURL, Content: info, }) } }) return results, nil } // executeWithRetry executes an HTTP request with exponential backoff retry logic func executeWithRetry(ctx context.Context, client *http.Client, req *http.Request, operation string) (*http.Response, error) { var resp *http.Response var err error // Retry logic with exponential backoff for attempt := 0; attempt <= maxRetries; attempt++ { resp, err = client.Do(req) if err == nil { return resp, nil } // Don't retry on the last attempt if attempt == maxRetries { break } // Exponential backoff: 1s, 2s, 4s backoffDuration := time.Duration(1<