use crate::progress::{self, FETCH_EMOJI, SEARCH_EMOJI}; use rig::tool::ToolExecutionError; use scraper::{Html, Selector}; use serde::Deserialize; const MAX_SEARCH_RESULTS: usize = 6; const MAX_PAGE_CHARS: usize = 6000; /// Local-only tool: scraping DuckDuckGo directly shares rate-limit fate with /// every other bot hitting it from this IP, and a rate-limited response /// looks identical to a genuine "no results" — which is exactly what took /// down a research run over a dozen turns without ever surfacing as an /// error. A self-hosted SearXNG instance has its own JSON API (so no HTML /// scraping) and spreads queries across multiple upstream engines instead /// of hammering one. This is deliberately not configurable beyond the env /// var below — this tool is never meant to run anywhere but this machine. fn searxng_base_url() -> String { std::env::var("SEARXNG_URL").unwrap_or_else(|_| "http://localhost:8080".to_string()) } #[derive(Deserialize)] struct SearxngResponse { results: Vec, } #[derive(Deserialize)] struct SearxngResult { title: String, url: String, #[serde(default)] content: String, } /// Searches the web via a local SearXNG instance's JSON API and returns each /// hit's title, URL, and snippet so the caller can decide which pages are /// worth fetching in full. #[rig::tool_macro( description = "Search the web for pages related to a query", required(query) )] pub(crate) async fn search_web( /// The search query query: String, ) -> Result { progress::set_activity(format!("{SEARCH_EMOJI} Searching: {query}")); let response = reqwest::Client::new() .get(format!("{}/search", searxng_base_url())) .query(&[("q", query.as_str()), ("format", "json")]) .send() .await .map_err(ToolExecutionError::from_error)?; let parsed: SearxngResponse = response .json() .await .map_err(ToolExecutionError::from_error)?; if parsed.results.is_empty() { return Ok("No results found.".to_string()); } Ok(parsed .results .into_iter() .take(MAX_SEARCH_RESULTS) .enumerate() .map(|(i, r)| format!("{}. {}\n {}\n {}", i + 1, r.title, r.url, r.content)) .collect::>() .join("\n\n")) } /// Fetches a page and returns its main text content, stripped of markup and /// truncated so a single fetch can't blow out the model's context window. #[rig::tool_macro( description = "Fetch a web page and return its readable text content", required(url) )] pub(crate) async fn fetch_page( /// The URL to fetch url: String, ) -> Result { progress::set_activity(format!("{FETCH_EMOJI} Fetching: {url}")); let response = reqwest::Client::new() .get(&url) .header("User-Agent", "Mozilla/5.0 (research-agent)") .send() .await .map_err(ToolExecutionError::from_error)?; let body = response .text() .await .map_err(ToolExecutionError::from_error)?; Ok(extract_readable_text(&body)) } fn extract_readable_text(html: &str) -> String { let document = Html::parse_document(html); let content_selector = Selector::parse("p, h1, h2, h3, h4, h5, li, td").expect("valid selector"); let mut text: String = document .select(&content_selector) .map(|el| el.text().collect::>().join(" ")) .collect::>() .join("\n"); if text.trim().is_empty() { text = document.root_element().text().collect::>().join(" "); } let collapsed = text.split_whitespace().collect::>().join(" "); collapsed.chars().take(MAX_PAGE_CHARS).collect() } #[cfg(test)] mod live_smoke_test { use super::*; /// Not run by default — this tool is local-only by design, so there's no /// CI environment where a SearXNG instance would exist to test against. /// Run manually with `cargo test -- --ignored` when SEARXNG_URL (or the /// localhost:8080 default) points at a running instance. #[tokio::test] #[ignore = "hits a real local SearXNG instance; run manually with --ignored"] async fn search_web_returns_real_results_from_local_searxng() { let output = search_web("uruguay senior software engineer hiring 2026".to_string()) .await .expect("search_web should succeed against a live local SearXNG instance"); println!("{output}"); assert_ne!(output, "No results found."); } }