From c150c67f1e505df2b695b5afba9d6df38994def7 Mon Sep 17 00:00:00 2001 From: Austin Schaefer Date: Tue, 18 Aug 2026 12:25:29 +0200 Subject: [PATCH] Switch search_web from DuckDuckGo HTML scraping to local SearXNG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DuckDuckGo's HTML endpoint rate-limits after enough requests, and a rate-limited response is indistinguishable from a genuine empty result — which is exactly what burned a full 12-turn research run on 13 consecutive "No results found" responses. Swapping to a local SearXNG instance's JSON API (no HTML scraping needed) fixes both problems: SearXNG spreads queries across multiple upstream engines instead of hammering one, and this machine already runs an instance. This tool is explicitly local-only and never released, so the base URL is a plain default (localhost:8080) overridable via SEARXNG_URL, not a general-purpose config surface. Evaluated the two third-party SearXNG crates on crates.io first (searxng, searxng-client) — both are single-maintainer v0.1.0 packages with no adoption signal and no official alternative exists, so a hand-rolled reqwest + serde call was the better bet for something this small. Drops the DuckDuckGo-specific HTML parsing (parse_search_results, resolve_ddg_redirect, the .result/.result__a/.result__snippet scraper selectors) entirely — fetch_page's extract_readable_text still needs scraper for arbitrary fetched pages, so that dependency stays. Adds an #[ignore]'d live smoke test (search_web_returns_real_results_from_local_searxng) for manually verifying against a running instance; not run by default since there's no CI environment with SearXNG available. --- deep_research/Cargo.toml | 2 +- deep_research/src/core.rs | 49 ++++++++------ deep_research/src/tools.rs | 131 ++++++++++++++++--------------------- 3 files changed, 87 insertions(+), 95 deletions(-) diff --git a/deep_research/Cargo.toml b/deep_research/Cargo.toml index 179c84c..18f0f38 100644 --- a/deep_research/Cargo.toml +++ b/deep_research/Cargo.toml @@ -9,7 +9,7 @@ chrono = "0.4.45" clap = { version = "4", features = ["derive"] } futures = { workspace = true } indicatif = "0.18.6" -reqwest = { workspace = true, features = ["query"] } +reqwest = { workspace = true, features = ["query", "json"] } rig = { workspace = true } schemars = "1" scraper = "0.27" diff --git a/deep_research/src/core.rs b/deep_research/src/core.rs index 1a31483..3c14a91 100644 --- a/deep_research/src/core.rs +++ b/deep_research/src/core.rs @@ -117,7 +117,7 @@ fn partial_findings_from_history(chat_history: &[Message]) -> String { let sections: Vec = chat_history .iter() .flat_map(|message| { - assistant_text(message) + extract_assistant_text(message) .into_iter() .chain(tool_result_text(message, MAX_TOOL_RESULT_CHARS)) }) @@ -131,13 +131,13 @@ fn partial_findings_from_history(chat_history: &[Message]) -> String { } /// Plain-text blocks from an assistant message, if any. -fn assistant_text(message: &Message) -> Vec { +fn extract_assistant_text(message: &Message) -> Vec { let Message::Assistant { content, .. } = message else { return Vec::new(); }; content .iter() - .filter_map(|item| match item { + .filter_map(|item: &AssistantContent| match item { AssistantContent::Text(text) => Some(text.text().to_string()), _ => None, }) @@ -151,26 +151,30 @@ fn tool_result_text(message: &Message, max_chars: usize) -> Vec { let Message::User { content } = message else { return Vec::new(); }; + content .iter() - .filter_map(|item| match item { - UserContent::ToolResult(result) => Some(result), - _ => None, - }) - .flat_map(|result| result.content.iter()) - .filter_map(|part| match part { - ToolResultContent::Text(text) => Some(truncate(text.text(), max_chars)), + .filter_map(|item: &UserContent| match item { + UserContent::ToolResult(tool_result) => Some(tool_result), _ => None, }) + .flat_map(|tool_result| tool_result.content.iter()) + .filter_map( + |tool_result_content: &ToolResultContent| match tool_result_content { + ToolResultContent::Text(text) => Some(truncate(text.text(), max_chars)), + _ => None, + }, + ) .collect() } fn truncate(text: &str, max_chars: usize) -> String { - if text.len() <= max_chars { - text.to_string() - } else { - format!("{} [...truncated]", &text[..max_chars]) + let mut result = text.to_string(); + if result.len() > max_chars { + result.truncate(max_chars); + result.push_str(" ...[truncated]"); } + result } /// Chronological transcript of a partial research run, annotated with tool @@ -182,7 +186,7 @@ fn annotated_transcript_from_history(chat_history: &[Message]) -> String { chat_history .iter() .flat_map(transcript_lines) - .collect::>() + .collect::>() .join("\n") } @@ -449,13 +453,16 @@ mod tests { #[test] fn truncate_cuts_long_text_and_marks_it() { - assert_eq!(truncate("hello world", 5), "hello [...truncated]"); + assert_eq!(truncate("hello world", 5), "hello ...[truncated]"); } #[test] fn assistant_text_extracts_text_blocks() { let message = Message::assistant("found it"); - assert_eq!(assistant_text(&message), vec!["found it".to_string()]); + assert_eq!( + extract_assistant_text(&message), + vec!["found it".to_string()] + ); } #[test] @@ -468,13 +475,13 @@ mod tests { serde_json::json!({ "query": "test" }), )), }; - assert!(assistant_text(&message).is_empty()); + assert!(extract_assistant_text(&message).is_empty()); } #[test] fn assistant_text_ignores_non_assistant_messages() { - assert!(assistant_text(&Message::user("hi")).is_empty()); - assert!(assistant_text(&Message::system("be careful")).is_empty()); + assert!(extract_assistant_text(&Message::user("hi")).is_empty()); + assert!(extract_assistant_text(&Message::system("be careful")).is_empty()); } #[test] @@ -488,7 +495,7 @@ mod tests { let long = Message::tool_result("call-2", "0123456789"); assert_eq!( tool_result_text(&long, 5), - vec!["01234 [...truncated]".to_string()] + vec!["01234 ...[truncated]".to_string()] ); } diff --git a/deep_research/src/tools.rs b/deep_research/src/tools.rs index 5a8d8c0..51237c2 100644 --- a/deep_research/src/tools.rs +++ b/deep_research/src/tools.rs @@ -1,13 +1,39 @@ use crate::progress::{self, FETCH_EMOJI, SEARCH_EMOJI}; use rig::tool::ToolExecutionError; use scraper::{Html, Selector}; +use serde::Deserialize; const MAX_SEARCH_RESULTS: usize = 6; const MAX_PAGE_CHARS: usize = 6000; -/// Searches the web via DuckDuckGo's HTML endpoint (no API key required) and -/// returns each hit's title, URL, and snippet so the caller can decide which -/// pages are worth fetching in full. +/// Local-only tool: scraping DuckDuckGo directly shares rate-limit fate with +/// every other bot hitting it from this IP, and a rate-limited response +/// looks identical to a genuine "no results" — which is exactly what took +/// down a research run over a dozen turns without ever surfacing as an +/// error. A self-hosted SearXNG instance has its own JSON API (so no HTML +/// scraping) and spreads queries across multiple upstream engines instead +/// of hammering one. This is deliberately not configurable beyond the env +/// var below — this tool is never meant to run anywhere but this machine. +fn searxng_base_url() -> String { + std::env::var("SEARXNG_URL").unwrap_or_else(|_| "http://localhost:8080".to_string()) +} + +#[derive(Deserialize)] +struct SearxngResponse { + results: Vec, +} + +#[derive(Deserialize)] +struct SearxngResult { + title: String, + url: String, + #[serde(default)] + content: String, +} + +/// Searches the web via a local SearXNG instance's JSON API and returns each +/// hit's title, URL, and snippet so the caller can decide which pages are +/// worth fetching in full. #[rig::tool_macro( description = "Search the web for pages related to a query", required(query) @@ -19,27 +45,27 @@ pub(crate) async fn search_web( progress::set_activity(format!("{SEARCH_EMOJI} Searching: {query}")); let response = reqwest::Client::new() - .get("https://html.duckduckgo.com/html/") - .query(&[("q", query.as_str())]) - .header("User-Agent", "Mozilla/5.0 (research-agent)") + .get(format!("{}/search", searxng_base_url())) + .query(&[("q", query.as_str()), ("format", "json")]) .send() .await .map_err(ToolExecutionError::from_error)?; - let body = response - .text() + let parsed: SearxngResponse = response + .json() .await .map_err(ToolExecutionError::from_error)?; - let results = parse_search_results(&body); - if results.is_empty() { + if parsed.results.is_empty() { return Ok("No results found.".to_string()); } - Ok(results + Ok(parsed + .results .into_iter() + .take(MAX_SEARCH_RESULTS) .enumerate() - .map(|(i, r)| format!("{}. {}\n {}\n {}", i + 1, r.title, r.url, r.snippet)) + .map(|(i, r)| format!("{}. {}\n {}\n {}", i + 1, r.title, r.url, r.content)) .collect::>() .join("\n\n")) } @@ -71,67 +97,6 @@ pub(crate) async fn fetch_page( Ok(extract_readable_text(&body)) } -struct SearchResult { - title: String, - url: String, - snippet: String, -} - -/// DuckDuckGo's HTML results page wraps each hit in a `.result` block; the -/// title/link lives in `.result__a` and links are redirected through -/// `duckduckgo.com/l/?uddg=`, so the real URL has to be pulled back -/// out of that query parameter rather than used as-is. -fn parse_search_results(body: &str) -> Vec { - let document = Html::parse_document(body); - let result_selector = Selector::parse(".result").expect("valid selector"); - let title_selector = Selector::parse(".result__a").expect("valid selector"); - let snippet_selector = Selector::parse(".result__snippet").expect("valid selector"); - - document - .select(&result_selector) - .filter_map(|result| { - let title_el = result.select(&title_selector).next()?; - let href = title_el.value().attr("href")?; - let url = resolve_ddg_redirect(href); - let title = title_el.text().collect::().trim().to_string(); - let snippet = result - .select(&snippet_selector) - .next() - .map(|el| el.text().collect::().trim().to_string()) - .unwrap_or_default(); - - if title.is_empty() || url.is_empty() { - None - } else { - Some(SearchResult { - title, - url, - snippet, - }) - } - }) - .take(MAX_SEARCH_RESULTS) - .collect() -} - -fn resolve_ddg_redirect(href: &str) -> String { - let full = if href.starts_with("//") { - format!("https:{href}") - } else { - href.to_string() - }; - - reqwest::Url::parse(&full) - .ok() - .and_then(|parsed| { - parsed - .query_pairs() - .find(|(k, _)| k == "uddg") - .map(|(_, v)| v.into_owned()) - }) - .unwrap_or(full) -} - fn extract_readable_text(html: &str) -> String { let document = Html::parse_document(html); let content_selector = @@ -150,3 +115,23 @@ fn extract_readable_text(html: &str) -> String { let collapsed = text.split_whitespace().collect::>().join(" "); collapsed.chars().take(MAX_PAGE_CHARS).collect() } + +#[cfg(test)] +mod live_smoke_test { + use super::*; + + /// Not run by default — this tool is local-only by design, so there's no + /// CI environment where a SearXNG instance would exist to test against. + /// Run manually with `cargo test -- --ignored` when SEARXNG_URL (or the + /// localhost:8080 default) points at a running instance. + #[tokio::test] + #[ignore = "hits a real local SearXNG instance; run manually with --ignored"] + async fn search_web_returns_real_results_from_local_searxng() { + let output = search_web("uruguay senior software engineer hiring 2026".to_string()) + .await + .expect("search_web should succeed against a live local SearXNG instance"); + + println!("{output}"); + assert_ne!(output, "No results found."); + } +}