deep_research/Cargo.toml

@@ -9,7 +9,7 @@ chrono = "0.4.45"
 clap = { version = "4", features = ["derive"] }
 futures = { workspace = true }
 indicatif = "0.18.6"
-reqwest = { workspace = true, features = ["query"] }
+reqwest = { workspace = true, features = ["query", "json"] }
 rig = { workspace = true }
 schemars = "1"
 scraper = "0.27"

deep_research/src/core.rs

@@ -117,7 +117,7 @@ fn partial_findings_from_history(chat_history: &[Message]) -> String {
     let sections: Vec<String> = chat_history
         .iter()
         .flat_map(|message| {
-            assistant_text(message)
+            extract_assistant_text(message)
                 .into_iter()
                 .chain(tool_result_text(message, MAX_TOOL_RESULT_CHARS))
         })
@@ -131,13 +131,13 @@ fn partial_findings_from_history(chat_history: &[Message]) -> String {
 }
 
 /// Plain-text blocks from an assistant message, if any.
-fn assistant_text(message: &Message) -> Vec<String> {
+fn extract_assistant_text(message: &Message) -> Vec<String> {
     let Message::Assistant { content, .. } = message else {
         return Vec::new();
     };
     content
         .iter()
-        .filter_map(|item| match item {
+        .filter_map(|item: &AssistantContent| match item {
             AssistantContent::Text(text) => Some(text.text().to_string()),
             _ => None,
         })
@@ -151,26 +151,30 @@ fn tool_result_text(message: &Message, max_chars: usize) -> Vec<String> {
     let Message::User { content } = message else {
         return Vec::new();
     };
+
     content
         .iter()
-        .filter_map(|item| match item {
+        .filter_map(|item: &UserContent| match item {
-            UserContent::ToolResult(result) => Some(result),
+            UserContent::ToolResult(tool_result) => Some(tool_result),
-            _ => None,
-        })
-        .flat_map(|result| result.content.iter())
-        .filter_map(|part| match part {
-            ToolResultContent::Text(text) => Some(truncate(text.text(), max_chars)),
             _ => None,
         })
+        .flat_map(|tool_result| tool_result.content.iter())
+        .filter_map(
+            |tool_result_content: &ToolResultContent| match tool_result_content {
+                ToolResultContent::Text(text) => Some(truncate(text.text(), max_chars)),
+                _ => None,
+            },
+        )
         .collect()
 }
 
 fn truncate(text: &str, max_chars: usize) -> String {
-    if text.len() <= max_chars {
+    let mut result = text.to_string();
-        text.to_string()
+    if result.len() > max_chars {
-    } else {
+        result.truncate(max_chars);
-        format!("{} [...truncated]", &text[..max_chars])
+        result.push_str(" ...[truncated]");
     }
+    result
 }
 
 /// Chronological transcript of a partial research run, annotated with tool
@@ -182,7 +186,7 @@ fn annotated_transcript_from_history(chat_history: &[Message]) -> String {
     chat_history
         .iter()
         .flat_map(transcript_lines)
-        .collect::<Vec<_>>()
+        .collect::<Vec<String>>()
         .join("\n")
 }
 
@@ -449,13 +453,16 @@ mod tests {
 
     #[test]
     fn truncate_cuts_long_text_and_marks_it() {
-        assert_eq!(truncate("hello world", 5), "hello [...truncated]");
+        assert_eq!(truncate("hello world", 5), "hello ...[truncated]");
     }
 
     #[test]
     fn assistant_text_extracts_text_blocks() {
         let message = Message::assistant("found it");
-        assert_eq!(assistant_text(&message), vec!["found it".to_string()]);
+        assert_eq!(
+            extract_assistant_text(&message),
+            vec!["found it".to_string()]
+        );
     }
 
     #[test]
@@ -468,13 +475,13 @@ mod tests {
                 serde_json::json!({ "query": "test" }),
             )),
         };
-        assert!(assistant_text(&message).is_empty());
+        assert!(extract_assistant_text(&message).is_empty());
     }
 
     #[test]
     fn assistant_text_ignores_non_assistant_messages() {
-        assert!(assistant_text(&Message::user("hi")).is_empty());
+        assert!(extract_assistant_text(&Message::user("hi")).is_empty());
-        assert!(assistant_text(&Message::system("be careful")).is_empty());
+        assert!(extract_assistant_text(&Message::system("be careful")).is_empty());
     }
 
     #[test]
@@ -488,7 +495,7 @@ mod tests {
         let long = Message::tool_result("call-2", "0123456789");
         assert_eq!(
             tool_result_text(&long, 5),
-            vec!["01234 [...truncated]".to_string()]
+            vec!["01234 ...[truncated]".to_string()]
         );
     }
 

deep_research/src/tools.rs

@@ -1,13 +1,39 @@
 use crate::progress::{self, FETCH_EMOJI, SEARCH_EMOJI};
 use rig::tool::ToolExecutionError;
 use scraper::{Html, Selector};
+use serde::Deserialize;
 
 const MAX_SEARCH_RESULTS: usize = 6;
 const MAX_PAGE_CHARS: usize = 6000;
 
-/// Searches the web via DuckDuckGo's HTML endpoint (no API key required) and
+/// Local-only tool: scraping DuckDuckGo directly shares rate-limit fate with
-/// returns each hit's title, URL, and snippet so the caller can decide which
+/// every other bot hitting it from this IP, and a rate-limited response
-/// pages are worth fetching in full.
+/// looks identical to a genuine "no results" — which is exactly what took
+/// down a research run over a dozen turns without ever surfacing as an
+/// error. A self-hosted SearXNG instance has its own JSON API (so no HTML
+/// scraping) and spreads queries across multiple upstream engines instead
+/// of hammering one. This is deliberately not configurable beyond the env
+/// var below — this tool is never meant to run anywhere but this machine.
+fn searxng_base_url() -> String {
+    std::env::var("SEARXNG_URL").unwrap_or_else(|_| "http://localhost:8080".to_string())
+}
+
+#[derive(Deserialize)]
+struct SearxngResponse {
+    results: Vec<SearxngResult>,
+}
+
+#[derive(Deserialize)]
+struct SearxngResult {
+    title: String,
+    url: String,
+    #[serde(default)]
+    content: String,
+}
+
+/// Searches the web via a local SearXNG instance's JSON API and returns each
+/// hit's title, URL, and snippet so the caller can decide which pages are
+/// worth fetching in full.
 #[rig::tool_macro(
     description = "Search the web for pages related to a query",
     required(query)
@@ -19,27 +45,27 @@ pub(crate) async fn search_web(
     progress::set_activity(format!("{SEARCH_EMOJI} Searching: {query}"));
 
     let response = reqwest::Client::new()
-        .get("https://html.duckduckgo.com/html/")
+        .get(format!("{}/search", searxng_base_url()))
-        .query(&[("q", query.as_str())])
+        .query(&[("q", query.as_str()), ("format", "json")])
-        .header("User-Agent", "Mozilla/5.0 (research-agent)")
         .send()
         .await
         .map_err(ToolExecutionError::from_error)?;
 
-    let body = response
+    let parsed: SearxngResponse = response
-        .text()
+        .json()
         .await
         .map_err(ToolExecutionError::from_error)?;
-    let results = parse_search_results(&body);
 
-    if results.is_empty() {
+    if parsed.results.is_empty() {
         return Ok("No results found.".to_string());
     }
 
-    Ok(results
+    Ok(parsed
+        .results
         .into_iter()
+        .take(MAX_SEARCH_RESULTS)
         .enumerate()
-        .map(|(i, r)| format!("{}. {}\n   {}\n   {}", i + 1, r.title, r.url, r.snippet))
+        .map(|(i, r)| format!("{}. {}\n   {}\n   {}", i + 1, r.title, r.url, r.content))
         .collect::<Vec<_>>()
         .join("\n\n"))
 }
@@ -71,67 +97,6 @@ pub(crate) async fn fetch_page(
     Ok(extract_readable_text(&body))
 }
 
-struct SearchResult {
-    title: String,
-    url: String,
-    snippet: String,
-}
-
-/// DuckDuckGo's HTML results page wraps each hit in a `.result` block; the
-/// title/link lives in `.result__a` and links are redirected through
-/// `duckduckgo.com/l/?uddg=<real-url>`, so the real URL has to be pulled back
-/// out of that query parameter rather than used as-is.
-fn parse_search_results(body: &str) -> Vec<SearchResult> {
-    let document = Html::parse_document(body);
-    let result_selector = Selector::parse(".result").expect("valid selector");
-    let title_selector = Selector::parse(".result__a").expect("valid selector");
-    let snippet_selector = Selector::parse(".result__snippet").expect("valid selector");
-
-    document
-        .select(&result_selector)
-        .filter_map(|result| {
-            let title_el = result.select(&title_selector).next()?;
-            let href = title_el.value().attr("href")?;
-            let url = resolve_ddg_redirect(href);
-            let title = title_el.text().collect::<String>().trim().to_string();
-            let snippet = result
-                .select(&snippet_selector)
-                .next()
-                .map(|el| el.text().collect::<String>().trim().to_string())
-                .unwrap_or_default();
-
-            if title.is_empty() || url.is_empty() {
-                None
-            } else {
-                Some(SearchResult {
-                    title,
-                    url,
-                    snippet,
-                })
-            }
-        })
-        .take(MAX_SEARCH_RESULTS)
-        .collect()
-}
-
-fn resolve_ddg_redirect(href: &str) -> String {
-    let full = if href.starts_with("//") {
-        format!("https:{href}")
-    } else {
-        href.to_string()
-    };
-
-    reqwest::Url::parse(&full)
-        .ok()
-        .and_then(|parsed| {
-            parsed
-                .query_pairs()
-                .find(|(k, _)| k == "uddg")
-                .map(|(_, v)| v.into_owned())
-        })
-        .unwrap_or(full)
-}
-
 fn extract_readable_text(html: &str) -> String {
     let document = Html::parse_document(html);
     let content_selector =
@@ -150,3 +115,23 @@ fn extract_readable_text(html: &str) -> String {
     let collapsed = text.split_whitespace().collect::<Vec<_>>().join(" ");
     collapsed.chars().take(MAX_PAGE_CHARS).collect()
 }
+
+#[cfg(test)]
+mod live_smoke_test {
+    use super::*;
+
+    /// Not run by default — this tool is local-only by design, so there's no
+    /// CI environment where a SearXNG instance would exist to test against.
+    /// Run manually with `cargo test -- --ignored` when SEARXNG_URL (or the
+    /// localhost:8080 default) points at a running instance.
+    #[tokio::test]
+    #[ignore = "hits a real local SearXNG instance; run manually with --ignored"]
+    async fn search_web_returns_real_results_from_local_searxng() {
+        let output = search_web("uruguay senior software engineer hiring 2026".to_string())
+            .await
+            .expect("search_web should succeed against a live local SearXNG instance");
+
+        println!("{output}");
+        assert_ne!(output, "No results found.");
+    }
+}