schaefera/doubleo7
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Switch search_web from DuckDuckGo HTML scraping to local SearXNG

Browse source 
DuckDuckGo's HTML endpoint rate-limits after enough requests, and a
rate-limited response is indistinguishable from a genuine empty result —
which is exactly what burned a full 12-turn research run on 13 consecutive
"No results found" responses. Swapping to a local SearXNG instance's JSON
API (no HTML scraping needed) fixes both problems: SearXNG spreads queries
across multiple upstream engines instead of hammering one, and this
machine already runs an instance.

This tool is explicitly local-only and never released, so the base URL is
a plain default (localhost:8080) overridable via SEARXNG_URL, not a
general-purpose config surface. Evaluated the two third-party SearXNG
crates on crates.io first (searxng, searxng-client) — both are
single-maintainer v0.1.0 packages with no adoption signal and no official
alternative exists, so a hand-rolled reqwest + serde call was the better
bet for something this small.

Drops the DuckDuckGo-specific HTML parsing (parse_search_results,
resolve_ddg_redirect, the .result/.result__a/.result__snippet scraper
selectors) entirely — fetch_page's extract_readable_text still needs
scraper for arbitrary fetched pages, so that dependency stays.

Adds an #[ignore]'d live smoke test (search_web_returns_real_results_from_local_searxng)
for manually verifying against a running instance; not run by default
since there's no CI environment with SearXNG available.
This commit is contained in:
Austin Schaefer 2026-08-18 12:25:29 +02:00
parent 21030462b1
commit c150c67f1e
3 changed files with 87 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
deep_research/Cargo.toml
View file

@ -9,7 +9,7 @@ chrono = "0.4.45"
clap = { version = "4", features = ["derive"] }
futures = { workspace = true }
indicatif = "0.18.6"
reqwest = { workspace = true, features = ["query"] }
reqwest = { workspace = true, features = ["query", "json"] }
rig = { workspace = true }
schemars = "1"
scraper = "0.27"

45
deep_research/src/core.rs
View file

@ -117,7 +117,7 @@ fn partial_findings_from_history(chat_history: &[Message]) -> String {
let sections: Vec<String> = chat_history
.iter()
.flat_map(|message| {
assistant_text(message)
extract_assistant_text(message)
.into_iter()
.chain(tool_result_text(message, MAX_TOOL_RESULT_CHARS))
})
@ -131,13 +131,13 @@ fn partial_findings_from_history(chat_history: &[Message]) -> String {
}
/// Plain-text blocks from an assistant message, if any.
fn assistant_text(message: &Message) -> Vec<String> {
fn extract_assistant_text(message: &Message) -> Vec<String> {
let Message::Assistant { content, .. } = message else {
return Vec::new();
};
content
.iter()
.filter_map(|item| match item {
.filter_map(|item: &AssistantContent| match item {
AssistantContent::Text(text) => Some(text.text().to_string()),
_ => None,
})
@ -151,26 +151,30 @@ fn tool_result_text(message: &Message, max_chars: usize) -> Vec<String> {
let Message::User { content } = message else {
return Vec::new();
};
content
.iter()
.filter_map(|item| match item {
UserContent::ToolResult(result) => Some(result),
.filter_map(|item: &UserContent| match item {
UserContent::ToolResult(tool_result) => Some(tool_result),
_ => None,
})
.flat_map(|result| result.content.iter())
.filter_map(|part| match part {
.flat_map(|tool_result| tool_result.content.iter())
.filter_map(
|tool_result_content: &ToolResultContent| match tool_result_content {
ToolResultContent::Text(text) => Some(truncate(text.text(), max_chars)),
_ => None,
})
},
)
.collect()
}
fn truncate(text: &str, max_chars: usize) -> String {
if text.len() <= max_chars {
text.to_string()
} else {
format!("{} [...truncated]", &text[..max_chars])
let mut result = text.to_string();
if result.len() > max_chars {
result.truncate(max_chars);
result.push_str(" ...[truncated]");
}
result
}
/// Chronological transcript of a partial research run, annotated with tool
@ -182,7 +186,7 @@ fn annotated_transcript_from_history(chat_history: &[Message]) -> String {
chat_history
.iter()
.flat_map(transcript_lines)
.collect::<Vec<_>>()
.collect::<Vec<String>>()
.join("\n")
}
@ -449,13 +453,16 @@ mod tests {
#[test]
fn truncate_cuts_long_text_and_marks_it() {
assert_eq!(truncate("hello world", 5), "hello [...truncated]");
assert_eq!(truncate("hello world", 5), "hello ...[truncated]");
}
#[test]
fn assistant_text_extracts_text_blocks() {
let message = Message::assistant("found it");
assert_eq!(assistant_text(&message), vec!["found it".to_string()]);
assert_eq!(
extract_assistant_text(&message),
vec!["found it".to_string()]
);
}
#[test]
@ -468,13 +475,13 @@ mod tests {
serde_json::json!({ "query": "test" }),
)),
};
assert!(assistant_text(&message).is_empty());
assert!(extract_assistant_text(&message).is_empty());
}
#[test]
fn assistant_text_ignores_non_assistant_messages() {
assert!(assistant_text(&Message::user("hi")).is_empty());
assert!(assistant_text(&Message::system("be careful")).is_empty());
assert!(extract_assistant_text(&Message::user("hi")).is_empty());
assert!(extract_assistant_text(&Message::system("be careful")).is_empty());
}
#[test]
@ -488,7 +495,7 @@ mod tests {
let long = Message::tool_result("call-2", "0123456789");
assert_eq!(
tool_result_text(&long, 5),
vec!["01234 [...truncated]".to_string()]
vec!["01234 ...[truncated]".to_string()]
);
}

131
deep_research/src/tools.rs
View file

@ -1,13 +1,39 @@
use crate::progress::{self, FETCH_EMOJI, SEARCH_EMOJI};
use rig::tool::ToolExecutionError;
use scraper::{Html, Selector};
use serde::Deserialize;
const MAX_SEARCH_RESULTS: usize = 6;
const MAX_PAGE_CHARS: usize = 6000;
/// Searches the web via DuckDuckGo's HTML endpoint (no API key required) and
/// returns each hit's title, URL, and snippet so the caller can decide which
/// pages are worth fetching in full.
/// Local-only tool: scraping DuckDuckGo directly shares rate-limit fate with
/// every other bot hitting it from this IP, and a rate-limited response
/// looks identical to a genuine "no results" — which is exactly what took
/// down a research run over a dozen turns without ever surfacing as an
/// error. A self-hosted SearXNG instance has its own JSON API (so no HTML
/// scraping) and spreads queries across multiple upstream engines instead
/// of hammering one. This is deliberately not configurable beyond the env
/// var below — this tool is never meant to run anywhere but this machine.
fn searxng_base_url() -> String {
std::env::var("SEARXNG_URL").unwrap_or_else(|_| "http://localhost:8080".to_string())
}
#[derive(Deserialize)]
struct SearxngResponse {
results: Vec<SearxngResult>,
}
#[derive(Deserialize)]
struct SearxngResult {
title: String,
url: String,
#[serde(default)]
content: String,
}
/// Searches the web via a local SearXNG instance's JSON API and returns each
/// hit's title, URL, and snippet so the caller can decide which pages are
/// worth fetching in full.
#[rig::tool_macro(
description = "Search the web for pages related to a query",
required(query)
@ -19,27 +45,27 @@ pub(crate) async fn search_web(
progress::set_activity(format!("{SEARCH_EMOJI} Searching: {query}"));
let response = reqwest::Client::new()
.get("https://html.duckduckgo.com/html/")
.query(&[("q", query.as_str())])
.header("User-Agent", "Mozilla/5.0 (research-agent)")
.get(format!("{}/search", searxng_base_url()))
.query(&[("q", query.as_str()), ("format", "json")])
.send()
.await
.map_err(ToolExecutionError::from_error)?;
let body = response
.text()
let parsed: SearxngResponse = response
.json()
.await
.map_err(ToolExecutionError::from_error)?;
let results = parse_search_results(&body);
if results.is_empty() {
if parsed.results.is_empty() {
return Ok("No results found.".to_string());
}
Ok(results
Ok(parsed
.results
.into_iter()
.take(MAX_SEARCH_RESULTS)
.enumerate()
.map(|(i, r)| format!("{}. {}\n {}\n {}", i + 1, r.title, r.url, r.snippet))
.map(|(i, r)| format!("{}. {}\n {}\n {}", i + 1, r.title, r.url, r.content))
.collect::<Vec<_>>()
.join("\n\n"))
}
@ -71,67 +97,6 @@ pub(crate) async fn fetch_page(
Ok(extract_readable_text(&body))
}
struct SearchResult {
title: String,
url: String,
snippet: String,
}
/// DuckDuckGo's HTML results page wraps each hit in a `.result` block; the
/// title/link lives in `.result__a` and links are redirected through
/// `duckduckgo.com/l/?uddg=<real-url>`, so the real URL has to be pulled back
/// out of that query parameter rather than used as-is.
fn parse_search_results(body: &str) -> Vec<SearchResult> {
let document = Html::parse_document(body);
let result_selector = Selector::parse(".result").expect("valid selector");
let title_selector = Selector::parse(".result__a").expect("valid selector");
let snippet_selector = Selector::parse(".result__snippet").expect("valid selector");
document
.select(&result_selector)
.filter_map(|result| {
let title_el = result.select(&title_selector).next()?;
let href = title_el.value().attr("href")?;
let url = resolve_ddg_redirect(href);
let title = title_el.text().collect::<String>().trim().to_string();
let snippet = result
.select(&snippet_selector)
.next()
.map(|el| el.text().collect::<String>().trim().to_string())
.unwrap_or_default();
if title.is_empty() || url.is_empty() {
None
} else {
Some(SearchResult {
title,
url,
snippet,
})
}
})
.take(MAX_SEARCH_RESULTS)
.collect()
}
fn resolve_ddg_redirect(href: &str) -> String {
let full = if href.starts_with("//") {
format!("https:{href}")
} else {
href.to_string()
};
reqwest::Url::parse(&full)
.ok()
.and_then(|parsed| {
parsed
.query_pairs()
.find(|(k, _)| k == "uddg")
.map(|(_, v)| v.into_owned())
})
.unwrap_or(full)
}
fn extract_readable_text(html: &str) -> String {
let document = Html::parse_document(html);
let content_selector =
@ -150,3 +115,23 @@ fn extract_readable_text(html: &str) -> String {
let collapsed = text.split_whitespace().collect::<Vec<_>>().join(" ");
collapsed.chars().take(MAX_PAGE_CHARS).collect()
}
#[cfg(test)]
mod live_smoke_test {
use super::*;
/// Not run by default — this tool is local-only by design, so there's no
/// CI environment where a SearXNG instance would exist to test against.
/// Run manually with `cargo test -- --ignored` when SEARXNG_URL (or the
/// localhost:8080 default) points at a running instance.
#[tokio::test]
#[ignore = "hits a real local SearXNG instance; run manually with --ignored"]
async fn search_web_returns_real_results_from_local_searxng() {
let output = search_web("uruguay senior software engineer hiring 2026".to_string())
.await
.expect("search_web should succeed against a live local SearXNG instance");
println!("{output}");
assert_ne!(output, "No results found.");
}
}