c150c67f1e
DuckDuckGo's HTML endpoint rate-limits after enough requests, and a rate-limited response is indistinguishable from a genuine empty result — which is exactly what burned a full 12-turn research run on 13 consecutive "No results found" responses. Swapping to a local SearXNG instance's JSON API (no HTML scraping needed) fixes both problems: SearXNG spreads queries across multiple upstream engines instead of hammering one, and this machine already runs an instance. This tool is explicitly local-only and never released, so the base URL is a plain default (localhost:8080) overridable via SEARXNG_URL, not a general-purpose config surface. Evaluated the two third-party SearXNG crates on crates.io first (searxng, searxng-client) — both are single-maintainer v0.1.0 packages with no adoption signal and no official alternative exists, so a hand-rolled reqwest + serde call was the better bet for something this small. Drops the DuckDuckGo-specific HTML parsing (parse_search_results, resolve_ddg_redirect, the .result/.result__a/.result__snippet scraper selectors) entirely — fetch_page's extract_readable_text still needs scraper for arbitrary fetched pages, so that dependency stays. Adds an #[ignore]'d live smoke test (search_web_returns_real_results_from_local_searxng) for manually verifying against a running instance; not run by default since there's no CI environment with SearXNG available.
22 lines
520 B
TOML
22 lines
520 B
TOML
[package]
|
|
name = "deep_research"
|
|
version = "0.1.0"
|
|
edition = "2024"
|
|
|
|
[dependencies]
|
|
anyhow = { workspace = true }
|
|
chrono = "0.4.45"
|
|
clap = { version = "4", features = ["derive"] }
|
|
futures = { workspace = true }
|
|
indicatif = "0.18.6"
|
|
reqwest = { workspace = true, features = ["query", "json"] }
|
|
rig = { workspace = true }
|
|
schemars = "1"
|
|
scraper = "0.27"
|
|
serde = { workspace = true }
|
|
tokio = { workspace = true }
|
|
tracing = { workspace = true }
|
|
tracing-subscriber = { workspace = true }
|
|
|
|
[dev-dependencies]
|
|
serde_json = "1"
|