doubleo7/src/tools.rs

use crate::progress::{self, FETCH_EMOJI, SEARCH_EMOJI};
use rig::tool::ToolExecutionError;
use scraper::{Html, Selector};
use serde::Deserialize;

const MAX_SEARCH_RESULTS: usize = 6;
const MAX_PAGE_CHARS: usize = 6000;

/// Local-only tool: scraping DuckDuckGo directly shares rate-limit fate with
/// every other bot hitting it from this IP, and a rate-limited response
/// looks identical to a genuine "no results" — which is exactly what took
/// down a research run over a dozen turns without ever surfacing as an
/// error. A self-hosted SearXNG instance has its own JSON API (so no HTML
/// scraping) and spreads queries across multiple upstream engines instead
/// of hammering one. This is deliberately not configurable beyond the env
/// var below — this tool is never meant to run anywhere but this machine.
fn searxng_base_url() -> String {
    std::env::var("SEARXNG_URL").unwrap_or_else(|_| "http://localhost:8080".to_string())
}

#[derive(Deserialize)]
struct SearxngResponse {
    results: Vec<SearxngResult>,
}

#[derive(Deserialize)]
struct SearxngResult {
    title: String,
    url: String,
    #[serde(default)]
    content: String,
}

/// Searches the web via a local SearXNG instance's JSON API and returns each
/// hit's title, URL, and snippet so the caller can decide which pages are
/// worth fetching in full.
#[rig::tool_macro(
    description = "Search the web for pages related to a query",
    required(query)
)]
pub(crate) async fn search_web(
    /// The search query
    query: String,
) -> Result<String, ToolExecutionError> {
    progress::set_activity(format!("{SEARCH_EMOJI} Searching: {query}"));

    let response = reqwest::Client::new()
        .get(format!("{}/search", searxng_base_url()))
        .query(&[("q", query.as_str()), ("format", "json")])
        .send()
        .await
        .map_err(ToolExecutionError::from_error)?;

    let parsed: SearxngResponse = response
        .json()
        .await
        .map_err(ToolExecutionError::from_error)?;

    if parsed.results.is_empty() {
        return Ok("No results found.".to_string());
    }

    Ok(parsed
        .results
        .into_iter()
        .take(MAX_SEARCH_RESULTS)
        .enumerate()
        .map(|(i, r)| format!("{}. {}\n   {}\n   {}", i + 1, r.title, r.url, r.content))
        .collect::<Vec<_>>()
        .join("\n\n"))
}

/// Fetches a page and returns its main text content, stripped of markup and
/// truncated so a single fetch can't blow out the model's context window.
#[rig::tool_macro(
    description = "Fetch a web page and return its readable text content",
    required(url)
)]
pub(crate) async fn fetch_page(
    /// The URL to fetch
    url: String,
) -> Result<String, ToolExecutionError> {
    progress::set_activity(format!("{FETCH_EMOJI} Fetching: {url}"));

    let response = reqwest::Client::new()
        .get(&url)
        .header("User-Agent", "Mozilla/5.0 (research-agent)")
        .send()
        .await
        .map_err(ToolExecutionError::from_error)?;

    let body = response
        .text()
        .await
        .map_err(ToolExecutionError::from_error)?;

    Ok(extract_readable_text(&body))
}

fn extract_readable_text(html: &str) -> String {
    let document = Html::parse_document(html);
    let content_selector =
        Selector::parse("p, h1, h2, h3, h4, h5, li, td").expect("valid selector");

    let mut text: String = document
        .select(&content_selector)
        .map(|el| el.text().collect::<Vec<_>>().join(" "))
        .collect::<Vec<_>>()
        .join("\n");

    if text.trim().is_empty() {
        text = document.root_element().text().collect::<Vec<_>>().join(" ");
    }

    let collapsed = text.split_whitespace().collect::<Vec<_>>().join(" ");
    collapsed.chars().take(MAX_PAGE_CHARS).collect()
}

#[cfg(test)]
mod live_smoke_test {
    use super::*;

    /// Not run by default — this tool is local-only by design, so there's no
    /// CI environment where a SearXNG instance would exist to test against.
    /// Run manually with `cargo test -- --ignored` when SEARXNG_URL (or the
    /// localhost:8080 default) points at a running instance.
    #[tokio::test]
    #[ignore = "hits a real local SearXNG instance; run manually with --ignored"]
    async fn search_web_returns_real_results_from_local_searxng() {
        let output = search_web("uruguay senior software engineer hiring 2026".to_string())
            .await
            .expect("search_web should succeed against a live local SearXNG instance");

        println!("{output}");
        assert_ne!(output, "No results found.");
    }
}