schaefera/doubleo7
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Write a partial report instead of erroring out when research hits max turns #8

Merged
schaefera merged 6 commits from worktree-deep-research-max-turns-report into master 2026-08-18 11:03:03 +00:00
Conversation 0 Commits 6 Files changed 16 +743 -280
16 changed files with 743 additions and 280 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock generated
View file

@ -1941,6 +1941,7 @@ dependencies = [
"schemars 1.2.2",
"scraper",
"serde",
"serde_json",
"tokio",
"tracing",
"tracing-subscriber",

5
deep_research/Cargo.toml
View file

@ -9,7 +9,7 @@ chrono = "0.4.45"
clap = { version = "4", features = ["derive"] }
futures = { workspace = true }
indicatif = "0.18.6"
reqwest = { workspace = true, features = ["query"] }
reqwest = { workspace = true, features = ["query", "json"] }
rig = { workspace = true }
schemars = "1"
scraper = "0.27"
@ -17,3 +17,6 @@ serde = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
[dev-dependencies]
serde_json = "1"

19
deep_research/src/cli.rs Normal file
View file

@ -0,0 +1,19 @@
use clap::Parser;
pub(crate) const DEFAULT_TOPIC: &str =
"What are the latest advances in running large language models locally, on consumer hardware?";
/// Deep research agentic loop over local Gemma models: a tool-calling agent
/// gathers and cross-checks web evidence, a reviewer agent gates it, and a
/// writer agent turns approved findings into a structured report.
#[derive(Parser)]
#[command(name = "doubleo7-research", version, about)]
pub(crate) struct Cli {
/// Research topic to investigate
pub(crate) topic: Option<String>,
/// Emit logs at this level (off by default; passing this also enables a
/// progress spinner to switch off, since the logs already show progress)
#[arg(short = 'l', long, value_name = "LEVEL")]
pub(crate) log_level: Option<tracing::Level>,
}

190
deep_research/src/core.rs
View file

@ -1,190 +0,0 @@
use crate::progress::{REJECTED_EMOJI, REPORT_EMOJI, RESEARCH_EMOJI, Spinner};
use crate::review::{self, Review};
use crate::stream::write_text_stream;
use crate::tools::{FetchPage, SearchWeb};
use clap::Parser;
use rig::client::{AgentClientExt, Nothing};
use rig::providers::ollama;
use rig::streaming::StreamingPrompt;
use std::io::Write;
/// The tool-calling research loop needs to reliably decide what to search
/// for, when a page is worth fetching, and when it has enough evidence —
/// that's a reasoning-heavy job best given to the largest local Gemma
/// variant. Turning the gathered notes into prose afterwards is comparatively
/// mechanical, so the smaller/faster variant handles that pass instead.
const RESEARCHER_MODEL: &str = "gemma4:26b";
const WRITER_MODEL: &str = "gemma4-e4b:latest";
const MAX_RESEARCH_TURNS: usize = 12;
/// Research/review rounds before giving up and writing the report from
/// whatever the last pass produced, rather than looping forever on a topic
/// the reviewer can never be satisfied with.
const MAX_RESEARCH_ROUNDS: usize = 3;
/// Deep research agentic loop over local Gemma models: a tool-calling agent
/// gathers and cross-checks web evidence, a reviewer agent gates it, and a
/// writer agent turns approved findings into a structured report.
#[derive(Parser)]
#[command(name = "doubleo7-research", version, about)]
pub(crate) struct Cli {
/// Research topic to investigate
pub(crate) topic: Option<String>,
/// Emit logs at this level (off by default; passing this also enables a
/// progress spinner to switch off, since the logs already show progress)
#[arg(short = 'l', long, value_name = "LEVEL")]
pub(crate) log_level: Option<tracing::Level>,
}
/// Only initializes a subscriber (and thus produces any log output at all)
/// when the caller opted in via `--log-level` — otherwise tracing's macros
/// are no-ops, leaving the terminal clean for the spinner.
pub(crate) fn initialize_observability(log_level: tracing::Level) {
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(log_level.to_string())),
)
.with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE)
.with_writer(std::io::stderr)
.init();
}
/// The least-agentic shape that fits: a plain Rust loop putting *this code*,
/// not a model, in charge of when to stop — re-running research with the
/// reviewer's feedback folded in until it approves or the round budget runs
/// out, then writing the report from whatever the last pass produced.
pub(crate) async fn research(topic: &str, show_progress: bool) -> anyhow::Result<String> {
let client = ollama::Client::new(Nothing)?;
let mut findings = String::new();
let mut feedback: Option<Review> = None;
for round in 1..=MAX_RESEARCH_ROUNDS {
findings = gather_findings(&client, topic, feedback.as_ref(), round, show_progress).await?;
let review = review::review_findings(&client, topic, &findings, show_progress).await?;
let approved = review.approved;
tracing::info!(round, approved, "review verdict");
if approved || round == MAX_RESEARCH_ROUNDS {
break;
}
if show_progress {
eprintln!("{REJECTED_EMOJI} Findings rejected — revising for round {}...", round + 1);
}
feedback = Some(review);
}
write_report(&client, topic, &findings, show_progress).await
}
/// Wraps the tool-calling research loop in its own span so it's visible as a
/// single unit in traces, distinct from the writing and review phases and
/// nesting rig's own per-turn `chat`/`execute_tool` spans underneath it.
#[tracing::instrument(skip(client, feedback), fields(gen_ai.agent.name = "researcher"))]
async fn gather_findings(
client: &ollama::Client,
topic: &str,
feedback: Option<&Review>,
round: usize,
show_progress: bool,
) -> anyhow::Result<String> {
let current_date = chrono::offset::Local::now().to_string();
let researcher = client
.agent(RESEARCHER_MODEL)
.name("researcher")
.preamble(format!(
"You are a meticulous research assistant. Use the search_web and fetch_page tools to \
investigate the user's topic: run several searches with varied phrasing, fetch the \
most promising pages, and cross-check claims across at least two sources before \
trusting them. Ensure sources are up to date: the current date is {current_date}. \
Once you are confident you have enough evidence, stop calling tools and reply with a \
plain-text dump of every fact you gathered, using footnote-style citations: write each \
fact followed by a bracketed number like [1], then at the end of your reply list a \
'Sources' section mapping each number to the exact URL it came from, one per line, e.g. \
'[1] https://example.com/page'. Reuse the same number when multiple facts come from the \
same URL — do not give one URL two different numbers. Also call out any open questions \
or contradictions between sources, citing the footnotes involved. This is raw research \
material for a writer, not a final report, so favor completeness over polish.")
.as_str(),
)
.tool(SearchWeb)
.tool(FetchPage)
.build();
let task = match feedback {
None => topic.to_string(),
Some(review) => format!(
"Topic: {topic}\n\n\
You already ran a research pass on this topic. A reviewer checked it against its \
cited sources and found it insufficient. Do more research to address the reviewer's \
feedback, then produce an updated findings dump: carry forward what's solid, and \
add, correct, or better-source whatever the gaps call for. Gaps include out-of-date,\
irrelevant, or clearly wrong information. The date is {current_date}.\n\n\
Solid findings from the last pass — keep and build on these:\n{}\n\n\
Gaps the reviewer found — conclusions not actually backed by their source, sources \
that don't line up with the conclusion drawn from them, or parts of the topic still \
uncovered:\n{}",
review.solid_findings, review.gaps
),
};
let spinner = Spinner::start(show_progress, format!("{RESEARCH_EMOJI} Researching..."));
let findings = researcher
.runner(task)
.max_turns(MAX_RESEARCH_TURNS)
.run()
.await?
.output;
drop(spinner);
tracing::info!(round, findings = %findings, "research phase complete");
Ok(findings)
}
#[tracing::instrument(skip(client, findings), fields(gen_ai.agent.name = "writer"))]
async fn write_report(
client: &ollama::Client,
topic: &str,
findings: &str,
show_progress: bool,
) -> anyhow::Result<String> {
let writer = client
.agent(WRITER_MODEL)
.name("writer")
.preamble(
"You turn raw research notes into a clear, well-organized report for the reader. The \
notes use footnote-style citations — a bracketed number like [1] after a fact, with a \
Sources section mapping numbers to URLs. Preserve this scheme in your report: keep the \
same [n] markers next to the claims they support (renumbering only if you drop unused \
sources), and end the report with a 'Sources' section listing every footnote number \
still in use next to its exact URL. Structure the body with headings, and call out any \
open questions or contradictions the research turned up. Do not invent facts beyond \
what the notes provide.",
)
.build();
let spinner = Spinner::start(show_progress, format!("{REPORT_EMOJI} Writing report..."));
let response_stream = writer
.stream_prompt(format!("Topic: {topic}\n\nResearch notes:\n{findings}"))
.await;
// Locked once for the whole stream rather than per chunk (as print!
// would do internally) — chunks arrive in a tight loop, so re-acquiring
// the lock on every one adds up. The spinner keeps running until the
// stream's first chunk arrives, so the terminal stays covered through
// the gap between sending the prompt and generation actually starting.
let stdout = std::io::stdout();
let mut handle = stdout.lock();
let report = write_text_stream(response_stream, &mut handle, spinner).await?;
writeln!(handle)?;
Ok(report)
}

292
deep_research/src/history.rs Normal file
View file

@ -0,0 +1,292 @@
use rig::completion::message::{ToolResultContent, UserContent};
use rig::completion::{AssistantContent, Message};
const MAX_TOOL_RESULT_CHARS: usize = 2000;
/// Best-effort reconstruction of research material from a chat history left
/// behind when the researcher hit its turn budget mid-investigation: whatever
/// prose the model wrote between tool calls, plus the raw text of every tool
/// result (search snippets, fetched pages), truncated so one huge page can't
/// crowd out everything else that was found.
pub(crate) fn partial_findings_from_history(chat_history: &[Message]) -> String {
let sections: Vec<String> = chat_history
.iter()
.flat_map(|message| {
extract_assistant_text(message)
.into_iter()
.chain(tool_result_text(message, MAX_TOOL_RESULT_CHARS))
})
.collect();
match sections.is_empty() {
true => "The researcher exhausted its turn budget before gathering any usable evidence."
.to_string(),
false => sections.join("\n"),
}
}
/// Plain-text blocks from an assistant message, if any.
fn extract_assistant_text(message: &Message) -> Vec<String> {
let Message::Assistant { content, .. } = message else {
return Vec::new();
};
content
.iter()
.filter_map(|item: &AssistantContent| match item {
AssistantContent::Text(text) => Some(text.text().to_string()),
_ => None,
})
.collect()
}
/// Plain-text tool results attached to a user message (that's where rig
/// places them), each truncated to `max_chars` so one huge fetched page
/// can't crowd out everything else that was found.
fn tool_result_text(message: &Message, max_chars: usize) -> Vec<String> {
let Message::User { content } = message else {
return Vec::new();
};
content
.iter()
.filter_map(|item: &UserContent| match item {
UserContent::ToolResult(tool_result) => Some(tool_result),
_ => None,
})
.flat_map(|tool_result| tool_result.content.iter())
.filter_map(
|tool_result_content: &ToolResultContent| match tool_result_content {
ToolResultContent::Text(text) => Some(truncate(text.text(), max_chars)),
_ => None,
},
)
.collect()
}
fn truncate(text: &str, max_chars: usize) -> String {
let mut result = text.to_string();
if result.len() > max_chars {
result.truncate(max_chars);
result.push_str(" ...[truncated]");
}
result
}
/// Chronological transcript of a partial research run, annotated with tool
/// calls (so a fetch's URL or a search's query stays attached to its
/// result) rather than just the bare result text `partial_findings_from_history`
/// collects — the summarizer agent needs that context to attribute facts to
/// the right source.
pub(crate) fn annotated_transcript_from_history(chat_history: &[Message]) -> String {
chat_history
.iter()
.flat_map(transcript_lines)
.collect::<Vec<String>>()
.join("\n")
}
fn transcript_lines(message: &Message) -> Vec<String> {
match message {
Message::Assistant { content, .. } => content
.iter()
.filter_map(|item| match item {
AssistantContent::Text(text) => Some(format!("Note: {}", text.text())),
AssistantContent::ToolCall(call) => Some(format!(
"Called {}({})",
call.function.name, call.function.arguments
)),
_ => None,
})
.collect(),
Message::User { content } => content
.iter()
.filter_map(|item| match item {
UserContent::ToolResult(result) => Some(result),
_ => None,
})
.flat_map(|result| result.content.iter())
.filter_map(|part| match part {
ToolResultContent::Text(text) => Some(format!(
"Result: {}",
truncate(text.text(), MAX_TOOL_RESULT_CHARS)
)),
_ => None,
})
.collect(),
Message::System { .. } => Vec::new(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn truncate_leaves_short_text_untouched() {
assert_eq!(truncate("hello", 10), "hello");
}
#[test]
fn truncate_leaves_exact_length_text_untouched() {
assert_eq!(truncate("hello", 5), "hello");
}
#[test]
fn truncate_cuts_long_text_and_marks_it() {
assert_eq!(truncate("hello world", 5), "hello ...[truncated]");
}
#[test]
fn assistant_text_extracts_text_blocks() {
let message = Message::assistant("found it");
assert_eq!(
extract_assistant_text(&message),
vec!["found it".to_string()]
);
}
#[test]
fn assistant_text_ignores_tool_calls() {
let message = Message::Assistant {
id: None,
content: rig::OneOrMany::one(AssistantContent::tool_call(
"call-1",
"search_web",
serde_json::json!({ "query": "test" }),
)),
};
assert!(extract_assistant_text(&message).is_empty());
}
#[test]
fn assistant_text_ignores_non_assistant_messages() {
assert!(extract_assistant_text(&Message::user("hi")).is_empty());
assert!(extract_assistant_text(&Message::system("be careful")).is_empty());
}
#[test]
fn tool_result_text_extracts_and_truncates() {
let short = Message::tool_result("call-1", "short result");
assert_eq!(
tool_result_text(&short, 100),
vec!["short result".to_string()]
);
let long = Message::tool_result("call-2", "0123456789");
assert_eq!(
tool_result_text(&long, 5),
vec!["01234 ...[truncated]".to_string()]
);
}
#[test]
fn tool_result_text_ignores_non_tool_result_content() {
assert!(tool_result_text(&Message::user("plain text, no tool result"), 100).is_empty());
assert!(tool_result_text(&Message::assistant("also ignored"), 100).is_empty());
}
#[test]
fn partial_findings_from_history_falls_back_when_nothing_usable() {
let history = vec![
Message::system("preamble"),
Message::Assistant {
id: None,
content: rig::OneOrMany::one(AssistantContent::tool_call(
"call-1",
"search_web",
serde_json::json!({ "query": "test" }),
)),
},
];
assert_eq!(
partial_findings_from_history(&history),
"The researcher exhausted its turn budget before gathering any usable evidence."
);
}
#[test]
fn partial_findings_from_history_collects_assistant_text_and_tool_results_in_order() {
let history = vec![
Message::assistant("Checking sources..."),
Message::tool_result("call-1", "Result A [1]"),
Message::assistant("Cross-checking..."),
Message::tool_result("call-2", "Result B [2]"),
];
assert_eq!(
partial_findings_from_history(&history),
"Checking sources...\nResult A [1]\nCross-checking...\nResult B [2]"
);
}
#[test]
fn transcript_lines_prefixes_assistant_notes() {
let message = Message::assistant("still checking this claim");
assert_eq!(
transcript_lines(&message),
vec!["Note: still checking this claim".to_string()]
);
}
#[test]
fn transcript_lines_renders_tool_calls_with_their_arguments() {
let message = Message::Assistant {
id: None,
content: rig::OneOrMany::one(AssistantContent::tool_call(
"call-1",
"search_web",
serde_json::json!({ "query": "uruguay senior engineers" }),
)),
};
assert_eq!(
transcript_lines(&message),
vec!["Called search_web({\"query\":\"uruguay senior engineers\"})".to_string()]
);
}
#[test]
fn transcript_lines_prefixes_and_truncates_tool_results() {
let short = Message::tool_result("call-1", "found via search [1]");
assert_eq!(
transcript_lines(&short),
vec!["Result: found via search [1]".to_string()]
);
}
#[test]
fn transcript_lines_ignores_system_messages() {
assert!(transcript_lines(&Message::system("be thorough")).is_empty());
}
#[test]
fn annotated_transcript_from_history_joins_calls_results_and_notes_in_order() {
let history = vec![
Message::Assistant {
id: None,
content: rig::OneOrMany::one(AssistantContent::tool_call(
"call-1",
"search_web",
serde_json::json!({ "query": "test" }),
)),
},
Message::tool_result("call-1", "1. Example\n https://example.com\n snippet"),
Message::assistant("that source looks solid"),
];
assert_eq!(
annotated_transcript_from_history(&history),
"Called search_web({\"query\":\"test\"})\n\
Result: 1. Example\n https://example.com\n snippet\n\
Note: that source looks solid"
);
}
#[test]
fn annotated_transcript_from_history_is_empty_with_no_usable_content() {
assert_eq!(
annotated_transcript_from_history(&[Message::system("preamble")]),
""
);
}
}

20
deep_research/src/main.rs
View file

@ -1,32 +1,36 @@
use clap::Parser;
mod cli;
mod history;
mod models;
mod observability;
mod progress;
mod research;
mod researcher;
mod review;
mod core;
mod stream;
mod summarizer;
mod tools;
pub(crate) const DEFAULT_TOPIC: &str =
"What are the latest advances in running large language models locally, on consumer hardware?";
mod writer;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let cli = core::Cli::parse();
let cli = cli::Cli::parse();
let show_progress = match cli.log_level {
Some(level) => {
core::initialize_observability(level);
observability::initialize_observability(level);
false
}
None => true,
};
let topic = cli.topic.unwrap_or_else(|| DEFAULT_TOPIC.to_string());
let topic = cli.topic.unwrap_or_else(|| cli::DEFAULT_TOPIC.to_string());
// The report streams to stdout as the writer generates it, so nothing
// left to print here — the return value only matters to callers that
// embed `research` rather than running it as this binary.
core::research(&topic, show_progress).await?;
research::research(&topic, show_progress).await?;
Ok(())
}

8
deep_research/src/models.rs Normal file
View file

@ -0,0 +1,8 @@
/// The tool-calling research loop needs to reliably decide what to search
/// for, when a page is worth fetching, and when it has enough evidence —
/// that's a reasoning-heavy job best given to the largest local Gemma
/// variant. Turning gathered notes into prose (writing, summarizing) is
/// comparatively mechanical, so the smaller/faster variant handles those
/// passes instead.
pub(crate) const RESEARCHER_MODEL: &str = "gemma4:26b";
pub(crate) const WRITER_MODEL: &str = "gemma4-e4b:latest";

13
deep_research/src/observability.rs Normal file
View file

@ -0,0 +1,13 @@
/// Only initializes a subscriber (and thus produces any log output at all)
/// when the caller opted in via `--log-level` — otherwise tracing's macros
/// are no-ops, leaving the terminal clean for the spinner.
pub(crate) fn initialize_observability(log_level: tracing::Level) {
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(log_level.to_string())),
)
.with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE)
.with_writer(std::io::stderr)
.init();
}

1
deep_research/src/progress.rs
View file

@ -10,6 +10,7 @@ pub(crate) const FETCH_EMOJI: &str = "📄";
pub(crate) const REVIEW_EMOJI: &str = "🧐";
pub(crate) const REJECTED_EMOJI: &str = "❌";
pub(crate) const REPORT_EMOJI: &str = "✍️";
pub(crate) const SUMMARIZE_EMOJI: &str = "🧩";
/// The spinner currently on screen, if any — set by `Spinner::start` and
/// cleared on drop. Tool implementations don't otherwise have a handle to

58
deep_research/src/research.rs Normal file
View file

@ -0,0 +1,58 @@
use crate::progress::REJECTED_EMOJI;
use crate::researcher::gather_findings;
use crate::review::{self, Review};
use crate::writer::write_report;
use rig::client::Nothing;
use rig::providers::ollama;
/// Research/review rounds before giving up and writing the report from
/// whatever the last pass produced, rather than looping forever on a topic
/// the reviewer can never be satisfied with.
const MAX_RESEARCH_ROUNDS: usize = 3;
/// The least-agentic shape that fits: a plain Rust loop putting *this code*,
/// not a model, in charge of when to stop — re-running research with the
/// reviewer's feedback folded in until it approves or the round budget runs
/// out, then writing the report from whatever the last pass produced.
pub(crate) async fn research(topic: &str, show_progress: bool) -> anyhow::Result<String> {
let client = ollama::Client::new(Nothing)?;
let mut findings = String::new();
let mut feedback: Option<Review> = None;
let mut incomplete = false;
for round in 1..=MAX_RESEARCH_ROUNDS {
let gathered =
gather_findings(&client, topic, feedback.as_ref(), round, show_progress).await?;
findings = gathered.findings;
incomplete = gathered.incomplete;
// The researcher ran out of turns mid-investigation rather than
// concluding on its own — another round would just repeat the same
// dead end, so stop and write up whatever was gathered.
if incomplete {
tracing::info!(round, "researcher exhausted its turn budget");
break;
}
let review = review::review_findings(&client, topic, &findings, show_progress).await?;
let approved = review.approved;
tracing::info!(round, approved, "review verdict");
if approved || round == MAX_RESEARCH_ROUNDS {
break;
}
if show_progress {
eprintln!(
"{REJECTED_EMOJI} Findings rejected — revising for round {}...",
round + 1
);
}
feedback = Some(review);
}
write_report(&client, topic, &findings, incomplete, show_progress).await
}

98
deep_research/src/researcher.rs Normal file
View file

@ -0,0 +1,98 @@
use crate::models::RESEARCHER_MODEL;
use crate::progress::{RESEARCH_EMOJI, Spinner};
use crate::review::Review;
use crate::summarizer::summarize_partial_history;
use crate::tools::{FetchPage, SearchWeb};
use rig::client::AgentClientExt;
use rig::completion::PromptError;
use rig::providers::ollama;
const MAX_RESEARCH_TURNS: usize = 12;
/// Findings gathered by a research pass, and whether the researcher was cut
/// off by the turn budget before it could conclude on its own.
pub(crate) struct GatheredFindings {
pub(crate) findings: String,
pub(crate) incomplete: bool,
}
/// Wraps the tool-calling research loop in its own span so it's visible as a
/// single unit in traces, distinct from the writing and review phases and
/// nesting rig's own per-turn `chat`/`execute_tool` spans underneath it.
#[tracing::instrument(skip(client, feedback), fields(gen_ai.agent.name = "researcher"))]
pub(crate) async fn gather_findings(
client: &ollama::Client,
topic: &str,
feedback: Option<&Review>,
round: usize,
show_progress: bool,
) -> anyhow::Result<GatheredFindings> {
let current_date = chrono::offset::Local::now().to_string();
let researcher = client
.agent(RESEARCHER_MODEL)
.name("researcher")
.preamble(format!(
"You are a meticulous research assistant. Use the search_web and fetch_page tools to \
investigate the user's topic: run several searches with varied phrasing, fetch the \
most promising pages, and cross-check claims across at least two sources before \
trusting them. Ensure sources are up to date: the current date is {current_date}. \
Once you are confident you have enough evidence, stop calling tools and reply with a \
plain-text dump of every fact you gathered, using footnote-style citations: write each \
fact followed by a bracketed number like [1], then at the end of your reply list a \
'Sources' section mapping each number to the exact URL it came from, one per line, e.g. \
'[1] https://example.com/page'. Reuse the same number when multiple facts come from the \
same URL — do not give one URL two different numbers. Also call out any open questions \
or contradictions between sources, citing the footnotes involved. This is raw research \
material for a writer, not a final report, so favor completeness over polish.")
.as_str(),
)
.tool(SearchWeb)
.tool(FetchPage)
.build();
let task = match feedback {
None => topic.to_string(),
Some(review) => format!(
"Topic: {topic}\n\n\
You already ran a research pass on this topic. A reviewer checked it against its \
cited sources and found it insufficient. Do more research to address the reviewer's \
feedback, then produce an updated findings dump: carry forward what's solid, and \
add, correct, or better-source whatever the gaps call for. Gaps include out-of-date,\
irrelevant, or clearly wrong information. The date is {current_date}.\n\n\
Solid findings from the last pass — keep and build on these:\n{}\n\n\
Gaps the reviewer found — conclusions not actually backed by their source, sources \
that don't line up with the conclusion drawn from them, or parts of the topic still \
uncovered:\n{}",
review.solid_findings, review.gaps
),
};
let spinner = Spinner::start(show_progress, format!("{RESEARCH_EMOJI} Researching..."));
let run_result = researcher
.runner(task)
.max_turns(MAX_RESEARCH_TURNS)
.run()
.await;
drop(spinner);
let gathered = match run_result {
Ok(response) => GatheredFindings {
findings: response.output,
incomplete: false,
},
Err(PromptError::MaxTurnsError { chat_history, .. }) => {
tracing::warn!(round, "researcher hit its turn budget before concluding");
GatheredFindings {
findings: summarize_partial_history(client, topic, &chat_history, show_progress)
.await,
incomplete: true,
}
}
Err(err) => return Err(err.into()),
};
tracing::info!(round, findings = %gathered.findings, incomplete = gathered.incomplete, "research phase complete");
Ok(gathered)
}

9
deep_research/src/review.rs
View file

@ -50,9 +50,14 @@ pub(crate) async fn review_findings(
.retries(2)
.build();
let spinner = Spinner::start(show_progress, format!("{REVIEW_EMOJI} Reviewing findings..."));
let spinner = Spinner::start(
show_progress,
format!("{REVIEW_EMOJI} Reviewing findings..."),
);
let review = reviewer
.extract(format!("Topic: {topic}\n\nResearch findings to review:\n{findings}"))
.extract(format!(
"Topic: {topic}\n\nResearch findings to review:\n{findings}"
))
.await?;
drop(spinner);

9
deep_research/src/stream.rs
View file

@ -64,7 +64,8 @@ mod tests {
let items = vec![text_item("Hello, "), text_item("world!")];
let mut written = Vec::new();
let accumulated = write_text_stream(stream::iter(items), &mut written, Spinner::start(false, ""))
let accumulated =
write_text_stream(stream::iter(items), &mut written, Spinner::start(false, ""))
.await
.unwrap();
@ -81,7 +82,8 @@ mod tests {
let items = vec![text_item("kept"), final_item];
let mut written = Vec::new();
let accumulated = write_text_stream(stream::iter(items), &mut written, Spinner::start(false, ""))
let accumulated =
write_text_stream(stream::iter(items), &mut written, Spinner::start(false, ""))
.await
.unwrap();
@ -97,7 +99,8 @@ mod tests {
let items = vec![text_item("kept"), Err(error)];
let mut written = Vec::new();
let result = write_text_stream(stream::iter(items), &mut written, Spinner::start(false, "")).await;
let result =
write_text_stream(stream::iter(items), &mut written, Spinner::start(false, "")).await;
assert!(result.is_err());
assert_eq!(String::from_utf8(written).unwrap(), "kept");

69
deep_research/src/summarizer.rs Normal file
View file

@ -0,0 +1,69 @@
use crate::history::{annotated_transcript_from_history, partial_findings_from_history};
use crate::models::WRITER_MODEL;
use crate::progress::{SUMMARIZE_EMOJI, Spinner};
use rig::client::AgentClientExt;
use rig::completion::{Message, Prompt};
use rig::providers::ollama;
/// Agentic alternative to the plain programmatic extraction in `history`:
/// hands the annotated transcript to a fresh model call and asks it to
/// reconstruct the same footnote-style findings dump the researcher would
/// have written itself, had it not run out of turns. This can dedupe
/// repeated URLs and restore correct `[n]` citation numbering in a way
/// string concatenation can't — but it's a model call like any other, so on
/// an empty transcript or a failure it falls back to
/// `partial_findings_from_history` rather than letting a second turn-budget
/// problem take down the one recovery path that's supposed to be
/// bulletproof.
#[tracing::instrument(skip(client, chat_history), fields(gen_ai.agent.name = "history-summarizer"))]
pub(crate) async fn summarize_partial_history(
client: &ollama::Client,
topic: &str,
chat_history: &[Message],
show_progress: bool,
) -> String {
let transcript = annotated_transcript_from_history(chat_history);
if transcript.is_empty() {
tracing::warn!("no usable transcript to summarize; skipping summarizer pass");
return partial_findings_from_history(chat_history);
}
let summarizer = client
.agent(WRITER_MODEL)
.name("history-summarizer")
.preamble(
"You are reconstructing research notes from a research session that was cut off \
before the researcher could write its own summary. You'll be given a raw transcript \
of tool calls (searches run, pages fetched), their results, and any interim comments \
the researcher made. Turn this into a footnote-style findings dump: write each fact \
the transcript actually supports, followed by a bracketed number like [1], then end \
with a 'Sources' section mapping each number to its exact URL, one per line. Reuse \
the same number for a URL that appears more than once. Do not invent facts beyond \
what the transcript shows, and note explicitly where it looks thin or cuts off \
mid-investigation.",
)
.build();
let spinner = Spinner::start(
show_progress,
format!("{SUMMARIZE_EMOJI} Reconstructing partial findings..."),
);
let result = summarizer
.prompt(format!(
"Topic: {topic}\n\nPartial research transcript:\n{transcript}"
))
.await;
drop(spinner);
match result {
Ok(findings) => {
tracing::info!(findings = %findings, "summarizer reconstructed partial findings");
findings
}
Err(error) => {
tracing::warn!(%error, "summarizer agent failed; falling back to programmatic extraction");
partial_findings_from_history(chat_history)
}
}
}

146
deep_research/src/tools.rs
View file

@ -1,14 +1,43 @@
use crate::progress::{self, FETCH_EMOJI, SEARCH_EMOJI};
use rig::tool::ToolExecutionError;
use scraper::{Html, Selector};
use serde::Deserialize;
const MAX_SEARCH_RESULTS: usize = 6;
const MAX_PAGE_CHARS: usize = 6000;
/// Searches the web via DuckDuckGo's HTML endpoint (no API key required) and
/// returns each hit's title, URL, and snippet so the caller can decide which
/// pages are worth fetching in full.
#[rig::tool_macro(description = "Search the web for pages related to a query", required(query))]
/// Local-only tool: scraping DuckDuckGo directly shares rate-limit fate with
/// every other bot hitting it from this IP, and a rate-limited response
/// looks identical to a genuine "no results" — which is exactly what took
/// down a research run over a dozen turns without ever surfacing as an
/// error. A self-hosted SearXNG instance has its own JSON API (so no HTML
/// scraping) and spreads queries across multiple upstream engines instead
/// of hammering one. This is deliberately not configurable beyond the env
/// var below — this tool is never meant to run anywhere but this machine.
fn searxng_base_url() -> String {
std::env::var("SEARXNG_URL").unwrap_or_else(|_| "http://localhost:8080".to_string())
}
#[derive(Deserialize)]
struct SearxngResponse {
results: Vec<SearxngResult>,
}
#[derive(Deserialize)]
struct SearxngResult {
title: String,
url: String,
#[serde(default)]
content: String,
}
/// Searches the web via a local SearXNG instance's JSON API and returns each
/// hit's title, URL, and snippet so the caller can decide which pages are
/// worth fetching in full.
#[rig::tool_macro(
description = "Search the web for pages related to a query",
required(query)
)]
pub(crate) async fn search_web(
/// The search query
query: String,
@ -16,31 +45,37 @@ pub(crate) async fn search_web(
progress::set_activity(format!("{SEARCH_EMOJI} Searching: {query}"));
let response = reqwest::Client::new()
.get("https://html.duckduckgo.com/html/")
.query(&[("q", query.as_str())])
.header("User-Agent", "Mozilla/5.0 (research-agent)")
.get(format!("{}/search", searxng_base_url()))
.query(&[("q", query.as_str()), ("format", "json")])
.send()
.await
.map_err(ToolExecutionError::from_error)?;
let body = response.text().await.map_err(ToolExecutionError::from_error)?;
let results = parse_search_results(&body);
let parsed: SearxngResponse = response
.json()
.await
.map_err(ToolExecutionError::from_error)?;
if results.is_empty() {
if parsed.results.is_empty() {
return Ok("No results found.".to_string());
}
Ok(results
Ok(parsed
.results
.into_iter()
.take(MAX_SEARCH_RESULTS)
.enumerate()
.map(|(i, r)| format!("{}. {}\n {}\n {}", i + 1, r.title, r.url, r.snippet))
.map(|(i, r)| format!("{}. {}\n {}\n {}", i + 1, r.title, r.url, r.content))
.collect::<Vec<_>>()
.join("\n\n"))
}
/// Fetches a page and returns its main text content, stripped of markup and
/// truncated so a single fetch can't blow out the model's context window.
#[rig::tool_macro(description = "Fetch a web page and return its readable text content", required(url))]
#[rig::tool_macro(
description = "Fetch a web page and return its readable text content",
required(url)
)]
pub(crate) async fn fetch_page(
/// The URL to fetch
url: String,
@ -54,71 +89,18 @@ pub(crate) async fn fetch_page(
.await
.map_err(ToolExecutionError::from_error)?;
let body = response.text().await.map_err(ToolExecutionError::from_error)?;
let body = response
.text()
.await
.map_err(ToolExecutionError::from_error)?;
Ok(extract_readable_text(&body))
}
struct SearchResult {
title: String,
url: String,
snippet: String,
}
/// DuckDuckGo's HTML results page wraps each hit in a `.result` block; the
/// title/link lives in `.result__a` and links are redirected through
/// `duckduckgo.com/l/?uddg=<real-url>`, so the real URL has to be pulled back
/// out of that query parameter rather than used as-is.
fn parse_search_results(body: &str) -> Vec<SearchResult> {
let document = Html::parse_document(body);
let result_selector = Selector::parse(".result").expect("valid selector");
let title_selector = Selector::parse(".result__a").expect("valid selector");
let snippet_selector = Selector::parse(".result__snippet").expect("valid selector");
document
.select(&result_selector)
.filter_map(|result| {
let title_el = result.select(&title_selector).next()?;
let href = title_el.value().attr("href")?;
let url = resolve_ddg_redirect(href);
let title = title_el.text().collect::<String>().trim().to_string();
let snippet = result
.select(&snippet_selector)
.next()
.map(|el| el.text().collect::<String>().trim().to_string())
.unwrap_or_default();
if title.is_empty() || url.is_empty() {
None
} else {
Some(SearchResult { title, url, snippet })
}
})
.take(MAX_SEARCH_RESULTS)
.collect()
}
fn resolve_ddg_redirect(href: &str) -> String {
let full = if href.starts_with("//") {
format!("https:{href}")
} else {
href.to_string()
};
reqwest::Url::parse(&full)
.ok()
.and_then(|parsed| {
parsed
.query_pairs()
.find(|(k, _)| k == "uddg")
.map(|(_, v)| v.into_owned())
})
.unwrap_or(full)
}
fn extract_readable_text(html: &str) -> String {
let document = Html::parse_document(html);
let content_selector = Selector::parse("p, h1, h2, h3, h4, h5, li, td").expect("valid selector");
let content_selector =
Selector::parse("p, h1, h2, h3, h4, h5, li, td").expect("valid selector");
let mut text: String = document
.select(&content_selector)
@ -133,3 +115,23 @@ fn extract_readable_text(html: &str) -> String {
let collapsed = text.split_whitespace().collect::<Vec<_>>().join(" ");
collapsed.chars().take(MAX_PAGE_CHARS).collect()
}
#[cfg(test)]
mod live_smoke_test {
use super::*;
/// Not run by default — this tool is local-only by design, so there's no
/// CI environment where a SearXNG instance would exist to test against.
/// Run manually with `cargo test -- --ignored` when SEARXNG_URL (or the
/// localhost:8080 default) points at a running instance.
#[tokio::test]
#[ignore = "hits a real local SearXNG instance; run manually with --ignored"]
async fn search_web_returns_real_results_from_local_searxng() {
let output = search_web("uruguay senior software engineer hiring 2026".to_string())
.await
.expect("search_web should succeed against a live local SearXNG instance");
println!("{output}");
assert_ne!(output, "No results found.");
}
}

77
deep_research/src/writer.rs Normal file
View file

@ -0,0 +1,77 @@
use crate::models::WRITER_MODEL;
use crate::progress::{REPORT_EMOJI, Spinner};
use crate::stream::write_text_stream;
use rig::client::AgentClientExt;
use rig::providers::ollama;
use rig::streaming::StreamingPrompt;
use std::io::Write;
#[tracing::instrument(skip(client, findings), fields(gen_ai.agent.name = "writer"))]
pub(crate) async fn write_report(
client: &ollama::Client,
topic: &str,
findings: &str,
incomplete: bool,
show_progress: bool,
) -> anyhow::Result<String> {
let writer = client
.agent(WRITER_MODEL)
.name("writer")
.preamble(
"You turn raw research notes into a clear, well-organized report for the reader. The \
notes use footnote-style citations — a bracketed number like [1] after a fact, with a \
Sources section mapping numbers to URLs. Preserve this scheme in your report: keep the \
same [n] markers next to the claims they support (renumbering only if you drop unused \
sources), and end the report with a 'Sources' section listing every footnote number \
still in use next to its exact URL. Structure the body with headings, and call out any \
open questions or contradictions the research turned up. Do not invent facts beyond \
what the notes provide.",
)
.build();
let task = if incomplete {
format!(
"Topic: {topic}\n\n\
The researcher ran out of turns before it could finish investigating, so these notes \
are an incomplete, best-effort reconstruction rather than the researcher's own \
conclusions — treat any citations already in them as tentative. Write up what the \
notes actually support, note explicitly where evidence is thin or missing, and open \
the report with a short callout that this research is inconclusive and further \
manual searching is needed.\n\n\
Research notes:\n{findings}"
)
} else {
format!("Topic: {topic}\n\nResearch notes:\n{findings}")
};
let spinner = Spinner::start(show_progress, format!("{REPORT_EMOJI} Writing report..."));
let response_stream = writer.stream_prompt(task).await;
// Locked once for the whole stream rather than per chunk (as print!
// would do internally) — chunks arrive in a tight loop, so re-acquiring
// the lock on every one adds up. The spinner keeps running until the
// stream's first chunk arrives, so the terminal stays covered through
// the gap between sending the prompt and generation actually starting.
let stdout = std::io::stdout();
let mut handle = stdout.lock();
// Guaranteed regardless of whether the writer model actually heeds the
// instruction above — the reader should never mistake a budget-exhausted
// run for a complete one just because the model forgot to say so.
let disclaimer = incomplete.then_some(
"> **Disclaimer:** incomplete research: the researcher exhausted its turn \
budget before it could finish investigating. The findings below are partial and may be \
incomplete or unbalanced; treat them as a starting point and verify further manually.\n\n",
);
if let Some(disclaimer) = disclaimer {
write!(handle, "{disclaimer}")?;
}
let report = write_text_stream(response_stream, &mut handle, spinner).await?;
writeln!(handle)?;
Ok(match disclaimer {
Some(disclaimer) => format!("{disclaimer}{report}"),
None => report,
})
}