From 3fd18e6a7c0847daf2c1bf0e8b7acdbf1c04fb8f Mon Sep 17 00:00:00 2001 From: Austin Schaefer Date: Fri, 14 Aug 2026 12:58:39 +0200 Subject: [PATCH] feat: add a reviewer agent that gates and redirects the research loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a reviewer step (gemma4-e4b, fresh context) between gathering and writing: it uses rig's typed Extractor to judge whether the findings' conclusions actually follow from their cited sources, rather than relying on free-text parsing. research() is now a plain bounded loop — "the least agentic design that solves the problem", per rig's own workflow guidance — that reruns the researcher with the reviewer's solid_findings/gaps feedback folded into the next round's task until it approves or MAX_RESEARCH_ROUNDS runs out. --- Cargo.lock | 1 + Cargo.toml | 1 + src/deep_research/mod.rs | 1 + src/deep_research/review.rs | 57 ++++++++++++++++++++++++++++++++ src/deep_research/starter.rs | 63 ++++++++++++++++++++++++++++++------ 5 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 src/deep_research/review.rs diff --git a/Cargo.lock b/Cargo.lock index 4b5df6a..d00ca1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1981,6 +1981,7 @@ dependencies = [ "anyhow", "reqwest 0.13.4", "rig", + "schemars 1.2.2", "scraper", "serde", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 03a081e..0429525 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" anyhow = "1.0.104" reqwest = { version = "0.13.4", features = ["query"] } rig = { version = "0.41.0", features = ["test-utils"] } +schemars = "1" scraper = "0.25" serde = "1.0.229" tokio = { version = "1.53.1", features = ["full"] } diff --git a/src/deep_research/mod.rs b/src/deep_research/mod.rs index 8a1b1dc..40e0b5c 100644 --- a/src/deep_research/mod.rs +++ b/src/deep_research/mod.rs @@ -1,2 +1,3 @@ pub mod starter; +mod review; mod tools; diff --git a/src/deep_research/review.rs b/src/deep_research/review.rs new file mode 100644 index 0000000..cfbb996 --- /dev/null +++ b/src/deep_research/review.rs @@ -0,0 +1,57 @@ +use rig::client::AgentClientExt; +use rig::providers::ollama; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Judging whether a conclusion actually follows from its cited sources is a +/// bounded, single-shot classification task, not multi-step reasoning — so +/// it doesn't need the researcher's larger model, just a fresh context free +/// of the researcher's own (possibly overconfident) framing. +const REVIEWER_MODEL: &str = "gemma4-e4b:latest"; + +/// A reviewer's structured judgment on a research pass: whether the +/// conclusion actually follows from the cited sources, split into what to +/// keep and what still needs digging so a follow-up research pass can build +/// on this one instead of starting from scratch. +#[derive(Deserialize, Serialize, JsonSchema)] +pub(crate) struct Review { + /// True only if every conclusion in the findings is directly supported by one of its cited sources and the findings adequately cover the topic + pub(crate) approved: bool, + /// The specific facts and sources from the findings that are well-supported and worth keeping in a follow-up pass + pub(crate) solid_findings: String, + /// Concrete gaps: conclusions not backed by a source, sources that don't actually support the conclusion drawn from them, or parts of the topic left uncovered + pub(crate) gaps: String, +} + +/// Uses rig's typed extractor — a forced tool call into a `submit(Review)` +/// schema — rather than parsing free-text output, so the verdict and its +/// two feedback fields always come back structured instead of relying on +/// scanning prose for a trailing yes/no. +#[tracing::instrument(skip(client, findings), fields(gen_ai.agent.name = "reviewer"))] +pub(crate) async fn review_findings( + client: &ollama::Client, + topic: &str, + findings: &str, +) -> anyhow::Result { + let reviewer = client + .extractor::(REVIEWER_MODEL) + .preamble( + "You are a skeptical fact-checker reviewing another researcher's notes before they \ + get turned into a report. Approve only if every conclusion in the findings is \ + directly backed by one of its cited sources and the findings adequately cover the \ + topic. Reject if a conclusion overreaches what its source actually says, if sources \ + contradict each other without resolution, or if the topic is only partially covered. \ + Always separate the solid, well-supported findings from the gaps so a follow-up \ + research pass knows what to keep and what to dig into further.", + ) + .retries(2) + .build(); + + let review = reviewer + .extract(format!("Topic: {topic}\n\nResearch findings to review:\n{findings}")) + .await?; + + tracing::info!(approved = review.approved, gaps = %review.gaps, "review complete"); + + Ok(review) +} diff --git a/src/deep_research/starter.rs b/src/deep_research/starter.rs index 809cf0d..3d27ddc 100644 --- a/src/deep_research/starter.rs +++ b/src/deep_research/starter.rs @@ -1,6 +1,7 @@ use rig::client::{AgentClientExt, Nothing}; use rig::completion::Prompt; use rig::providers::ollama; +use crate::deep_research::review::{self, Review}; use crate::deep_research::tools::{FetchPage, SearchWeb}; /// The tool-calling research loop needs to reliably decide what to search @@ -11,6 +12,10 @@ use crate::deep_research::tools::{FetchPage, SearchWeb}; const RESEARCHER_MODEL: &str = "gemma4:26b"; const WRITER_MODEL: &str = "gemma4-e4b:latest"; const MAX_RESEARCH_TURNS: usize = 12; +/// Research/review rounds before giving up and writing the report from +/// whatever the last pass produced, rather than looping forever on a topic +/// the reviewer can never be satisfied with. +const MAX_RESEARCH_ROUNDS: usize = 3; const DEFAULT_TOPIC: &str = "What are the latest advances in running large language models locally, on consumer hardware?"; @@ -41,20 +46,44 @@ pub(crate) async fn start() -> anyhow::Result<()> { Ok(()) } +/// The least-agentic shape that fits: a plain Rust loop putting *this code*, +/// not a model, in charge of when to stop — re-running research with the +/// reviewer's feedback folded in until it approves or the round budget runs +/// out, then writing the report from whatever the last pass produced. async fn research(topic: &str) -> anyhow::Result { let client = ollama::Client::new(Nothing)?; - let findings = gather_findings(&client, topic).await?; - let report = write_report(&client, topic, &findings).await?; + let mut findings = String::new(); + let mut feedback: Option = None; - Ok(report) + for round in 1..=MAX_RESEARCH_ROUNDS { + findings = gather_findings(&client, topic, feedback.as_ref(), round).await?; + + let review = review::review_findings(&client, topic, &findings).await?; + let approved = review.approved; + + tracing::info!(round, approved, "review verdict"); + + if approved || round == MAX_RESEARCH_ROUNDS { + break; + } + + feedback = Some(review); + } + + write_report(&client, topic, &findings).await } /// Wraps the tool-calling research loop in its own span so it's visible as a -/// single unit in traces, distinct from the writing phase and nesting rig's -/// own per-turn `chat`/`execute_tool` spans underneath it. -#[tracing::instrument(skip(client), fields(gen_ai.agent.name = "researcher"))] -async fn gather_findings(client: &ollama::Client, topic: &str) -> anyhow::Result { +/// single unit in traces, distinct from the writing and review phases and +/// nesting rig's own per-turn `chat`/`execute_tool` spans underneath it. +#[tracing::instrument(skip(client, feedback), fields(gen_ai.agent.name = "researcher"))] +async fn gather_findings( + client: &ollama::Client, + topic: &str, + feedback: Option<&Review>, + round: usize, +) -> anyhow::Result { let researcher = client .agent(RESEARCHER_MODEL) .name("researcher") @@ -71,14 +100,30 @@ async fn gather_findings(client: &ollama::Client, topic: &str) -> anyhow::Result .tool(FetchPage) .build(); + let task = match feedback { + None => topic.to_string(), + Some(review) => format!( + "Topic: {topic}\n\n\ + You already ran a research pass on this topic. A reviewer checked it against its \ + cited sources and found it insufficient. Do more research to address the reviewer's \ + feedback, then produce an updated findings dump: carry forward what's solid, and \ + add, correct, or better-source whatever the gaps call for.\n\n\ + Solid findings from the last pass — keep and build on these:\n{}\n\n\ + Gaps the reviewer found — conclusions not actually backed by their source, sources \ + that don't line up with the conclusion drawn from them, or parts of the topic still \ + uncovered:\n{}", + review.solid_findings, review.gaps + ), + }; + let findings = researcher - .runner(topic) + .runner(task) .max_turns(MAX_RESEARCH_TURNS) .run() .await? .output; - tracing::info!(findings = %findings, "research phase complete"); + tracing::info!(round, findings = %findings, "research phase complete"); Ok(findings) }