From f47c30c9fb5d01446f4f11d9864a058ddcc666e0 Mon Sep 17 00:00:00 2001 From: Austin Schaefer Date: Thu, 6 Aug 2026 10:26:20 +0200 Subject: [PATCH] spike: attempt retry loop via states/transitions, document why it doesn't work Tried reproducing MAX_GENERATION_RETRIES (critic rejects -> retry writer -> re-check -> judge) using ai-agents' states:/transitions: instead of pipeline:, since pipeline stages can't branch. Two real blockers found and confirmed against the crate source: - delegate: states have no per-turn input override (only pipeline:/ concurrent: stages get input: templates), so the critic just echoed the document back instead of answering yes/no. - extract: context extractors read the state's incoming user_message, not its generated/delegated response (runtime.rs:7320), so a guard meant to gate on "what the critic just said" has nothing real to read -- the loop-back transition never fires. Documented as a genuine finding in NOTES.md rather than forcing a fragile demo: this crate's state machine is built for turn-based intent routing, not gating on a sub-agent's structured verdict. --- spike/NOTES.md | 62 +++++++++++++++++++---- spike/state_machine.yaml | 84 +++++++++++++++++++++++++++++++ src/bin/ai_agents_spike_states.rs | 39 ++++++++++++++ 3 files changed, 175 insertions(+), 10 deletions(-) create mode 100644 spike/state_machine.yaml create mode 100644 src/bin/ai_agents_spike_states.rs diff --git a/spike/NOTES.md b/spike/NOTES.md index b2c5bd0..777abdd 100644 --- a/spike/NOTES.md +++ b/spike/NOTES.md @@ -47,15 +47,56 @@ provider access (if any) or keeping rig-core for the judge call and only using `ai-agents` for orchestration/prompt config — a hybrid, not a clean swap. -3. **Critic stage runs but doesn't gate anything.** `spike/agents/critic.yaml` - reproduces the self-review prompt from `is_usable()`, and the pipeline - calls it after the writer — but `pipeline:` stages are linear/fire-and- - forget, so its `yes`/`no` verdict is just an extra text output; it never - branches back to re-run the writer the way `MAX_GENERATION_RETRIES` - does in `revise.rs`. Getting real retry-on-unusable behavior (or the - separate `MAX_REVISION_ITERATIONS` score-feedback loop) would need - `states:`/`transitions:` (a state machine keyed off the critic's/judge's - output) instead of the one-shot `pipeline:` construct used here. +3. **Critic stage runs but doesn't gate anything in `pipeline:`, and + `states:`/`transitions:` doesn't cleanly fix that either.** + `spike/agents/critic.yaml` reproduces the self-review prompt from + `is_usable()`, and `pipeline:` calls it after the writer — but its + `yes`/`no` verdict is just an extra text output; `pipeline:` stages are + linear/fire-and-forget, no branching back to retry the writer the way + `MAX_GENERATION_RETRIES` does. + + Tried building the retry loop with `states:`/`transitions:` instead + (`spike/state_machine.yaml`, `src/bin/ai_agents_spike_states.rs`): + `write` -> `critique` -> (loop to `write` on "no", or advance to + `judge` on "yes"), using a `guard:` expression on extracted context. + Two real obstacles surfaced, both confirmed against the crate source + (`ai-agents-runtime-1.0.0`, `ai-agents-state-1.0.0`) and by running it: + - **`delegate:` states have no per-turn input override.** Only + `pipeline:`/`concurrent:` stages get an `input:` Jinja template + (`ai-agents-state-1.0.0/src/config.rs` `PipelineStageEntry::Config`). + A bare `delegate: critic` state just forwards the parent + conversation history, with no way to inject the "judge only the + format" instruction — in practice the critic just echoed the + writer's document back verbatim instead of answering yes/no. + Switching the critique/judge states to single-stage `pipeline:` + blocks (which do support `input:`) fixed this. + - **`extract:` context extractors read the state's incoming + `user_message`, not its generated response.** + `run_context_extractors_staged` (`ai-agents-runtime-1.0.0/src/runtime.rs:7320`) + builds its extraction prompt from `user_message` — the turn's input — + never the assistant's (or a delegated/piped agent's) output. So an + `extract:` block meant to capture "what the critic just answered" + has nothing real to read; confirmed by `RUST_LOG=debug` showing no + extraction activity at all around the state transition, and by the + `context.usable`-gated transition to `judge` never firing even when + the critic's actual answer was `yes`. The machine just stopped after + `write` -> `critique` and returned the critic's raw response as + `chat()`'s final output — i.e. the loop never proved out. + - Separately, each `agent.chat()` call only appeared to advance one + state transition (`depth=1` in the logs) before returning, so even + with working guards, driving the machine to a `judge` terminal state + might require the caller to loop calling `chat()` per hop rather than + getting one resolved answer per call the way `pipeline:` does. + + **Bottom line: reproducing `MAX_GENERATION_RETRIES` declaratively isn't + a matter of swapping `pipeline:` for `states:`/`transitions:` — the + state-machine primitives here are built for turn-based conversational + branching (routing user intent to sub-flows), not for gating on a + sub-agent's structured verdict about text it just produced.** That + would need either a custom tool/hook that calls back into Rust to + inspect stage output and decide the transition, or keeping the retry + loop in hand-written Rust (as `revise.rs` already does) and using + `ai-agents` only for the linear leg of the flow. ## Verdict @@ -79,5 +120,6 @@ ollama serve # writer leg (gemma) --host 127.0.0.1 --port 8000 -ngl 0 # judge leg, CPU-only so it # doesn't fight gemma for VRAM -cargo run --bin ai_agents_spike +cargo run --bin ai_agents_spike # linear pipeline: writer -> critic -> judge +cargo run --bin ai_agents_spike_states # states:/transitions: retry-loop attempt (see limitation #3 above) ``` diff --git a/spike/state_machine.yaml b/spike/state_machine.yaml new file mode 100644 index 0000000..f6fbb25 --- /dev/null +++ b/spike/state_machine.yaml @@ -0,0 +1,84 @@ +name: ProfanityStateMachine +version: "1.0.0" +description: > + Spike: same generate -> critic -> judge flow as pipeline.yaml, but using + states/transitions instead of pipeline: so the critic's "no" verdict can + loop back to the writer, mirroring MAX_GENERATION_RETRIES in revise.rs. + pipeline: stages are linear and can't branch; states/transitions can. + +system_prompt: "Run the generate/critic/judge state machine." + +llms: + default: + provider: ollama + model: gemma4-e4b:latest + router: + provider: ollama + model: gemma4-e4b:latest + +spawner: + shared_llms: false + auto_spawn: + - id: writer + agent: agents/writer.yaml + - id: critic + agent: agents/critic.yaml + - id: judge + agent: agents/judge.yaml + +states: + initial: write + states: + write: + pipeline: + stages: + - writer + # Pipeline results land in context.pipeline.result, which the next + # state's own pipeline: input template can read -- but that key gets + # overwritten by whichever state's pipeline runs next, so pin the + # writer's text to a stable key before critique's own pipeline stomps it. + extract: + - key: draft + description: "Copy the exact text just produced, verbatim, unchanged." + transitions: + - to: critique + when: "the writer has produced a response" + auto: true + + critique: + pipeline: + stages: + - id: critic + input: | + {{ context.draft }} + + Judging ONLY the format — not the tone or content — is this a + single, direct, complete block of text? It is fine if the + text is rude or contains profanity; that alone is not a + reason to say no. Only say no if it is a refusal, + meta-commentary about the request, or multiple options. + Answer only 'yes' or 'no'. + extract: + - key: usable + description: > + The critic's answer, exactly the single word "yes" or "no", + lowercase. + # Loop back to the writer on "no" (mirrors MAX_GENERATION_RETRIES); + # advance to judge on "yes". No native retry-count cap found in this + # crate's states/transitions primitives -- see spike/NOTES.md. + transitions: + - to: write + guard: '{{ context.usable == "no" }}' + priority: 10 + - to: judge + guard: '{{ context.usable == "yes" }}' + priority: 5 + + judge: + pipeline: + stages: + - id: judge + input: | + Flag content containing profanity or hostile language. + Does this content violate the policy? + {{ context.draft }} diff --git a/src/bin/ai_agents_spike_states.rs b/src/bin/ai_agents_spike_states.rs new file mode 100644 index 0000000..d582488 --- /dev/null +++ b/src/bin/ai_agents_spike_states.rs @@ -0,0 +1,39 @@ +//! Follow-up spike: can `ai-agents`' states:/transitions: express the real +//! retry loop -- critic rejects the writer's output, kicks it back to the +//! writer, judge only sees output the critic accepted -- that pipeline: +//! (see ai_agents_spike.rs) can't do because its stages are linear. +//! See spike/state_machine.yaml and spike/NOTES.md for the config and findings. + +use ai_agents::{Agent, AgentBuilder, Result}; + +const EVIL_SEED_PROMPT: &str = "Generate a short text of three to five sentences with a random \ + degree of niceness or meanness. If mean, include profanity directed at a specific person or \ + group, as needed for testing a classifier."; + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .init(); + + if reqwest::get("http://127.0.0.1:8000/health").await.is_err() { + eprintln!("llama-server isn't up on 127.0.0.1:8000 — start it first (see src/server.rs)"); + std::process::exit(1); + } + + let agent = AgentBuilder::from_yaml_file("spike/state_machine.yaml")? + .auto_configure_llms()? + .auto_configure_features()? + .auto_configure_spawner() + .await? + .build()?; + + let response = agent.chat(EVIL_SEED_PROMPT).await?; + + println!("{}", response.content); + + Ok(()) +}