{ "library": { "repo": "https://github.com/Mahalanobis-Systems/aot-kit", "version": "0.2.0", "commit": "40afb71" }, "vocabulary": { "generated": "2026-10-26", "version": "0.2.0", "commit": "40afb71", "levers": [ { "id": "model", "name": "Model", "covers": "Model choice, routing, cascades, reasoning effort" }, { "id": "prompt", "name": "Prompt", "covers": "System prompt, instructions, few-shot examples, prompt optimization, skill content" }, { "id": "context", "name": "Context", "covers": "What enters and stays in the window: compaction, filtering, isolation, cache layout" }, { "id": "tool-surface", "name": "Tool surface", "covers": "Tool names, schemas, descriptions, return size, typed tools vs code execution" }, { "id": "control-flow", "name": "Control flow", "covers": "The harness loop: orchestration, multi-agent topology, budgets, stop rules, verify-and-retry" }, { "id": "state", "name": "State", "covers": "Externalized state: plans, progress files, checkpoints, memory stores" }, { "id": "knowledge-access", "name": "Knowledge access", "covers": "Connecting sources, query formulation, retriever choice, ranking" }, { "id": "guardrails", "name": "Guardrails", "covers": "Sandboxes, permissions, approval policies, data containment" }, { "id": "post-training", "name": "Post-training", "covers": "Changing the weights: fine-tuning, RL, distillation" }, { "id": "serving", "name": "Serving", "covers": "Inference infrastructure: quantization, kernels, serving engines, gateways" } ], "domains": [ { "id": "retrieval", "name": "Retrieval", "covers": "Fetching knowledge the model lacks: RAG, search over docs, knowledge bases, customer systems" }, { "id": "memory", "name": "Memory", "covers": "Remembering across turns and sessions: memory stores, cheatsheets, episodic memory" }, { "id": "tool-use", "name": "Tool use", "covers": "Choosing and calling tools, APIs and MCP servers correctly" }, { "id": "code-generation", "name": "Code generation", "covers": "Writing, editing, repairing software" }, { "id": "data-analysis", "name": "Data analysis", "covers": "Querying structured data: text-to-SQL, semantic layers, BI, spreadsheets" }, { "id": "document-extraction", "name": "Document extraction", "covers": "Reading documents: contracts, legal review, PDFs, forms" }, { "id": "web-research", "name": "Web research", "covers": "Browsing the open web to answer or research" }, { "id": "computer-use", "name": "Computer use", "covers": "Operating GUIs, browsers, operating systems" }, { "id": "conversation", "name": "Conversation", "covers": "Multi-turn dialogue with end users: customer support, CRM assistants" }, { "id": "planning", "name": "Planning", "covers": "Long-horizon, multi-step execution; staying coherent over many steps" }, { "id": "reasoning", "name": "Reasoning", "covers": "Math, proofs, science, abstract puzzles" }, { "id": "judging", "name": "Judging", "covers": "Evaluating, grading, verifying outputs: LLM-as-judge, graders, decision models" }, { "id": "incident-response", "name": "Incident response", "covers": "On-call and SRE work: triage, root cause, remediation" }, { "id": "structured-output", "name": "Structured output", "covers": "Schema-valid typed output" }, { "id": "commerce", "name": "Commerce", "covers": "Transacting autonomously" } ], "tiers": [ { "tier": 1, "name": "Hygiene", "objective": "Remove waste and defects that can be seen without measuring quality at scale: no-regret changes that cannot alter what the model sees or decides.", "gate": "An instrumented agent and about 10 hand-labeled cases." }, { "tier": 2, "name": "Reliability", "objective": "Remove hard failures (crashes, timeouts, tool failures, injection, runs that exhaust their budget and return nothing) so the agent finishes, safely, under real load.", "gate": "Failure evidence (traces, a load-test stream or safety-eval results) with enough occurrences to measure before and after." }, { "tier": 3, "name": "Tuning", "objective": "Improve quality, speed and cost by changing components within the existing design: model, prompt, context, tools. Real trade-offs, so every change is measured against a baseline.", "gate": "A calibrated eval (about 200 human-validated cases across the agent's main workflows) and a baseline." }, { "tier": 4, "name": "Restructuring", "objective": "Change the structure of the agent: how many agents, how work flows between them, where state lives, which harness it runs on.", "gate": "Production-scale cost baselines and a production test (A/B or blue/green)." } ], "categories": [ { "id": "models", "number": 1, "name": "Models", "summary": null, "items": [ { "id": "models.workload-fit", "letter": "a", "name": "Benchmarking workloads", "summary": "Analyze workloads and test models across each", "lever": "model" }, { "id": "models.specialists", "letter": "b", "name": "Finding specialists", "summary": null, "lever": "model" }, { "id": "models.cheap-verifiable", "letter": "c", "name": "Using cheaper models", "summary": "Cheaper models with verifiable outputs", "lever": "model" } ] }, { "id": "tools", "number": 2, "name": "Tools", "summary": null, "items": [ { "id": "tools.refactor", "letter": "a", "name": "Refactoring", "summary": "Refactor & combine", "lever": "tool-surface" }, { "id": "tools.misuse", "letter": "b", "name": "Avoiding misuse", "summary": null, "lever": "tool-surface" }, { "id": "tools.lean-returns", "letter": "c", "name": "Trimming results", "summary": "Return only what the model needs", "lever": "tool-surface" } ] }, { "id": "prompts", "number": 3, "name": "Prompts", "summary": null, "items": [ { "id": "prompts.structure", "letter": "a", "name": "Structuring", "summary": "Clarify structure and intent", "lever": "prompt" }, { "id": "prompts.caching", "letter": "b", "name": "Caching", "summary": "Design for caching and reuse", "lever": "context" }, { "id": "prompts.reasoning-effort", "letter": "c", "name": "Sizing reasoning effort", "summary": "Match reasoning effort to the task", "lever": "model" } ] }, { "id": "memory", "number": 4, "name": "Memory", "summary": null, "items": [ { "id": "memory.externalize", "letter": "a", "name": "Externalizing state", "summary": "Externalize state beyond the context window", "lever": "state" }, { "id": "memory.resume", "letter": "b", "name": "Resuming", "summary": "Recover and resume rather than restart", "lever": "state" }, { "id": "memory.reuse", "letter": "c", "name": "Reusing work", "summary": "Reuse prior work across runs", "lever": "state" } ] }, { "id": "context", "number": 5, "name": "Context", "summary": null, "items": [ { "id": "context.lean-window", "letter": "a", "name": "Compacting", "summary": "Keep the window lean as the task grows", "lever": "context" }, { "id": "context.filter-input", "letter": "b", "name": "Filtering input", "summary": "Filter what enters the context", "lever": "context" }, { "id": "context.isolate", "letter": "c", "name": "Isolating subtasks", "summary": "Isolate sub-tasks in their own contexts", "lever": "context" } ] }, { "id": "topologies", "number": 6, "name": "Topologies", "summary": null, "items": [ { "id": "topologies.escalate", "letter": "a", "name": "Escalating", "summary": "Escalate from cheap to capable models", "lever": "model" }, { "id": "topologies.parallel", "letter": "b", "name": "Parallelizing", "summary": "Decompose work across parallel agents", "lever": "control-flow" }, { "id": "topologies.trust", "letter": "c", "name": "Separating privileges", "summary": "Separate roles by trust and privilege", "lever": "guardrails" } ] }, { "id": "orchestration", "number": 7, "name": "Orchestration", "summary": null, "items": [ { "id": "orchestration.route", "letter": "a", "name": "Routing", "summary": "Route work to the right path up front", "lever": "control-flow" }, { "id": "orchestration.bound", "letter": "b", "name": "Bounding execution", "summary": "Bound execution and fail gracefully", "lever": "control-flow" }, { "id": "orchestration.extra-compute", "letter": "c", "name": "Scaling compute", "summary": "Spend extra compute only where it pays", "lever": "control-flow" }, { "id": "orchestration.loop", "letter": "d", "name": "Shaping the loop", "summary": "Choose the shape of the agent loop", "lever": "control-flow" } ] }, { "id": "retrieval", "number": 8, "name": "Retrieval", "summary": "Retrieval & knowledge access", "items": [ { "id": "retrieval.sources", "letter": "a", "name": "Expanding sources", "summary": "Connect and expand the sources the task needs", "lever": "knowledge-access" }, { "id": "retrieval.query", "letter": "b", "name": "Querying precisely", "summary": null, "lever": "knowledge-access" }, { "id": "retrieval.results", "letter": "c", "name": "Ranking results", "summary": "Return what is relevant and trustworthy", "lever": "knowledge-access" } ] }, { "id": "verification", "number": 9, "name": "Verification", "summary": null, "items": [ { "id": "verification.external", "letter": "a", "name": "Checking externally", "summary": "Check work against sound external signals", "lever": "control-flow" }, { "id": "verification.completion", "letter": "b", "name": "Confirming completion", "summary": "Confirm the task is actually done before stopping", "lever": "control-flow" }, { "id": "verification.spend", "letter": "c", "name": "Sizing verification", "summary": "Match verification effort to what it catches", "lever": "control-flow" } ] }, { "id": "safety", "number": 10, "name": "Safety", "summary": null, "items": [ { "id": "safety.sandbox", "letter": "a", "name": "Sandboxing", "summary": "Constrain what the agent can touch", "lever": "guardrails" }, { "id": "safety.approval", "letter": "b", "name": "Gating risky actions", "summary": "Gate risky actions behind policy or a human", "lever": "guardrails" }, { "id": "safety.data", "letter": "c", "name": "Containing data", "summary": "Keep sensitive data and untrusted input contained", "lever": "guardrails" } ] } ] }, "optimizations": [ { "id": "1.b.i", "legacy_id": "T65", "former_ids": [], "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class", "kind": "technique", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "model per call site within the existing design", "proposed": false }, "levers": [ "model", "prompt" ], "domains": [ "general" ], "evidence_domains": [ "data-analysis", "code-generation", "reasoning", "web-research", "tool-use" ], "categories": [ "Finding specialists", "Benchmarking workloads", "Sizing reasoning effort" ], "category_ids": [ "models.specialists", "models.workload-fit", "prompts.reasoning-effort" ], "summary": "Pick the model per role or call site rather than one model everywhere, and set its reasoning effort and sampling parameters per site; tune each role's prompt to its model (GEPA raised every model tested at lower cost). Give the agent explicit effort budgets by query class (e.g. one agent and 3-10 tool calls for simple fact-finding) so simple queries are not over-served. Upgrade the model generation before doubling token budgets. A prompt that forces stepwise reasoning is the same lever from the prompt side and hurt on GPQA; prefer the model's own effort setting.", "how": [ "From traces, list call sites and their token, cost and error profile.", "For each, run a small model grid (cheaper, specialist, effort levels) on the eval slice that exercises it. Sweep effort rather than max it, since forced extra thinking cut GSM8K accuracy from 87.3% to 70.3% (1,100 to 15,980 thinking tokens), majority-voted parallel traces beat one long trace at the same budget (+10.1%), and stopping once a candidate answer appears cut reasoning tokens by 24.5% for a 2.9-point accuracy drop.", "Optimize each site's prompt for its model.", "Add per-query-class budgets.", "Paired eval per call site." ], "detect": { "static": "one model and one effort setting for every call site; no per-query-class budget", "traces": "simple queries using as many steps, tokens or sub-agents as complex ones; a call site whose cost dominates with no quality reason; retries exhausted or timeouts at one site", "evals": "quality and cost per call site, per model; price not predicting quality across the measured configurations (in an email-triage agent we measured: Spearman 0.12)" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "Genie: 81.4% to 91.6% with a model per role (May 2026); in an email-triage agent we measured, the best recall came from gpt-5-nano at low effort", "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "large", "when": "Genie 2.13x to 0.95x cost", "basis": "evidence" }, { "dimension": "performance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval per call site and per query class" } ], "evidence": [ { "source": "The Databricks AI Research Team, \"Pushing the Frontier for Data Agents with Genie\"", "url": "https://www.databricks.com/blog/pushing-frontier-data-agents-genie", "date": "2026-05-08", "locator": "Multi-LLM section; Figure 1; Figure 3", "task": "Assign a different LLM to each sub-agent role", "effect": "Figure 1, internal benchmark: + Multi-LLM (GPT-5.4 + Opus 4.6) on top of parallel thinking 81.4% -> 91.6% (+10.2pp) while relative cost falls 2.13x -> 0.95x (baseline code agent = 1.00x). Bundled with GEPA-optimized prompts (\"Multi-LLM design with optimized prompts\").", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "The Databricks AI Research Team, \"Pushing the Frontier for Data Agents with Genie\"", "url": "https://www.databricks.com/blog/pushing-frontier-data-agents-genie", "date": "2026-05-08", "locator": "Multi-LLM section; Figure 6", "task": "Optimize each sub-agent prompt per model with GEPA", "effect": "Figure 6, end-to-end accuracy of the Multi-LLM agent vs relative token cost (Sonnet 4.5 baseline = 1.0), approximate reads, baseline -> GEPA-optimized table-search sub-agent: Sonnet 4.5 ~73.1% @1.00 -> ~78.2% @~0.80; GPT-5.4 ~71.8% @~0.76 -> ~78.2% @~0.74; GPT-5.4-mini ~75.3% @~0.72 -> ~76.9% @~0.66; Opus 4.6 ~66.7% @~0.98 -> ~74.4% @0.90. Every model gained accuracy and lost cost; weakest baseline (Opus 4.6 as table-search sub-agent) gained most.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 7.2; Observation 3; Table 12 \"Model-Family Prompt Matrix\"", "task": "Dispatch distinct base prompts per model family", "effect": "OpenCode \"nine base prompts by model-id substring\"; Codex per-generation prompts delivered as server-side catalog data; Hermes \"15 guidance blocks gated by model family (tool-use enforcement blocks are sent to GPT/Codex/Gemini/Grok/Qwen/DeepSeek models and never to Claude, with code comments citing observed per-model failures).\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 7.1; Sec 12.3; Recommendation 2", "task": "Keep a per-model metadata registry", "effect": "Aider \"per-model metadata for 350+ models\"; \"This data-driven approach enables support for new models without code changes\"; Pi \"20 per-model compat quirk flags\"; OpenCode 1,400-line per-vendor transform matrix.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 8.4; Observation 4; Sec 15.4; Table 11 \"Polymorphic Edits\"", "task": "Select edit format / edit toolset per model", "effect": "Aider picks among 13 edit formats via Coder.create() from model metadata; OpenCode \"GPT-family models receive a port of Codex's apply_patch DSL and lose edit/write entirely\"; OpenHands \"model-specific tool presets (default, gemini, gpt5, planning)\". The paper calls editing strategy \"one of the most important determinants of code-modification accuracy\" (unmeasured).", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Benefits of a multi-agent system", "task": "Upgrade the model generation before doubling token budget", "effect": "\"upgrading to Claude Sonnet 4 is a larger performance gain than doubling the token budget on Claude Sonnet 3.7\" (BrowseComp analysis; no absolute numbers).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Implementation Details, Experimental Setup", "task": "Use a non-reasoning model as the tool-calling policy", "effect": "\"The reasoning LLM (DeepSeek-R1) is not chosen for AutoLink's policy due to observed instruction following degradation ... which occasionally leads to deviations from the required thought and action format.\" Not quantified.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 13", "task": "GPQA", "effect": "Train 0.61 to 0.58 (regressed) in phase 1.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper §4.3, Table 10", "task": "GAIA/FACTS/SimpleQA (Knight agent)", "effect": "Knight's 140-line prompt bakes in CoT, ReAct and self-verification; it is harder to improve but far more stable than Pawn.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, VeRO code release", "url": "https://github.com/scaleapi/vero", "date": null, "locator": "figure_01", "task": "five follow-up benchmarks", "effect": "8 to 14 of 20 runs per benchmark.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Prompt engineering: Scale effort to query complexity", "task": "Embed effort-scaling rules keyed to query complexity", "effect": "\"Simple fact-finding requires just 1 agent with 3-10 tool calls, direct comparisons might need 2-4 subagents with 10-15 calls each, and complex research might use more than 10 subagents\". Fixes errors \"like spawning 50 subagents for simple queries\"; over-investment \"was a common failure mode in our early versions.\"", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Albert Chen, Manas Bundele, Gaurav Ahlawat, et al. (LinkedIn), \"Text-to-SQL for Enterprise Data Analytics\"", "url": "https://arxiv.org/abs/2507.14372", "date": "2025-07-18", "locator": "§2.3.1", "task": "Skip reflection on easy questions and prefetch metadata", "effect": "Stated for speed only: \"it evaluates question difficulty, bypasses self-reflection for simple queries, and pre-fetches table metadata to minimize routine tool calls.\" Not quantified.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Xiaofang Yang, Lijun Li, Heng Zhou, Tong Zhu, Xiaoye Qu, Yuchen Fan, et al. (Shanghai AI Lab et al.), \"Toward Efficient Agents: A Survey of Memory, Tool Use, and Planning\"", "url": "https://arxiv.org/abs/2601.14192", "date": "2026-01-20", "locator": "§3.3.1 Rule-enhanced Retrieval; Hierarchical Retrieval", "task": "Scale retrieval depth to query difficulty", "effect": "The rationale is to control 'online access cost through query-adaptive retrieval'. No numbers are quoted.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Xiaofang Yang, Lijun Li, Heng Zhou, Tong Zhu, Xiaoye Qu, Yuchen Fan, et al. (Shanghai AI Lab et al.), \"Toward Efficient Agents: A Survey of Memory, Tool Use, and Planning\"", "url": "https://arxiv.org/abs/2601.14192", "date": "2026-01-20", "locator": "§4.2 Cost-Aware Tool Calling; §4.3 Selective Invocation; §4.4 'Knowing When Not to Call Tools'", "task": "Call tools only when model confidence is low", "effect": "Xu et al.'s method 'could reduce the number of tool calls'. Framing: 'an efficient tool user... calls tools only when the expected reduction in uncertainty or error justifies the cost'.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Xiaofang Yang, Lijun Li, Heng Zhou, Tong Zhu, Xiaoye Qu, Yuchen Fan, et al. (Shanghai AI Lab et al.), \"Toward Efficient Agents: A Survey of Memory, Tool Use, and Planning\"", "url": "https://arxiv.org/abs/2601.14192", "date": "2026-01-20", "locator": "§4.2 Efficient Test-Time Scaling", "task": "Pick respond/retrieve/tool/verify/stop by estimated utility", "effect": "Described only qualitatively. The survey's point is that efficient tool calling must control 'when to stop, verify, or search'.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Xiaofang Yang, Lijun Li, Heng Zhou, Tong Zhu, Xiaoye Qu, Yuchen Fan, et al. (Shanghai AI Lab et al.), \"Toward Efficient Agents: A Survey of Memory, Tool Use, and Planning\"", "url": "https://arxiv.org/abs/2601.14192", "date": "2026-01-20", "locator": "§5.1 Adaptive Budgeting and Control; §7 (MLLM fast-slow)", "task": "Default to a fast policy, escalate to slow planning", "effect": "SwiftSage reports token use and time steps, but the survey quotes no numbers. The rationale is 'reserving expensive reasoning for difficult decisions'.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Lennart Meincke, Ethan Mollick, Lilach Mollick, Dan Shapiro (Wharton Generative AI Labs), \"Prompting Science Report 2: The Decreasing Value of Chain of Thought in Prompting\"", "url": "https://arxiv.org/abs/2506.07142", "date": "2025-06-08", "locator": "Summary; Results, Figures 1-3; Figure S2; Discussion and Conclusion", "task": "Adds a \"Think step by step\" prompt, against an answer-directly prompt and a no-instruction default, on reasoning and non-reasoning models, 25 trials per question", "effect": "Reasoning models, average rating against direct: o3-mini RD = 0.029 (p = .024), o4-mini RD = 0.031 (p = .003), Gemini 2.5 Flash RD = -0.033 (p = .005), and Flash 2.5 lost 0.131 on the 25-of-25-correct metric; \"CoT requests took between 20-80% (10-20 seconds) longer than direct requests.\" Non-reasoning models gained against direct (Gemini 2.0 Flash RD = 0.135, Sonnet 3.5 RD = 0.117); against the no-instruction default, Gemini 2.0 Flash still gained significantly (RD = 0.062, p < .001) while Sonnet 3.5 did not (RD = -0.019, p = 0.189). \"For dedicated reasoning models, the added benefits of explicit CoT prompting appear negligible and may not justify the substantial increase in processing time.\"", "direction": "supports", "measured": true, "conditions": "GPQA Diamond, 198 questions x 25 trials per condition; reasoning: o3-mini, o4-mini, Gemini 2.5 Flash; non-reasoning: Claude 3.5 Sonnet, Gemini 2.0 Flash, GPT-4o-mini, GPT-4o, Gemini 1.5 Pro; a plain step-by-step prompt, not a composed structure", "older": true }, { "source": "Soumya Suvra Ghosal, Souradip Chakraborty, Avinash Reddy, Yifu Lu, Mengdi Wang, Dinesh Manocha, Furong Huang, Mohammad Ghavamzadeh, Amrit Singh Bedi, \"Does Thinking More always Help? Mirage of Test-Time Scaling in Reasoning Models\"", "url": "https://arxiv.org/abs/2506.04210", "date": "2025-06-04", "locator": "§2.3 Figure 2; §4 Figures 6-7", "task": "Reasoning models made to keep re-checking by suppressing the end-of-thinking token and appending \"Wait\", against parallel thinking: the same token budget spread over independent traces with a majority vote", "effect": "DeepSeek-R1-Distill-Qwen-1.5B on GSM8K: \"accuracy increases from 82.2% to 87.3% as the average number of thinking tokens increases from 385 to 1100\", then \"pushing the average thinking token count from 1100 to 15980 reduces accuracy from 87.3% to 70.3%\". At a 16,000-token budget, \"Wait & Think More\" gave \"a 11.8% performance drop\" against standard thinking, while parallel thinking with the same budget \"boosts accuracy by 10.1%\". The authors report \"a consistent non-monotonic trend in performance across multiple tasks and datasets\".", "direction": "supports", "measured": true, "conditions": "DeepSeek-R1-Distill-Qwen-1.5B, -Qwen-7B and -Llama-8B; GSM8K (1,320), MATH-500, AIME 2024 (30)", "older": true }, { "source": "Liwei Kang, Yue Deng, Yao Xiao, Zhanfeng Mo, Wee Sun Lee, Lidong Bing, \"First Try Matters: Revisiting the Role of Reflection in Reasoning Models\"", "url": "https://arxiv.org/abs/2510.08308", "date": "2025-10-09", "locator": "§2.2 Figures 3-4; §4 Table 2", "task": "Stops reasoning early once candidate answers appear, after finding most later reflections only confirm the first answer", "effect": "\"over 90% of the reflections are confirmatory\"; corrective ones (wrong to right) are \"mostly less than 2%\". Reflections after the first candidate took 16.8% to 47.8% of the tokens for a gain of 1.4% to 3.5%. Stopping early on Qwen3-8B cut reasoning tokens by 24.5% for a 2.9-point average accuracy drop (84.7 to 81.8).", "direction": "supports", "measured": true, "conditions": "eight reasoning models from 7B to 685B (DeepSeek-R1-0528, gpt-oss-20b, Magistral-Small-2506, MiMo-7B-RL, Qwen3-8B, three R1 distills); AIME 2024 and 2025, AMC, Olympiad Bench, MATH-500; 3,427 rollouts per model; answers checked with Math-Verify", "older": true } ], "caveats": "The model checkpoint itself is off-limits under VeRO's rules; only its call parameters move.", "also_covers": [ { "name": "Chain-of-thought" }, { "name": "Routing to specialists" } ], "related": [ { "id": "9.c.ii", "slug": "samples-before-verification", "name": "Spend extra samples before extra verification: parallel attempts with a vote or judge; verify only where it pays" }, { "id": "6.b.iii", "slug": "multi-agent-for-breadth", "name": "Use multi-agent orchestration only for breadth-first, independent sub-tasks" }, { "id": "6.c.i", "slug": "code-then-execute", "name": "Code-then-execute (CaMeL)" }, { "id": "4.c.i", "slug": "episodic-memory", "name": "Episodic memory retrieval" }, { "id": "7.c.iv", "slug": "graph-of-thought", "name": "Graph-of-Thought" }, { "id": "7.d.ii", "slug": "inversion-of-control", "name": "Inversion of control" }, { "id": "5.c.ii", "slug": "isolated-subagent-context", "name": "Sub-agent architectures with clean contexts" }, { "id": "7.c.v", "slug": "language-agent-tree-search", "name": "Language Agent Tree Search (LATS)" }, { "id": "6.b.ii", "slug": "llm-map-reduce", "name": "LLM map-reduce" }, { "id": "7.c.vi", "slug": "multi-agent-debate", "name": "Multi-agent debate" }, { "id": "6.a.i", "slug": "oracle-worker-escalation", "name": "Oracle-worker multi-model" }, { "id": "6.c.ii", "slug": "plan-then-execute", "name": "Plan-then-execute" }, { "id": "7.c.i", "slug": "reflection-loop", "name": "Reflection loop" }, { "id": "3.c.i", "slug": "self-discover-reasoning", "name": "Self-Discover reasoning structures" }, { "id": "7.c.vii", "slug": "tree-of-thought", "name": "Tree-of-Thought" } ], "url": "https://mahalsystems.ai/aot/optimizations/model-and-effort-per-call-site/", "markdown": "https://mahalsystems.ai/aot/optimizations/model-and-effort-per-call-site.md" }, { "id": "2.a.i", "legacy_id": "T60", "former_ids": [], "slug": "reduce-tool-surface", "name": "Reduce the tool surface: remove and consolidate tools, then defer the rest behind search", "kind": "technique", "status": "card", "tier": { "tier": 1, "name": "Hygiene", "why": "removing tools that are never called, verified unchanged on the hand-labeled set", "proposed": false }, "levers": [ "tool-surface", "context" ], "domains": [ "tool-use" ], "evidence_domains": [ "tool-use", "code-generation" ], "categories": [ "Refactoring", "Filtering input" ], "category_ids": [ "tools.refactor", "context.filter-input" ], "summary": "Offer the agent only the tools the task needs. First remove tools no task uses; then consolidate tools with overlapping scopes (read and write tools with a method parameter, tools grouped around jobs), because rewording cannot separate tools that genuinely overlap. Then keep a small core always loaded and defer the rest: names and one-line purposes in the prompt, full definitions loaded on demand through tool search, per-server files or expandable groups. Defer once definitions exceed about 10k tokens or 10 tools; keep deferred groups under about 10 tools each. Every tool offered has a cost even when not called: one irrelevant tool cut accuracy from 98.2% to 63.5%. Deferral costs extra calls (2-3x) and time (about 50%), so consolidate first.", "how": [ "From traces over the eval set, list each tool's call count and the cases where it was needed.", "Remove tools with no calls and no needing case.", "Find overlapping pairs (similar names or descriptions, or cases routed to the wrong one of the pair) and merge them; a method parameter often replaces several tools.", "If what remains exceeds about 10 tools or 10k tokens, keep a core loaded and defer the rest behind search or expandable groups.", "Run the paired eval, old set against new, and compare coverage, success, calls and latency." ], "detect": { "static": "more than about 10 tools, or more than about 10k tokens of tool definitions, in every prompt; duplicate or near-duplicate tool names or descriptions (73% of audited MCP servers have repeated names); tools the agent's credentials cannot use", "traces": "tools never called across the eval runs; tool-use coverage below the eval's needs (share of needing cases where the right tool was loaded and chosen; GitHub measured 69% for a static list); calls to a tool where none was needed; failure rows labeled incorrect-tool-selection concentrated on one pair of tools", "evals": "accuracy falling as tools are added (an ablation over catalog size); paired eval, current tool set against a reduced one" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "GitHub +2-5 points on SWE-bench with 40 to 13 tools (Nov 2025); Anthropic tool search 49% to 74% (Opus 4); one irrelevant tool cost 34.7 points on questions answerable without tools (Sep 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "large", "when": "definitions -85% (Anthropic); -46.9% tokens (Cursor A/B); deferral alone raises call count", "basis": "evidence" }, { "dimension": "performance", "effect": "mixed", "size": "medium", "when": "consolidation faster (GitHub -400 ms); deferral about 50% slower with 2-3x calls (Speakeasy)", "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "verify-fix", "tier": 1, "how": "replay the hand-labeled set after removing unused tools" }, { "method": "experiment", "tier": 3, "how": "paired eval, old tool set against new: success, coverage, calls, latency" } ], "evidence": [ { "source": "Jediah Katz (Cursor), \"Dynamic context discovery\"", "url": "https://cursor.com/blog/dynamic-context-discovery", "date": "2026-01-06", "locator": "4. Efficiently loading only the MCP tools needed; footnote 1", "task": "Sync MCP tool descriptions to per-server folders", "effect": "\"In an A/B test, we found that in runs that called an MCP tool, this strategy reduced total agent tokens by 46.9% (statistically significant, with high variance based on the number of MCPs installed).\" No n, model or quality metric given.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Adam Jones, Conor Kelly (Anthropic), \"Code execution with MCP: Building more efficient agents\"", "url": "https://www.anthropic.com/engineering/code-execution-with-mcp", "date": "2025-11-04", "locator": "Code execution with MCP improves context efficiency", "task": "Expose MCP tools as a code-API filesystem", "effect": "\"This lets the agent load only the definitions it needs for the current task. This reduces the token usage from 150,000 tokens to 2,000 tokens, a time and cost saving of 98.7%.\" (single Google Drive to Salesforce example; model not stated.) Cloudflare reports similar findings (\"Code Mode\").", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Adam Jones, Conor Kelly (Anthropic), \"Code execution with MCP: Building more efficient agents\"", "url": "https://www.anthropic.com/engineering/code-execution-with-mcp", "date": "2025-11-04", "locator": "Benefits: Progressive disclosure", "task": "Add a search_tools tool with detail levels", "effect": "\"Including a detail level parameter in the search_tools tool that allows the agent to select the level of detail required ... also helps the agent conserve context and find tools efficiently.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Drew Breunig, \"How Long Contexts Fail / How to Fix Your Context\", https://www.dbreunig.com/2025/06/22/how-contexts-fail-and-how-to-fix-them.html ; https://www.dbreunig.com/2025/06/26/how-to-fix-your-context.html (2025-06-26)", "url": null, "date": "2025-06-22", "locator": "06-26 post, \"Tool Loadout\"", "task": "Select a per-task tool loadout via retrieval", "effect": "RAG-MCP (Gan & Sun), DeepSeek-v3: selection \"becomes critical when you have more than 30 tools\"; \"Beyond 100 tools, the model was virtually guaranteed to fail\"; RAG-selecting under 30 tools gave \"as much as 3x better tool selection accuracy\" and shorter prompts. \"Less is More\": Llama 3.1 8b fails with 46 tools, succeeds with 19; dynamic tool selection improved Llama 3.1 8b by 44% on BFCL; where accuracy did not improve, it saved 18% power and 77% time.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Xiaofang Yang, Lijun Li, Heng Zhou, Tong Zhu, Xiaoye Qu, Yuchen Fan, et al. (Shanghai AI Lab et al.), \"Toward Efficient Agents: A Survey of Memory, Tool Use, and Planning\"", "url": "https://arxiv.org/abs/2601.14192", "date": "2026-01-20", "locator": "§4.1 Tool Selection (External Retriever, MLC); §4.4", "task": "Retrieve top-k tools instead of listing all", "effect": "TinyAgent (DeBERTa-v3 small multi-label selector, >50% threshold) 'reduces nearly half of the prompt size'. The other methods give no numbers. Trade-off: aggressive selection 'may filter out necessary tools'.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 8.3; Recommendation 4; Table 11 \"Deferred Loading\"; Appendix table (Claude Code row)", "task": "Defer tool loading behind a tool-search tool", "effect": "\"Claude Code's shouldDefer flag + ToolSearchTool reduces the initial prompt by 40%\" (43 tools; no further conditions given); Hermes collapses catalogs \"once schemas would exceed 10% of the context window\"; \"Below 15 tools the indirection is not worth the added complexity.\"", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 8.3", "task": "Front a large MCP catalog with a code-mode execute tool", "effect": "OpenCode \"treats its experimental code mode as MCP-catalog deferral: instead of exposing every MCP tool, one execute tool runs model-written scripts against the catalog.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Lance Martin, \"Agent Design Patterns\"", "url": "https://rlancemartin.github.io/2026/01/09/agent_design/", "date": "2026-01-09", "locator": "Progressive Disclosure", "task": "Index tool definitions and retrieve them on demand", "effect": "\"some agents index tool definitions and retrieve tools on demand, potentially using a search tool to encapsulate this process.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Lance Martin, \"Agent Design Patterns\"", "url": "https://rlancemartin.github.io/2026/01/09/agent_design/", "date": "2026-01-09", "locator": "Progressive Disclosure", "task": "List CLI utilities in instructions; agent reads --help", "effect": "Manus \"just provides a list of available utilities in the agent instructions. Then agent uses its bash tool to call the --help flag ... to learn about any one of them if needed.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Lance Martin, \"Agent Design Patterns\"", "url": "https://rlancemartin.github.io/2026/01/09/agent_design/", "date": "2026-01-09", "locator": "Progressive Disclosure", "task": "Package skills as SKILL.md folders, frontmatter only", "effect": "\"The YAML frontmatter of each file is loaded into agent instructions, letting the agent decide to read SKILL.md only if needed.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Mak et al., \"Is Bash All You Need? An Empirical Study of Tool Interfaces for Enterprise Digital Worker Agents\"", "url": "https://arxiv.org/abs/2609.11999", "date": "2026-09-10", "locator": null, "task": "Give a bash-only interface rather than typed tools for enterprise digital-worker tasks", "effect": "Opus-4.8 and GPT-5.5 on TheAgentCompany and APEX-Agents, five interfaces compared: bash-only beats typed tools by 21.8-24.5 pp on TheAgentCompany and 4.8-7.4 pp on APEX-Agents, with 19-72% fewer tokens. Adding typed or agent-synthesized tools on top of bash gives no measurable gain; programmatic tool calling saves tokens but scores below bash.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Repantis et al., \"How Many Tools Should an LLM Agent See? A Chance-Corrected Answer\"", "url": "https://arxiv.org/abs/2605.24660", "date": "2026-05-23", "locator": null, "task": "Show an adaptive shortlist of tools rather than a fixed top-k", "effect": "Registries of 20-3,251 tools, chance-corrected coverage metric, RL-trained adaptive-depth selector: BFCL (370 tools) 90.3% coverage while showing about 7 tools, against 90.8% at fixed depth; hard ToolBench queries (3,251 tools) 16.7% where fixed top-5 gets 0. With Claude Sonnet 4.6 choosing: 93.1% vs 87.1% selection accuracy, 76.8% vs 60.9% on medium queries.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Paturi et al., \"When Tools Get in the Way: The Effect of Unnecessary Tool Availability on LLM Answering\"", "url": "https://arxiv.org/abs/2609.14157", "date": "2026-09-12", "locator": null, "task": "Do not offer tools the task does not need", "effect": "500 query pairs, 10 domains, 6 LLMs, every query answerable from the model's own knowledge: accuracy falls from 98.2% to 63.5% once an irrelevant tool is offered, whether or not the model calls it.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Wang et al., \"From Docs to Descriptions: Smell-Aware Evaluation of MCP Server Descriptions\"", "url": "https://arxiv.org/abs/2602.18914", "date": "2026-02-21", "locator": null, "task": "Fix description defects: purpose, functionality and accuracy", "effect": "18 description smells audited over 10,831 MCP servers; 73% have repeated tool names. Controlled mutations: functionality and accuracy defects shift selection most (+11.6%, +8.8%, p<0.001). Among functionally equivalent servers, a standard-compliant description is picked 72% of the time against a 20% baseline.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Zhou et al., \"A Single Rewrite Suffices: Empirical Lessons from Production Skill Description Optimization\"", "url": "https://arxiv.org/abs/2606.30775", "date": "2026-06-29", "locator": null, "task": "Consolidate tools whose scopes genuinely overlap", "effect": "Same study: where two skills genuinely overlap in scope, rewording cannot separate them; the paper gives a signal for detecting such pairs, which need consolidation instead.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Shiyang Chen, \"Looking Is Not Picking: An Attention-Segment Account of Tool-Selection Failures in LLM Agents\"", "url": "https://arxiv.org/abs/2606.16364", "date": "2026-06-15", "locator": null, "task": "Do not assume pruning the input fixes wrong-tool selection", "effect": "3B-32B open models: attention lands most on the correct tool 80% of the time (21% by chance), so failures happen at the decision step. Input fixes recover at most 23% of failures; decision-step interventions 59-91%. Training-free attention-based selector +11.9 pts function-name accuracy on BFCL. Single-turn only.", "direction": "mixed", "measured": true, "conditions": null, "older": false }, { "source": "Xiao et al., \"SING: Synthetic Intention Graph for Scalable Active Tool Discovery in LLM Agents\"", "url": "https://arxiv.org/abs/2606.16591", "date": "2026-06-15", "locator": null, "task": "Retrieve tools from an intention and co-usage graph", "effect": "7,471 tools, three benchmarks: up to +59.8% Global Recall@5 and +28.9% downstream success over retrieval baselines, exposing 99.8% less tool schema.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Suresh Babu and Ganesh Iyer, \"ToolChoiceConfusion: Causal Minimal Tool Filtering for Reliable LLM Agents\"", "url": "https://arxiv.org/abs/2606.06284", "date": "2026-06-04", "locator": null, "task": "Filter tools per step by precondition and effect", "effect": "102 tasks, 100 tools, 4 model backends, 2,448 runs: matches stronger causal baselines while showing about 1 tool per step instead of 100, ~90% fewer tokens than loading all tools.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Li et al., \"Scalable LLM Agent Tool Access in the Cloud\"", "url": "https://arxiv.org/abs/2607.15593", "date": "2026-07-17", "locator": null, "task": "Front a large tool catalog with hybrid retrieval", "effect": "Production MCP gateway over 3,000+ tools: 98% top-15 recall, 8.9x faster tool selection and 23.8x fewer tokens than mounting all tools. Systems metrics, not task success.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Li et al., \"ComplexMCP: Evaluation of LLM Agents in Dynamic, Interdependent, and Large-Scale Tool Sandbox\"", "url": "https://arxiv.org/abs/2605.10787", "date": "2026-05-11", "locator": null, "task": "Expect performance to fall as more tools are available", "effect": "300+ tools across 7 stateful sandboxes: best models under 60% against 90% for humans; names tool retrieval saturation (performance falls as tools grow), over-confidence and strategic defeatism as the main failure modes. No count-vs-success curve in the abstract.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Bandi et al. (Scale AI), \"MCP-Atlas: A Large-Scale Benchmark for Tool-Use Competency with Real MCP Servers\"", "url": "https://arxiv.org/abs/2602.00933", "date": "2026-01-31", "locator": null, "task": "Measure selection with plausible distractor tools", "effect": "1,000 tasks, 36 real servers, 220 tools, 6-37 offered per task (mean 15.2) with plausible distractors, 20 models: best pass rate 82.2%; 63.3% of diagnosed failures are reasoning failures, not tool-call failures.", "direction": "mixed", "measured": true, "conditions": null, "older": false }, { "source": "Anisha Agarwal and Connor Peet (GitHub), \"How we're making GitHub Copilot smarter with fewer tools\"", "url": "https://github.blog/ai-and-ml/github-copilot/how-were-making-github-copilot-smarter-with-fewer-tools/", "date": "2025-11-19", "locator": null, "task": "Cut the default toolset and group the rest behind expandable virtual tools", "effect": "Default built-in tools cut from about 40 to 13; the rest in 4 expandable virtual-tool groups, grouped by embedding similarity. Tool-use coverage 94.5% (embeddings) vs 87.5% (LLM grouping) vs 69% (static list). SWE-Lancer and SWE-bench Verified +2-5 points with GPT-5 and Sonnet 4.5; average response time -400 ms, time to first token -190 ms. An oversized toolset made the agent ignore instructions and call unneeded tools.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Bin Wu et al. (Anthropic), \"Introducing advanced tool use on the Claude Developer Platform\"", "url": "https://www.anthropic.com/engineering/advanced-tool-use", "date": "2025-11-24", "locator": null, "task": "Defer tool loading behind tool search", "effect": "Tool Search with defer_loading: tool-definition tokens ~77k to 8.7k (-85%); internal MCP evals 49% to 74% (Opus 4) and 79.5% to 88.1% (Opus 4.5). Recommended when definitions exceed 10k tokens or there are 10+ tools. Internal evals only.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Eric Kim (Amplitude), \"Bring headless Amplitude where you work\"", "url": "https://amplitude.com/blog/headless-amplitude", "date": "2026-09-16", "locator": null, "task": "Consolidate an MCP server's tools around jobs, and regression-test the new set", "effect": "MCP server cut from 100+ tools to about 40, grouped around jobs; paired regression over 80+ scenarios: task completion matched or improved in every paired suite. No per-suite numbers.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Ryan Caldwell and Bhavya U (Microsoft), \"Improving token efficiency for GitHub Copilot in VS Code\"", "url": "https://code.visualstudio.com/blogs/2026/06/17/improving-token-efficiency-in-github-copilot", "date": "2026-06-17", "locator": null, "task": "Keep a core toolset loaded and defer the rest behind server-side search", "effect": "Deferred tools shown by name and description only, a curated always-loaded core, embedding tool search for Anthropic models: server-side tool search cut total tokens about 11% per turn; user error rate -4.01% with Sonnet 4.6.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "OpenAI, \"Tool search (API guide)\"", "url": "https://developers.openai.com/api/docs/guides/tools-tool-search", "date": "2026-09-23", "locator": null, "task": "Group deferred tools into small namespaces with use-case descriptions", "effect": "Vendor guidance, not measured: namespaces under 10 functions with short use-case descriptions; defer when the catalog is large and each task needs few tools, load eagerly when small; mixing eager and deferred generally not recommended. Living doc; date is the access date.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Chase Crumbaugh (Speakeasy), \"Reducing MCP token usage by 100x — you don't need code mode\"", "url": "https://www.speakeasy.com/blog/how-we-reduced-token-usage-by-100x-dynamic-toolsets-v2", "date": "2025-11-18", "locator": null, "task": "Replace the catalog with search, describe and execute meta-tools", "effect": "Sonnet 4.5 over 40-400 tools, vendor benchmark: 100% success; input tokens -96.7% (simple) and -91.2% (complex). Cost: 2-3x more tool calls (6-8 vs 3) and about 50% slower.", "direction": "mixed", "measured": true, "conditions": null, "older": false }, { "source": "GitHub, \"GitHub MCP Server changelogs (Oct 2025, Jan 2026)\"", "url": "https://github.blog/changelog/2026-01-28-github-mcp-server-new-projects-tools-oauth-scope-filtering-and-new-features/", "date": "2026-01-28", "locator": null, "task": "Merge related tools into read and write tools with a method parameter", "effect": "Consolidated many tools into read/write tools with a method parameter (issue_read, issue_write); smaller default toolset; server instructions. Projects consolidation saved about 23k tokens (~50%). Tools the user's token scopes cannot use are hidden. Tokens only, no accuracy data.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Matt Carey (Cloudflare), \"Code Mode: give agents an entire API in 1,000 tokens\"", "url": "https://blog.cloudflare.com/code-mode-mcp/", "date": "2026-02-20", "locator": null, "task": "Expose a large API through search and execute in code", "effect": "2,500+ API endpoints behind search() and execute(); the model writes code against the typed spec in a sandbox. About 1k tokens against 1.17M, calculated not tested.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Guillaume Lebedel (StackOne), \"Comparing BM25, TF-IDF, and Hybrid Search for MCP Tool Discovery\"", "url": "https://www.stackone.com/blog/mcp-tool-search-bm25-tfidf-hybrid/", "date": "2026-02-17", "locator": null, "task": "Choose the tool-search method by how many results the agent sees", "effect": "270 tools, 2,700 queries: top-1 retrieval BM25 14%, hybrid 21%, embeddings 38%; top-5 85-90% for all three. Whether a tool is found depends on distinctive terms in its description. Retrieval only, not task success.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Lily Chen et al. (Datadog), \"Gain end-to-end visibility into MCP clients with Datadog Agent Observability\"", "url": "https://www.datadoghq.com/blog/mcp-client-monitoring/", "date": "2025-12-01", "locator": null, "task": "Trace tool listing and calls, linked to the model call that chose them", "effect": "Traces tools/list and call_tool, each linked to the LLM call that chose the tool; per-tool and per-server error rate, retries and p95 latency; large registries add latency. No wrong-tool or unused-tool metric. Not measured.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Grace et al. (Anthropic), \"Demystifying evals for AI agents\"", "url": "https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents", "date": "2026-01-09", "locator": null, "task": "Grade outcomes, not exact tool-call sequences", "effect": "Warns against grading exact tool-call sequences as too rigid and brittle; grade the outcome and read transcripts; production monitoring is the ground truth after launch. Not about tool counts.", "direction": "mixed", "measured": false, "conditions": null, "older": false } ], "caveats": null, "also_covers": [ { "name": "Consolidate tools into higher-level operations" }, { "name": "Progressive tool discovery" } ], "related": [ { "id": "2.a.ii", "slug": "rewrite-tool-surface", "name": "Rewrite the tool surface: names, schemas, descriptions" }, { "id": "2.b.ii", "slug": "tool-use-guidance", "name": "Tell the agent when to use each tool, in context it always sees" }, { "id": "2.a.iii", "slug": "choose-action-interface", "name": "Choose how the agent acts: typed tools, code execution, or shell, matched to the model and task" }, { "id": "5.c.ii", "slug": "isolated-subagent-context", "name": "Sub-agent architectures with clean contexts" }, { "id": "3.b.ii", "slug": "mask-tools-not-remove", "name": "Mask tools instead of removing them" }, { "id": "4.c.ii", "slug": "skill-library", "name": "Skill library evolution" }, { "id": "8.a.ii", "slug": "surface-unreadable-information", "name": "Surface information the source holds but the agent cannot read" } ], "url": "https://mahalsystems.ai/aot/optimizations/reduce-tool-surface/", "markdown": "https://mahalsystems.ai/aot/optimizations/reduce-tool-surface.md" }, { "id": "2.a.ii", "legacy_id": "T35", "former_ids": [], "slug": "rewrite-tool-surface", "name": "Rewrite the tool surface: names, schemas, descriptions", "kind": "technique", "status": "card", "tier": { "tier": 1, "name": "Hygiene", "why": "when the hand-labeled set shows behavior unchanged", "proposed": false }, "levers": [ "tool-surface" ], "domains": [ "tool-use" ], "evidence_domains": [ "tool-use", "code-generation", "web-research" ], "categories": [ "Refactoring", "Avoiding misuse" ], "category_ids": [ "tools.refactor", "tools.misuse" ], "summary": "Change what the model sees of each tool without touching its implementation: a distinct name, a strict schema (enums, required fields), and a description with fixed sections (what it is for and when to use it versus sibling tools, defaults and limits, error modes, side effects), plus a usage example for complex parameters. Write for the model: distinctive terms, since search finds tools by them; error messages that say what to do next. One rewrite fed the misrouted cases matches hand-tuning; richer descriptions add steps, so slim them after. Where two tools genuinely overlap, rewording cannot separate them: consolidate instead (2.a.i).", "how": [ "From traces, collect misrouted cases per tool (called when it should not have been, and not called when it should).", "Audit descriptions for missing purpose, missing when-to-use, and duplicate names.", "Rewrite once with an LLM, given the misrouted cases; add a usage example for complex parameters.", "Slim the result and check step count.", "Replay the hand-labeled set, or run the paired eval; pairs that stay confused go to 2.a.i for consolidation." ], "detect": { "static": "descriptions that do not state the tool's purpose (56% of audited MCP tools) or when to use it; duplicate or overlapping names; schemas without enums or required fields; no example for complex parameters", "traces": "misrouted cases per tool (false positives and false negatives); failure rows labeled incorrect-tool-selection, malformed-arguments or suboptimal-arguments; per-tool error rate and retries (Datadog tracks these per tool and server)", "evals": "failures attributed to tool selection or arguments; step count per task after a description change" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "+5.85 pp median success (Feb 2026); complex-parameter accuracy 72% to 90% with examples (Anthropic); a compliant description picked 72% vs 20% among equivalents", "basis": "evidence" }, { "dimension": "cost", "effect": "mixed", "size": "medium", "when": "augmented descriptions added 67% more steps; slim variants keep most of the gain", "basis": "evidence" }, { "dimension": "performance", "effect": "mixed", "size": "small", "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "verify-fix", "tier": 1, "how": "replay the hand-labeled set" }, { "method": "experiment", "tier": 3, "how": "paired eval, old against new descriptions" } ], "evidence": [ { "source": "Scale AI, VeRO code release", "url": "https://github.com/scaleapi/vero", "date": null, "locator": "figure_01", "task": "five follow-up benchmarks", "effect": "7 to 18 of 20 runs per benchmark.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 11", "task": "GAIA/FACTS/SimpleQA", "effect": "The Tool-Centric template is built around docstring guidance and parameter design.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Prompt engineering: Tool design and selection are critical", "task": "Give explicit tool-selection heuristics in the prompt", "effect": "\"an agent searching the web for context that only exists in Slack is doomed from the start\" ... \"Bad tool descriptions can send agents down completely wrong paths, so each tool needs a distinct purpose and a clear description.\"", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Prompt engineering: Let agents improve themselves", "task": "Use a tool-testing agent to rewrite tool descriptions", "effect": "\"This process for improving tool ergonomics resulted in a 40% decrease in task completion time for future agents using the new description, because they were able to avoid most mistakes.\" (no benchmark, n or model beyond \"Claude 4 models\")", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Fan et al., \"An Empirical Study of Harness Design for Coding Agents\"", "url": "https://arxiv.org/abs/2609.20804", "date": "2026-09-17", "locator": "App 8, Figs 21-31", "task": "Document each tool with When/Protocol/Errors/Side-effects", "effect": "Every tool description in the study uses this template. Not ablated.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Xiaofang Yang, Lijun Li, Heng Zhou, Tong Zhu, Xiaoye Qu, Yuchen Fan, et al. (Shanghai AI Lab et al.), \"Toward Efficient Agents: A Survey of Memory, Tool Use, and Planning\"", "url": "https://arxiv.org/abs/2601.14192", "date": "2026-01-20", "locator": "§4.1 External Retriever; §4.2 Cost-Aware Tool Calling", "task": "Rewrite and compress tool documentation for retrieval", "effect": "Described as reducing token overhead and redundant calls. No numbers are quoted.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 7.3 \"Tool-use philosophy\"", "task": "Prefer dedicated tools over shell in instructions", "effect": "Claude Code \"Do NOT use the Bash tool to run commands when a relevant dedicated tool is provided ... allows the user to better understand and review your work.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 7.2 (Pi)", "task": "Generate tool guidance from the active toolset", "effect": "Pi \"each tool contributes promptSnippet/promptGuidelines, so the prompt co-varies with the active toolset\"; whole prompt is 30–40 lines.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Guo et al., \"Learning to Rewrite Tool Descriptions for Reliable LLM-Agent Tool Use (Trace-Free+)\"", "url": "https://arxiv.org/abs/2602.20426", "date": "2026-02-23", "locator": null, "task": "Rewrite tool descriptions with a trained rewriter", "effect": "StableToolBench with 150+ candidate tools: a curriculum-trained description rewriter, needing no per-tool traces, cuts the accuracy lost as the catalog grows by 29.23% and raises average query-level success by 60.89%; complementary to agent fine-tuning. Requires training the rewriter.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Wang et al., \"From Docs to Descriptions: Smell-Aware Evaluation of MCP Server Descriptions\"", "url": "https://arxiv.org/abs/2602.18914", "date": "2026-02-21", "locator": null, "task": "Fix description defects: purpose, functionality and accuracy", "effect": "18 description smells audited over 10,831 MCP servers; 73% have repeated tool names. Controlled mutations: functionality and accuracy defects shift selection most (+11.6%, +8.8%, p<0.001). Among functionally equivalent servers, a standard-compliant description is picked 72% of the time against a 20% baseline.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Hasan et al., \"Model Context Protocol (MCP) Tool Descriptions Are Smelly!\"", "url": "https://arxiv.org/abs/2602.14878", "date": "2026-02-16", "locator": null, "task": "Augment tool descriptions, then slim them", "effect": "856 tools on 103 MCP servers: 97.1% of descriptions have at least one smell; 56% do not state their purpose. Augmented descriptions: +5.85 pp median success, +15.12% partial completion, but +67.46% execution steps and 16.67% of cases worse. Slimmer variants keep most of the gain.", "direction": "mixed", "measured": true, "conditions": null, "older": false }, { "source": "Zhou et al., \"A Single Rewrite Suffices: Empirical Lessons from Production Skill Description Optimization\"", "url": "https://arxiv.org/abs/2606.30775", "date": "2026-06-29", "locator": null, "task": "Rewrite descriptions once from misrouted cases", "effect": "Production router (9 skills, 372 test cases) and ToolBench (16k tools): one LLM rewrite fed the false-positive and false-negative cases reaches 79.2% F1 against 79.4% hand-tuned, at 32x less effort. Iteration budget, feedback mix and training-set size each move F1 by under 0.5%.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Zhou et al., \"A Single Rewrite Suffices: Empirical Lessons from Production Skill Description Optimization\"", "url": "https://arxiv.org/abs/2606.30775", "date": "2026-06-29", "locator": null, "task": "Consolidate tools whose scopes genuinely overlap", "effect": "Same study: where two skills genuinely overlap in scope, rewording cannot separate them; the paper gives a signal for detecting such pairs, which need consolidation instead.", "direction": "mixed", "measured": true, "conditions": null, "older": false }, { "source": "Bin Wu et al. (Anthropic), \"Introducing advanced tool use on the Claude Developer Platform\"", "url": "https://www.anthropic.com/engineering/advanced-tool-use", "date": "2025-11-24", "locator": null, "task": "Add usage examples to tool definitions", "effect": "Internal testing: examples in tool definitions raised accuracy on complex parameters from 72% to 90%.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Rizel Scarlett (Block), \"8 Things You Didn't Know About Code Mode\"", "url": "https://goose-docs.ai/blog/2026/02/06/8-things-you-didnt-know-about-code-mode/", "date": "2026-02-06", "locator": null, "task": "Use code mode for multi-tool tasks only", "effect": "About 30% fewer tokens on identical tasks (one author's test, Opus 4.5); little benefit on single-tool tasks; model-dependent. A description mentioning 'long-running task' pulled in the wrong extension; explicit imports prevented it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Guillaume Lebedel (StackOne), \"Comparing BM25, TF-IDF, and Hybrid Search for MCP Tool Discovery\"", "url": "https://www.stackone.com/blog/mcp-tool-search-bm25-tfidf-hybrid/", "date": "2026-02-17", "locator": null, "task": "Choose the tool-search method by how many results the agent sees", "effect": "270 tools, 2,700 queries: top-1 retrieval BM25 14%, hybrid 21%, embeddings 38%; top-5 85-90% for all three. Whether a tool is found depends on distinctive terms in its description. Retrieval only, not task success.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Sattanathan, Byrro and Tahler (Arcade.dev), \"54 Patterns for Building Better MCP Tools\"", "url": "https://www.arcade.dev/blog/mcp-tool-patterns/", "date": "2026-02-09", "locator": null, "task": "Write descriptions and parameters for the model; errors that say what to do next", "effect": "From building 8,000+ tools, not measured: descriptions and parameter names written for the model, lenient input handling, error messages that tell the agent what to do next, consistent response shapes, batch operations instead of one call per item.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Lily Chen et al. (Datadog), \"Gain end-to-end visibility into MCP clients with Datadog Agent Observability\"", "url": "https://www.datadoghq.com/blog/mcp-client-monitoring/", "date": "2025-12-01", "locator": null, "task": "Trace tool listing and calls, linked to the model call that chose them", "effect": "Traces tools/list and call_tool, each linked to the LLM call that chose the tool; per-tool and per-server error rate, retries and p95 latency; large registries add latency. No wrong-tool or unused-tool metric. Not measured.", "direction": "supports", "measured": false, "conditions": null, "older": false } ], "caveats": null, "also_covers": [ { "name": "Low-friction tool input formats" }, { "name": "Poka-yoke tool design" }, { "name": "Tool namespacing" } ], "related": [ { "id": "2.b.ii", "slug": "tool-use-guidance", "name": "Tell the agent when to use each tool, in context it always sees" }, { "id": "2.a.i", "slug": "reduce-tool-surface", "name": "Reduce the tool surface: remove and consolidate tools, then defer the rest behind search" }, { "id": "2.b.i", "slug": "fail-soft-tools", "name": "Make tool calls and tool results fail soft, and retry transient failures" }, { "id": "7.b.ii", "slug": "keep-errors-in-context", "name": "Keep errors in context" }, { "id": "3.b.ii", "slug": "mask-tools-not-remove", "name": "Mask tools instead of removing them" } ], "url": "https://mahalsystems.ai/aot/optimizations/rewrite-tool-surface/", "markdown": "https://mahalsystems.ai/aot/optimizations/rewrite-tool-surface.md" }, { "id": "2.a.iii", "legacy_id": "T61", "former_ids": [], "slug": "choose-action-interface", "name": "Choose how the agent acts: typed tools, code execution, or shell, matched to the model and task", "kind": "technique", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes the agent's tools within the existing design", "proposed": false }, "levers": [ "tool-surface", "model" ], "domains": [ "tool-use" ], "evidence_domains": [ "code-generation", "tool-use", "data-analysis", "web-research", "reasoning", "retrieval" ], "categories": [ "Refactoring", "Benchmarking workloads" ], "category_ids": [ "tools.refactor", "models.workload-fit" ], "summary": "Decide the action interface deliberately, by model and by task, and provision what it needs. The rule the evidence supports: (1) weaker models get typed tools (read, write, edit, grep, a narrow calculator); a general code interpreter made a weak agent write boilerplate and fail until it was replaced with typed functions. (2) Structured data gets a typed query tool, whatever the model: SQL scored 100% against 52.7% for bash at a seventh of the tokens, and SQL plus bash also reached 100%. (3) Strong models on shell-heavy or general office work get bash: on enterprise digital-worker tasks bash-only beat typed tools by 21.8-24.5 points with fewer tokens, and typed tools on top of bash added nothing. (4) Work that chains many calls or processes bulk data gets code execution, because intermediate results never pass through the model; calling tools from code matched or beat JSON calling for 11 of 14 models on BFCL v4, and a narrow calculator did as well as full code execution on maths. Whichever you choose: start minimal and add tools per observed failure; for code or shell, provision the packages and fixtures the agent expects, allow concurrent commands, and sandbox it (10.a.i).", "how": [ "Classify the model's shell and coding ability from a probe run (typed tools against bash-only on 20 cases).", "Pick the interface by the table in the summary; for code execution, install the packages the tasks need and set up fixtures.", "Add tools only when a trace shows a failure a tool would have prevented.", "Paired eval per model: success, cost, steps." ], "detect": { "static": "the interface in use (typed tools, code interpreter, bash), the model it serves, and whether the environment has the packages and fixtures the tasks need", "traces": "bash-only runs emitting tools that do not exist; typed-tool runs re-patching the same file; code errors in routine operations (reading a file, searching); import-error or command-not-found spans at the start of runs; calculations done in the model's own text; independent commands serialized", "evals": "success and cost by model under each interface" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "large", "when": "typed tools +15.0 points for a weak model; bash-only +3.6 for a strong one (Fan, Sep 2026); typed functions in place of an interpreter took GAIA from 0.07 to 0.24 (VeRO)", "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "large", "when": "bash-only -53% cost for a bash-capable model; chaining tools in code 150k to 2k tokens in one example", "basis": "evidence" }, { "dimension": "performance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval per model, interface A against B: success, cost, steps" } ], "evidence": [ { "source": "Fan et al., \"An Empirical Study of Harness Design for Coding Agents\"", "url": "https://arxiv.org/abs/2609.20804", "date": "2026-09-17", "locator": "§2.2 Table 1; §3.2 'The predefined tool set scaffolds weaker models...', Fig 7; §4 Table 7; App 9.3.1 Table 10, 9.5 Tables 11-12", "task": "Match action space to model's shell proficiency and task", "effect": "T4/128k, planning on. Nemotron-3 30B: typed tools +15.0 pts SWE (25.2 vs 10.2) and +10.1 TB. 66% of its bash-only TB runs end after emitting tools that don't exist (trajectory 71->15 turns). 120B: tools +1.6 SWE / +4.5 TB and shorter runs (101->77, 96->70 turns) at no extra cost. 550B: bash-only +3.6 SWE / +5.6 TB with cost -53% / -30% and calls -32% / -24%. Mistral: tools +23.2 on SWE (bash-only 45.4 vs 68.6; 32.8% of bash-only runs never edit vs 1.2%; file-localization failures 16.0%->41.4%), but bash-only +6.7 on TB, where 71.9% of its actions were already bash. Bash-only cuts re-patching for all models (e.g. 550B 4.6->1.5) and raises create/replace share on TB (e.g. 28%->64% for 30B). The ablation bundles tool availability, prompts, file-state tracking and diagnostics.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Andrew Qu (Vercel), \"We removed 80% of our agent's tools\"", "url": "https://vercel.com/blog/we-removed-80-percent-of-our-agents-tools", "date": "2025-12-22", "locator": "v2: The file system is the agent; 3.5x faster table", "task": "Replace specialized text-to-SQL tools with bash plus SQL", "effect": "Claude Opus 4.5, 5 representative queries, old vs new: avg time 274.8s -> 77.4s (\"3.5x faster\"); success 4/5 -> 5/5; avg tokens ~102k -> ~61k (\"37% fewer\"); avg steps ~12 -> ~7 (\"42% fewer\"). Worst case: old \"724 seconds, 100 steps, and 145,463 tokens before failing\"; new \"141 seconds with 19 steps and 67,483 tokens\" and succeeded. v1 code lists 17 tools. Bundled with the context and prompt removals below, so not a single-variable result.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Andrew Qu (Vercel), \"We removed 80% of our agent's tools\"", "url": "https://vercel.com/blog/we-removed-80-percent-of-our-agents-tools", "date": "2025-12-22", "locator": "Getting out of the model's way; Lessons learned", "task": "Drop reasoning-constraining prompts and pre-filtered context", "effect": "Part of the bundled v1 -> v2 change (5 queries, Opus 4.5, numbers above). \"We were constraining reasoning because we didn't trust the model to reason. With Opus 4.5, that constraint became a liability.\" Both code samples show instructions: \"\", so the size of the prompt change is not visible.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Recommendation 3 (Sec 16.3); Sec 13.4; Sec 15.2", "task": "Start with a bash-only tool, add tools per failure", "effect": "\"Mini-SWE-Agent's single-tool design reports 74%+ on SWE-Bench with essentially no tool infrastructure (self-reported)\"; the paper says it is not comparable head-to-head with Codex's 69.1% (different models, runs, sandboxing).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Recommendation 17 (Sec 16.9)", "task": "Consolidate APIs instead of 1-to-1 wrapper tools", "effect": "Rec 17 quoting [18]: \"a common error we've observed is tools that merely wrap existing software functionality or API endpoints.\" Consolidate instead.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Lance Martin, \"Agent Design Patterns\"", "url": "https://rlancemartin.github.io/2026/01/09/agent_design/", "date": "2026-01-09", "locator": "Give Agents A Computer", "task": "Give the agent a filesystem and shell", "effect": "\"The filesystem gives agents access to persistent context. The shell lets agents run built-in utilities, CLIs, provided scripts, or code they write.\" Examples: Claude Code, Manus virtual computer.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Lance Martin, \"Agent Design Patterns\"", "url": "https://rlancemartin.github.io/2026/01/09/agent_design/", "date": "2026-01-09", "locator": "Multi-Layer Action Space", "task": "Keep few atomic tools; push actions into bash", "effect": "Rationale: the GitHub MCP server \"has 35 tools with ~26K tokens of tool definitions\"; overlapping tools confuse models. Claude Code uses \"around a dozen tools\", Manus \"fewer than 20\"; Manus action space is \"a hierarchy\".", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Li et al., \"Beyond Semantic Similarity: Rethinking Retrieval for Agentic Search via Direct Corpus Interaction\"", "url": "https://arxiv.org/abs/2605.05242", "date": "2026-05-03", "locator": "Sec 4.3 RQ6; Table 5", "task": "Give a read plus grep tool pair as minimum retrieval surface", "effect": "BrowseComp-Plus 100-q subset, GPT-5.4 nano, L4 context policy: read+grep 61% acc, 19 tool calls, $0.0355/q vs Qwen3-Embedding-8B retrieval 45%, 18 calls, $0.0498/q vs BM25 32%, 19 calls, $0.0527/q.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Li et al., \"Beyond Semantic Similarity: Rethinking Retrieval for Agentic Search via Direct Corpus Interaction\"", "url": "https://arxiv.org/abs/2605.05242", "date": "2026-05-03", "locator": "Sec 4.3 RQ6, Table 5; Appendix B.1, Fig 6; Fig 4 (right)", "task": "Allow piped shell commands for chained lexical search", "effect": "BrowseComp-Plus 100-q subset, GPT-5.4 nano: open bash 73% vs read+grep 61% (+12), but 35 vs 19 tool calls and $0.1021 vs $0.0355 per q. Lite bash usage (3,168 commands): rg | head 56.2%, rg | rg 20.6%, wc 7.8%, single rg 6.3%, ls 4.3%, python -c 2.4%, cat 0.1%. CC: Bash 62.4% / Grep 33.0% of calls; within Bash, chained search 22.3%, local context peek 18.0%, regex 17.0%, file localization 14.0%, full-document read 9.1%.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 12", "task": "MATH", "effect": "Validation 0.88 to 0.92 (best of run) and test 0.86 to 0.88 with T07; train stayed at 0.81.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 12", "task": "MATH", "effect": "Added in phase 5 (train 0.90, validation 0.90) then removed in phase 6; no lasting gain over T08.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 14 (Default, Sonnet 4.5)", "task": "GAIA", "effect": "The seed's CodeInterpreter tool was replaced with plain function tools (see T15) and that was the biggest gain of the run.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 14", "task": "GAIA", "effect": "Validation 0.07 to 0.24 in one phase; the largest single jump in any example trajectory. Final 0.30, the best GAIA result in Table 2.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 10", "task": "GAIA/FACTS/SimpleQA", "effect": "Knight 12+ packages vs Pawn 3 (math, datetime, json, re).", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, VeRO code release", "url": "https://github.com/scaleapi/vero", "date": null, "locator": "figure_04", "task": "Terminal-Bench and others", "effect": "MAX_CONCURRENT_SHELLS raised twice, never lowered.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, VeRO code release", "url": "https://github.com/scaleapi/vero", "date": null, "locator": "figure_01", "task": "five follow-up benchmarks", "effect": "3 to 13 of 20 runs per benchmark (13 on GAIA-Shell, whose seed is empty).", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Mak et al., \"Is Bash All You Need? An Empirical Study of Tool Interfaces for Enterprise Digital Worker Agents\"", "url": "https://arxiv.org/abs/2609.11999", "date": "2026-09-10", "locator": null, "task": "Give a bash-only interface rather than typed tools for enterprise digital-worker tasks", "effect": "Opus-4.8 and GPT-5.5 on TheAgentCompany and APEX-Agents, five interfaces compared: bash-only beats typed tools by 21.8-24.5 pp on TheAgentCompany and 4.8-7.4 pp on APEX-Agents, with 19-72% fewer tokens. Adding typed or agent-synthesized tools on top of bash gives no measurable gain; programmatic tool calling saves tokens but scores below bash.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Patel et al., \"The Bitter Lesson of Tool Calling\"", "url": "https://arxiv.org/abs/2608.06370", "date": "2026-08-06", "locator": null, "task": "Call tools from code rather than JSON", "effect": "BFCL v4, 14 models: programmatic (code) tool calling matches or beats JSON tool calling in 11; more robust under degraded context, where the JSON baseline loses 2.3% on average.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Bin Wu et al. (Anthropic), \"Introducing advanced tool use on the Claude Developer Platform\"", "url": "https://www.anthropic.com/engineering/advanced-tool-use", "date": "2025-11-24", "locator": null, "task": "Call tools programmatically from code", "effect": "Programmatic tool calling cut tokens 37% (43.6k to 27.3k) on complex research tasks.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Matt Carey (Cloudflare), \"Code Mode: give agents an entire API in 1,000 tokens\"", "url": "https://blog.cloudflare.com/code-mode-mcp/", "date": "2026-02-20", "locator": null, "task": "Expose a large API through search and execute in code", "effect": "2,500+ API endpoints behind search() and execute(); the model writes code against the typed spec in a sandbox. About 1k tokens against 1.17M, calculated not tested.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Rizel Scarlett (Block), \"8 Things You Didn't Know About Code Mode\"", "url": "https://goose-docs.ai/blog/2026/02/06/8-things-you-didnt-know-about-code-mode/", "date": "2026-02-06", "locator": null, "task": "Use code mode for multi-tool tasks only", "effect": "About 30% fewer tokens on identical tasks (one author's test, Opus 4.5); little benefit on single-tool tasks; model-dependent. A description mentioning 'long-running task' pulled in the wrong extension; explicit imports prevented it.", "direction": "mixed", "measured": false, "conditions": null, "older": false }, { "source": "Ankur Goyal (Braintrust) and Andrew Qu (Vercel), \"Testing if 'bash is all you need'\"", "url": "https://vercel.com/blog/testing-if-bash-is-all-you-need", "date": "2026-01-22", "locator": null, "task": "Give a typed query tool for structured data, not bash alone", "effect": "Structured-data questions, Sonnet 4.5: SQL tool 100% accurate, bash 52.7%, basic file tools 63%; bash used 7x the tokens and cost 6.5x more. SQL plus bash reached 100% with the agent checking its own answers, at about 2x SQL's tokens. Found and fixed 5 wrong expected answers in the benchmark.", "direction": "mixed", "measured": true, "conditions": null, "older": false } ], "caveats": "The paper flags this as intuitively helpful but the test gain is within noise; MATH baseline 0.87 is near the target model's ceiling.", "also_covers": [ { "name": "CodeAct" } ], "related": [ { "id": "2.a.i", "slug": "reduce-tool-surface", "name": "Reduce the tool surface: remove and consolidate tools, then defer the rest behind search" }, { "id": "6.c.i", "slug": "code-then-execute", "name": "Code-then-execute (CaMeL)" }, { "id": "10.a.i", "slug": "constrain-agent-reach", "name": "Constrain what the agent can touch" }, { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "2.c.i", "slug": "bound-tool-output", "name": "Bound tool output: cap it, or write it to a file the agent can read" }, { "id": "3.b.ii", "slug": "mask-tools-not-remove", "name": "Mask tools instead of removing them" }, { "id": "6.b.i", "slug": "parallel-read-only-tools", "name": "Parallel tool execution for read-only calls" }, { "id": "7.d.iii", "slug": "react-loop", "name": "ReAct loop" }, { "id": "4.c.ii", "slug": "skill-library", "name": "Skill library evolution" } ], "url": "https://mahalsystems.ai/aot/optimizations/choose-action-interface/", "markdown": "https://mahalsystems.ai/aot/optimizations/choose-action-interface.md" }, { "id": "2.b.i", "legacy_id": "T16", "former_ids": [], "slug": "fail-soft-tools", "name": "Make tool calls and tool results fail soft, and retry transient failures", "kind": "technique", "status": "card", "tier": { "tier": 2, "name": "Reliability", "why": "removes crashes, a hard failure", "proposed": false }, "levers": [ "tool-surface", "control-flow" ], "domains": [ "tool-use" ], "evidence_domains": [ "web-research", "code-generation", "tool-use", "reasoning" ], "categories": [ "Avoiding misuse", "Bounding execution" ], "category_ids": [ "tools.misuse", "orchestration.bound" ], "summary": "Keep a failed call from ending the run: validate tool calls and model responses, return tool errors to the agent as results, and retry only failures that are transient and safe to repeat. Concretely, guard the parser against calls in an unexpected shape (a string where a dict is expected, a missing required argument), catch errors inside tools (bad file format, failed fetch, extraction failure) and return a usable message instead of raising, replace custom extraction code (HTML, tables) with a standard parser, and retry rate limits, timeouts and server errors with backoff and a bounded count, returning exhaustion to the loop as a result. The harness often decides the outcome: under injected faults, tool-call faults that tool error handling caught cost under 5 points while unchecked truncated arguments cost one pipeline 52.63 points, and in real benchmark runs a repeat detector that ended runs after repeated malformed calls, with no final message to the model, ended 55 runs, only one of which scored above zero. The limits: fixing crashes rarely turned them into passes (3 and 1 of 11 in one study), a tool fix can break the agent outright, 91.3% of failures mined from MCP server issues were permanent so retrying them only burns tokens, and blindly retrying a write whose reply was lost duplicated it in 53.33% of trials, so retry writes only with server-side idempotency or after checking whether the first call landed.", "how": [ "Start from the harness as it is: from traces, list unhandled exceptions by tool and by parser, and runs the harness ended rather than the model (a repeat detector after malformed calls, a provider stream error, the output cap); then inject faults at the tool and model-API boundaries (explicit errors, timeouts, rate limits, empty and truncated responses, malformed arguments) for a baseline, since in two fault-injection studies robustness varied with the agent system more than with the model.", "Validate every call and response before using it: add type guards where calls and results are built (a tool call emitted as a string rather than a dict crashed one agent), check each model response's finish_reason, emptiness and tool-call arguments against the tool's schema, and return the validation error as the tool result (its wording is 7.b.ii). Empty or truncated responses arrive as successful calls and bypass error handling: tool-call faults that error handling caught cost under 5 points of pass@1, while a pipeline that passed truncated arguments on unchecked lost 52.63 points. When the same malformed call repeats, warn the model before a repeat detector (7.b.i) ends the run: a detector that ended runs after repeated malformed edit calls, with no final message, ended 55 runs and one scored above zero, while the same model repaired such failures in other harnesses.", "Wrap each tool body; on error return a short result marked as an error that the model can act on (what the error says is 7.b.ii).", "Replace hand-written HTML or table extraction with a library parser.", "Retry in the harness only what is transient: rate limits, timeouts, server errors and dropped streams from the model provider or a read-only tool, with backoff and a bounded count (provider stream errors cut off turns in 25 of one model's 62 runs in one harness, all scored as failures). Resume a turn cut off by the output cap instead of ending the run: one harness resumed 100 such runs and 48 went on to score above zero. Return permanent errors at once, marked as not retryable: 91.3% of failures mined from MCP server issues were permanent, and in the traces examined, agents left to decide never told permanent from transient failures and retried a failing tool up to 12 times in a row. When the count runs out, return that to the loop as a result.", "For tools with side effects, retry automatically only if the call failed before it was sent, the server honors an idempotency key, or a check shows the first call did not land; gate irreversible writes with 10.b.i. Blind retries (up to 3, with exponential backoff) of writes whose reply was lost duplicated them in 53.33% of trials; client-side keys helped only on endpoints that deduplicate; server-side idempotency on every endpoint gave 97.24% recovery with no duplicates, and checking before retrying raised recovery from 43.13% to 75.00% on four composite workflows.", "Re-run the errored cases first (a tool fix can crash the agent outright), then the full set, then paired runs with the same injected faults: task success, runs ended by the harness, recovery per fault type, retries per failure, duplicate side effects, and LLM calls and tokens per task." ], "detect": { "static": "parsing that assumes a fixed call shape; model responses used without checking finish_reason, empty content or tool-call arguments against the schema; tool bodies with no try/except; custom HTML or table extraction code; retry wrappers that retry every error, including permanent ones and writes without an idempotency key the server honors", "traces": "runs ending in an unhandled exception (AttributeError, parse or fetch errors) rather than an answer; tool spans returning empty or garbled text; failure rows labeled tool-recovery-failure or observation-failure; requests failing on rate limits or transient server errors with no retry, or with retries exhausted; failure rows labeled service-failure or recovery-failure-external; runs ended by the harness rather than the model (a repeat detector after malformed calls, a provider stream error, the output cap); three or more consecutive calls to a tool that keeps failing; a write retried after a timeout and committed twice", "evals": "crash outcomes as a share of failures; request-failure outcomes", "probes": "Faults at the tool and model-API boundaries: malformed calls, errors, timeouts, empty and truncated responses, permanent failures, and lost replies on writes." }, "impact": { "measured_basis": true, "primary": [ "quality" ], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "tool-call faults caught by tool error handling cost under 5 points of pass@1, while a pipeline that passed truncated arguments on unchecked lost 52.63 points (Tan, Aug 2026); a repeat detector ending runs on malformed calls ended 55 runs, one scoring above zero, and resuming output-capped turns let 48 of 100 runs go on to score (Li, Oct 2026); after lost replies on writes, server-side idempotency gave 97.24% recovery against 33.20%, and verify-before-retry 75.00% against 43.13% (Sah, Oct 2026); limits: VeRO removed all 11 parser crashes in 2 of 3 runs but only 3 and 1 became passes, and a table-extraction fix crashed one agent (validation 0.10) until a standard parser took it to 0.86 (VeRO, Feb 2026); calibrated MCP faults still cut task-completion score from 8.08 to 3.23 with errors returned to the model (Iannillo, Oct 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "small", "when": "no extra calls when nothing fails; under injected faults, retries raised AutoGen's LLM calls per task from 1.72 to 7.3 (4.24x) while a pipeline that failed early used fewer (0.71x) (Tan, Aug 2026); agents retried a failing MCP tool three or more times in a row in 38 runs, the costliest of these runs reaching 52 tool calls and 107,849 tokens (Iannillo, Oct 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "none", "size": null, "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "mixed", "size": "medium", "when": "blind retries (up to 3, with exponential backoff) of writes whose reply was lost duplicated the external effect in 53.33% of trials, 65.62% with Gemini 3.8 Flash and 51.25% with GLM-5.2; server-side idempotency took duplicates to 0.00%, and retries of calls that failed before dispatch caused none in 3,028 capable trials (Sah, Oct 2026)", "basis": "evidence" }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 2, "how": "paired runs on the same tasks and model, the harness as is against the change, with faults injected at the tool and model-API boundaries at fixed steps: explicit errors, timeouts, rate limits, empty and truncated responses, malformed or schema-violating arguments, permanent as well as transient failures (most real MCP failures are permanent), and lost replies on write calls; report task success, runs ended by the harness rather than the model, recovery per fault type, retries per failure, duplicate external effects, and LLM calls and tokens per task; re-run the errored cases first, since a tool fix can crash the agent" }, { "method": "monitor", "tier": null, "how": "in production traces: unhandled-exception and crash outcomes; runs ended by the harness (repeat detector, provider stream error, output cap); schema-validation failures of tool calls per model and provider; request failures and retries per request by error class, and retry exhaustion; consecutive calls to a failing tool; writes committed twice (same idempotency key or same arguments)" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 14", "task": "GAIA", "effect": "\"Robust file/Wikipedia tools\" flat (0.24 to 0.22); the follow-up \"fix Wikipedia tool file errors\" 0.22 to 0.26.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 15", "task": "SimpleQA", "effect": "A Wikipedia table-extraction fix crashed the agent (validation to 0.10) before being replaced by T18.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper §A.4, Table 16", "task": "TerminalBench-2 (Terminus-KIRA on Haiku 4.5)", "effect": "Eliminated all 11 baseline AttributeError crashes in two of three runs; only 3 and 1 of those samples became passes, the rest failed differently.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper §5.4", "task": "TerminalBench-2", "effect": "Found independently by the Tools-B=89 and Filesystem-B=178 runs. Tools-B=178 guarded three result builders but not the parser and left the crash intact, yet had the best pass rate.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 15", "task": "SimpleQA", "effect": "Validation 0.10 to 0.86 and test 0.58 to 0.86; the best SimpleQA result in Table 2 (0.86 max).", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Antonio Ken Iannillo, Joshua Segun Owotogbe, Roberto Natella, Francesco Avallone, Kristina Kudryavtseva, Indika Kumara, \"Tool-Calling LLM Agents Under Realistic MCP Faults: Scale Does Not Purchase Fault Tolerance\"", "url": "https://arxiv.org/abs/2610.32555", "date": "2026-10-23", "locator": "Abstract; §4 (Profile synthesis, What is calibrated, Table 2); §5.2-5.3; §6.1-6.3; §7; App. F Table 6; App. G", "task": "Inject MCP tool faults drawn from a profile calibrated on 57,240 issue and pull-request threads from 1,774 MCP server repositories (explicit errors, wrong or invalid output, delays, silent drops; permanent, intermittent or transient) into three agent stacks that all return tool errors to the model, and record quality, completion and what the agent does after each failure", "effect": "Profile: \"91.3% permanent, 8.0% intermittent, 0.75% transient; EXPLICIT_ERROR/PERMANENT alone carries 65.6%\" (N=13,824 classified failures). Pooled, \"task completion score drops 60% (8.08 to 3.23)\"; completion fell from 95-100% without faults to 88-97%. LangChain absorbed faults best; MCPBench lost a further 0.93 points relative to LangChain (p=0.048) and the OpenAI Agents SDK a further 0.84 (p=0.074), which the authors \"treat as suggestive rather than established\". After a failure the next action was a different tool 57% of the time, a final answer 22% and a retry 21%; \"Precise fault diagnosis, for instance distinguishing a permanent error from a transient one, is absent in all examined traces.\" 38 runs made three or more consecutive calls to the same failing tool; the longest had \"12 consecutive retries over 22 tool calls and 47,761 tokens\", the costliest \"52 tool calls, 107,849 tokens\". The 120B model kept no larger share of its fault-free score than the 20B (p=0.96).", "direction": "mixed", "measured": true, "conditions": "gpt-oss-20B and gpt-oss-120B served locally (MXFP4, 16K context); MCP-Bench plan-and-execute loop, OpenAI Agents SDK and LangChain create_agent with loop bounds of 40, 40 and 25; 25 single-server MCP-Bench tasks, 3 seeds, 1,350 cells; gpt-4o-mini judge; GitHub issues over-represent persistent failures, so the profile is a stress test, and only 9 of 1,905 fired faults were transient or intermittent; the stack effect is confounded with fault exposure", "older": false }, { "source": "Dolly Sah, Tanmay Sah, Harshul Jain, Tanya Sah, \"UndoBench: Separating Task Competence from Recovery Capability in Tool-Using AI Agents\"", "url": "https://arxiv.org/abs/2610.05622", "date": "2026-10-04", "locator": "Abstract; §3 (Recovery Baselines); §4.1-4.3, Tables 1-2; §4.5 Table 3; §5 Table 4; §6; Limitations", "task": "Inject a fault after a write tool's mutation commits but before its reply arrives (and, in extensions, before dispatch and mid-mutation) and compare recovery policies: naive retry, client-side idempotency keys, a zero-privilege mutation journal, server-side idempotency on every endpoint, and verify-before-retry", "effect": "Lost-reply faults, 2,880 paired trials: nominal success 83.54% against 46.72% conditional recovery. Naive retry (\"re-issuing tool calls up to 3 times with exponential backoff upon receiving any error\"): 42.77% recovery and duplicate external effects in 53.33% of trials; client-side idempotency keys: 53.49% and 45.42%, since only three endpoints deduplicated and \"retrying unacknowledged calls blindly produced duplicates in 100% of trials across five workflows\". Commercial models under naive retry: duplicates in 65.62% (Gemini 3.8 Flash) and 51.25% (GLM-5.2) of trials, recovery 21.08% and 21.89%. Server-side idempotency on every endpoint: 97.24% recovery and 0.00% duplicates, against 33.20% with standard keys. Faults before dispatch: naive retry recovered 78.44% with zero duplicate effects among 3,028 nominally capable trials. On four composite workflows verify-before-retry recovered 75.00% after commit against 43.13% for naive retry, and 25.89% against 0.00% mid-mutation.", "direction": "mixed", "measured": true, "conditions": "Llama-3.1-8B and Llama-3.2-3B (Q4) under direct tool calling and LangGraph (equivalent within 10 points), plus Gemini 3.8 Flash and GLM-5.2 in a secondary extension; 12 held-out workflows in 8 domains on mocked APIs (SQLite, local Git, mocked Stripe and AWS), one fault per run, 20 seeds; the +10.72-point gain from client-side keys had a bootstrap CI of 0 to +24.23 and adjusted p=1.00; verify-before-retry was evaluated post hoc; the authors' own journaling baseline ran at parity with naive retry", "older": false }, { "source": "Yixuan Li, Yiyun Zhou, Yao Long Teng, Fuchao Yang, Yanchen Deng, Zhiyi Lyu, Xuyu Dong, Feng Chen, Bo An, \"Finding the Right Fit: Model-Harness Interactions across Agent Tasks\"", "url": "https://arxiv.org/abs/2610.00917", "date": "2026-10-01", "locator": "Abstract; §3.2; Table 1; §5.1; App. A Table 5", "task": "Cross four configurable harnesses with five current models on three agent benchmarks, and code how each harness handed failures back (malformed calls, provider stream errors, output-capped turns) in matched trajectories", "effect": "In ten matched Terminal-Bench 4 pairs, \"Almost all responses were model-initiated (180 of 192)\" and diagnosis or a targeted patch resolved the signal in 116 of 133 cases; \"The decisive difference, however, was usually not whether the model could repair an error but whether the harness returned the failure as usable feedback.\" OpenHands' repeat detector \"ended 55 runs, 48 of them Kimi re-issuing an edit call without its required content argument, and only one of these runs scored above zero.\" openJiuwen re-prompts the model after a turn is cut by the output cap (\"100 runs resumed, 48 of which went on to score above zero\"). \"Turns cut off by provider stream errors (25 of Kimi's 62 PI runs on TB4) and sandbox stalls are scored as failures\". Kimi K3 on Terminal-Bench 4 scored 11.11% (OpenHands), 17.46% (DeepSeek Harness), 11.11% (PI) and 28.57% (openJiuwen).", "direction": "supports", "measured": true, "conditions": "Claude Opus 5, GPT-6 Astra, GLM-5.3, Kimi K3 and DeepSeek V4 Pro via OpenRouter pinned to first-party providers, high reasoning effort; OpenHands, DeepSeek Harness, PI and openJiuwen, plus Codex and Claude Code natively; TUA-Bench (120), ALE-CLI (99) and Terminal-Bench 4 (63); 66 configurations, 6,204 scored trajectories; retries and limits at each harness's defaults; the failure-handling counts come from the authors' coding of final attempts, and harness score gaps bundle error handling with tools, prompts and limits", "older": false }, { "source": "Gou Tan, Zhensu Sun, Jieke Shi, et al., \"AgentChaos: Chaos Engineering for Agent Systems via Programmatic Fault Injection\"", "url": "https://arxiv.org/abs/2608.06790", "date": "2026-08-07", "locator": "Abstract; §4.3; §5.1-5.3, Tables 3, 4, 6, 7; §6.1-6.2", "task": "Inject faults into model API responses at runtime (errors, timeouts, empty, truncated, corrupted and schema-violating content or tool-call arguments) across five agent systems and four backbones, and measure the drop in pass@1, call counts and how failures surface", "effect": "All systems degraded, pass@1 dropping \"from 0.87% (Mini-SE on SWE-bench Pro) to 49.66% (MapCoder on HumanEval+ under GPT-5.2)\", and the ranking held across backbones. With Claude-Sonnet-4.5: \"Most tool call faults cause Δpass@1 below 5% (e.g., error ranges from -12.5% to 4.05%), because tool error handling catches them before they propagate to subsequent agents\", but tool-call truncation cost MapCoder 52.63 points \"because its pipeline passes corrupted tool call arguments to the next stage without validation\"; on MAD, \"empty content causes a 38.46% drop, close to error content (37.5%) and higher than timeout content (22.33%)\". Under faults AutoGen went from 1.72 to 7.3 LLM calls (4.24x) and from 0.47 to 3.07 tool calls (6.55x) \"because faulty responses trigger repeated retries\"; silent failures (plausible wrong output with no warning) were 21.82% to 65.71% of failed tasks. The authors advise \"output validation after each LLM API call, such as checking finish_reason, verifying code syntax completeness, and confirming that tool call arguments match the expected schema.\"", "direction": "supports", "measured": true, "conditions": "AutoGen, MAD, MapCoder, EvoMAC and Mini-SE reimplemented on Google ADK; Claude-Sonnet-4.5, GPT-5.2, DeepSeek-V3.2 and Seed-1.8 at temperature 0.7; HumanEval, MBPP, HumanEval+, MBPP+, MMLU-Pro, MATH-500 and SWE-bench Pro; 65 fault configurations, about 4.6 triggered tasks each; error and timeout faults are written into the body of a successful response, so provider-level HTTP retries are not exercised", "older": false } ], "caveats": "Fail-soft handling is necessary but not sufficient: in VeRO removing all 11 crashes turned only 3 and 1 of them into passes, a tool fix crashed one agent outright, and even with errors returned to the model, calibrated MCP faults cut task-completion scores by 60% (Iannillo, Oct 2026). Retries help only where failures are transient and the call is safe to repeat: 91.3% of failures mined from MCP issue threads were permanent (a stress profile, since issues over-represent reproducible bugs), agents retried permanent failures on their own, and blind retries of writes duplicated them in 53.33% of trials in a benchmark with mocked APIs and one fault per run; no study here tests marking errors as not retryable. The fault-injection studies are small or synthetic (about 4.6 triggered tasks per fault configuration in one, two gpt-oss models on an easy benchmark in another, 12 workflow clusters with verify-before-retry tested post hoc in a third), and harness comparisons bundle error handling with tools, prompts and limits, so measure the effect in your own agent.", "also_covers": [], "related": [ { "id": "7.b.ii", "slug": "keep-errors-in-context", "name": "Keep errors in context" }, { "id": "7.b.i", "slug": "detect-loops-and-stalls", "name": "Detect stalls and loops: warn, then stop early" }, { "id": "7.b.iii", "slug": "size-execution-budgets", "name": "Size run budgets so productive runs aren't cut off: turn caps, deadlines, command timeouts" }, { "id": "9.b.i", "slug": "extractable-final-answer", "name": "Make the final answer reliably extractable: state the format, extract robustly, force an answer at the limit" }, { "id": "2.a.ii", "slug": "rewrite-tool-surface", "name": "Rewrite the tool surface: names, schemas, descriptions" }, { "id": "10.b.i", "slug": "risky-action-policy", "name": "Gate irreversible actions: allow, ask or deny by side effect" }, { "id": "4.b.i", "slug": "filesystem-checkpoints", "name": "Filesystem-based agent state and checkpoints" }, { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "3.b.ii", "slug": "mask-tools-not-remove", "name": "Mask tools instead of removing them" }, { "id": "6.b.i", "slug": "parallel-read-only-tools", "name": "Parallel tool execution for read-only calls" } ], "url": "https://mahalsystems.ai/aot/optimizations/fail-soft-tools/", "markdown": "https://mahalsystems.ai/aot/optimizations/fail-soft-tools.md" }, { "id": "2.b.ii", "legacy_id": "T09", "former_ids": [], "slug": "tool-use-guidance", "name": "Tell the agent when to use each tool, in context it always sees", "kind": "technique", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes what the model sees", "proposed": false }, "levers": [ "tool-surface", "prompt" ], "domains": [ "tool-use" ], "evidence_domains": [ "reasoning", "web-research", "code-generation", "tool-use" ], "categories": [ "Avoiding misuse" ], "category_ids": [ "tools.misuse" ], "summary": "For each tool or skill that is under-used or misused, add an explicit trigger rule where the model always sees it (system prompt, AGENTS.md): the situation that calls for the tool, and what to use it instead of (e.g. the dedicated read or edit tool rather than bash). Test the wording, since results swing with it. It will not make a model that rarely calls tools start calling them. The cheapest form is one sentence per tool stating what it is for: that alone recovered most of a 34.7-point loss caused by offering the model an irrelevant tool.", "how": [ "From traces, list tools with a low call rate on cases where they would have helped, and tools chosen wrongly.", "For each, write one rule: when to use it, and instead of what.", "Put the rules in always-loaded context, not in an optional skill.", "A/B two wordings; keep the one with the higher call rate and pass rate." ], "detect": { "static": "tools or skills whose intended use is stated nowhere the model always sees; overlapping tools with no rule for choosing between them", "traces": "tool spans: a tool called on under about 5% of cases where its trigger condition held (e.g. a docs skill never invoked in 56% of runs); bash spans where a dedicated tool exists; failure rows labeled incorrect-tool-selection", "evals": "failures where the right tool existed but was not called" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "large", "when": "Vercel: pass rate 53% to 79% once told when to invoke the docs skill (Jan 2026); VeRO mixed: -5 and +4.5 points; one sentence of tool scope recovered most of a 34.7-point loss from an irrelevant tool (Paturi et al., Sep 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "performance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval with and without the rule, plus the tool's call rate in traces" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 12 (Orchestrator, Sonnet 4.5; 'Early Peaker')", "task": "MATH", "effect": "Phase 3, 'Enhanced tool usage instructions', added right after phase 2 added a SymPy calculator tool: validation 0.917 to 0.867 (regressed), train flat at 0.813. The figure gives only the label, not the instruction text.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 15 (Orchestrator, Opus 4.5; 'Best SimpleQA')", "task": "SimpleQA", "effect": "Phase 1, 'Improved prompt, tool descriptions', bundled with prompt changes: validation 0.577 to 0.622, train 0.456 to 0.587. Not separable from the description changes (T35).", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Jude Gao (Vercel), \"AGENTS.md outperforms skills in our agent evals\"", "url": "https://vercel.com/blog/agents-md-outperforms-skills-in-our-agent-evals", "date": "2026-01-27", "locator": "Explicit instructions helped, but wording was fragile", "task": "Tell the agent in AGENTS.md when to invoke a skill", "effect": "Trigger rate to 95%+, pass rate 53% -> 79% (+26pp); Build/Lint/Test 95/100/84. Still 21pp below the AGENTS.md index. Vercel calls the wording sensitivity \"brittle for production use\".", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Jude Gao (Vercel), \"AGENTS.md outperforms skills in our agent evals\"", "url": "https://vercel.com/blog/agents-md-outperforms-skills-in-our-agent-evals", "date": "2026-01-27", "locator": "Skills weren't being triggered reliably", "task": "Do not rely on the agent choosing to invoke a skill", "effect": "Skill never invoked in 56% of eval cases; pass rate 53% = baseline 53% (+0pp); on tests 58% vs baseline 63%, lint 89% vs 95%.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 7.3 \"Tool-use philosophy\"", "task": "Prefer dedicated tools over shell in instructions", "effect": "Claude Code \"Do NOT use the Bash tool to run commands when a relevant dedicated tool is provided ... allows the user to better understand and review your work.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 7.2 (Pi)", "task": "Generate tool guidance from the active toolset", "effect": "Pi \"each tool contributes promptSnippet/promptGuidelines, so the prompt co-varies with the active toolset\"; whole prompt is 30–40 lines.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Prompt engineering: Tool design and selection are critical", "task": "Give explicit tool-selection heuristics in the prompt", "effect": "\"an agent searching the web for context that only exists in Slack is doomed from the start\" ... \"Bad tool descriptions can send agents down completely wrong paths, so each tool needs a distinct purpose and a clear description.\"", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Chen et al., \"BrowseComp-Plus: A More Fair and Transparent Evaluation Benchmark of Deep-Research Agent\"", "url": "https://arxiv.org/abs/2508.06600", "date": "2025-08-08", "locator": "Sec 4.3; Sec 4.6; Sec 4.8.1", "task": "Prompting weak models to search does not make them search", "effect": "Same prompt explicitly telling the model to use the tool: Qwen3-32B 0.89-0.94 search calls/q, Search-R1-32B 1.68-1.78, vs gpt-5 / o3 >20. With oracle evidence in context Qwen3-32B reaches 83.25% (gpt-4.1 93.49%), vs 10.36% when it must search.", "direction": "contradicts", "measured": true, "conditions": null, "older": true }, { "source": "Paturi et al., \"When Tools Get in the Way: The Effect of Unnecessary Tool Availability on LLM Answering\"", "url": "https://arxiv.org/abs/2609.14157", "date": "2026-09-12", "locator": null, "task": "State in one sentence what each tool is for", "effect": "Same setup: a one-sentence instruction stating the tool's scope recovers most of the lost answers.", "direction": "supports", "measured": true, "conditions": null, "older": false } ], "caveats": null, "also_covers": [ { "name": "Tool docstrings with use / do-not-use guidance and examples" } ], "related": [ { "id": "2.a.ii", "slug": "rewrite-tool-surface", "name": "Rewrite the tool surface: names, schemas, descriptions" }, { "id": "2.a.i", "slug": "reduce-tool-surface", "name": "Reduce the tool surface: remove and consolidate tools, then defer the rest behind search" }, { "id": "6.b.i", "slug": "parallel-read-only-tools", "name": "Parallel tool execution for read-only calls" }, { "id": "3.a.iv", "slug": "restructure-system-prompt", "name": "Restructure the system prompt: every requirement once, up front, in named sections; nothing the model doesn't need" }, { "id": "4.c.ii", "slug": "skill-library", "name": "Skill library evolution" } ], "url": "https://mahalsystems.ai/aot/optimizations/tool-use-guidance/", "markdown": "https://mahalsystems.ai/aot/optimizations/tool-use-guidance.md" }, { "id": "2.c.i", "legacy_id": "T02", "former_ids": [], "slug": "bound-tool-output", "name": "Bound tool output: cap it, or write it to a file the agent can read", "kind": "technique", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes what the model sees", "proposed": false }, "levers": [ "tool-surface", "context" ], "domains": [ "general" ], "evidence_domains": [ "code-generation", "tool-use" ], "categories": [ "Trimming results", "Filtering input" ], "category_ids": [ "tools.lean-returns", "context.filter-input" ], "summary": "Cap the characters of tool output kept per call (VeRO settled near 15-20 KB on TerminalBench), or, better, write long output to a file and return its path with a short head, so the agent can read more if it needs to. When the agent runs code, filter and aggregate results inside the execution environment and return only the slice it needs.", "how": [ "From traces, find the largest tool results and the share of context they take.", "Set a cap from the distribution of results on successful cases; or switch to file offload for that tool.", "Re-run cases that previously overflowed or truncated." ], "detect": { "static": "tool results passed to the model whole, with no cap or file offload", "traces": "a single tool result over about 10% of the context window; context-overflow outcomes; a truncated result followed by re-running the same command", "evals": "context-overflow outcomes as a share of failures" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "small", "when": "a cap can drop what the agent needed; file offload avoids that (Cursor, unmeasured)", "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "medium", "when": "VeRO: overflow errors 13 to 1 with a lower cap", "basis": "evidence" }, { "dimension": "performance", "effect": "improves", "size": "small", "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval at the old and new cap, or cap against file offload; count overflow failures" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper §A.4", "task": "TerminalBench-2", "effect": "30KB to 15KB (Tools-B=89), 20KB (Filesystem-B=178), 17KB (Tools-B=178, after probing 15KB on 16 samples and seeing some tasks needed more). Filesystem run cut ContextLengthExceeded from 13 to 1.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, VeRO code release", "url": "https://github.com/scaleapi/vero", "date": null, "locator": "figure_04 (100 runs, five benchmarks)", "task": "mixed", "effect": "MAX_TOOL_OUTPUT_CHARS lowered 16 times, raised 13. The only frequently-touched constant lowered more often than raised.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Jediah Katz (Cursor), \"Dynamic context discovery\"", "url": "https://cursor.com/blog/dynamic-context-discovery", "date": "2026-01-06", "locator": "1. Turning long tool responses into files", "task": "Write long tool outputs to files, don't truncate", "effect": "Truncation \"can lead to data loss\"; the file approach \"has resulted in fewer unnecessary summarizations when reaching context limits.\" No number.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Adam Jones, Conor Kelly (Anthropic), \"Code execution with MCP: Building more efficient agents\"", "url": "https://www.anthropic.com/engineering/code-execution-with-mcp", "date": "2025-11-04", "locator": "Benefits: Context efficient tool results", "task": "Filter and aggregate tool results in code", "effect": "Illustrative: a 10,000-row sheet filtered to pending orders; \"The agent sees five rows instead of 10,000.\" Not measured.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Adam Jones, Conor Kelly (Anthropic), \"Code execution with MCP: Building more efficient agents\"", "url": "https://www.anthropic.com/engineering/code-execution-with-mcp", "date": "2025-11-04", "locator": "Intermediate tool results consume additional tokens; Privacy-preserving operations", "task": "Pass data between tools in code, not context", "effect": "With direct calls \"the full call transcript flows through twice. For a 2-hour sales meeting, that could mean processing an additional 50,000 tokens\" (estimate). Also: \"models may be more likely to make mistakes when copying data between tool calls.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Adam Jones, Conor Kelly (Anthropic), \"Code execution with MCP: Building more efficient agents\"", "url": "https://www.anthropic.com/engineering/code-execution-with-mcp", "date": "2025-11-04", "locator": "Benefits: More powerful and context-efficient control flow", "task": "Express loops, polling and conditionals in code", "effect": "\"more efficient than alternating between MCP tool calls and sleep commands through the agent loop\" and \"saves on 'time to first token' latency: rather than having to wait for a model to evaluate an if-statement\".", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Lance Martin, \"Agent Design Patterns\"", "url": "https://rlancemartin.github.io/2026/01/09/agent_design/", "date": "2026-01-09", "locator": "Multi-Layer Action Space", "task": "Chain actions by writing and executing code", "effect": "Cites CodeAct and Anthropic, \"highlighting the token savings benefit because the agent does not process the intermediate tool results.\"", "direction": "supports", "measured": false, "conditions": null, "older": false } ], "caveats": "Samples 16, 17 and 47 regressed under every run; output-length changes are the usual suspect.", "also_covers": [ { "name": "Token-efficient tool responses" } ], "related": [ { "id": "5.a.i", "slug": "staged-compaction", "name": "Compact conversation history in stages: elide stale tool outputs, then keep an anchored summary" }, { "id": "2.a.iii", "slug": "choose-action-interface", "name": "Choose how the agent acts: typed tools, code execution, or shell, matched to the model and task" }, { "id": "5.c.ii", "slug": "isolated-subagent-context", "name": "Sub-agent architectures with clean contexts" }, { "id": "7.b.ii", "slug": "keep-errors-in-context", "name": "Keep errors in context" } ], "url": "https://mahalsystems.ai/aot/optimizations/bound-tool-output/", "markdown": "https://mahalsystems.ai/aot/optimizations/bound-tool-output.md" }, { "id": "3.a.i", "legacy_id": "T11", "former_ids": [], "slug": "few-shot-examples", "name": "Add worked examples to the prompt (few-shot), or curated example queries for data agents", "kind": "technique", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes what the model sees", "proposed": false }, "levers": [ "prompt" ], "domains": [ "general" ], "evidence_domains": [ "data-analysis", "reasoning", "tool-use" ], "categories": [ "Structuring" ], "category_ids": [ "prompts.structure" ], "summary": "Put a few solved examples of the task in the prompt (the input and the exact output wanted), or give a data agent commented example queries for its custom metrics and an index of past queries it retrieves per question. The clear gains are on knowledge the model cannot infer, mostly in data agents: one Genie space went from 54% to 77% on 13 questions, removing LinkedIn's example-query index cut correct answers from 49% to 24%, and retrieving past queries raised one text-to-SQL pipeline from 54.27% to 60.95%. On strong and reasoning models examples mostly fix the output format rather than the reasoning: Qwen2.5-72B scored 95.83% zero-shot against 95.75% with eight examples on GSM8K once the answer extractor was fixed, and five examples moved o1-preview by -2.2 to +1.2 points across six medical benchmarks. Start from a zero-shot baseline, add a few examples only where a paired eval shows a gain, and keep fixed examples in the cached prefix with retrieved ones after it.", "how": [ "Start from the baseline: the same model zero-shot, with a plain instruction, an explicit output format and a robust answer extractor (9.b.i), on held-out cases. Examples often look like a gain only because they fix the format: Qwen2.5-72B went from 91.58% zero-shot to 95.75% with eight examples on GSM8K, but scored 95.83% zero-shot once the extractor read the boxed answer. On a reasoning model begin with no examples: five examples moved o1-preview by -2.2 to +1.2 points across six medical benchmarks, while a tailored prompt and ensembling helped.", "Add examples only where failures cluster on what the instruction and schema cannot carry: a custom metric, business logic, a join path, a value convention, an exact output shape. For a data agent, write one example query per custom metric with comments that explain the logic (54% to 77% on 13 questions in one Genie space), and build an example-query index from queries in code, wikis and the query log, each with a description embedded for retrieval and refreshed as needed (removing such an index cut correct answers from 49% to 24%). With dozens of metrics, define them once in a view or semantic layer instead (7.a.i): in the same Genie space a view holding the definitions also scored 100%.", "Make each example the full input and the full desired output, in exactly the target format: in the math study the format benefit fell as more of each example was masked, while the reasoning content was largely ignored (in most settings, replacing half of the example tokens with noise barely changed accuracy). Include worked reasoning only for small or older models, which did gain from it (LLaMA3.2-1B, LLaMA2-7B, Qwen-7B); for strong models even reasoning written by DeepSeek-R1 rarely beat zero-shot. Vary the examples' form so the model does not copy one pattern (3.a.ii).", "Use a few examples, retrieved per question when the pool is large: on underspecified text-to-SQL, retrieving past queries from the same database raised execution accuracy from 54.27% to 60.95% (67.34% with the right ones), peaking at four examples (61.31%, against 58.36% with one), which the authors attribute to noise from over-retrieval. Retrieve broadly, then filter and rerank for precision: whole-question similarity alone reached 59.15%, retrieval precision was 35.53%, and in one failure the model copied a retrieved pattern that did not fit.", "Keep the prefix cacheable (3.b.i): put fixed examples in the static part of the prompt and append retrieved ones after it, next to the question. In many-shot prompts the edge of similarity selection fades as examples are added, and caching a large fixed set while adding 20 retrieved examples per query matched similarity-only selection at an estimated half the cost at 50 examples and a tenth at 200. Retrieval is not always better: on a tool-selection benchmark, similarity-retrieved examples did not beat zero-shot while random ones did.", "Keep the pool curated, as the tested setups did: final, successful queries only, filtered by execution count and certification. If the examples come from the agent's own past runs, that is episodic memory (4.c.i): verify them before reuse. Do not add examples where the problem calls for validation (one of VeRO's anti-patterns).", "Measure with paired runs on held-out cases (never the example pool): zero-shot, fixed examples and retrieved examples, per model and again after each model change; report accuracy, output-format errors, input tokens, cache hit rate and latency (retrieval with LLM reranking took about 6 s per question in one test)." ], "detect": { "static": "A prompt with no worked examples on a task with a fixed output shape; a data agent with no example queries for its custom metrics; examples kept on a reasoning or strong model with no zero-shot comparison; examples retrieved per request placed before static prompt content, so the cached prefix changes on every call.", "traces": "Outputs that miss the expected shape or metric definition; repeated queries for the same custom metric; retrieved examples with low similarity to the question, or answers that copy a retrieved pattern that does not fit.", "evals": "Failures that cluster on custom metrics, business logic or an output format the examples would show; no paired zero-shot baseline for the examples in use." }, "impact": { "measured_basis": true, "primary": [ "quality" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "data agents asked about custom metrics or underspecified business logic: 54% to 77% on 13 questions (Databricks, Feb 2026), 49% to 24% without the example-query index (Chen, Jul 2025), 54.27% to 60.95% with retrieved past queries and 61.31% at four examples (Zhou, Aug 2026); strong models on math: 95.75% with eight examples against 95.83% zero-shot once the extractor was fixed (Cheng, Jun 2025), MATH train 0.81 to 0.83 with validation flat (VeRO, Feb 2026); o1-preview -2.2 to +1.2 points across six medical benchmarks (Nori, Nov 2024)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "small", "when": "more input tokens per call; examples retrieved per query change the prompt and defeat prefix caching, while caching a large fixed set and adding 20 retrieved examples cut estimated attention cost about 2x at 50 examples and 10x at 200 against similarity-only selection (Golchin, Jul 2025); few-shot o1-preview used more tokens than a tailored prompt for lower accuracy (Nori, Nov 2024); indexing a query log took about 5.2 s of LLM time per query (Zhou, Aug 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "small", "when": "fixed examples add only input tokens; retrieving and reranking examples per question with an LLM added about 6 s per question with serial API calls (Zhou, Aug 2026)", "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired runs on held-out cases, never the example pool: zero-shot with an explicit output format and a robust extractor, against fixed examples and against examples retrieved per query (1 to 5), per model and again after each model change; report accuracy (for data agents, sliced by questions on custom metrics or business logic), output-format errors, input tokens, cache hit rate and latency, over several runs" }, { "method": "monitor", "tier": null, "how": "per request: which examples were retrieved and their similarity, whether the answer followed a retrieved pattern, output-format errors, input tokens and cache hit rate; weekly: failed questions with no relevant example (gaps in the index) and examples that no longer run after schema or metric changes" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 12", "task": "MATH, phase 4", "effect": "Adding few-shot maths examples: train 0.81 to 0.83, validation flat at 0.87.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Databricks, \"How to Build Production-Ready Genie Spaces and Build Trust Along the Way\"", "url": "https://www.databricks.com/blog/how-build-production-ready-genie-spaces-and-build-trust-along-way", "date": "2026-02-06", "locator": "Iteration 4: Defining Custom Metrics; Iteration 5; (Optional) Iteration 6", "task": "Commented example SQL for custom metrics", "effect": "Adding example SQL queries for custom metrics such as CTR (comments in the SQL recommended) raised benchmark accuracy from 54% to 77% on 13 questions, \"the largest single accuracy improvement thus far\". Text instructions then reached 100%, and replacing base tables with a view that pre-defines the metrics also scored 100%.", "direction": "supports", "measured": true, "conditions": "One Genie space on the authors' marketing data; 13 benchmark questions with validated SQL; the 54% came after table and column descriptions, join keys and value sampling; vendor evaluating its own product", "older": false }, { "source": "Albert Chen, Manas Bundele, Gaurav Ahlawat, et al. (LinkedIn), \"Text-to-SQL for Enterprise Data Analytics\"", "url": "https://arxiv.org/abs/2507.14372", "date": "2025-07-18", "locator": "Section 2.1 (Example query index); Section 3.3.1, Table 2 (A.4 to A.3)", "task": "Remove the example-query index", "effect": "Removing the example-query index (human-authored queries from code repos and wikis, filtered by execution count and certification) cut answers rated correct or close to correct from 49% to 24%, and table recall from 76% to 60%.", "direction": "supports", "measured": true, "conditions": "133 expert-written questions over 10 product areas; gpt-4o-2024-05-13 agents at temperature 0, scored by a gpt-4o judge (share of answers rated 4 or 5), one run per configuration; measured after domain knowledge had already been removed", "older": true }, { "source": "Yunfan Zhou, Qiming Shi, Yizhou Yang, Di Weng, Yingcai Wu, \"BIRD-History: A Benchmark for History-Driven Text-to-SQL with Fine-Grained Knowledge Annotations\"", "url": "https://arxiv.org/abs/2608.29345", "date": "2026-08-29", "locator": "Section 5.2 Tables 2, 3 and 5; Section 5.3; App. B.2 Table 13; App. B.3 Table 14", "task": "Retrieve past SQL queries from the same database's query log as examples for underspecified questions whose business logic, join paths or value conventions are not in the schema: no retrieval, each pipeline's own whole-question similarity retrieval, and the authors' retriever (query-level plus clause-level matching, then LLM filtering and reranking); vary the maximum number of examples", "effect": "OpenSearch-SQL execution accuracy: 54.27% with no retrieval, 59.15% with its own similarity retrieval, 60.95% with the authors' retriever, 67.34% with the gold examples. DAIL-SQL rose from 40.78% to 51.33%, N-rep from 54.49% to 57.79%, CodeS-7B from 12.92% to 18.95%. Capping retrieval at 1, 2, 3, 4 and 5 examples gave 58.36%, 59.30%, 59.58%, 61.31% and 60.01% (uncapped 60.95% at 4.57 examples on average), \"suggesting overly permissive retrieval introduces noise\". Retrieval precision was 35.53%; in one failure the model chose an oversimplified retrieved pattern. Online retrieval took about 6 s per question (1,918 ms keyword extraction, 54 ms embedding search, 4,062 ms LLM filter and rerank, serial API calls); indexing took 5,181 ms of LLM time per query.", "direction": "supports", "measured": true, "conditions": "1,393 tasks over 11 BIRD databases, built so each question needs knowledge from the query log; the log is BIRD's development-set gold SQL (final, successful queries only); qwen-plus-2025-07-28 at temperature 0 for generation and reranking, all-mpnet-base-v2 embeddings; one run per setting; the authors' own benchmark and retriever", "older": false }, { "source": "Shahriar Golchin, Yanfei Chen, Rujun Han, Manan Gandhi, Tianli Yu, Swaroop Mishra, Mihai Surdeanu, Rishabh Agarwal, Chen-Yu Lee, Tomas Pfister, \"Towards Compute-Optimal Many-Shot In-Context Learning\"", "url": "https://arxiv.org/abs/2507.16217", "date": "2025-07-22", "locator": "Section 1; Sections 4.1 to 4.4; Figures 2, 4 and 5", "task": "Many-shot prompts (50 to 200 examples): a fixed random set, a fixed k-means-selected set, examples retrieved by similarity for each test case, and two hybrids that add 20 retrieved examples per case after a larger fixed set that is cached", "effect": "\"the influence of selection criteria (e.g., similarity) on performance diminishes as the number of demonstrations increases substantially\"; on TREC similarity selection barely changed past 50 examples. \"Both of our selection strategies match the similarity-based approach in cases where it delivers the highest performance\" and beat it on ANLI and MetaTool, at an estimated inference cost \"nearly two times lower in 50-shot settings and ten times lower in 200-shot settings\". On MetaTool, similarity-retrieved examples did not improve on zero-shot while random examples did. With a 100-example pool, adjusting the share of similar examples gained about 3% to 6% over using the whole pool.", "direction": "supports", "measured": true, "conditions": "gemini-1.5-pro-002 and gemini-2.0-flash-001 at temperature 0; ANLI, TREC, GSM Plus and MetaTool (up to 1,000 test cases each) and three BBH tasks (150 each); Gecko embeddings; cost is a proportional attention cost computed from complexity, not measured spend or latency; results shown in figures only; the authors' own method", "older": true }, { "source": "Xiang Cheng, Chengyan Pan, Minjun Zhao, Deyang Li, Fangchao Liu, Xinyu Zhang, Xiao Zhang, Yong Liu, \"Revisiting Chain-of-Thought Prompting: Zero-shot Can Be Stronger than Few-shot\"", "url": "https://arxiv.org/abs/2506.14641", "date": "2025-06-17", "locator": "Abstract; Section 4; Sections 5.1 to 5.3; Section 6.1; App. B.1 Table 1; App. B.2 Tables 2 and 3", "task": "Compare zero-shot chain of thought with few-shot chain-of-thought examples (fixed, or chosen by seven selection methods, 1 to 8 examples, and examples written by Qwen2.5-Max or DeepSeek-R1) on math, after fixing the answer extractor to read the boxed answer", "effect": "\"for recent strong models such as the Qwen2.5 series, adding traditional CoT exemplars does not improve reasoning performance compared to Zero-Shot CoT. Instead, their primary function is to align the output format with human expectations.\" Qwen2.5-72B-Instruct on GSM8K: 91.58% zero-shot with the default extractor, 95.75% with 8 examples, 95.83% zero-shot with the fixed extractor; on MATH 81.30% with 8 examples against 81.64% zero-shot. Retrieved examples were \"comparable to or worse than the Zero-shot baseline\", the best gains about 0.2 points; examples written by stronger models beat standard 8-shot but rarely zero-shot; in most settings, replacing 50% of example tokens with noise caused no significant drop. Weak or older models (LLaMA3.2-1B, LLaMA2-7B, Qwen-7B) did gain from examples. In an appendix, few-shot prompts fell far below zero-shot on LSAT and CommonsenseQA (LSAT-LR 41.55% against 84.65% for Qwen2.5-72B), with \"repetitive or illogical outputs\".", "direction": "contradicts", "measured": true, "conditions": "Instruction-tuned open models: Qwen2.5 0.5B to 72B, LLaMA3 1B to 70B, Gemma2, Ministral-8B, plus older LLaMA2-7B and Qwen-7B; full GSM8K and MATH test sets; greedy decoding, one run, no variance reported; no API or thinking models; main results in figures, numbers from the appendix tables", "older": true }, { "source": "Harsha Nori, Naoto Usuyama, Nicholas King, Scott Mayer McKinney, Xavier Fernandes, Sheng Zhang, Eric Horvitz (Microsoft), \"From Medprompt to o1: Exploration of Run-Time Strategies for Medical Challenge Problems and Beyond\"", "url": "https://arxiv.org/abs/2411.03590", "date": "2024-11-06", "locator": "Abstract; Section 4.3 (Figures 5 and 6); Section 4.5; App. 8.1 Figure 8", "task": "Test Medprompt's components on a reasoning model: 5 random or 5 nearest-neighbor question-answer examples, a tailored prompt and answer-shuffled ensembles, against a minimal zero-shot prompt, with o1-preview on six medical benchmarks", "effect": "\"Few-shot prompting was inconsistent and, on average, hurt performance.\" Change against the zero-shot baseline, JMLE-2024 / MMLU / MedMCQA / MedQA / USMLE Sample Exam / USMLE Self Assessment: 5 random examples +1.2 / -1.1 / -0.0 / -1.4 / -0.4 / -1.0 points; 5 nearest-neighbor examples +0.6 / -0.1 / +1.2 / -2.2 / -0.3 / -0.6. A 15-run ensemble gained +0.6 to +2.0 on every benchmark and a tailored prompt -0.0 to +1.6. \"o1-preview with minimal or few-shot prompting employs more tokens but performs worse than with tailored prompts.\"", "direction": "contradicts", "measured": true, "conditions": "o1-preview only (temperature fixed at 1.0), three runs per evaluation; zero-shot baselines 81.4% to 95.6%, near ceiling on most sets; numbers from the Figure 8 heatmap; the examples follow the authors' own Medprompt method", "older": true } ], "caveats": "The large gains are from data agents asked about custom metrics or underspecified logic, on small or purpose-built sets: 13 questions in one Genie space, 133 questions scored by an LLM judge in one run at LinkedIn, and a benchmark built so that past queries are needed, run with one model. On strong instruction-tuned and reasoning models examples mostly align the format and can hurt: eight examples tied zero-shot on math for Qwen2.5-72B, few-shot prompts collapsed on LSAT and CommonsenseQA in the same study's appendix, and five examples lowered o1-preview on four of six medical benchmarks, though neither study tests a 2026 model. Retrieved examples are not reliably better than fixed ones (they did not beat zero-shot on a tool-selection benchmark, and low-precision retrieval misled the model), and the caching savings are estimated from attention complexity, not measured bills. No source here measures examples for multi-step agent tool calling, and VeRO lists adding examples where the problem calls for validation as an anti-pattern.", "also_covers": [], "related": [ { "id": "3.a.ii", "slug": "vary-few-shot-examples", "name": "Avoid few-shot ruts" }, { "id": "3.a.iv", "slug": "restructure-system-prompt", "name": "Restructure the system prompt: every requirement once, up front, in named sections; nothing the model doesn't need" }, { "id": "7.a.i", "slug": "semantic-layer-routing", "name": "Route metric questions through a deterministic semantic layer; fall back to text-to-SQL" }, { "id": "5.b.i", "slug": "ground-queries-in-schema", "name": "Give the model the schema it queries, in context: descriptions, keys, codes, sample values" }, { "id": "4.c.i", "slug": "episodic-memory", "name": "Episodic memory retrieval" }, { "id": "3.b.i", "slug": "cache-stable-prefix", "name": "Keep the prompt prefix cache-stable" }, { "id": "9.b.i", "slug": "extractable-final-answer", "name": "Make the final answer reliably extractable: state the format, extract robustly, force an answer at the limit" } ], "url": "https://mahalsystems.ai/aot/optimizations/few-shot-examples/", "markdown": "https://mahalsystems.ai/aot/optimizations/few-shot-examples.md" }, { "id": "3.a.ii", "legacy_id": "P27", "former_ids": [], "slug": "vary-few-shot-examples", "name": "Avoid few-shot ruts", "kind": "pattern", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes what the model sees", "proposed": true }, "levers": [ "prompt" ], "domains": [ "general" ], "evidence_domains": [ "tool-use", "code-generation", "planning", "document-extraction", "judging" ], "categories": [ "Structuring" ], "category_ids": [ "prompts.structure" ], "summary": "An agent's own history, and any examples in its prompt, act as few-shot demonstrations: when the context fills with near-identical action-observation pairs (twenty similar items in a row, a retried call), the model tends to copy their surface form instead of judging the current item. The change is to break that uniformity: vary the serialization, phrasing and order of examples and of newly appended actions and observations, describe repeated or failed actions instead of replaying them verbatim, or move independent items into separate contexts (5.c.ii). The measured support is mostly for the mechanism and for diverse examples: a failed call's verbatim string accounted for 83% of small models' pull toward repeating it, batching 20 studies flipped 5.6-26.7% of one model's screening decisions against one-at-a-time runs, and diverse demonstrations beat the most similar ones by up to 23 points. No study tests Manus-style serialization noise on an agent end to end, and equivalent formats alone moved accuracy by up to 76 points, so measure the variation instead of assuming it helps.", "how": [ "Start from the baseline: run the agent with its uniform template on a batch of near-identical items (for example 20 resumes or abstracts in one context), plant an outlier late in the batch (position 15 of 20), run every item alone in a fresh context as the reference, and log accuracy and decisions by position, repeated actions and repeats of failed calls.", "Find where uniformity enters: tool results and actions serialized with one fixed template, harness messages with identical wording, few-shot examples that share one format and one structure, many similar items in one context, and failed calls replayed verbatim.", "Diversify the examples in the prompt: choose examples that differ in structure and cover edge cases instead of the most similar ones; if you also vary their formatting (field names, separators, layout), pin the required output format with an instruction or a schema, since examples are what the model copies the format from.", "Vary newly appended history, deterministically: rotate two or three equivalent templates and phrasings for observations and harness messages, and the order of fields that carry no meaning, choosing the variant from the item index or a seed at append time; never rewrite earlier entries, so the context stays append-only and the prefix cache-stable (see 3.b.i).", "Do not replay repeated or failed actions verbatim (see 7.b.ii): replace a failed call with a short runtime-generated description of what failed, or block exact repeats at the decoder (see 7.b.i); an instruction not to repeat did not reduce repeats, and deleting the failed step made them worse.", "Where items are independent, break the repetition structurally instead: process each item or small batches in fresh contexts (5.c.ii), shuffle item order across batches, and in proposal loops reject proposals too similar to recently accepted ones; raising the temperature did not add real diversity.", "Measure with paired runs on the same items, uniform against varied (and against per-item contexts): accuracy and outlier detection by position, decision flip rate against the single-item runs, exact-repeat rate, output-format errors, tokens and cache hit rate, over several template sets and per model, reporting the spread rather than one number." ], "detect": { "static": "Every action and tool result serialized with one fixed template; harness messages with identical wording; few-shot examples that share one format and one structure or were picked as the most similar to the input; many similar items processed in one context; failed calls replayed verbatim in the history.", "traces": "Identical or near-identical consecutive actions; repeats of a call that already failed; later outputs that copy wording or structure from earlier items; decisions that drift with position in the batch or toward the batch majority, or that differ from the same item run alone.", "evals": "Accuracy and outlier detection by position in the batch; decision flip rate against each item run alone; spread across several equivalent templates.", "judge": "Detect over-generalization: later outputs that carry over content or conclusions from earlier items instead of the current one.", "probes": "A batch of similar items with one outlier." }, "impact": { "measured_basis": true, "primary": [ "quality" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "diverse demonstrations beat the most similar ones by up to 23 points, 50.3 to 73.5, on compositional semantic parsing (Levy, Dec 2022); mixing formats across few-shot examples raised one task's worst-format accuracy from 0.139 to 0.712 but lost on three task-model pairs (Ngweta, Apr 2025); describing a failed call instead of replaying it cut exact repeats from 31% to 16% while task success went from 42% to 33%, within noise (Gumaan, Aug 2026); rejecting near-duplicate proposals and reweighting toward unused edit types raised audited faithfulness from 0.44 to 0.71 where a higher temperature added 0.02-0.03 (He, Aug 2026); equivalent formats alone moved accuracy by up to 76 points (Sclar, Oct 2023)", "basis": "evidence" }, { "dimension": "cost", "effect": "mixed", "size": "small", "when": "replacing a verbatim call with a description cost no generated tokens (Gumaan, Aug 2026); varying entries already in the history breaks the prompt cache, which the same Manus post warns against (Ji, Jul 2025); one item per call gives up the savings that are the reason to batch (Hida, Aug 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "none", "size": null, "when": "no study reports latency; serialization changes add no generated tokens (Gumaan, Aug 2026); per-item contexts add calls that can run in parallel (see 5.c.ii)", "basis": "evidence" }, { "dimension": "security", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "governance", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "brand", "effect": "mixed", "size": "small", "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired runs on one batch of near-identical items with an outlier planted late (position 15 of 20), each item also run alone as the reference: uniform template against varied templates, phrasing and order (and against per-item contexts); accuracy and outlier detection by position, decision flip rate against the single-item runs, exact-repeat rate of actions, output-format errors, tokens and cache hit rate; repeat over at least three template sets and report the spread, per model" }, { "method": "monitor", "tier": null, "how": "per batch in production: share of consecutive identical or near-identical actions, repeats of a call that already failed, decision rates by position against the first items of the batch and drift toward the batch majority; on a weekly sample, re-run items alone and track the decision flip rate" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file master-agent-optimization-cookbook.md", "task": "Lists this practice in its agent-optimization pattern library (Production group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Yichao 'Peak' Ji (Manus), \"Context Engineering for AI Agents: Lessons from Building Manus\"", "url": "https://manus.im/blog/Context-Engineering-for-AI-Agents-Lessons-from-Building-Manus", "date": "2025-07-18", "locator": "Don't Get Few-Shotted", "task": "Add structured variation to actions and observations so the agent does not copy its own past pattern", "effect": "Manus \"introduces small amounts of structured variation in actions and observations—different serialization templates, alternate phrasing, minor noise in order or formatting.\" Example of drift while reviewing 20 resumes. No numbers.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Esmail Gumaan, \"Feedback That Backfires: Why Small Language Model Agents Repeat the Call They Just Watched Fail\"", "url": "https://arxiv.org/abs/2608.23651", "date": "2026-08-24", "locator": "Abstract; §1; §6.4; §7; §8.3 Table 8; §11", "task": "Measure whether a failed tool call kept verbatim in the transcript makes the agent repeat it, and compare harnesses that change only how the failure is serialized: the verbatim call plus error, a runtime-generated description instead of the call, the failed step deleted (clean retry), an added \"do not repeat\" instruction, and a decoder ban on previously failed strings", "effect": "For every model \"the normalised probability of repeating the failed call rises from 0.06 to 0.54\", and greedy decoding repeated it token for token on 19% of items against 0% before. \"The failed call's surface form accounts for 83% of the damage.\" A description instead of the call \"removes 76% of the inversion at no token cost\"; the instruction left it where it was, and deleting the failed step was \"the worst harness we measured for repetition\". The authors' rule: \"the context after a failure must differ from the context before it, and the difference must not be the failed action.\" In end-to-end rollouts the description cut exact repeats from 31% to 16% (paired difference -14 [-32, +2] points) and the decoder ban to 8%, but \"Neither surface-form intervention improves task success\": success went from 0.42 to 0.33 with the description. Within a family, \"Larger models copy less\".", "direction": "mixed", "measured": true, "conditions": "6 instruction-tuned checkpoints of 135M-1.7B (Llama-3.2, Qwen2.5, Qwen3, SmolLM2); the authors' simulated tool-calling environment and MBPP program repair; rollouts on 24 tasks with Qwen2.5-0.5B and greedy decoding; frontier-scale models untested", "older": false }, { "source": "Akshit Sinha, Arvindh Arun, Shashwat Goel, Steffen Staab, Jonas Geiping, \"The Illusion of Diminishing Returns: Measuring Long Horizon Execution in LLMs\"", "url": "https://arxiv.org/abs/2509.09677", "date": "2025-09-11", "locator": "§3.2 Results 3-5, Figures 5-6; Appendix C, Figures 9-10", "task": "Inject a controlled rate of errors into the model's own prior turns of a long, simple execution task (look up values in an in-context dictionary and keep a running sum) and measure accuracy at a later turn", "effect": "\"as we increase the rate of injected errors into the context, accuracy at turn 100 consistently degrades further\"; the authors tie this to in-context learning, since here \"the model conditions on its own past behaviour, which could be wrong\". Large non-thinking models \"remain susceptible to self-conditioning\". \"Qwen3 models with thinking enabled no longer self-condition, even when the entire prior history has wrong answers.\" Keeping only recent turns \"reduces the self-conditioning effect, but relies on the Markovian nature of our task\", and \"Self-verification does not fix self-conditioning\". Results are in figures.", "direction": "mixed", "measured": true, "conditions": "Synthetic key-value running-sum task, 100 samples per setting; Qwen3 4-32B and Gemma3 4-27B, with Kimi-K2, DeepSeek-V3 and Qwen3-235B-Instruct-2507 for scale; the errors in history are injected, and the tested fix shrinks history rather than varying it", "older": true }, { "source": "Gilberto Sussumu Hida, Danilo Monteiro Ribeiro, Clayton Suguio Hida, \"Class Imbalance and Batch Effects in LLM-Based Screening for Systematic Reviews\"", "url": "https://arxiv.org/abs/2608.14737", "date": "2026-08-13", "locator": "§3.3; §4.3 RQ2; §4.4 Table 2; §4.5 RQ4; §5", "task": "Screen studies for systematic reviews one per call, against the same studies shuffled into batches of 20 in one prompt, with and without a sentence giving the expected inclusion rate", "effect": "For Llama, batching changed the decision on 5.6% (rsl_1) to 26.7% (rsl_4) of studies, significant by McNemar (p < 0.001) in all five reviews; for GPT-5-mini 1.3% to 14.0%, significant in four of five (mean flip rate 0.160 for Llama against 0.057). Batching made Llama more inclusive in the four reviews with prevalence of 20% or less (net shift +3.5 to +10.5 points) and more restrictive in the balanced one (-25.3), which the authors read as \"a possible tendency toward implicit normalization of decisions within the batch\". F2 changes ranged from -0.101 to +0.096; in one review F2 rose from 0.37 to 0.46 while at item level \"29 studies improved and 85 worsened\". Stating the expected inclusion rate did not correct the shift in the balanced review for either model.", "direction": "mixed", "measured": true, "conditions": "Llama-3.3-70B-Instruct-Turbo and GPT-5-mini; five software-engineering reviews from SESR-Eval (prevalence 2.9% to 53%); one batch arrangement and one run per condition; compares batching with one study per call (isolation), not varied serialization", "older": false }, { "source": "Tianxi Li, Jie Ding, \"Trustworthy Method Comparison with AI Judges: Estimation and Design under Order, Batch, and Aggregation Effects\"", "url": "https://arxiv.org/abs/2610.07755", "date": "2026-10-06", "locator": "§2 (Figures 1-2); §2.3; §7", "task": "Three LLM judges each rate the same 30 research proposals accept or reject, all 30 in one context, in 200 randomly shuffled orders; model how each rating depends on the ratings before it", "effect": "Gemini 2.5 Flash accepted an item 73.0% of the time right after a rejection and 66.6% after an acceptance, a difference of -6.4% (95% interval [-8.9%, -3.9%]). \"The lag-1 contrast is negative and supported by the bootstrap for all three judges, indicating that an acceptance lowers the probability of a subsequent acceptance\"; GPT-5.4-mini also showed a lag-3 effect and Claude Haiku 4.5 significant negative contrasts at all three lags. The dependence is contrast between neighbors, not imitation. The authors \"recommend GLMM inference with a random-shuffle batch design\".", "direction": "mixed", "measured": true, "conditions": "Gemini 2.5 Flash, GPT-5.4-mini, Claude Haiku 4.5; 6,000 binary ratings per judge plus 100 new shuffled batches for out-of-sample checks; one pool of 30 items", "older": false }, { "source": "Bowei He, Weixu Zhang, Yili Jin, Xue Liu, \"Beneath the Diff: Diagnosing and Mitigating Algorithmic Mode Collapse in Code-Level Autonomous Research Loops\"", "url": "https://arxiv.org/abs/2609.00077", "date": "2026-08-31", "locator": "Abstract; §4.2 Table 2; §4.5 Table 5; Appendix A (F3)", "task": "Track the diversity of an autonomous research agent's accepted code edits over hundreds of iterations, then add diversity-aware proposal sampling (reweighting toward under-used edit types, a memory that rejects proposals too similar to recently accepted ones, and an audit-based revert gate), compared with raising the proposer's temperature from 0.7 to 1.2", "effect": "\"surface-level edit diversity remains stable, but semantic and mechanism-level diversity collapse: the agent continues to edit different lines of code while repeatedly proposing the same kinds of algorithmic changes\", and in-loop gains do not transfer (audited faithfulness 0.38-0.49). Higher temperature gave 0.43, 0.49, 0.40 and 0.51 against 0.41, 0.46, 0.38 and 0.49: \"raising temperature is not a remedy\". The two diversity components alone raised average faithfulness from 0.44 to 0.71 and cut semantic-cluster decay from 0.68 to 0.28 at 0.986 of the in-loop gain; the full method reached 0.74-0.82.", "direction": "supports", "measured": true, "conditions": "Claude Opus 4.7 proposer (GPT-5.2 and Llama-3.3-70B in robustness runs); four single-GPU tasks (small-LM pretraining, instruction tuning, GSM8K tuning, prompt optimization); 300 iterations, 3 seeds (5 for one task); autoresearch fork and Aider. The authors' \"self-imitation through history conditioning\" is part of a conceptual model, not isolated; the fix acts on proposal sampling, not on how history is serialized", "older": false }, { "source": "Lilian Ngweta, Kiran Kate, Jason Tsay, Yara Rizk, \"Towards LLMs Robustness to Changes in Prompt Format Styles\"", "url": "https://arxiv.org/abs/2504.06969", "date": "2025-04-09", "locator": "§3.2; Table 1; Figure 3; Appendix A Table 2", "task": "Mixture of Formats: present each few-shot example in a different format style (field names, separators, casing) and have the model rewrite each example in another style, instead of one shared format; compare accuracy across 10 format variants", "effect": "\"MOF prompting effectively reduces style-induced prompt brittleness across several datasets and LLMs, with a notable 46% reduction in task280 using Llama-2-13b.\" For llama-3-70b-instruct on task317, worst/best-format accuracy rose from 0.139/0.229 to 0.712/0.795. Mean accuracy for Llama-2-13b rose on task280 (0.635 to 0.842) but fell on task1612 (0.564 to 0.505), and \"exceptions occur in task190 (llama-3-70b-instruct), task1612 (llama-2-13b-chat), and task320 (falcon-11B), where traditional prompts perform better.\"", "direction": "supports", "measured": true, "conditions": "16 randomly chosen Super-NaturalInstructions tasks, 1,000 samples each, 5-shot, 10 format variants per arm built with FormatSpread; falcon-11B, Llama-2-13b, Llama-2-13b-chat, llama-3-70b-instruct; short paper calling its results preliminary, no variance or significance reported", "older": true }, { "source": "Itay Levy, Ben Bogin, Jonathan Berant, \"Diverse Demonstrations Improve In-context Compositional Generalization\"", "url": "https://arxiv.org/abs/2212.06800", "date": "2022-12-13", "locator": "Figure 1; §4.4 Table 2", "task": "Choose few-shot demonstrations that together cover the structures the output needs (or are dissimilar from one another), instead of the top-k most similar ones, which \"yields repetitive demonstrations\"", "effect": "Diverse demonstrations \"improve performance by up to 23 absolute points (e.g., 50.3 → 73.5 on SMCalFlow-CS) compared to a baseline that retrieves demonstrations according to similarity alone\"; all diversity methods beat top-k \"in 7 out of 8 compositional splits\". The coverage method failed on the zero-shot cross-domain split (0.3 against 17.0 for top-k).", "direction": "supports", "measured": true, "conditions": "Codex (code-davinci-002), 24 demonstrations, 100 test examples x 3 seeds per split; GeoQuery, SMCalFlow-CS and COVR-10 semantic parsing; also T5-large fine-tuned with demonstrations", "older": true }, { "source": "Melanie Sclar, Yejin Choi, Yulia Tsvetkov, Alane Suhr, \"Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting\"", "url": "https://arxiv.org/abs/2310.11324", "date": "2023-10-17", "locator": "Abstract; §1; §4.2; §4.5 (Figure 4)", "task": "Measure few-shot accuracy across meaning-preserving prompt formats (separators, casing, spacing, field names)", "effect": "Accuracy varied \"in up to 76 points for LLaMA-2-13B between equivalent formats\", about 10 points on average across 50+ tasks and several models, and \"this variance is not eliminated by adding few-shot examples, increasing model size, or instruction tuning\". GPT-3.5 showed \"a spread up to 56 accuracy points with a median spread of 6.4\". Format performance \"only weakly correlates between models\"; LLaMA-2-13B and -70B swapped order with probability 0.141 when the format changed.", "direction": "mixed", "measured": true, "conditions": "53 Super-NaturalInstructions classification tasks; LLaMA-2 7B/13B/70B, Falcon-7B and -7B-Instruct, GPT-3.5-Turbo; 1- and 5-shot; one format per prompt (formats are not mixed within a prompt)", "older": true }, { "source": "Manus, \"Wide Research: Beyond the Context Window\"", "url": "https://manus.im/blog/manus-wide-research-solve-context-problem", "date": "2025-10-29", "locator": "The Fabrication Threshold; Why Bigger Context Windows Can't Fix This", "task": "Describes quality decay when one agent context handles many similar items in sequence, and Manus's later answer: one sub-agent with a fresh context per item", "effect": "\"by the eighth or ninth item in a multi-subject research task, the AI starts fabricating.\" \"Items 6-8: The quality begins to subtly degrade. Descriptions become slightly more generic. The model starts relying more on prior patterns than fresh research.\" The fabrications \"follow the established format perfectly\" and are \"stylistically consistent with the earlier, legitimate entries.\" Manus attributes this to context load and answers it with fresh-context sub-agents, not with variation. No numbers.", "direction": "mixed", "measured": false, "conditions": "Vendor product post; no method, model or sample reported", "older": false }, { "source": "Anthropic, \"Prompting best practices\" (Claude Docs)", "url": "https://docs.claude.com/en/docs/build-with-claude/prompt-engineering/multishot-prompting", "date": "2024-06-21", "locator": "Use examples effectively", "task": "Vendor guidance on writing few-shot examples", "effect": "Make examples \"Diverse: Cover edge cases and vary enough that Claude doesn't pick up unintended patterns.\" Examples \"are one of the most reliable ways to steer Claude's output format, tone, and structure.\"", "direction": "supports", "measured": false, "conditions": "Living documentation page; the date is the earliest archived capture of the multishot-prompting page, which already carried the same \"Diverse\" guidance; no measurements", "older": true }, { "source": "Google, \"Prompt design strategies\" (Gemini API docs)", "url": "https://ai.google.dev/gemini-api/docs/prompting-strategies", "date": "2024-07-21", "locator": "Zero-shot vs few-shot prompts: Optimal number of examples; Consistent formatting", "task": "Vendor guidance on formatting few-shot examples", "effect": "\"Make sure that the structure and formatting of few-shot examples are the same to avoid responses with undesired formats. ... it is essential to ensure a consistent format across all examples, especially paying attention to XML tags, white spaces, newlines, and example splitters.\" The same page warns that with too many examples \"the model may start to overfit the response to the examples.\"", "direction": "contradicts", "measured": false, "conditions": "Living documentation page; the date is the earliest archived capture found containing the passage; no measurements. Contradicts varying the serialization of examples; agrees that models over-fit to their examples", "older": true } ], "caveats": "No study tests Manus-style serialization noise on an agent end to end: the measured support is for the mechanism (models copy what repeats in their context) and for diverse few-shot examples, often on small or older models (135M-1.7B for the failed-call study, Codex for the demonstration study). Variation is not free: equivalent formats moved accuracy by up to 76 points, mixing formats lost on some task-model pairs, Google advises identical formatting across examples, and a serialization fix that cut repeats did not raise task success. The effect depends on the model and task: thinking models did not self-condition on errors in their history, larger models copied a failed call less, and a batch-judging study found contrast between neighbors rather than imitation. Varying entries already in the history also breaks the append-only, cache-stable context, so vary only new entries, deterministically.", "also_covers": [], "related": [ { "id": "3.a.i", "slug": "few-shot-examples", "name": "Add worked examples to the prompt (few-shot), or curated example queries for data agents" }, { "id": "5.c.ii", "slug": "isolated-subagent-context", "name": "Sub-agent architectures with clean contexts" }, { "id": "7.b.ii", "slug": "keep-errors-in-context", "name": "Keep errors in context" }, { "id": "7.b.i", "slug": "detect-loops-and-stalls", "name": "Detect stalls and loops: warn, then stop early" }, { "id": "3.b.i", "slug": "cache-stable-prefix", "name": "Keep the prompt prefix cache-stable" }, { "id": "7.b.iv", "slug": "track-task-progress", "name": "Track explicit progress: a task list with a status per item, re-injected each turn" }, { "id": "3.a.iv", "slug": "restructure-system-prompt", "name": "Restructure the system prompt: every requirement once, up front, in named sections; nothing the model doesn't need" } ], "url": "https://mahalsystems.ai/aot/optimizations/vary-few-shot-examples/", "markdown": "https://mahalsystems.ai/aot/optimizations/vary-few-shot-examples.md" }, { "id": "3.a.iii", "legacy_id": "P02", "former_ids": [], "slug": "inject-runtime-context", "name": "Dynamic context injection", "kind": "pattern", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes what the model sees", "proposed": true }, "levers": [ "prompt", "context" ], "domains": [ "general" ], "evidence_domains": [ "conversation", "reasoning", "code-generation" ], "categories": [ "Structuring" ], "category_ids": [ "prompts.structure" ], "summary": "Put the runtime facts the model cannot know into its context each run: today's date and timezone, the working directory, OS and git state, the tools available, the user and locale. Inject them after the cached prefix, as a date rather than a timestamp, and refresh them when they change. The measured gains are on time-sensitive work: absolute dates beat relative ones, and stating the time remaining each turn raised deal closure under deadlines. On tasks that do not depend on time, the date only adds variance.", "how": [ "List the runtime facts your tasks depend on: date and IANA timezone, working directory, OS and shell, git status, the tools or skills available, the user and locale.", "Inject them after the cached prefix, in a trailing block or a message in the conversation, never under a cache breakpoint; keep the date to the day, and serve time of day through a clock tool or the shell.", "State dates as absolute values, and resolve relative dates (\"last quarter\") to absolute ones before the model sees them.", "Refresh on change: resend the block when a value changes and after compaction or resume; if the run has a deadline, inject the time remaining each turn.", "Fix the date in evals and report it; run a time-sensitive slice with and without the injection, and watch cache-read tokens." ], "detect": { "static": "Prompt templates contain runtime placeholders (date, time, user, locale, tool availability). Absence where tasks are time- or user-sensitive is the signal.", "traces": "Model guesses the date or year, asks the user for information the system already has, builds queries with stale dates.", "evals": "Failures concentrated on time-sensitive or user-specific items.", "judge": "Flag outputs that assume stale state.", "probes": "Tasks whose answer depends on today's date, relative dates, locale or the current user." }, "impact": { "measured_basis": true, "primary": [ "quality" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "time-sensitive work: deal closure 0.04 to 0.32 with the time remaining stated each turn (Sehgal, Jan 2026); answers 24-35% worse with relative than with absolute dates (Wallat, 2024). On tasks that do not depend on time, the date moved scores by up to 14% with no good date to fix (Sanz-Guerrero, Sep 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "none", "size": null, "when": "if the block sits after the cached prefix; a date in the system prompt cost Claude Code cache hits until it was moved out (Feb 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "none", "size": null, "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "mixed", "size": "small", "when": "Injected state is a small injection surface.", "basis": "assessment" }, { "dimension": "governance", "effect": "mixed", "size": "small", "when": "User data in the prompt needs retention rules.", "basis": "assessment" }, { "dimension": "brand", "effect": "improves", "size": "small", "when": "Fewer obviously wrong dates.", "basis": "assessment" } ] }, "measure": [ { "method": "before-after", "tier": 3, "how": "run a time-sensitive slice with and without the injected block, with the date fixed; compare cache-read tokens per call before and after" }, { "method": "monitor", "tier": null, "how": "questions asking the user for facts the system holds, date errors in outputs, cache-hit rate" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file master-agent-optimization-cookbook.md", "task": "Lists this practice in its agent-optimization pattern library (Context Engineering group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Anthropic, \"System Prompts\" (release notes)", "url": "https://docs.claude.com/en/release-notes/system-prompts", "date": null, "locator": "Page intro", "task": "Anthropic injects runtime state such as the current date into the system prompt of its own apps", "effect": "Claude's web and mobile apps \"use a system prompt to provide up-to-date information, such as the current date, to Claude at the start of every conversation.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Zhao, Brumbaugh, Wang, Hajishirzi, Smith, \"Set the Clock: Temporal Alignment of Pretrained Language Models\"", "url": "https://arxiv.org/abs/2402.16797", "date": "2024-02-26", "locator": "App. A, Table 6", "task": "State the reference year in the prompt (\"As of year 2022\") for time-sensitive questions", "effect": "F1 on questions about 2022 rose from 17.2 to 24.4 with the year in a 5-shot prompt. The authors also note that \"prompting LMs to be aligned to a more recent year could lead to more QA errors.\"", "direction": "supports", "measured": true, "conditions": "LLaMa2-70B; time-sensitive QA, target years 2021 and 2022", "older": true }, { "source": "Wallat, Jatowt, Anand, \"Temporal Blind Spots in Large Language Models\"", "url": "https://arxiv.org/abs/2401.12078", "date": "2024-01-22", "locator": "Figure 2 and surrounding text", "task": "Ask with relative time references (\"3 years ago\") instead of absolute years", "effect": "Performance dropped 24-35% for text-davinci-003 and 23-30% for alpaca-7B with relative references; both models answered 2021 when asked what year it is.", "direction": "supports", "measured": true, "conditions": "27k ArchivalQA question-answer pairs", "older": true }, { "source": "Sehgal, Guntuku, Ungar, \"Real-Time Deadlines Reveal Fragile Temporal Adaptation in LLM Strategic Dialogues\"", "url": "https://arxiv.org/abs/2601.13206", "date": "2026-01-19", "locator": "Figure 2; Table 2", "task": "Inject the time remaining into each turn of deadline-bound negotiations between agents", "effect": "Deal closure rose from 0.04 (12/300) to 0.32 (97/300) for GPT-5.1; offers were over six times more likely to be accepted (OR = 6.38). GPT-4.1 went from 44.7% to 72.0%. \"only 5 of 10 comparisons reach Bonferroni-corrected significance.\"", "direction": "supports", "measured": true, "conditions": "GPT-5.1-chat-latest, N=300 per condition; several other models; revised 2026-08-30", "older": false }, { "source": "Sanz-Guerrero, Bui, Mager, von der Wense, \"Dating the Model: Hidden Dates in System Prompts Affect LLM Evaluation\"", "url": "https://arxiv.org/abs/2609.36931", "date": "2026-09-29", "locator": "Abstract; Table 3; App. B", "task": "Vary the current date in the system prompt on benchmarks that do not depend on time", "effect": "Performance \"varies solely with the current date, with deltas of up to 6% on MCQA, 14% on math reasoning, 7% on code generation, and 2.84 BLEU\"; \"chain-of-thought even amplifies it\". With no date, scores fall within the range seen across dates: the date adds variance, and there is no good date to fix.", "direction": "mixed", "measured": true, "conditions": "9 LLMs, 6 datasets, every date in 2024", "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 7.2", "task": "Inject environment, OS and git state into the prompt", "effect": "Mini-SWE-Agent template \"includes OS-specific instructions (e.g., sed -i '' for macOS)\"; Mistral Vibe Git context block \"(output of git log --oneline -N --decorate and git status --porcelain, cached)\"; Hermes keeps \"a date-only timestamp\" so the prefix stays byte-identical.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Anthropic, Claude Code changelog, version 2.1.42", "url": "https://github.com/anthropics/claude-code/blob/main/CHANGELOG.md", "date": "2026-02-13", "locator": "Version 2.1.42", "task": "Move the date out of the system prompt", "effect": "\"Improved prompt cache hit rates by moving date out of system prompt.\" No numbers.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "OpenAI Codex CLI, \"feat: add local date/timezone to turn environment context\"", "url": "https://github.com/openai/codex/pull/12947", "date": "2026-02-26", "locator": "PR description", "task": "Add the local date and IANA timezone to the per-turn environment context", "effect": "Adds current_date and timezone to the environment_context block and \"treats date/timezone changes as environment updates, so prompt caching and context refresh logic do not silently reuse stale time context.\"", "direction": "supports", "measured": false, "conditions": null, "older": false } ], "caveats": "Few studies isolate the date on agent tasks; the measured support is indirect: a stated year, absolute against relative dates, the time remaining in a negotiation. On tasks that do not depend on time the date only adds variance, and chain-of-thought amplifies it, so fix the date in evals. Some providers already add a date server-side, so check the model is not seeing two. Any per-request value placed before the cached prefix turns every call into a cache write (3.b.i).", "also_covers": [], "related": [ { "id": "3.b.i", "slug": "cache-stable-prefix", "name": "Keep the prompt prefix cache-stable" } ], "url": "https://mahalsystems.ai/aot/optimizations/inject-runtime-context/", "markdown": "https://mahalsystems.ai/aot/optimizations/inject-runtime-context.md" }, { "id": "3.a.iv", "legacy_id": "T04", "former_ids": [], "slug": "restructure-system-prompt", "name": "Restructure the system prompt: every requirement once, up front, in named sections; nothing the model doesn't need", "kind": "technique", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes what the model sees", "proposed": false }, "levers": [ "prompt" ], "domains": [ "general" ], "evidence_domains": [ "code-generation", "reasoning" ], "categories": [ "Structuring" ], "category_ids": [ "prompts.structure" ], "summary": "Rewrite the system prompt so that each requirement the task has (constraints, the answer format, the completion check, how to work efficiently, the domain procedure) is stated once, up front, in a named section, and remove instructions the model does not need. Adding structure and removing bulk are the same edit seen from two sides: what stays is what the eval shows the model needs. A domain procedure section is worth adding only when failures cluster on a step an expert would take; write it from those failures, from a domain expert, or by searching prompt variants against the eval (GEPA).", "how": [ "List the requirements the task actually has, from the task spec and from failures.", "Put each in one named section; put the answer format and completion check in their own sections.", "Cut everything else, and shorten tool descriptions and checklists with it (the TerminalBench run cut the prompt from 7 lines to 3).", "Add a domain procedure only if failures cluster on a step; derive it from those failures or an expert, or generate and test variants with GEPA.", "Run the paired eval on development and held-out cases; keep only sections that earn their place." ], "detect": { "static": "system prompt with no named sections; requirements repeated, contradictory, or absent (no answer format, no completion criteria); prompt plus tool descriptions over about a quarter of the context window", "traces": "failure rows labeled instruction-following-failure or instruction-grader-mismatch where the requirement was never in the prompt; prompt tokens the largest share of input per step; context-overflow outcomes", "evals": "an ablation that removes a section and shows no loss (cut it), or failures that cluster on one step of the task procedure (add it)" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "small", "when": "adds: MAST +4.7 to +5.0 points on GSM-Plus (2025), VeRO MATH validation +0.10 with train -0.07; cuts: TerminalBench +6.8 points where the prompt crowded out context, GPQA train -0.04 where the prompt carried procedure", "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "small", "when": "fewer prompt tokens", "basis": "evidence" }, { "dimension": "performance", "effect": "improves", "size": "small", "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval, old against restructured prompt, checked on held-out cases; count context-overflow failures too" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper §A.4", "task": "TerminalBench-2", "effect": "8 lines to 23 lines in Filesystem-B=178. Net-zero pass rate (6 new passes, 6 regressions) although the run had the largest crash reduction (46.1% to 30.3%).", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Huang et al., \"Large Language Models Cannot Self-Correct Reasoning Yet\"", "url": "https://arxiv.org/abs/2310.01798", "date": "2023-10-03", "locator": "Sec. 5; Table 8", "task": "Move self-correction criteria into the initial prompt", "effect": "Table 8, CommonGen-Hard concept coverage, gpt-3.5-turbo-0613: Madaan et al. reported standard 44.0 -> self-correct 67.0 (7 calls); replication 53.0 -> 61.1; adding \"Write a reasonable paragraph that includes *ALL* of the above concepts\" gives 81.8 in 1 call, and self-correcting that drops it to 75.1.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Kamoi et al., \"When Can LLMs Actually Correct Their Own Mistakes? A Critical Survey of Self-Correction of LLMs\"", "url": "https://arxiv.org/abs/2406.01297", "date": "2024-06-03", "locator": "Sec. 4 Weak Initial Responses; Table 4", "task": "Put every task requirement in the initial prompt", "effect": "Self-Refine used wrong few-shot labels or mismatched instructions only for initial generation (Tables 9-10); detoxification studies often do not ask for harmless output initially.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Cemri et al., \"Why Do Multi-Agent LLM Systems Fail?\"", "url": "https://arxiv.org/abs/2503.13657", "date": "2025-03-17", "locator": "App. H.1, App. L.1-L.2, Table 5", "task": "Restructure prompt with a dedicated verification section", "effect": "Table 5, GSM-Plus (200 sampled problems, 6 repetitions): GPT-4 84.75±1.94 -> 89.75±1.44; GPT-4o 84.25±1.86 -> 89.00±1.38 (Wilcoxon p=0.03 vs baseline for both).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Cemri et al., \"Why Do Multi-Agent LLM Systems Fail?\"", "url": "https://arxiv.org/abs/2503.13657", "date": "2025-03-17", "locator": "App. G.1, Table 4", "task": "Specify each agent's role and task clearly", "effect": "Proposed tactic (App. L.2 is the example; measured forms are the ChatDev/AG2 entries above). Such tactics \"often prove inconsistent\".", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper §A.4", "task": "TerminalBench-2", "effect": "Five-line block in Tools-B=89 alongside T01 and T02; that commit went 30.3% to 33.7% pass rate.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper §A.4, Table 16", "task": "TerminalBench-2", "effect": "\"Tools-B=178: prompt 7 to 3 lines, tool descriptions to one phrase each, checklist 10 to 4 lines, cap 17KB. Best pass rate of the study, 37.1% (+6.8pp), with ContextLengthExceeded 13 to 2, without fixing the parser crash.\"", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 12 (Orchestrator, Sonnet 4.5)", "task": "MATH", "effect": "Validation 0.78 to 0.88 in phase 1; train dropped 0.85 to 0.78 in the same phase.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 12", "task": "MATH", "effect": "Flat on validation (0.87).", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 13 (Resources Only, Sonnet 4.5)", "task": "GPQA", "effect": "Train 0.58 to 0.54 (regressed).", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper §A.4", "task": "TerminalBench-2", "effect": "As part of T06, the best result in that study.", "direction": "supports", "measured": null, "conditions": null, "older": false } ], "caveats": "Intermediate 20-task subset showed 45% and misled the optimizer; full evaluation regressed previously-passing tasks. 12 new passes against 6 regressions; the same run left 11 AttributeError crashes in place. Helps when the prompt was crowding out task context; hurts on reasoning tasks where the prompt carried procedure.", "also_covers": [ { "name": "System prompt altitude and sectioned structure" } ], "related": [ { "id": "3.a.ii", "slug": "vary-few-shot-examples", "name": "Avoid few-shot ruts" }, { "id": "2.b.ii", "slug": "tool-use-guidance", "name": "Tell the agent when to use each tool, in context it always sees" }, { "id": "9.b.i", "slug": "extractable-final-answer", "name": "Make the final answer reliably extractable: state the format, extract robustly, force an answer at the limit" }, { "id": "3.b.i", "slug": "cache-stable-prefix", "name": "Keep the prompt prefix cache-stable" }, { "id": "3.a.i", "slug": "few-shot-examples", "name": "Add worked examples to the prompt (few-shot), or curated example queries for data agents" }, { "id": "3.c.i", "slug": "self-discover-reasoning", "name": "Self-Discover reasoning structures" } ], "url": "https://mahalsystems.ai/aot/optimizations/restructure-system-prompt/", "markdown": "https://mahalsystems.ai/aot/optimizations/restructure-system-prompt.md" }, { "id": "3.b.i", "legacy_id": "T63", "former_ids": [], "slug": "cache-stable-prefix", "name": "Keep the prompt prefix cache-stable", "kind": "technique", "status": "card", "tier": { "tier": 1, "name": "Hygiene", "why": "a reordering that leaves what the model sees unchanged, verified on the hand-labeled set", "proposed": false }, "levers": [ "context" ], "domains": [ "general" ], "evidence_domains": [ "code-generation", "data-analysis" ], "categories": [ "Caching" ], "category_ids": [ "prompts.caching" ], "summary": "Put static content (identity, rules, tool definitions) before a boundary and per-turn content (date, state, memory) after it; keep the prefix bit-for-bit identical across turns; strip thinking text from persisted history; and track per-turn cache-miss cost. Check the provider's caching minimum: a prefix below it never caches.", "how": [ "Read the prompt assembly code: find timestamps, per-turn state, or non-deterministic serialization (dict order, ids) early in the prompt.", "Move them after the static block; sort and fix serialization.", "Confirm the static prefix exceeds the provider's minimum cacheable length.", "Replay the hand-labeled set; compare cache-hit rate before and after." ], "detect": { "static": "a timestamp, session id or per-turn state before the static block; non-deterministic serialization; a marked-for-caching prefix shorter than the provider's minimum (in one image-labeling agent, 504 tokens against Haiku 4.5's 4,096)", "traces": "cache_read tokens near zero across calls that share a prefix; cache-hit rate below about 50% on multi-turn runs" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "none", "size": null, "when": null, "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "large", "when": "cached input is billed at a fraction of full price; RETRACE's cache hits rose 45.6% to 90.8% as a side effect (Aug 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "improves", "size": "small", "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "verify-fix", "tier": 1, "how": "replay the hand-labeled set; compare cache-hit rate" }, { "method": "monitor", "tier": null, "how": "cache-hit rate and cache-miss cost per turn" } ], "evidence": [ { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 7.2; Observation 2; Table 11 \"Prompt Caching\"", "task": "Split the system prompt at a static/dynamic cache boundary", "effect": "Claude Code SYSTEM_PROMPT_DYNAMIC_BOUNDARY; the static prefix is \"cached via Blake2b hashing with scope 'global'\" and \"achieves high cache hit rates across turns and even across sub-agents\"; OpenHands CacheTier.STATIC/DYNAMIC arrived at the same design independently; Codex prompt_cache_key derived from ThreadId. No hit-rate numbers.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 7.2 (Hermes); Observation 2", "task": "Normalize the prompt prefix bit-for-bit for cache hits", "effect": "Hermes \"prompt prefix is bit-perfect normalized (sorted compact re-serialization of tool-call JSON, a date-only timestamp, a frozen memory snapshot) so that even local llama.cpp and vLLM KV caches hit across turns\"; places four cache_control breakpoints (\"system_and_3\").", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 7.2; Observation 2; Table 12 \"Cache-Dialect Fanout\"", "task": "Emit every provider's cache dialect at once", "effect": "OpenCode \"emits six providers' cache dialects simultaneously and shapes its system prompt to at most two messages (one by default) to match the cache slots.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 7.4; Sec 15.1", "task": "Strip extended-thinking text from persisted history", "effect": "Gemini CLI \"Keeps extended-thinking parts out of persisted history at response-recording time so they don't pollute future cache lookups\"; cited as a mitigation for \"Multi-agent systems use ~15x more tokens than chat\" [17].", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 7.5; Observation 2", "task": "Audit per-turn cache-miss cost as a metric", "effect": "Pi \"audits per-turn cache-miss dollar waste as a first-class metric\"; \"a per-turn cache-miss dollar-waste audit that prices every avoidable cache invalidation.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Lance Martin, \"Agent Design Patterns\"", "url": "https://rlancemartin.github.io/2026/01/09/agent_design/", "date": "2026-01-09", "locator": "Cache Context", "task": "Maximize prompt-cache hit rate; track it as metric", "effect": "Manus called cache hit rate \"the most important metric for production agents\": \"a higher capacity ($/token) model with caching can actually be cheaper than a lower cost model without it.\" Coding agents \"would be cost-prohibitive without caching.\" No numbers.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Chenglin Li, Yisen Xu, Zehao Wang, Shin Hwei Tan, Tse-Hsun (Peter) Chen, \"Independent Patch Verification for Coding Agents with a Bidirectional Reconstruct-and-Verify Framework\"", "url": "https://arxiv.org/abs/2608.08950", "date": "2026-08-09", "locator": "Cost analysis; Table 4", "task": "Share issue and trajectory prefix across verifier calls", "effect": "Table 4, average per issue, SWE-bench Verified: GPT-5 mini baseline 26.3K input, 45.6% cache hit, $0.07 vs RETRACE 60.6K input, 90.8% cache hit, $0.06; output tokens 4.2K -> 1.2K. MiniMax M2.5 baseline 123.3K input, 94.6% cache hit, 0.07 vs RETRACE 78.0K, 96.7%, 0.05. Cost ratio below 1.0x on both backbones.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Michael Rumiantsau, Ivan Fokeev (Cube), \"Semantic Layers for Reliable LLM-Powered Data Analytics: A Paired Benchmark of Accuracy and Hallucination Across Three Frontier Models\"", "url": "https://arxiv.org/abs/2604.25149", "date": "2026-04-28", "locator": "Section 5.3 Cost and Latency", "task": "Cache the schema and semantic-layer prefix", "effect": "Argued, not measured: the document adds ~2-4K input tokens per query (Table 1); with caching \"the marginal cost ... is effectively zero\".", "direction": "supports", "measured": false, "conditions": null, "older": false } ], "caveats": null, "also_covers": [ { "name": "KV-cache-friendly prompt layout" } ], "related": [ { "id": "3.a.iv", "slug": "restructure-system-prompt", "name": "Restructure the system prompt: every requirement once, up front, in named sections; nothing the model doesn't need" }, { "id": "3.a.i", "slug": "few-shot-examples", "name": "Add worked examples to the prompt (few-shot), or curated example queries for data agents" }, { "id": "5.b.i", "slug": "ground-queries-in-schema", "name": "Give the model the schema it queries, in context: descriptions, keys, codes, sample values" }, { "id": "3.a.iii", "slug": "inject-runtime-context", "name": "Dynamic context injection" }, { "id": "7.c.v", "slug": "language-agent-tree-search", "name": "Language Agent Tree Search (LATS)" }, { "id": "3.b.ii", "slug": "mask-tools-not-remove", "name": "Mask tools instead of removing them" }, { "id": "6.a.i", "slug": "oracle-worker-escalation", "name": "Oracle-worker multi-model" }, { "id": "3.c.i", "slug": "self-discover-reasoning", "name": "Self-Discover reasoning structures" }, { "id": "3.a.ii", "slug": "vary-few-shot-examples", "name": "Avoid few-shot ruts" } ], "url": "https://mahalsystems.ai/aot/optimizations/cache-stable-prefix/", "markdown": "https://mahalsystems.ai/aot/optimizations/cache-stable-prefix.md" }, { "id": "3.b.ii", "legacy_id": "P23", "former_ids": [], "slug": "mask-tools-not-remove", "name": "Mask tools instead of removing them", "kind": "pattern", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes which tools the model can choose", "proposed": true }, "levers": [ "context", "tool-surface" ], "domains": [ "general" ], "evidence_domains": [ "tool-use", "web-research", "code-generation" ], "categories": [ "Caching", "Avoiding misuse" ], "category_ids": [ "prompts.caching", "tools.misuse" ], "summary": "When the tools an agent may use change mid-task (by mode, phase or permission), declare every tool once, keep the definitions byte-stable, and restrict the choice another way: OpenAI's allowed_tools or tool_choice \"none\", Anthropic's appended tool_removal and tool_addition blocks, a decode-time logit mask or grammar on self-hosted models, and a harness check that rejects calls outside the allowed set. It pays in long sessions with cached prefixes, because editing the tool list invalidates everything after it; on the newest Claude models an edited tool list also makes replayed thinking blocks fail with an error. The evidence is vendor guidance plus indirect measurements: caching cut agent cost 41-80% against a prefix broken on every call, and schema-enforced tool calling nearly eliminated calls to nonexistent tools (3 against 34). No study compares masking with removal directly, and keeping disallowed tools in view has its own measured cost (one unneeded tool cut answer rates from 98.2% to 63.5%).", "how": [ "Start from the baseline: find where the harness rebuilds the tool list per turn (by mode, phase, permission or connected MCP servers) and log a hash of the tools array on every request with the cache read and write tokens, so each mid-session change and the cache miss after it is visible.", "Declare the full tool set once at session start, in a fixed order with deterministic serialization, and give tools that switch on and off together a shared name prefix (browser_, shell_) so a group can be allowed or blocked as one. Add tools unknown at the start by appending them (tool search with defer_loading, OpenAI's additional_tools input item, Anthropic's tool_addition block), never by editing the array.", "Write availability as a state machine in the harness (state to allowed set) and enforce it without touching the definitions: on OpenAI, tool_choice {\"type\": \"allowed_tools\", ...} or \"none\"; on Claude, an appended system message with tool_removal or tool_addition blocks (beta), because a tool_choice change there re-reads the message history and forced tool choice and prefill are rejected on recent models; on self-hosted models, a logit mask or grammar over the allowed names, or a prefill up to the shared name prefix.", "Keep a harness-side check whatever the provider: validate every call against the current allowed set and the full registry, return an error result that names what is allowed instead of executing or silently dropping the call, and leave earlier calls to now-unavailable tools in the history.", "Keep the model able to stop and to decline: do not force a tool call on every turn, and keep a final-answer or no-tool path inside any mask.", "If tools stay visible but disallowed for long stretches, check accuracy, since a visible unneeded tool can lower answer rates: add a one-sentence note that tools are optional and narrow in scope, and move groups that are rarely allowed behind deferred loading (see the tool-surface entry, 2.a.i).", "Measure with paired runs on sessions that change availability midway, the list rebuilt per state against the stable list plus a restriction: cache read share, cost per task, time to first token, calls to unavailable or undefined tools, task success and turns to finish, per model and provider." ], "detect": { "static": "Tool list rebuilt per turn from mode, phase, permissions or connected servers (a tools array filtered by state, or prompt text that co-varies with the active toolset); no allowed-tools or tool_choice restriction, appended tool_removal or tool_addition blocks, decode-time mask, or harness check of each call against the allowed set.", "traces": "A different tools-array hash between turns of one session, followed by cache reads near zero (provider cache diagnostics name the tools as the point of divergence); on Claude, tool_choice changing between turns; calls to tools no longer defined or not allowed; on recent Claude models, 400 errors on replayed thinking blocks after a tool edit.", "evals": "Cache hit rate; invalid tool call rate.", "probes": "Sessions where a tool becomes unavailable midway." }, "impact": { "measured_basis": true, "primary": [ "cost" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "schema-enforced calling cut calls to nonexistent tools from 34 to 3 across ten models (Iyer, Sep 2026); but forcing a call on every turn made weak local models loop for 600-900 turns (Tang, Sep 2026), one visible unneeded tool cut answer rates from 98.2% to 63.5% (Paturi, Sep 2026), and per-step pruning beat a full list with the retrieved tools emphasized on small models (Patel, Dec 2025); no study compares masking with removal", "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "medium", "when": "avoids a full-prefix recompute at each tool-set change; caching cut agent cost 41.4-79.6% against a prefix broken on every call (Lumer, Jan 2026); cached reads cost 0.1x input on most GPT-5.6+ models (OpenAI docs, Oct 2026); on Claude a tool_choice change still re-reads the message history (Anthropic docs, Oct 2026); the saving scales with how often the tool set changes", "basis": "evidence" }, { "dimension": "performance", "effect": "improves", "size": "small", "when": "time to first token 6.1-30.9% better with caching than with a prefix broken on every call (Lumer, Jan 2026); a mask that forces a call every turn can stop a weak model from finishing (Tang, Sep 2026)", "basis": "evidence" }, { "dimension": "security", "effect": "improves", "size": "small", "when": "Constraints enforced at decode time.", "basis": "assessment" }, { "dimension": "governance", "effect": "improves", "size": "small", "when": null, "basis": "assessment" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired runs on probe sessions where the allowed tool set changes midway (a mode or phase switch, a revoked permission, a tool the model used earlier becoming unavailable): the tool list rebuilt per state against stable definitions with the provider's restriction (allowed_tools or tool_choice, tool_removal blocks, or a decode-time mask) plus a harness check; compare cache read share and cost per task, time to first token, calls to unavailable or undefined tools, task success and turns to finish, per model and provider" }, { "method": "monitor", "tier": null, "how": "per session: a hash of the tools array on every request, with cache read and write tokens on the turn after any change (or the provider's cache diagnostics); calls rejected for naming an unavailable or undefined tool; sessions that hit the turn cap while a tool call was forced on every turn" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file master-agent-optimization-cookbook.md", "task": "Lists this practice in its agent-optimization pattern library (Production group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Yichao 'Peak' Ji (Manus), \"Context Engineering for AI Agents: Lessons from Building Manus\"", "url": "https://manus.im/blog/Context-Engineering-for-AI-Agents-Lessons-from-Building-Manus", "date": "2025-07-18", "locator": "Mask, Don't Remove", "task": "Mask token logits to restrict tool choice instead of removing tool definitions mid-task", "effect": "Advises \"avoid dynamically adding or removing tools mid-iteration\" because changes \"invalidate the KV-cache\" and leave references to undefined tools; Manus instead \"masks the token logits during decoding\". No numbers.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "OpenAI, \"Prompt caching\" and \"Function calling\" (API guides), https://platform.openai.com/docs/guides/prompt-caching ;", "url": "https://platform.openai.com/docs/guides/function-calling", "date": "2026-10-17", "locator": "Prompt caching: settings table and \"Manage tools with append-only updates\"; Function calling: Tool choice, \"When to use allowed_tools\"", "task": "Keep the tools list stable and restrict which tools are callable per request with tool_choice \"none\" or an allowed_tools list", "effect": "Prompt caching guide: a change to tools (\"tool names, descriptions, schemas, ordering, or tool-specific instructions\") breaks prefix reuse. \"When the tools your application needs vary between requests, change which tools are callable while keeping their definitions stable to preserve reusable prefixes\": set tool_choice to \"none\" \"instead of removing the tool definitions\", use allowed_tools \"to restrict which tools are callable while keeping the supplied `tools` list stable\", and with tool search \"Discovered tools are appended at the end of context, preserving earlier reusable content.\" Function calling guide: configure allowed_tools to \"not modify the list of tools you pass in, so you can maximize savings from prompt caching.\" Vendor guidance, no numbers.", "direction": "supports", "measured": false, "conditions": "Living docs; date is the access date. Same guide: cache reads cost 0.1x the input rate on most GPT-5.6 and later models", "older": false }, { "source": "Anthropic, \"Prompt caching\" and \"Tool use with prompt caching\" (Claude API docs), https://docs.claude.com/en/docs/build-with-claude/prompt-caching ;", "url": "https://docs.claude.com/docs/en/agents-and-tools/tool-use/tool-use-with-prompt-caching", "date": "2026-10-17", "locator": "Prompt caching: What invalidates the cache (table); Tool use with prompt caching: defer_loading and cache preservation; What invalidates your cache", "task": "Which changes to tool definitions and tool settings invalidate the prompt cache", "effect": "The cache follows tools, then system, then messages: \"Modifying tool definitions (names, descriptions, parameters) invalidates the entire cache\", while \"Changes to `tool_choice` parameter only affect message blocks\" (the tools and system caches stay valid, the messages cache does not); changing disable_parallel_tool_use also invalidates the messages cache. Deferred tools \"are not included in the system-prompt prefix\" and a discovered one is appended inline, so \"adding tools dynamically through tool search does not break your cache.\" No numbers.", "direction": "mixed", "measured": false, "conditions": "Living docs; date is the access date. Restricting choice with tool_choice keeps the tool and system cache but re-reads the conversation history on Claude", "older": false }, { "source": "Anthropic, \"Mid-conversation system messages and tool changes\" (Claude API docs)", "url": "https://docs.claude.com/docs/en/build-with-claude/mid-conversation-system-messages", "date": "2026-07-24", "locator": "Mid-conversation tool changes; Define tools in a message (beta); Limitations; API release notes July 24 and September 22, 2026", "task": "Declare the full tool set up front and withdraw or re-offer tools mid-conversation with appended tool_removal and tool_addition blocks", "effect": "\"The `tools` array sits even earlier in the hashed request prefix than the top-level `system` field, so editing it invalidates the prompt cache for the entire conversation.\" Instead, \"declare the full tool set in `tools` up front, then use `tool_addition` and `tool_removal` blocks to offer a tool to the model, or withdraw it, from a specific point in the conversation onward. The `tools` array itself never changes, so the cached prefix stays intact.\" Since September 22, 2026 a tool_addition block can carry a full definition for a tool unknown at the start. The docs do not say whether a call to a withdrawn tool is blocked. No numbers.", "direction": "supports", "measured": false, "conditions": "Beta (mid-conversation-tool-changes-2026-07-01, then inline-tools-2026-09-15 headers), Claude API; date is the beta launch in the API release notes", "older": false }, { "source": "Anthropic, \"What's new in Claude Opus 5.5\" and \"Migration guide\" (Claude API docs), https://docs.claude.com/docs/en/models/opus-5-5/whats-new-opus-5-5 ;", "url": "https://docs.claude.com/docs/en/models/opus-5-5/migration-guide", "date": "2026-09-22", "locator": "What's new: Breaking changes (Forced tool use is not supported; Thinking blocks are tied to the model and the conversation); Migration guide: Prefill removal", "task": "What happens on the newest Claude models when tools are edited mid-conversation, and which restriction mechanisms remain", "effect": "The API \"checks whether anything before a Claude Opus 5.5 thinking block (the `system` prompt, the `tools`, or an earlier message) has changed since the block was produced\"; for accounts created on or after August 31, 2026, replaying a block after such a change \"returns a 400 error.\" Advice: \"Keep the conversation append-only so the question never arises: change instructions or tools with mid-conversation system messages rather than edits.\" Two masking routes are gone on this model: tool_choice \"any\" or \"tool\" \"returns a 400 `invalid_request_error`\" (auto and none still work), and \"Prefilling assistant messages returns a 400 error on Claude Opus 4.6 and later Opus models\".", "direction": "supports", "measured": false, "conditions": "Claude Opus 5.5 (the forced-tool rule also applies to Claude Fable 5.1); Claude API and cloud platforms; vendor documentation, no numbers", "older": false }, { "source": "Elias Lumer, Faheem Nizar, Akshaya Jangiti, Kevin Frank, Anmol Gulati, Mandar Phadate, Vamse Kumar Subbiah, \"Don't Break the Cache: An Evaluation of Prompt Caching for Long-Horizon Agentic Tasks\"", "url": "https://arxiv.org/abs/2601.06007", "date": "2026-01-09", "locator": "§3.2; §4.1 Table 1; §5.1; §6.1", "task": "Measure what prompt caching saves in a multi-turn tool-calling agent against a baseline that breaks the prefix at the start of the system prompt on every call", "effect": "With the best cache mode per model, cost fell 79.6% (GPT-5.2), 78.5% (Claude Sonnet 4.5), 41.4% (Gemini 2.5 Pro) and 45.9% (GPT-4o), and time to first token 13.0%, 22.9%, 6.1% and 30.9%; at 50,000-token prompts GPT-5.2 saved 89% and Claude Sonnet 4.5 88%. The authors argue, without testing it, that \"any change to the available tool set invalidates the cached prefix\" and suggest \"a fixed set of general-purpose, reusable functions\" with dynamic capabilities done through code generation.", "direction": "supports", "measured": true, "conditions": "DeepResearch Bench, 40 sessions per model and cache condition, 10,000-token system prompt; the baseline breaks the cache on every request (worst case), not one tool change; no tool-list change was tested", "older": false }, { "source": "Laxmipriya Ganesh Iyer, \"Closed-World Resolution Against Tool Hallucination in LLM Agents\"", "url": "https://arxiv.org/abs/2609.19425", "date": "2026-09-16", "locator": "§VI-A; Table I", "task": "Compare a schema-enforced tool-calling API with a raw-JSON surface (tool catalog described in the prompt, calls parsed by a bridge) on probes that invite calls to nonexistent tools and undeclared arguments", "effect": "\"across all ten models the schema surface emitted just 3 H1 calls, against 34 on raw-JSON\" (H1 = a fabricated tool), and every schema-surface one came from Llama-3.1-8B or Ministral-3-8B; on raw-JSON \"the same prompts induce hallucination at 13–57% across all ten models\". Claude Opus and Sonnet emitted no call on 70-75% of the adversarial schema probes, so their low rate is partly refusal.", "direction": "supports", "measured": true, "conditions": "Ten Amazon Bedrock models (Claude Opus 4.8, Sonnet 4.6, Haiku 4.5, Nova Premier, Nova 2 Lite, GPT-OSS-120B, Llama-3.1 8B and 70B, Ministral-3-8B, Mistral-Large-3); 10 tools exposed from a 100-tool registry; 60 adversarial probes per model per surface; single author; not a mid-task tool change", "older": false }, { "source": "Lijuan Tang, Yuemeng Zheng, \"Measuring the Serving Stack Instead of the Model: Hidden Confounds in Local Tool-Use Evaluation\"", "url": "https://arxiv.org/abs/2609.26693", "date": "2026-09-22", "locator": "§4 Results (\"Constrained decoding removes the protocol failures, at a cost\"); Limitations", "task": "Constrain decoding to a JSON schema whose name field is an enum of the available tools, so the model must emit a valid call", "effect": "\"On a single tool-call step, all nine local models ... emit a valid in-schema call on 8 of 8 trials\", but \"Forced to emit a tool call on every turn, the weaker models never terminate, producing 600–900-turn loops within a single episode (Qwen-0.5B reached 869 assistant turns in one seed before we cut it off).\" The constraint \"trades an unparseable-call failure for a non-termination failure on weak models\".", "direction": "mixed", "measured": true, "conditions": "Nine small local models on Ollama (including Qwen-0.5B, Llama-3.2, Phi-3, Gemma-3-270m); coding-agent tasks; 8 trials; the constrained condition was run single-step only because the agentic version did not terminate", "older": false }, { "source": "Saanvi Paturi, Arsen Kenzhebayev, Arham Sethi, Vyas Raina, Ivaxi Sheth, Vatsal Raina, \"When Tools Get in the Way: The Effect of Unnecessary Tool Availability on LLM Answering\"", "url": "https://arxiv.org/abs/2609.14157", "date": "2026-09-12", "locator": "Abstract; §6 Table 2; §6.3 Table 4", "task": "Make a related but unnecessary tool available for questions the model can answer from its own knowledge (the cost of a visible tool that should not be used)", "effect": "\"Across 3,000 baseline trials the pooled answer rate is 98.2%. When an unnecessary tool is available it falls to 63.5%\"; Gemini 2.5 Flash-Lite fell from 99.4% to 23.4% while calling the tool in 7.8% of trials, so the drop \"cannot be explained by unnecessary tool invocation alone\". A one-sentence scope-aware system instruction raised the tool-available answer rate by up to 45.6 percentage points, at some cost to tool use where the tool was needed.", "direction": "mixed", "measured": true, "conditions": "500 query pairs, 10 domains, one tool per domain; Gemini 2.5 Flash-Lite, Flash and Pro, GPT-OSS 20B, Grok 4.1 Fast, Llama 3.3 70B; LLM judge; tests availability, not masking: whether a visible but disallowed tool has the same effect is untested", "older": false }, { "source": "Bhrij Patel, Davide Belli, Amir Jalalirad, Maximilian Arnold, Aleksandr Ermolov, Bence Major, \"Dynamic Tool Dependency Retrieval for Lightweight Function Calling\"", "url": "https://arxiv.org/abs/2512.17052", "date": "2025-12-18", "locator": "Abstract; §5 ICL methods; §6 \"How to encode tools in prompt?\", Figure 3", "task": "Re-retrieve the tools at every step from the query and the calls so far, and compare showing only the retrieved tools (\"Hard Mask\": \"completely exclude the remaining tools from the prompt\") with showing the full set and emphasizing the retrieved ones (\"Soft Mask\")", "effect": "\"Hard Masking achieves the highest accuracy for smaller models by directly pruning the set of available tools presented to the LLM\"; \"Pruning irrelevant tools from the prompt has a greater impact for smaller models that cannot handle longer contexts.\" Per-step retrieval \"improves function calling success rates between 23% and 104% compared to state-of-the-art static retrievers.\" Efficiency is measured as prompt length only; cache reuse is not measured.", "direction": "contradicts", "measured": true, "conditions": "Qwen3 0.6B-14B, GPT-4o, Gorilla-V2; TinyAgent and three TaskBench sets; the soft mask is a prompt emphasis, not a decode-time mask; hard against soft numbers only in Figure 3", "older": false } ], "caveats": "No study measures masking against removing tools on the same agent: the cache case rests on vendor documentation and on measurements of caching in general, and the saving scales with how often the tool set changes and how long the prefix is. Masking keeps disallowed tools in view, and visibility has a measured cost: one unneeded tool cut answer rates from 98.2% to 63.5% across six models, and per-step pruning beat a full list with the retrieved tools emphasized on small on-device models. Mechanisms differ by provider: on Claude a tool_choice change still re-reads the message history, and forced tool choice and prefill are rejected on recent models, so Manus's prefill recipe now applies mainly to self-hosted models, where forcing a call every turn made weak models loop. The constrained-decoding results come from small open models, adversarial probes and single-step runs.", "also_covers": [], "related": [ { "id": "3.b.i", "slug": "cache-stable-prefix", "name": "Keep the prompt prefix cache-stable" }, { "id": "2.a.i", "slug": "reduce-tool-surface", "name": "Reduce the tool surface: remove and consolidate tools, then defer the rest behind search" }, { "id": "10.b.i", "slug": "risky-action-policy", "name": "Gate irreversible actions: allow, ask or deny by side effect" }, { "id": "2.b.i", "slug": "fail-soft-tools", "name": "Make tool calls and tool results fail soft, and retry transient failures" }, { "id": "2.a.ii", "slug": "rewrite-tool-surface", "name": "Rewrite the tool surface: names, schemas, descriptions" }, { "id": "2.a.iii", "slug": "choose-action-interface", "name": "Choose how the agent acts: typed tools, code execution, or shell, matched to the model and task" } ], "url": "https://mahalsystems.ai/aot/optimizations/mask-tools-not-remove/", "markdown": "https://mahalsystems.ai/aot/optimizations/mask-tools-not-remove.md" }, { "id": "3.c.i", "legacy_id": "P35", "former_ids": [], "slug": "self-discover-reasoning", "name": "Self-Discover reasoning structures", "kind": "pattern", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes the reasoning prompt", "proposed": true }, "levers": [ "model", "prompt" ], "domains": [ "reasoning" ], "evidence_domains": [ "reasoning", "planning" ], "categories": [ "Sizing reasoning effort" ], "category_ids": [ "prompts.reasoning-effort" ], "summary": "Self-Discover has the model compose a reasoning structure once per task type (select useful modules from 39 seed heuristics, adapt them to the task, write them as a step-by-step plan), then follow that cached structure on every instance, so each instance costs one call. On non-reasoning models it beat CoT (GPT-4 on BBH 75% to 81%, on T4D 52% to 85%) and, in a third-party run, 5-sample self-consistency at under half the cost; but it trailed CoT on Claude 3 Sonnet and on multistep arithmetic, and trailed zero-shot prompting on long-form essays. On reasoning models with thinking on, the evidence is thin and leans negative: no study runs the original method, a plan-first prompt changed mean accuracy over 10 benchmarks by at most 0.51 points, and Anthropic advises general instructions over hand-written step-by-step plans.", "how": [ "Start from the cheap baselines on your own labeled items: plain CoT for a non-reasoning model, the model's own thinking at a chosen effort setting for a reasoning model (1.b.i), and self-consistency at the same token budget (9.c.ii). Expect little on reasoning models with thinking on (a plan-first prompt moved mean accuracy by 0.51 points or less; an induced task strategy moved o3-mini by -1.42 to +3.20); the clearest gains were on non-reasoning and smaller models, on task types the baselines get wrong.", "Group traffic into task types that are solved the same way, and pick a handful of unlabeled examples for each (the follow-up studies used 3 to 10).", "Run discovery offline, once per task type, with the strongest model you have: SELECT the useful modules from the paper's 39 seed modules, ADAPT each to the task, and IMPLEMENT them as a step-by-step plan, showing one human-written plan for another task as the format example; that is 3 calls per task type. Small models wrote poor structures, while structures from a stronger model transferred (GPT-4's raised Llama2-70B on disambiguation QA from 42% with CoT to 52%). Also try letting the model write its own task-specific modules instead of picking from the 39 seeds: in a rival team's rerun (Auto-Evolve), that reached 65.4% on BBH for Claude 2.0 against 58.7% for Self-Discover and 53.7% for direct prompting.", "Check each structure before caching it: would following it solve a few labeled examples? In the original study 12.5% of MATH structures were wrong, and an instance that follows a wrong structure fails.", "Cache the structure as a versioned block in that task type's prompt, in the stable prefix (3.b.i; the rest of the system prompt is 3.a.iv), followed by the instance and the paper's instruction to fill in each step by reasoning about the task, not to rephrase the keys, so each instance costs one call; keep the final answer extractable (9.b.i). On Claude with thinking on, give the structure as worked examples (problem, method, answer) rather than asking for it as visible output, and test it against thinking alone. Prefer a natural-language plan to JSON (it won on every benchmark, e.g. 75.50% vs 63.50% on MATH, in a per-question study with no CoT baseline), and leave out task types where the structure lost to the baseline, such as multistep arithmetic (70 vs 92 for CoT on GPT-4) and long-form rubric-graded answers (46.32 vs 57.35 zero-shot).", "Measure on held-out labeled items per task type: accuracy against CoT, the model's own thinking at matched tokens and self-consistency at matched cost, plus tokens, cost and p50 and p95 latency per item. Rediscover and re-test the structures when the model changes: the gain over CoT disappeared on a stronger model (Claude 3 Sonnet 67.7 vs 68.3)." ], "detect": { "static": "A module-selection or planning step before solving, or a reasoning structure (key-value or numbered steps) per task type in the prompt. Flag a structure regenerated for every instance where one cached per task type would do, JSON structures where a natural-language plan would do, a structure stacked on a reasoning model with thinking on that was never compared against thinking alone, and no CoT or self-consistency baseline.", "traces": "Structure id and version per call and the task type it was discovered for; modules selected; whether the output fills every key or rephrases or skips them; tokens and latency per call against plain CoT.", "evals": "Accuracy per task type against CoT, the model's own thinking at matched tokens and self-consistency at matched cost; computation-heavy split from knowledge-heavy, short answers from long-form.", "judge": "Before caching, check each discovered structure: would following it solve labeled examples of the task? (12.5% of MATH structures were wrong in the original study.)", "probes": "A mix of task types needing different reasoning." }, "impact": { "measured_basis": true, "primary": [ "quality" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "non-reasoning models: BBH 75% to 81% and T4D 52% to 85% over CoT on GPT-4 (Zhou, Feb 2024); BBH 91.0 vs 86.0 for CoT and 89.6 for self-consistency on DeepSeek-V3 (Fan, Apr 2026); below CoT on Claude 3 Sonnet, 67.7 vs 68.3 (Aswani, Oct 2024), and on GPT-4 multistep arithmetic, 70 vs 92 (Zhou, Feb 2024); below zero-shot on long-form CFA essays, 46.32 vs 57.35 (Shetty, Jun 2025); JSON plans 63.50 vs 75.50 for natural-language plans on MATH, per question (Gunasekara, Jul 2025); reasoning models: a plan-first prompt within 0.51 pp of direct prompting (Preet, May 2026), an induced task strategy -1.42 to +3.20 on o3-mini (Chen, May 2026), a hand-written structure 74.3 to 83.3 for o1 on obfuscated Mystery Blocksworld, while o1-mini fell 19.1 to 16.83 (Goh, Feb 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "medium", "when": "3 task-level calls per task type, then one call per instance with longer input and output (Zhou, Feb 2024); 2.7x zero-shot cost per multiple-choice task ($0.686 vs $0.255) against 6.7x for 3-sample self-consistency ($1.701) (Shetty, Jun 2025); normalized cost 5.0 vs 2.5 for CoT and 12.0 for 5-sample self-consistency on DeepSeek-V3 (Fan, Apr 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "medium", "when": "one call per instance after offline discovery, but longer outputs: 14.9 s vs 6.0 s zero-shot per multiple-choice task and 25.27 s vs 15.18 s per essay, against 37.9 s for 3-sample self-consistency (Shetty, Jun 2025); a step-by-step prompt made reasoning models 20-80% slower (Meincke, Jun 2025)", "basis": "evidence" }, { "dimension": "security", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "governance", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval per task type on held-out labeled items: the cached structure against CoT, against the model's own thinking or a higher effort setting at matched tokens (1.b.i), and against self-consistency at matched cost (9.c.ii); accuracy split by task type (computation-heavy vs knowledge-heavy, short answer vs long-form), tokens, cost and p50 and p95 latency per item; repeat on every model change" }, { "method": "monitor", "tier": null, "how": "per call: which structure was used (task type, version, discovering model), whether the output fills every key or rephrases or skips them, answer-extraction failures, tokens and latency; accuracy by task type on a regularly labeled sample, and the share of traffic that matches no task type with a structure" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Reasoning & Search group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Zhou et al., \"Self-Discover: Large Language Models Self-Compose Reasoning Structures\"", "url": "https://arxiv.org/abs/2402.03620", "date": "2024-02-06", "locator": "Abstract; §2, Figures 2-3; §3.3; Table 1; §4.1-4.3, Figures 4-5; §5.1-5.2, Figures 8-9 and footnote 3; Appendix A (Table 2, Stage 2 prompt); Appendix C, Table 3; Appendix D", "task": "Stage 1, once per task: from 39 seed reasoning modules and a few unlabeled task examples, three meta-prompts SELECT the useful modules, ADAPT them to the task and IMPLEMENT them as a key-value (JSON) reasoning structure, with a human-written structure for another task as the example. Stage 2: the structure is appended to every instance and the model fills in each value to reach the answer", "effect": "Table 1, CoT -> Self-Discover (plan-and-solve in brackets): BBH PaLM 2-L 60% -> 67% (61%), GPT-4 75% -> 81% (73%); T4D 40% -> 69% (42%), 52% -> 85% (53%); MATH 42% -> 50.5% (49%), 71% -> 73% (70%). The abstract's \"by as much as 32% compared to Chain of Thought\" is T4D; it beats CoT-Self-Consistency \"by more than 20%, while requiring 10-40x fewer inference compute\", a comparison (Figure 5) of GPT-4 on 2 BBH tasks that counts inference calls per instance: 1 plus 3 task-level calls for Self-Discover, 10 for CoT-Self-Consistency, 40 for majority voting over every module. \"We acknowledge that Self-Discover input and output are longer than CoT and Direct prompting, increasing cost.\" Gains were largest on world-knowledge tasks and moderate on algorithmic ones; per task, GPT-4 with Self-Discover trailed CoT on multistep arithmetic (70 vs 92), shuffled-object tracking (68 vs 80), web of lies (71 vs 80) and word sorting (85 vs 90). Each of the three actions added accuracy in the ablation. Transfer: PaLM 2-L structures used by GPT-4 beat OPRO prompts on 3 of 4 tasks (OPRO used 20% of the data); GPT-4 structures gave Llama2-70B 52% vs 42% for CoT on disambiguation QA and GPT-3.5-turbo 56% vs 51% on geometry (3-shot); \"We tried zero-shot meta prompting Llama2 but observed low-quality structure outputs.\" On MATH, 87.5% of structures were correct and 74.7% of errors were computation errors.", "direction": "supports", "measured": true, "conditions": "GPT-4 (gpt-4-turbo-preview), PaLM 2-L, GPT-3.5-turbo, Llama2-70B; BBH (23 tasks), T4D, 200-item MATH subsample with instance-level structures from a one-shot demonstration; zero-shot baselines direct, CoT and plan-and-solve; CoT-Self-Consistency (10 samples) and per-module voting on 2 BBH tasks only", "older": true }, { "source": "Krishna Aswani, Huilin Lu, Pranav Patankar, Priya Dhalwani, Iris Tan, Jayant Ganeshmohan, Simon Lacasse, \"Auto-Evolve: Enhancing Large Language Model's Performance via Self-Reasoning Framework\"", "url": "https://arxiv.org/abs/2410.06328", "date": "2024-10-08", "locator": "§4.3-4.4; §5.1, Tables 1-2; §5.6; Appendix D, Figure 9; Appendix H", "task": "Re-runs Self-Discover (39 seed modules) as a baseline next to direct prompting and zero-shot CoT on BBH, and proposes Auto-Evolve, in which the model writes its own task-specific modules and refines the plan iteratively", "effect": "BBH average over 23 tasks (Table 1), direct / CoT / Self-Discover / Auto-Evolve: Claude 2.0 53.7% / 55.0% / 58.7% / 65.4%; Claude 3 Sonnet 68.6% / 68.3% / 67.7% / 71.6%; Mistral Large 61.9% / 67.3% / 72.7% / 75.4%. GPT-4, gain over direct (Table 2): CoT +16.6%, Self-Discover +20.3%, Auto-Evolve +22.9%. Llama 3.1 70B: disambiguation QA direct 22.4%, CoT 60.4%, Self-Discover 56.8%, Auto-Evolve 72.0%; causal judgement 27.3%, 62.6%, 64.7%, 65.3%. \"Self-Discover only uses a few reasoning seed modules in solving the BBH tasks (out of 39)\". Calls for a 250-question task: CoT 250, Self-Discover 253, CoT with self-consistency 2,500.", "direction": "mixed", "measured": true, "conditions": "Claude 2.0, Claude 3 Sonnet, Mistral Large, GPT-4, Llama 3.1 70B; BBH, 23 tasks; each experiment run three times and averaged; written by the authors of a competing method", "older": true }, { "source": "Pranam Shetty, Abhisek Upadhayaya, Parth Mitesh Shah, Srikanth Jagabathula, Shilpi Nayak, Anna Joo Fee, \"Advanced Financial Reasoning at Scale: A Comprehensive Evaluation of Large Language Models on CFA Level III\"", "url": "https://arxiv.org/abs/2507.02954", "date": "2025-06-29", "locator": "§2.3; §3.2, Tables 1, 3, 4; Appendix B.3 (v2)", "task": "Compares zero-shot, CoT with self-consistency (N = 3 and 5) and a one-prompt, per-question Self-Discover variant (select modules, adapt them, execute, evaluate) across 23 reasoning and non-reasoning models on mock CFA Level III questions, with cost and latency", "effect": "Means across models (Tables 3-4), multiple choice: accuracy zero-shot 61.8%, Self-Discover 68.6%, CoT-SC N=3 69.1%, N=5 69.6%; cost per task $0.255, $0.686, $1.701, $2.840; latency 6.0, 14.9, 37.9, 63.1 s. Essays (LLM-graded): 57.35, 46.32, 60.40, 61.03; latency 15.18, 25.27, 69.83, 130.48 s. \"Self-Discover prompting presents a compelling middle ground for MCQ questions, delivering a substantial 6.8 percentage point accuracy improvement over Zero Shot while increasing costs by only 2.7x. However, for essay evaluation, Self-Discover significantly underperforms across all metrics\". Table 1 lists Self-Discover as the best multiple-choice strategy for o4-mini (75.00%), Claude 3.7 Sonnet (73.33%) and DeepSeek-R1 (71.67%), tied with zero-shot for Gemini 2.5 Pro (76.67%), whose Self-Discover essay score was 61.74 against 75.17 with CoT-SC.", "direction": "mixed", "measured": true, "conditions": "60 multiple-choice and 43 essay questions from AnalystPrep mock exams; 23 models incl. o3-mini, o4-mini, DeepSeek-R1, Gemini 2.5 Pro and Flash, Claude 3.7 Sonnet, Sonnet 4 and Opus 4 with thinking; essays graded by GPT-4.1; no single-sample CoT arm; the prompt names example modules instead of the 39 seeds and runs per question in one call", "older": true }, { "source": "Sachith Gunasekara, Yasiru Ratnayake (Surge Global), \"Effects of structure on reasoning in instance-level Self-Discover\"", "url": "https://arxiv.org/abs/2507.03347", "date": "2025-07-04", "locator": "§4.3; §5.1-5.3; Table 1", "task": "Reimplements task-level Self-Discover (10 unlabeled examples per task) and compares it with an instance-level variant that discovers a plan per question, written either as a JSON structure or as a natural-language plan", "effect": "Table 1, LLaMA-3.1-405B / Mistral-Large: task-level Self-Discover BBH 86.59 / 82.04, T4D 100.00 / 96.63; instance-level JSON (0-shot) BBH 85.05 / 83.14, T4D 73.23 / 76.06, MATH 63.50 / 67.50; instance-level natural language (0-shot) BBH 87.27 / 85.57, T4D 78.90 / 82.09, MATH 75.50 / 76.50; natural language with 5 unlabeled examples T4D 86.35 / 88.29. Natural-language plans beat JSON plans by up to 18.90% relative on MATH; task-level discovery won on the homogeneous T4D, instance-level on the diverse BBH with Mistral (+4.30% relative; +0.79% with LLaMA).", "direction": "mixed", "measured": true, "conditions": "LLaMA-3.1-405B-Instruct and Mistral-Large; BBH (27 tasks), T4D rebuilt from ToMi, 200-item MATH subsample with a one-shot plan demonstration (per instance, as in the original); no CoT or self-consistency baseline", "older": true }, { "source": "Fulong Fan, Peilin Liu, Fengzhe Liu, Shuyan Yang, Gang Yan, \"Self-Awareness before Action: Mitigating Logical Inertia via Proactive Cognitive Awareness\"", "url": "https://arxiv.org/abs/2604.20413", "date": "2026-04-22", "locator": "§4.1 Baselines, Implementation Details; §4.2, Table 2; Appendix A.1, Table 5", "task": "Runs Self-Discover as a baseline next to direct prompting, CoT, self-consistency (k = 5), Self-Refine, CRITIC, GoT and the authors' SABA, reporting accuracy and an inference cost normalized to direct prompting (T)", "effect": "DeepSeek-V3 (Table 2), direct / CoT / SC(k=5) / Self-Discover / GoT / SABA: BBH 78.7 / 86.0 / 89.6 / 91.0 / 90.7 / 93.2; StrategyQA 82.0 / 87.6 / 90.1 / 92.5 / 91.7 / 94.4; HotpotQA answer EM 62.4 / 68.4 / 72.0 / 75.6 / 78.0 / 78.6; T 1.0 / 2.5 / 12.0 / 5.0 / 35.7 / 9.2. Gemini-1.5-Flash (Table 5): BBH 68.7 / 75.0 / 79.5 / 83.4 / 82.7 / 86.1; T 1.0 / 2.4 / 12.1 / 4.8 / 35.5 / 9.8.", "direction": "supports", "measured": true, "conditions": "DeepSeek-V3 and Gemini-1.5-Flash, temperature 0, three runs; BBH, StrategyQA, HotpotQA, Detective Puzzle; the Self-Discover setup is not described and T is not defined beyond \"average relative cost\"; written by the authors of a competing method", "older": false }, { "source": "Lennart Meincke, Ethan Mollick, Lilach Mollick, Dan Shapiro (Wharton Generative AI Labs), \"Prompting Science Report 2: The Decreasing Value of Chain of Thought in Prompting\"", "url": "https://arxiv.org/abs/2506.07142", "date": "2025-06-08", "locator": "Summary; Results, Figures 1-3; Figure S2; Discussion and Conclusion", "task": "Adds a \"Think step by step\" prompt, against an answer-directly prompt and a no-instruction default, on reasoning and non-reasoning models, 25 trials per question", "effect": "Reasoning models, average rating against direct: o3-mini RD = 0.029 (p = .024), o4-mini RD = 0.031 (p = .003), Gemini 2.5 Flash RD = -0.033 (p = .005), and Flash 2.5 lost 0.131 on the 25-of-25-correct metric; \"CoT requests took between 20-80% (10-20 seconds) longer than direct requests.\" Non-reasoning models gained against direct (Gemini 2.0 Flash RD = 0.135, Sonnet 3.5 RD = 0.117); against the no-instruction default, Gemini 2.0 Flash still gained significantly (RD = 0.062, p < .001) while Sonnet 3.5 did not (RD = -0.019, p = 0.189). \"For dedicated reasoning models, the added benefits of explicit CoT prompting appear negligible and may not justify the substantial increase in processing time.\"", "direction": "contradicts", "measured": true, "conditions": "GPQA Diamond, 198 questions x 25 trials per condition; reasoning: o3-mini, o4-mini, Gemini 2.5 Flash; non-reasoning: Claude 3.5 Sonnet, Gemini 2.0 Flash, GPT-4o-mini, GPT-4o, Gemini 1.5 Pro; a plain step-by-step prompt, not a composed structure", "older": true }, { "source": "Inder Preet, Shuxin Lin, Dhaval Patel, \"ReasonLab: A Controlled and Auditable Evaluation of Prompting Techniques for Multiple-Choice QA\"", "url": "https://arxiv.org/abs/2607.14109", "date": "2026-05-07", "locator": "§1.1; §3.2-3.3, Table 2; §4.1, Table 3 (v2)", "task": "Holds model, dataset and answer extraction fixed and varies 8 prompting techniques, including Plan-and-Solve (\"Gather information, devise a plan, answer step by step\") and CoT, on models with reasoning off and on", "effect": "Matched comparison (Table 3). Reasoning off (8 configurations, 80 cells): direct 48.27, Plan-and-Solve 52.88 (+4.61 pp), CoT 52.68 (+4.41 pp), Self-Analogical 52.96 (+4.69 pp); \"the differences among these five are not significant\". Reasoning on (GPT-OSS-20B and 120B, Qwen3-8B, Qwen3-30B-A3B-Thinking; 40 cells): direct 70.84, CoT 70.69, Plan-and-Solve 70.32 (p = 0.266); \"no technique differs from direct prompting by more than 0.51 pp\". Enabling reasoning added 12.74 pp for GPT-5.4 and 2.10 pp for Claude Sonnet 4; an eightfold budget increase from low to medium added 0.48 to 2.10 pp.", "direction": "contradicts", "measured": true, "conditions": "10 MCQA datasets (about 4,300 questions, incl. AIME 2025 QA, MedQA, MMLU-Pro, SuperGPQA), 27 model configurations, temperature 0, one generation per cell; techniques from published templates without per-model tuning; Self-Discover itself not tested; vendor-hosted reasoning models ran with the direct prompt only", "older": false }, { "source": "Po-Chun Chen, Hen-Hsen Huang, Hsin-Hsi Chen, \"Strategy-Induct: Task-Level Strategy Induction for Instruction Generation\"", "url": "https://arxiv.org/abs/2605.20924", "date": "2026-05-20", "locator": "§2; §3.3-3.4; §4.1, Table 1; §5.2.3-5.2.4", "task": "Induces one reusable task-level instruction from strategies the model writes for 3 unlabeled questions, then applies it to every question of the task; compared with zero-shot CoT, per-instance Strategic CoT (SCoT) and INDUCT on 18 models, including o3-mini at three effort levels", "effect": "Table 1, zero-shot CoT -> Strategy-Induct, BBH-Induct / Evals-Induct / Shift Cipher: o3-mini low 85.91 -> 88.81 / 78.29 -> 77.26 / 93.60 -> 96.80; medium 88.70 -> 89.74 / 83.60 -> 82.18 / 97.44 -> 97.44; high 88.87 -> 91.30 / 85.02 -> 85.08 / 98.40 -> 98.40; Gemini 2.0 Flash Thinking 87.25 -> 88.99 / 74.46 -> 74.16 / 80.00 -> 86.88; Gemini 2.0 Flash 78.33 -> 85.45 on BBH-Induct. Win-tie-lose against zero-shot CoT 50-3-7 over 60 settings. Per-instance SCoT broke o3-mini's output format on Shift Cipher (24.64 at high effort against 98.40 for CoT). Instructions induced by Llama 3.1 405B helped o3-mini (low, medium) more than its own; GPT-4o's instructions raised Llama 3.1 8B, 70B and 405B by 7.88%, 5.33% and 3.95% on BBH-Induct.", "direction": "mixed", "measured": true, "conditions": "25 sampled questions per task, temperature 0; BBH-Induct (23 tasks), Evals-Induct (24 tasks), Shift Cipher (25 shifts); 16 LLMs plus Gemini 2.0 Flash Thinking and o3-mini (low, medium, high)", "older": false }, { "source": "Erik Goh, John Kos, Ashok Goel, \"Knowledge Model Prompting Increases LLM Performance on Planning Tasks\"", "url": "https://arxiv.org/abs/2602.03900", "date": "2026-02-03", "locator": "§3.2; §4, Table 2; §4.1", "task": "Prompts models with a hand-written Task-Method-Knowledge (TMK) structure for Blocksworld planning (what to do, how, and why each step is taken), against plain-text prompts, on non-reasoning and reasoning models", "effect": "Table 2, plain text -> TMK, Classic / Mystery / Random: o1 95.7 -> 98.5 / 74.3 -> 83.3 / 31.5 -> 97.33; GPT-5 99.3 -> 99.7 / 98.1 -> 98.3 / 92.5 -> 99.0; o1-mini 56.7 -> 57 / 19.1 -> 16.83 / 9.33 -> 27.0; GPT-4o 35.5 -> 45.3 / 0 -> 5.5 / 0.83 -> 4.83.", "direction": "mixed", "measured": true, "conditions": "PlanBench Blocksworld (Classic, and the obfuscated Mystery and Random variants); TMK one-shot against the best of zero- and one-shot plain text; the authors updated answer extraction for Random Blocksworld, so its larger jump may be an extraction artifact; one domain; a fixed expert structure, not a self-discovered one", "older": false }, { "source": "Anthropic, \"Prompting best practices\" (Claude API docs)", "url": "https://platform.claude.com/docs/en/build-with-claude/prompt-engineering/claude-prompting-best-practices", "date": "2026-10-21", "locator": "Thinking and reasoning: Leverage thinking & interleaved thinking capabilities", "task": "Vendor guidance on prompting Claude models that think before answering", "effect": "\"Prefer general instructions over prescriptive steps. A prompt like \"think thoroughly\" often produces better reasoning than a hand-written step-by-step plan. Claude's reasoning frequently exceeds what a human would prescribe.\" Also: \"Multishot examples work with thinking ... Present each example as a problem, the method to apply, and the expected answer\"; on the newest models, rely on thinking rather than manual chain-of-thought prompts, \"at a lower effort level if cost or latency matters\". On Opus 5.5, Fable 5.x and other recent models, prompts asking for written-out reasoning \"may be declined\". No numbers.", "direction": "contradicts", "measured": false, "conditions": "Claude models with adaptive or extended thinking; living documentation, read 2026-10-21", "older": false } ], "caveats": "The original's compute advantage counts inference calls on two BBH tasks, not tokens, and its authors note longer inputs and outputs; the headline 32% is one task (T4D), and on MATH it built a structure per instance. Most follow-ups are small or come from papers proposing a rival method (Auto-Evolve, SABA): the CFA study's Self-Discover is a one-prompt, per-question imitation scored on 60 multiple-choice questions with no single-sample CoT arm, the instance-level replication had no CoT baseline, and Strategy-Induct sampled 25 questions per task. The reasoning-model evidence comes mostly from neighboring prompts (CoT, plan-and-solve, induced instructions, a hand-written TMK structure) rather than Self-Discover itself, and it disagrees: no gain on multiple-choice benchmarks, a gain on one obfuscated planning task, where o1-mini nonetheless fell from 19.1% to 16.83% on the Mystery variant. Per-question strategy prompts broke a reasoning model's answer format (o3-mini on Shift Cipher, 24.64 vs 98.40 for CoT), which favors one cached strategy per task type.", "also_covers": [], "related": [ { "id": "1.b.i", "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class" }, { "id": "3.a.iv", "slug": "restructure-system-prompt", "name": "Restructure the system prompt: every requirement once, up front, in named sections; nothing the model doesn't need" }, { "id": "9.c.ii", "slug": "samples-before-verification", "name": "Spend extra samples before extra verification: parallel attempts with a vote or judge; verify only where it pays" }, { "id": "6.c.ii", "slug": "plan-then-execute", "name": "Plan-then-execute" }, { "id": "3.b.i", "slug": "cache-stable-prefix", "name": "Keep the prompt prefix cache-stable" }, { "id": "9.b.i", "slug": "extractable-final-answer", "name": "Make the final answer reliably extractable: state the format, extract robustly, force an answer at the limit" }, { "id": "7.c.vii", "slug": "tree-of-thought", "name": "Tree-of-Thought" } ], "url": "https://mahalsystems.ai/aot/optimizations/self-discover-reasoning/", "markdown": "https://mahalsystems.ai/aot/optimizations/self-discover-reasoning.md" }, { "id": "4.b.i", "legacy_id": "P54", "former_ids": [], "slug": "filesystem-checkpoints", "name": "Filesystem-based agent state and checkpoints", "kind": "pattern", "status": "card", "tier": { "tier": 2, "name": "Reliability", "why": "operational hardening: checkpoint and resume", "proposed": true }, "levers": [ "state" ], "domains": [ "general" ], "evidence_domains": [ "code-generation", "tool-use" ], "categories": [ "Resuming" ], "category_ids": [ "memory.resume" ], "summary": "Save the agent's state at each step boundary (the conversation, the workspace files, process state when it matters, and a ledger of external side effects) so a crashed or stuck run resumes or rolls back instead of starting over. With one crash injected per task, restoring all of that recovered every task within 1.9% of fault-free time, while resuming from the conversation alone recovered 8 to 13% of Terminal-Bench tasks and restarting cost up to 1.67x the time; in one study, letting the agent roll back workspace and conversation together, with a note on what failed, raised success on long engineering tasks from 62.2% to 87.8%. The risks are side effects and partial restores: widely used framework checkpointers re-ran completed steps after a real crash, restores that bring back some state but not the rest let approvals and validations carry over to the wrong action, and rewinding the conversation without the files did worse than not rewinding.", "how": [ "Start from the baseline: restart the run from scratch on a crash, and resume from the conversation history alone (what most framework checkpointers and hosted agents restore). On 100 solved tasks per setup with one injected crash each, restart recovered every task but added up to 1.52x (Terminal-Bench) and 1.67x (SWE-bench) to completion time, and conversation-only resume recovered 8 to 13% of Terminal-Bench tasks and 9% of SWE-bench tasks (Wu, Apr 2026).", "Checkpoint what the next step depends on, not only the transcript: at each turn boundary (after tool results, before the next model call) record the conversation and the workspace together, for example a commit per step in a bare git repository kept outside the project's own history, with secrets excluded and caches dropped (Zhuang, Aug 2026), and add a process or sandbox snapshot when the task leaves services running or packages installed: conversation plus files recovered 100% on SWE-bench but only 28 to 42% on Terminal-Bench, where processes matter (Wu, Apr 2026). Always restore conversation and workspace from the same point: inconsistent restored state appeared only in the two tools that let files and conversation be restored separately, and a workspace rollback that kept the agent's passing malware scan released the uncleaned repository (Wu, Aug 2026).", "Size checkpoints to what changed: skip turns with no net file or process change (up to 87% of turns), take file snapshots (about 20 to 100 ms) where only files changed and process snapshots (700 to 1,000 ms) only where processes changed, and run them during the next model call; that kept a crashed run within 1.9% of fault-free time, while a full checkpoint every turn was 3.78x slower at 96 sandboxes per host, worse than restarting (Wu, Apr 2026).", "Treat external side effects (payments, messages, deploys, approvals) as the hard part: record each call and its result in a durable ledger keyed by the step, outside the checkpoint, and on resume return the recorded result instead of calling again. Test this with a real kill, not a raised exception: LangGraph 1.2.9 re-ran a completed step whose result was durably written after a SIGKILL, CrewAI 1.15.2's checkpoint restore re-ran a completed effect while its state looked correct, and an embedded DBOS engine ran each step's effect once (Khan, Aug 2026). Ordering the effect and the checkpoint write does not close the gap: with a failure between them, execute-then-commit repeated or skipped the effect in 90 of 96 workflows and commit-then-execute in 93 (Wu, Aug 2026); use idempotency keys or a transactional effect record (2.b.i).", "Make resumes single-shot and re-check what a restore carries forward: claim a parked step or approval in the shared store with a compare-and-swap before running it (when 2 to 16 processes resumed one parked approval, every racer fired the gated effect in 36 of 40 cells) (Khan, Aug 2026). After any restore, re-validate approvals, checks and external bindings (the active account or mailbox, recipients) against the live world, and store concrete values (timestamps, object IDs, versions) rather than references such as \"now\" or \"current state\": rollback re-ran a test mail-forwarding rule on a live mailbox and paid an invoice twice under one approval (Wu, Aug 2026); gate irreversible actions (10.b.i).", "Use checkpoints for rollback, not only crash recovery: give the agent a rewind tool that restores workspace and conversation to an earlier step and adds a short note on what failed. On 82 long engineering tasks with GPT-5.4 this raised success from 62.2% (continue) and 78.0% (restart with notes) to 87.8%, and recovered 30% of 50 stuck runs against 8%; rewinding the conversation without the workspace scored 43.9%, below not rewinding (Zhuang, Aug 2026). External effects are not rewound, so keep steps 4 and 5; for branching search over snapshots see 7.c.v.", "Measure with crash-injection runs on tasks the agent already solves: one fail-stop per task at a random step, recovered by restart, transcript-only resume and your checkpoints, scoring recovery correctness, redo time and tokens, duplicate effects counted in a ledger the agent cannot see, and checkpoint latency exposed in fault-free runs; for rollback, compare runs with and without the rewind tool at a fixed token budget." ], "detect": { "static": "What a checkpoint covers (transcript only, workspace files, sandbox processes, a ledger of external effects) and when it is written; whether resume serves recorded tool results or re-executes them; whether conversation and workspace can be restored from different points; whether a parked step or approval is claimed atomically before it runs; resume logic on start.", "traces": "Resume and rollback events; steps re-executed after a resume; an external effect issued twice for one step; checkpoint latency exposed on the critical path; rollbacks per run and what followed them.", "evals": "Recovery from injected crashes (a real kill, not only an exception) on tasks the agent already solves, against restart and transcript-only resume: recovery correctness, redo time and tokens, duplicate effects in an external ledger.", "probes": "Kill mid-run and resume." }, "impact": { "measured_basis": true, "primary": [ "quality", "cost" ], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "after one injected crash, restoring files and processes recovered 100% of solved tasks against 28 to 42% for conversation plus files and 8 to 13% for conversation only on Terminal-Bench (Wu, Apr 2026); a rewind tool restoring workspace and conversation raised success from 62.2% (continue) and 78.0% (restart with notes) to 87.8% on 82 long tasks with GPT-5.4, and from 78.7% to 83.1% on Terminal-Bench 2.0, but rewinding the conversation without the workspace scored 43.9% (Zhuang, Aug 2026); framework checkpointers re-ran completed steps after a crash (Khan, Aug 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "medium", "when": "a crash with no checkpoint re-runs prior work: restart added up to 1.52x to 1.67x completion time against within 1.9% with checkpoints, and forking from a checkpoint instead of re-running a shared prefix cut RL rollout tokens 40.0 to 64.2% (Wu, Apr 2026); the rollback study reports no token cost and set no budget (Zhuang, Aug 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "mixed", "size": "medium", "when": "checkpointing only turns with net changes and overlapping snapshots with the model call kept a crashed run within 1.9% of fault-free time, with p95 exposed delay of 0.44% at 64 sandboxes per host; a full file-and-process checkpoint every turn was 3.06x and 3.78x slower at 64 and 96 sandboxes, worse than restarting (up to 1.52x); process snapshots took 0.7 to 1.0 s against 20 to 100 ms for file snapshots (Wu, Apr 2026)", "basis": "evidence" }, { "dimension": "security", "effect": "mixed", "size": "medium", "when": "restores combine state from different points: incomplete internal state in 67.2% of 1,735 executions across five frameworks, and attacks that kept a passing malware scan for an uncleaned repository, re-ran a mail-forwarding rule on a live mailbox and paid twice under one approval (Wu, Aug 2026); 2 to 16 processes resuming one parked approval each fired its effect (Khan, Aug 2026); rollback also restores a known-good state after a destructive action, such as overwritten test fixtures (Zhuang, Aug 2026)", "basis": "evidence" }, { "dimension": "governance", "effect": "mixed", "size": "medium", "when": "per-step commits and checkpoints give a reviewable history and an undo (Young, Nov 2025; Zhuang, Aug 2026), but framework records can misstate what happened: CrewAI's restore re-ran an effect while its state looked correct, and LangGraph persisted schema-invalid state silently and served a second, different approval the first answer's branch (Khan, Aug 2026)", "basis": "evidence" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 2, "how": "crash-injection runs on tasks the agent already solves: one fail-stop per task at a random step (SIGKILL of the agent process or sandbox, not a raised exception), recovered by restart, transcript-only resume, transcript plus files, and the full checkpoint; score recovery correctness, redo time and tokens, duplicate external effects counted in a ledger the agent cannot see, and exposed checkpoint latency in fault-free runs; add probes that kill right after each durable write and between an effect and its checkpoint, resume one parked approval from two processes, and roll back after a validation; for rollback, paired runs with and without a rewind tool on long tasks at a fixed token budget (success, tokens, rewinds per run)" }, { "method": "monitor", "tier": null, "how": "resume and rollback events per run, steps re-executed per resume, external effects issued more than once per step key, approvals consumed more than once, checkpoint latency exposed on the critical path, checkpoint size and retention, and task outcomes after each rewind" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Context & Memory group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Justin Young (Anthropic), \"Effective harnesses for long-running agents\"", "url": "https://www.anthropic.com/engineering/effective-harnesses-for-long-running-agents", "date": "2025-11-26", "locator": "The long-running agent problem; Incremental progress", "task": "Each session leaves a progress log and git commits so the next session can resume from a known state", "effect": "The initializer sets up \"an init.sh script, a claude-progress.txt file that keeps a log of what agents have done, and an initial git commit\"; later sessions \"make incremental progress, then leave structured updates.\" Committing progress to git with descriptive messages \"allowed the model to use git to revert bad code changes and recover working states of the code base.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Guanlong Wu, Dahui Li, Ke Jiang, Jianyu Niu, Cong Wang, Yinqian Zhang, \"Safe to Resume? Breaking Execution Continuity of Agent Execution via Rollback\"", "url": "https://arxiv.org/abs/2608.29381", "date": "2026-08-29", "locator": "§II Table I; §IV; §V-A to V-C; §VI-C Table II; §VI-D Tables III and IV", "task": "Security study of checkpoint and rollback in five agent systems (LangGraph, CrewAI, Hermes, Cline, E2B): five failure modes, three end-to-end attacks, a rollback-validated prevalence study, and a microbenchmark of effect and checkpoint ordering", "effect": "Across 1,735 framework-task executions (one random recovery point each): incomplete internal state coverage in 1,166 (67.2%), inconsistent checkpoint state in 489, external state mismatch in 198, divergent nondeterministic replay in 1,174 (67.7%), unrecorded external effects in 40 (2.3%). E2B's full sandbox snapshots showed no incomplete or inconsistent internal state but still allowed external mismatch and replay divergence; Cline and Hermes, which restore files and conversation separately, were the only systems with inconsistent restored state (249 and 240 executions). Attacks: a Hermes workspace rollback kept a passing malware scan and released the uncleaned repository; a Cline task-history-only restore re-ran a forwarding rule on the live mailbox the service had switched to; a LangGraph payment node re-ran after a failure between payment and checkpoint, paying twice under one approval. With a failure injected between an external effect and its checkpoint write, execute-then-commit repeated or skipped the effect in 90 of 96 workflows (93.8%) and commit-then-execute in 93 (96.9%). 1,357 context-dependent references (\"now\", \"current state\") in checkpointed state; 30 of 30 sampled findings were exploitable.", "direction": "mixed", "measured": true, "conditions": "347 traces (241 Terminal-Bench, 106 AgentBench OS, DB and ALFWorld), DeepSeek-v4 as the agent and in the authors' detection pipeline (98.7% precision, 99.9% recall on manual inspection); a failure counts only when marked security-relevant; LangGraph and CrewAI ran one fixed agent-tool workflow; the authors say counts are not a security ranking of frameworks", "older": false }, { "source": "Yu Zhuang, Kefei Chen, Yitong Duan, Shuxin Zheng, Jian Li, Xu-Yao Zhang, \"AgentRewind: Recoverable Execution for Long-Horizon LLM Agents\"", "url": "https://arxiv.org/abs/2608.14380", "date": "2026-08-14", "locator": "AgentRewind Framework section; Tables 1 to 5; App. B Table 13; App. C.4 and C.5; App. D", "task": "Let the agent rewind to an earlier step: restore the workspace (a git commit per model or tool call in a bare repository outside the project) and the conversation together, serve the retained prefix from the log, and add a note on the abandoned attempt; compared with continuing, restarting with notes from failed attempts, and per-action safety review", "effect": "MettleBench (82 tasks), GPT-5.4 with mini-SWE-agent, success / checklist progress: continue 62.2% / 81.4%, restart with experiences 78.0% / 88.8%, safety review 34.1% / 54.4%, AgentRewind 87.8% / 94.3%; GPT-5.4 mini success: 33.7%, 43.1%, 36.2%, 51.2%. Other harnesses with GPT-5.4, continue against AgentRewind: FnCallAgent 58.5% / 81.7%, smolagents CodeAgent 67.1% / 82.9%. Terminal-Bench 2.0 (89 tasks): continue 78.7%, restart with experiences 70.8%, AgentRewind 83.1%. From 50 stuck endpoints, recovery 30.0% against 8.0% for continuing. Ablation: without workspace rewind 43.9%, without conversation rewind 65.9%, without the note 51.2%. Network requests and external-service calls are not undone; the retained prefix is served from the log, so they do not fire again.", "direction": "supports", "measured": true, "conditions": "Authors' own benchmark (criteria written by an LLM under their protocol) and runtime; temperature 0, three runs, 74.7% of task-strategy-model triples agree across runs; no step, time or token limit and no cost reported; the success gain over restart with experiences is not significant for GPT-5.4 mini after Holm correction", "older": false }, { "source": "Sajjad Khan, \"Resume Means Resume: A Machine-Checked Conformance Contract for Checkpoint, Interrupt, and Resume Semantics in Workflow Persistence Layers\"", "url": "https://arxiv.org/abs/2608.03836", "date": "2026-08-04", "locator": "Abstract; §II Table I; §VI-A to VI-G; Tables V and VI", "task": "Define six resume properties (including effect exactly-once and consume-once), model-check them, and measure five agent workflow frameworks' checkpoint, interrupt and resume APIs with a deterministic LLM-free harness and an external effect ledger, plus live-model runs", "effect": "LangGraph 1.2.9 (in-memory, SQLite and PostgreSQL savers): after a SIGKILL following a durable write, resume re-executed the completed task (effect count 1 before, 2 after), and killing after each of five persistence writes re-executed completed work at all four incomplete boundaries (\"exactly-once across interrupts, at-least-once across crashes\"); a second resume with a different approval value was served the first decision; schema-invalid state was persisted silently. CrewAI 1.15.2 checkpoint restore re-ran a completed effect-bearing method (effect counter 2) while its state, rebuilt from initial values, looked correct. pydantic-graph 1.x refused to restore after a crash inside a node. Concurrent resumes of one parked interrupt (2 to 16 racers): 2,628 gated effects where exactly-once predicts 400 over 40 cells, every racer firing in 36 of 40 cells, 16 fires from one approval in 10 of 10 runs at 16 racers; a compare-and-swap gate in the shared store served one racer. An embedded DBOS 2.27.0 worker conformed on every probed cell, running each step's effect once across process death; an OpenAI Agents SDK file-backed session restore did not re-invoke a tool in a live run.", "direction": "mixed", "measured": true, "conditions": "Pinned releases; pure-Python probes with no model calls (crashes by exception and by SIGKILL), plus 240 live runs on claude-haiku-4-5, claude-sonnet-4-6, gpt-4o-mini and gpt-4.1-mini; single independent author who also ships a repair layer (Remit); existence results, no prevalence claimed", "older": false }, { "source": "Tianyuan Wu, Chaokun Chang, Lunxi Cao, Wei Gao, Wei Wang, \"Crab: A Semantics-Aware Checkpoint/Restore Runtime for Agent Sandboxes\"", "url": "https://arxiv.org/abs/2604.28138", "date": "2026-04-30", "locator": "§1; §3.1 Figure 1; §3.2; §7.2 Figures 12, 13 and 15; §7.3 Figure 17; §7.4; §7.5", "task": "Inject one crash per task and compare recovery for sandboxed agents: restart from scratch, conversation only, conversation plus files, a full file-and-process checkpoint every turn, and Crab (classifies each turn's net file and process changes, checkpoints only what changed at turn boundaries, overlaps snapshots with the model call)", "effect": "100 solved tasks per setup, one crash each: Crab, restart and every-turn full checkpoints recovered 100%; conversation plus files 100% on SWE-bench but 28% (Claude Code) and 42% (iFlow-cli) on Terminal-Bench; conversation only 13% and 8% on Terminal-Bench and 9% on SWE-bench. Crab stayed within 0 to 1.9% of fault-free time; restart added up to 1.52x (Terminal-Bench) and 1.67x (SWE-bench); full checkpoints every turn were 3.06x and 3.78x slower at 64 and 96 sandboxes per host (Claude Code, Terminal-Bench). Up to 87% of turns needed no checkpoint; file snapshots took about 20 to 100 ms and process snapshots 700 to 1,000 ms; p95 exposed checkpoint delay 0.44% of task time at 64 sandboxes. Live-model pilot (50 tasks): conversation only 28%, conversation plus files 34% (48% under replay) as the agent's corrective actions perturbed the restored run; restart 1.55x median time. Forking from a checkpoint cut RL rollout tokens 40.0 to 64.2%; in a counterfactual two-run case study a rollback tool shortened one run 29%.", "direction": "supports", "measured": true, "conditions": "Claude Code (Claude Opus 4.6) and iFlow-cli (iFlow-ROME) on Terminal-Bench, SWE-agent (MiniMax-M2.7) on SWE-bench; deterministic replay of recorded traces of tasks each agent had solved; 16 to 96 sandboxes on one 128-core host; authors' own runtime; the abstract's \"checkpoint traffic\" cut of up to 87% is the share of turns skipped", "older": false } ], "caveats": "The quality gains come from the authors' own systems: rollback was tested on the authors' own 82-task benchmark with one model family, no token or step limit and no cost reported, and 74.7% of task-strategy-model triples agreed across three runs (Zhuang, Aug 2026); the crash-recovery comparison replays recorded traces of tasks the agents had already solved (Wu, Apr 2026). A restored run is not a replay: with a live model, conversation plus files recovered 34% against 48% under replay because the agent's corrective actions perturbed the run (Wu, Apr 2026), and widely used framework checkpointers differ in what they re-execute, so a framework's exactly-once claim has to be tested with real kills (Khan, Aug 2026). Checkpoints cannot undo external effects and ordering the effect and the checkpoint write does not fix that (90 and 93 of 96 workflows still repeated or skipped the effect); the security study's prevalence counts come from its own model-built detection pipeline (98.7% precision on manual inspection) at one recovery point per run (Wu, Aug 2026).", "also_covers": [], "related": [ { "id": "4.b.iii", "slug": "multi-window-task-state", "name": "Keep plan, progress and results in files when work outlives one context window" }, { "id": "4.b.ii", "slug": "initializer-maintainer-agents", "name": "Initializer-maintainer dual agent" }, { "id": "7.d.i", "slug": "continuous-task-loop", "name": "Continuous autonomous task loop" }, { "id": "2.b.i", "slug": "fail-soft-tools", "name": "Make tool calls and tool results fail soft, and retry transient failures" }, { "id": "10.b.i", "slug": "risky-action-policy", "name": "Gate irreversible actions: allow, ask or deny by side effect" }, { "id": "10.a.i", "slug": "constrain-agent-reach", "name": "Constrain what the agent can touch" }, { "id": "7.c.v", "slug": "language-agent-tree-search", "name": "Language Agent Tree Search (LATS)" }, { "id": "7.b.i", "slug": "detect-loops-and-stalls", "name": "Detect stalls and loops: warn, then stop early" }, { "id": "4.c.i", "slug": "episodic-memory", "name": "Episodic memory retrieval" } ], "url": "https://mahalsystems.ai/aot/optimizations/filesystem-checkpoints/", "markdown": "https://mahalsystems.ai/aot/optimizations/filesystem-checkpoints.md" }, { "id": "4.b.ii", "legacy_id": "P46", "former_ids": [], "slug": "initializer-maintainer-agents", "name": "Initializer-maintainer dual agent", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it splits the work across two agents", "proposed": true }, "levers": [ "state" ], "domains": [ "planning", "code-generation" ], "evidence_domains": [ "code-generation", "planning" ], "categories": [ "Resuming" ], "category_ids": [ "memory.resume" ], "summary": "Give the first session of a multi-session build a setup-only prompt (expand the spec into a JSON list of end-to-end tests, all failing, and write an init script, a progress file and a first commit) and every later session a maintainer prompt that reads that state, re-tests what passed, builds one failing item, verifies it end to end and commits. A separate up-front spec step pays when the model would otherwise start building and under-scope, and it is cheap: Anthropic kept its planner on Opus 4.6 for that reason at $0.46 of a $124.70 run, and in one controlled study a planning call before a single development pass raised a game-building score from 49.58 to 59.71. Freezing that first plan is the weak part: in the same study a plan written once and built over three fresh sessions scored 63.39, above 58.24 for one session continued for three passes but below 71.52 when each session replanned from test results, and Cursor dropped its one-time planner as too rigid. No study compares the split with one agent keeping the same state files (4.b.iii), Anthropic's two \"agents\" differ only in their first prompt, and strong models narrow the need: one Codex session ran about 25 hours on spec, plan and status files.", "how": [ "Start from one agent with one prompt that keeps the 4.b.iii state files (spec, plan, progress log, git history) in one continuous session with compaction, and log what goes wrong across a multi-session build: it starts building before it specs and under-scopes, tries to one-shot and runs out of context mid-feature, declares the job done early, or breaks features that had passed. A strong model may need nothing more: one Codex session with a spec, a milestone plan it wrote first, a runbook and a status file ran about 25 hours.", "If it under-scopes or one-shots, give the first session a setup-only prompt on the same harness, tools and system prompt (in Anthropic's version the first user prompt is the only difference), chosen by the harness when no feature list exists yet. It expands the spec into a JSON list of end-to-end tests ordered by priority, each with steps and all failing (over 200 for Anthropic's claude.ai clone), and writes an init script that installs, starts the app and runs a smoke test, a progress file and a first git commit. Keep it at product level rather than implementation detail, since spec errors cascade downstream; ask for a concrete number of items (\"20-100 tasks\", not \"many\"); and review the spec itself, since a weak one set Cursor's first browser on an architecture that could not grow into a full browser.", "Give every later session a maintainer prompt with a fixed contract: read the progress file, the list and the git log; run the init script and re-test one or two core passing items, marking any regression failing and fixing it first; take the highest-priority failing item and only that; verify it end to end as a user would (browser automation for a web app); change only that item's status; commit, update the progress file and leave a merge-ready state.", "Let the plan move with the evidence instead of freezing it: allow each session, or a planning step at its start that cannot edit code, to append items for defects and requirements the build exposes and to re-prioritize, never to delete or weaken an item. In one study a plan frozen after the first loop scored 63.39 against 71.52 when it was revised each loop from test results.", "Keep each role narrow: check each session's work outside the maintainer's own pass flags (7.c.iii, 9.b.iii), and do not give one agent planning, coding, review, merging and the completion decision at once; Cursor's agent that held all of them stalled and claimed premature completion.", "Measure with paired runs on the same specs and model at matched sessions and tokens: one agent with the 4.b.iii files, setup plus maintainer, and setup plus a maintainer that may append items; score with an end-to-end suite the agents never see (features passing, spec features never built, regressions reopened), plus tokens, the setup session's share and wall time; re-run after a model upgrade, since Anthropic dropped context resets for Opus 4.5 and sprints for Opus 4.6 but kept the planner." ], "detect": { "static": "Two entry points; a progress file and feature list; bootstrap and per-session flows.", "traces": "Session handoffs; progress file updates; whether each session starts by reading state and running the smoke test; items removed or reworded in the feature list; sessions declaring the project done with items failing; a first-session plan never revised after tests expose new defects.", "evals": "Multi-session completion rate.", "probes": "A multi-session project." }, "impact": { "measured_basis": true, "primary": [ "quality" ], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "multi-session builds where the model under-scopes or one-shots: a separate planning call before one development pass 49.58 to 59.71 on 45 game builds; a plan frozen after the first loop and built in fresh sessions 63.39 against 58.24 for one session continued for the same three passes, and 71.52 when each loop replanned from test results (Yan, Sep 2026); without its planner Opus 4.6 under-scoped (Rajasekaran, Mar 2026); one-time up-front planning was too rigid to readjust and was removed (Lin, Feb 2026); no study against one agent keeping the same state files", "basis": "evidence" }, { "dimension": "cost", "effect": "mixed", "size": "small", "when": "the setup session is a small share: planner $0.46 of $124.70 (Rajasekaran, Mar 2026); a planning call before one pass 2.88M against 2.59M tokens per task; three fresh-session loops 8.41M (frozen plan 7.56M) against 6.33M for three passes of one continued session, but 3.77 against 2.32 points per extra million tokens, and rebuilding from scratch each loop instead of continuing raised tokens to 11.12M (Yan, Sep 2026); planner, sprints and evaluator together cost $200 against $9 for a solo run (Rajasekaran, Mar 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "small", "when": "the setup session adds minutes, not hours: planner 4.7 min of a 3 hr 50 min run (Rajasekaran, Mar 2026); Anthropic's quickstart warns the first session takes 10-20+ minutes and each coding session 5-15 (Young, Nov 2025); every later session re-reads state and re-runs a smoke test before new work; no source compares wall time with one agent", "basis": "assessment" }, { "dimension": "security", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "governance", "effect": "improves", "size": "medium", "when": "a commit and progress-log entry per session and an issue record make each session's work reviewable; a 70-loop build kept 81 issues on record, 17 of them reopened after later changes broke verified behavior (Yan, Sep 2026)", "basis": "assessment" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired runs on the same specs and model at matched sessions and tokens: one agent with the 4.b.iii state files in one continuous session; a setup-only first session plus maintainer sessions; the same with the maintainer allowed to append items from test results; score with an end-to-end suite the agents never see (features passing, spec features never built, regressions reopened), plus total tokens, the setup session's share and wall time; per model, and again after an upgrade" }, { "method": "monitor", "tier": null, "how": "per session: whether it read the progress file, list and git log and ran the smoke test before new work; regressions found at session start; items marked passing that fail the hidden suite; items removed or reworded in the list (should be zero); items added from test findings; sessions per item; sessions that declare the project done with items still failing" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Multi-Agent group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Justin Young (Anthropic), \"Effective harnesses for long-running agents\"", "url": "https://www.anthropic.com/engineering/effective-harnesses-for-long-running-agents", "date": "2025-11-26", "locator": "Intro; The long-running agent problem; Getting up to speed; Future work; footnote 1; accompanying autonomous-coding quickstart", "task": "Introduces the two-agent harness: an initializer agent sets up the environment once, a coding agent makes incremental progress each session", "effect": "\"an initializer agent that sets up the environment on the first run, and a coding agent that is tasked with making incremental progress in every session, while leaving clear artifacts for the next session.\" The two are one harness: \"We refer to these as separate agents in this context only because they have different initial user prompts. The system prompt, set of tools, and overall agent harness was otherwise identical.\" Working on one feature at a time \"turned out to be critical\"; no numbers are reported, and \"it's still unclear whether a single, general-purpose coding agent performs best across contexts, or if better performance can be achieved through a multi-agent architecture.\"", "direction": "supports", "measured": false, "conditions": "Failure modes shown with Opus 4.5 on the Claude Agent SDK; the accompanying quickstart defaults to Sonnet 4.5 and asks the initializer for at least 200 end-to-end tests; a claude.ai clone web app; internal experiments, no comparison reported", "older": false }, { "source": "Haoyang Yan, Min-Le Su, Hangfan Zhang, Zhanhao Li, Chen Zhang, Shao Zhang, Yang Chen, Lei Bai, Shuyue Hu (Shanghai AI Laboratory), \"Harness-of-Harness: Multi-Day Autonomous Software Development with Continual Improvement\"", "url": "https://arxiv.org/abs/2609.01481", "date": "2026-09-01", "locator": "§4.2 Table 1; §4.3 Tables 2-3; Appendix B.2 Table 7; B.6 Table 13; C.3; C.4 Tables 22-23; §5.2", "task": "Wrap an existing coding harness in repeated loops of a planner (writes a bounded development document from the spec and the last test report, cannot edit code), a developer (builds the increment on the previous artifact) and an independent QA tester, each a fresh invocation with state carried in files; compared with one development pass and with the same session continued by an extra iteration prompt; ablations freeze the first plan, withhold test evidence from the planner, or rebuild from an empty workspace each loop", "effect": "GameCraft-Bench Overall, Codex with GPT-5.5 (high), 45 tasks: one pass 49.58 (2.59M tokens per task); after the first loop 59.71 (2.88M); one session continued for two and three passes 54.99 (4.56M) and 58.24 (6.33M); HoH after two and three loops 64.84 (5.67M) and 71.52 (8.41M), 3.77 against 2.32 points per extra million tokens. First plan frozen for later loops 63.39 (7.56M); replanning without test evidence 65.23 (7.46M); rebuilding from an empty workspace each loop 63.67 at 11.12M tokens. After three loops HoH beat one pass under all three harness-model pairs (GameCraft-Bench +16.62 to +22.08 points, ProgramBench +6.09 to +16.85). A 70-loop FPS build closed 65 of 81 recorded issues; 17 were reopened after later changes broke verified behavior.", "direction": "mixed", "measured": true, "conditions": "Codex CLI + GPT-5.5 (high), OpenCode + DeepSeek-V4-Pro, Pi + MiniMax-M3; GameCraft-Bench (45 of 140 tasks), FrontierSWE (15 of 17), ProgramBench; the pass-matched comparison and ablations on Codex + GPT-5.5 only; one run per task and condition at default temperature; the authors' own framework; its roles add an independent tester and per-loop planning, so it does not test initializer and maintainer alone; multi-day case on GPT-5.6-Sol", "older": false }, { "source": "Prithvi Rajasekaran (Anthropic), \"Harness design for long-running application development\"", "url": "https://www.anthropic.com/engineering/harness-design-long-running-apps", "date": "2026-03-24", "locator": "The architecture; Running the harness; Iterating on the harness; Removing the sprint construct; Results from the updated harness", "task": "A planner agent expands a 1-4 sentence prompt into a product spec once, before a generator builds and an evaluator tests; harness components removed one at a time after Opus 4.6", "effect": "The planner survived the ablation: \"Without the planner, the generator under-scoped: given the raw prompt, it would start building without first speccing its work, and end up creating a less feature-rich application than the planner did.\" It was prompted to stay \"focused on product context and high level technical design rather than detailed technical implementation\" because otherwise \"the errors in the spec would cascade into the downstream implementation\". Browser DAW on Opus 4.6: planner 4.7 min and $0.46 of 3 hr 50 min and $124.70. Earlier, on Opus 4.5, solo 20 min and $9 against 6 hr and $200 for planner, sprints and evaluator; in the solo app \"the actual game was broken.\" Context resets were dropped for Opus 4.5 and sprints for Opus 4.6.", "direction": "supports", "measured": false, "conditions": "Claude Opus 4.5 and Opus 4.6 on the Claude Agent SDK; one prompt per app, one run each; quality judged by the author; the planner ablation is described, not scored; the $9 vs $200 contrast adds planner, sprints and evaluator at once", "older": false }, { "source": "Derrick Choi (OpenAI), \"Run long horizon tasks with Codex\"", "url": "https://developers.openai.com/blog/run-long-horizon-tasks-with-codex", "date": "2026-02-23", "locator": "My setup for the test; The key idea: durable project memory; Takeaways for long-horizon Codex tasks", "task": "One Codex session builds a design tool from a blank repo with no separate setup agent and no context resets: an author-written spec file, a milestone plan the opening prompt had Codex generate from it, a runbook file, and a status and decisions file it kept updated, with tests, lint, typecheck and build after each milestone", "effect": "\"Codex ran for about 25 hours uninterrupted, used about 13M tokens, and generated about 30k lines of code.\" \"The most important technique was durable project memory. I wrote the spec, plan, constraints, and status in markdown files that Codex could revisit repeatedly. That prevented drift and kept a stable definition of “done.”\" The result \"was not perfect or production-ready, but it was real and testable.\"", "direction": "mixed", "measured": false, "conditions": "GPT-5.3-Codex at Extra High reasoning in Codex; one run, no comparison; judged by the author, a vendor demonstrating its own model", "older": false }, { "source": "Wilson Lin (Cursor), \"Towards self-driving codebases\"", "url": "https://cursor.com/blog/self-driving-codebases", "date": "2026-02-05", "locator": "Background; Adding structure and roles; Continuous executor; Pathological behaviors; The final system design; Specifying intent to agents; Optimizing prompts", "task": "Successive harnesses for building a web browser over weeks: one agent nudged to \"keep going\" on its own plan; equal agents sharing a state file; an up-front planner handing its plan to an executor, workers and a judge; a continuous executor that also planned; and finally recursive planners that never code plus workers that hand results back to them", "effect": "One agent on its own plan: \"The model lost track of what it was doing, frequently stopped to proclaim success despite being far from it\". With a separate planner, \"Doing all planning upfront also made it hard for the system to dynamically readjust as new issues were discovered\", and \"The next version removed the independent planner.\" The executor that then planned, spawned, reviewed, merged and judged would \"sleep randomly, stop running agents, do work itself ... and claim premature completion\"; \"it was being given too many roles\". The final design keeps planners that never code and keep receiving worker handoffs. The first browser \"converged on an architecture that was unfit to evolve into a full browser. This was a failure of the initial specification.\" \"Generate 20-100 tasks\" instead of \"generate many tasks\" gave \"very different wider behavior\".", "direction": "mixed", "measured": false, "conditions": "Opus 4.5 at first, then GPT-5.1 and GPT-5.2; one research project (a browser) run with hundreds of concurrent agents; no controlled comparison or quality numbers, judged by the author; parallel agents rather than sequential sessions", "older": false } ], "caveats": "No study isolates the split as Anthropic built it, two first prompts on one harness, against one agent keeping the same state files (4.b.iii); the one controlled comparison is one framework, one model setup and 45 game-building tasks with one run each, and its roles add an independent tester and per-loop planning. The vendor accounts (Anthropic, Cursor, OpenAI) are single builds judged by their authors, mostly on the vendors' own models, with no quality numbers. The advice conflicts on freezing the plan: Anthropic forbids editing the test list, while a frozen first plan lost 8.13 points to per-loop replanning and Cursor dropped up-front planning as too rigid; an append-only list sits between them and is untested. The need shifts with the model: Anthropic dropped resets for Opus 4.5 and sprints for Opus 4.6 but kept the planner, and one Codex session ran about 25 hours without a separate setup role.", "also_covers": [], "related": [ { "id": "4.b.iii", "slug": "multi-window-task-state", "name": "Keep plan, progress and results in files when work outlives one context window" }, { "id": "5.c.i", "slug": "discrete-phase-separation", "name": "Discrete phase separation" }, { "id": "7.c.iii", "slug": "evaluator-optimizer-loop", "name": "Evaluator-optimizer loop" }, { "id": "9.b.iii", "slug": "verify-completion", "name": "Verify completion against the environment before the agent stops" }, { "id": "7.a.ii", "slug": "prompt-chaining-with-gates", "name": "Prompt chaining with gate checks" }, { "id": "7.b.iv", "slug": "track-task-progress", "name": "Track explicit progress: a task list with a status per item, re-injected each turn" }, { "id": "6.b.iii", "slug": "multi-agent-for-breadth", "name": "Use multi-agent orchestration only for breadth-first, independent sub-tasks" }, { "id": "4.b.i", "slug": "filesystem-checkpoints", "name": "Filesystem-based agent state and checkpoints" }, { "id": "7.d.i", "slug": "continuous-task-loop", "name": "Continuous autonomous task loop" } ], "url": "https://mahalsystems.ai/aot/optimizations/initializer-maintainer-agents/", "markdown": "https://mahalsystems.ai/aot/optimizations/initializer-maintainer-agents.md" }, { "id": "4.b.iii", "legacy_id": "P21", "former_ids": [], "slug": "multi-window-task-state", "name": "Keep plan, progress and results in files when work outlives one context window", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it moves task state outside the context window", "proposed": true }, "levers": [ "state" ], "domains": [ "planning" ], "evidence_domains": [ "web-research", "code-generation", "document-extraction", "tool-use", "conversation", "planning" ], "categories": [ "Resuming", "Externalizing state" ], "category_ids": [ "memory.resume", "memory.externalize" ], "summary": "When a task spans context resets (compaction, a truncation limit, a new session), have the agent keep what the next window needs in files and read them back at the start of each window: the plan and a progress list, decisions and open items, intermediate results too large to hold in context, and for coding a structured test list and an init script. In measured studies, mostly of deep-research and long-horizon coding agents, state kept outside the history beat a growing or compacted one: Anthropic's memory tool plus context editing gained 39% on agentic search against 29% for context editing alone. The limits: model-written notes were incomplete or never read back, a stale file misled agents more than no file, and a stronger model needed resets less. Within one window this adds nothing a re-injected task list (7.b.iv) doesn't.", "how": [ "Start from the harness as it is (compaction or truncation only) on tasks that really run past one window, and log what breaks after each reset: work redone, decisions or user constraints lost, work declared done too early.", "In the first window, before real work, have the agent create the state files: a structured list of features or tests (JSON, every item failing, with an instruction that items are never removed or rewritten, only marked passing), a free-text progress log, a decisions and open-items file that also holds the user's constraints, and for coding an init script that starts the app and runs a smoke test; commit them to git.", "Write large intermediate results (sources, query results, documents, logs) to files and keep only the path and a one-line note in context.", "Start every later window with a fixed resume step written into the prompt: read the progress log, the decisions file, the test list and the git log, run the init script and the smoke test, then take one open item. Do not leave reading optional; agents write notes they never read.", "Update at fixed points: when an item passes an end-to-end check, mark it, append to the log and commit; before a reset, write the handoff. Rewrite the files in place so they describe the present and delete stale entries, because agents follow a written file over the code.", "Keep harness-side safety nets instead of relying on model-written notes alone: keep compaction or the harness's own state capture, re-append the constraint list after every compaction, and choose a fresh window or compaction per model.", "Measure with paired runs that force a reset midway, with and without the files and the resume step: completion, steps redone, constraint violations after the reset, tokens and wall time, per model; re-run after a model change." ], "detect": { "static": "A plan or todo file, a progress log, a decisions and open-items file, and for coding a structured test list and an init script, kept outside the context window, plus a start-of-window step that reads them back.", "traces": "Context resets followed by re-exploration or redone steps; notes written but never read back; user constraints or decisions violated after compaction; work declared done after a reset; state files older than the code they describe.", "evals": "Completion rate and redone work on tasks longer than one window, with a reset forced midway.", "probes": "Tasks longer than one context window." }, "impact": { "measured_basis": true, "primary": [ "quality" ], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "tasks past one window: memory tool plus context editing +39% against +29% for context editing alone (Anthropic, Sep 2025); a report rewritten into each rebuilt context +12.7 (o3) and +19.2 (DeepSeek-V3.1) points on BrowseComp over ReAct (Chen, Nov 2025); without todo, checklist and log files 52.76 to 48.69 RACE on 10 queries (Zhu, Feb 2026); model-written notes alone degraded Devin (Cognition, Sep 2025); a stale convention file cut better-form decisions from 33% with no file to 0% (Mohammadi, Aug 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "mixed", "size": "small", "when": "file I/O under 0.03% of wall time (Zhu, Feb 2026); resets add token overhead (Rajasekaran, Mar 2026); an agent sometimes spent more tokens writing summaries than solving the problem (Cognition, Sep 2025); no study reports tokens for the files alone", "basis": "evidence" }, { "dimension": "performance", "effect": "mixed", "size": "small", "when": "with a state-recovery memory, minutes per DeepSWE task rose from 42.0 to 54.2 under one harness and fell from 103.8 to 75.3 under another (Xu, Oct 2026); resets add latency (Rajasekaran, Mar 2026)", "basis": "evidence" }, { "dimension": "security", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "governance", "effect": "improves", "size": "small", "when": null, "basis": "assessment" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired runs on tasks longer than one window, forcing a reset (or capping the window) midway: the harness as is against the same harness with state files and a fixed resume step; completion, steps redone after the reset, constraint violations after the reset, tokens and wall time, per model; re-run after a model change" }, { "method": "monitor", "tier": null, "how": "per reset: whether the first actions read the state files, steps redone, items marked done that fail their test, state files older than the code or plan they describe, and the share of tokens spent writing notes" } ], "evidence": [ { "source": "Lance Martin, \"Agent Design Patterns\"", "url": "https://rlancemartin.github.io/2026/01/09/agent_design/", "date": "2026-01-09", "locator": "Offload Context", "task": "Write plan to file and re-read it periodically", "effect": "\"Some agents write a plan to a file and read it back into context periodically to reinforce objectives and / or verify its work.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Architecture overview (process diagram caption); Appendix: Long-horizon conversation management", "task": "Save the plan to external memory before truncation", "effect": "\"saving its plan to Memory to persist the context, since if the context window exceeds 200,000 tokens it will be truncated and it is important to retain the plan.\"", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file master-agent-optimization-cookbook.md", "task": "Lists this practice in its agent-optimization pattern library (Long-Horizon group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Justin Young (Anthropic), \"Effective harnesses for long-running agents\"", "url": "https://www.anthropic.com/engineering/effective-harnesses-for-long-running-agents", "date": "2025-11-26", "locator": "The long-running agent problem", "task": "Keep a progress file and git history so a fresh context window can pick up the work", "effect": "\"The key insight here was finding a way for agents to quickly understand the state of work when starting with a fresh context window, which is accomplished with the claude-progress.txt file alongside the git history.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Anthropic, \"Managing context on the Claude Developer Platform\"", "url": "https://www.anthropic.com/news/context-management", "date": "2025-09-29", "locator": "Performance improvements with context management", "task": "Give the agent a file-based memory tool (create, read, update and delete files in a memory directory) alongside context editing that clears stale tool results", "effect": "\"combining the memory tool with context editing improved performance by 39% over baseline. Context editing alone delivered a 29% improvement.\" The 84% token reduction in the same post is for context editing alone, on a 100-turn web search evaluation.", "direction": "supports", "measured": true, "conditions": "Internal agentic-search evaluation set; model, n and baseline not stated; vendor evaluating its own feature", "older": true }, { "source": "Guoxin Chen, Zile Qiao, Xuanzhong Chen, Donglei Yu, Haotian Xu, Wayne Xin Zhao, Ruihua Song, Wenbiao Yin, et al., \"IterResearch: Rethinking Long-Horizon Agents with Interaction Scaling\"", "url": "https://arxiv.org/abs/2511.07327", "date": "2025-11-10", "locator": "Abstract; §4.3 Table 2; §4.5 Figure 4", "task": "Each round, rebuild the context from the question, an evolving report the agent rewrites as its memory, and only the last action and its result, instead of one growing history", "effect": "As a prompting strategy with no training, it beat ReAct on BrowseComp by 12.7 points with o3 and 19.2 with DeepSeek-V3.1. With identical training data, the iterative agent averaged 49.1 against 36.5 for a single-context agent across six benchmarks (+12.6), although the single-context agent had a 64K window against 40K.", "direction": "supports", "measured": true, "conditions": "Trained agent on Qwen3-30B-A3B; prompting with o3 and DeepSeek-V3.1; HLE, BrowseComp, BrowseComp-zh, GAIA, Xbench-DeepSearch, SEAL-0; the report is carried in the prompt, not a file", "older": false }, { "source": "Chenglin Yu, Yuchen Wang, Songmiao Wang, Hongxia Yang, Ming Li, \"InfiAgent: An Infinite-Horizon Framework for General-Purpose Autonomous Agents\"", "url": "https://arxiv.org/abs/2601.03204", "date": "2026-01-06", "locator": "§5.2 Table 1 and Ablation analysis; §7 Limitations", "task": "Keep task state in files and rebuild each step's context from a workspace snapshot plus a fixed window of recent actions; ablate by replacing the file state with compressed long-context prompts", "effect": "Average coverage on an 80-paper literature review: 67.1 (GPT-OSS-20B), 80.0 (Gemini-3-Flash) and 80.0 (Claude-4.5-Sonnet) with file-centric state; 3.2, 21.1 and 27.7 with \"No File State (Compressed Context)\". Claude Code with Claude-4.5-Sonnet averaged 29.1. The authors note latency overhead, and that wrong information a sub-agent writes to the files can propagate.", "direction": "supports", "measured": true, "conditions": "80-paper review task, repeated runs (count not stated); authors' own framework; DeepResearch benchmark score 41.45 with gpt-oss-20b", "older": false }, { "source": "Chiwei Zhu, Benfeng Xu, Mingxuan Du, Shaohan Wang, Xiaorui Wang, Zhendong Mao, Yongdong Zhang, \"FS-Researcher: Test-Time Scaling for Long-Horizon Research Tasks with File-System-Based Agents\"", "url": "https://arxiv.org/abs/2602.01566", "date": "2026-02-02", "locator": "§3 (control files); §4.2 Table 5; Appendix M Table 15", "task": "Deep research over several sessions on a file workspace: a todo file with a status per item, a fixed checklist and a log read at each session start, plus a knowledge base of notes and archived sources; ablate by removing the control files and replacing the structured workspace with a flat note", "effect": "Removing the persistent workspace dropped RACE from 52.76 to 48.69, with Insight -7.95 and Comprehensiveness -3.58. File I/O took under 0.03% of wall-clock time.", "direction": "supports", "measured": true, "conditions": "GPT-5; 10 sampled DeepResearch Bench queries for the ablation; 3 rounds for the note-building agent", "older": false }, { "source": "Hongming Xu, Le Zhou, ZhongHe Jin, Xiang Zhang, Bo Tang, Zhiyu Li, Xuanhe Zhou, Juncheng Zhang, \"MemTrace: State-Consistent Memory for Long-Horizon Coding Agents\"", "url": "https://arxiv.org/abs/2610.04838", "date": "2026-10-04", "locator": "Abstract; Table 2; Table 3; §4.3 State–memory alignment; Appendix A.2 Table 6", "task": "Store execution history outside the window as traces anchored to files, symbols and tests; after a context refresh, rebuild what is done and what remains from them and check stored evidence against the current repository before reuse", "effect": "Against each harness's native context management: under Codex CLI, DeepSWE pass@1 35.4 to 56.6, SWE-EVO resolved 47.8 to 69.6, SWE-Milestone score 28.6 to 46.4; under mini-swe-agent, DeepSWE 48.7 to 66.4. A flat trace store without the structure scored 32.1 on DeepSWE, below native. 57.0% of 9,226 file-bound traces saw a later change to a bound file. Mean minutes per DeepSWE task: 42.02 to 54.17 under Codex CLI, 103.77 to 75.30 under mini-swe-agent.", "direction": "supports", "measured": true, "conditions": "deepseek-v4-flash, 200K window; DeepSWE 113 tasks, SWE-EVO 46, SWE-Milestone 98 milestones; one model; authors' own system", "older": false }, { "source": "Zhiqi Wang, Yichi Zhang, Dongwon Lee, Nathalie Baracaldo, Yuchen Yang, \"Lost in Context Compaction: When LLM Systems Drop Side Constraints\"", "url": "https://arxiv.org/abs/2608.11242", "date": "2026-07-31", "locator": "Abstract; §2; §5.1; §5.4 Table 4", "task": "Keep user-issued constraints in a registry outside the summary and append it after every compaction, against compaction alone", "effect": "Compactors retained 17% of injected constraints on average; in an inbox test the agent deleted emails without approval in 30/100 sessions after compaction, against none before. The registry kept 90.3%, 93.7% and 95.1% of constraints (WildChat, Hermes Agent, OpenResearcher), and appending it raised compliance by 39.5-42.6 percentage points.", "direction": "supports", "measured": true, "conditions": "100K-token contexts, N=750 per dataset; compactors gpt-oss-120b, Qwen3-30B-A3B, Gemma-4-E4B, GPT-5.4-mini; extractor Qwen3.5-9B; compliance probed with multiple-choice questions", "older": false }, { "source": "Maciej Besta et al., \"Affordable AI Assistants with Knowledge Graph of Thoughts\"", "url": "https://arxiv.org/abs/2504.02670", "date": "2025-04-03", "locator": "§5.1; §5.3; §5.6; Appendix Table 2 (GAIA, per model)", "task": "The agent keeps its task state in an external knowledge graph that its tool calls (web, Python, file readers) keep extending and that it queries (Cypher, SPARQL or Python) to answer, instead of holding that state in context", "effect": "GAIA validation, GPT-4o mini, tasks solved of 165: single knowledge-graph variants 36 to 44, \"between 42% to 62% lower costs\", against 30 and 33 for the same pipeline without the graph (\"No KG\") and 35 for Hugging Face Agents. A fusion of all graph variants solved 71, against 40 for the no-graph fusion. Open models (Neo4j + Cypher variant): DeepSeek-R1-7B solved 13 zero-shot but 6 with the graph, R1-32B 14 against 21, R1-70B 20 against 22.", "direction": "mixed", "measured": true, "conditions": "GAIA validation (165) and test (301); GPT-4o mini, Qwen2.5, QwQ-32B and DeepSeek-R1 distills; single runs; fusion runs \"simulate the effect from KGoT runs with both graph backends available simultaneously\"; a graph store, not files", "older": true }, { "source": "Axel Backlund, Lukas Petersson, \"Vending-Bench: A Benchmark for Long-Term Coherence of Autonomous Agents\"", "url": "https://arxiv.org/abs/2502.15840", "date": "2025-02-20", "locator": "§2.1; §3.2.2; §3.5.2 Figures 12-13; §3.6 Table 9", "task": "Long-running agent with only the last 30,000 tokens in context plus a scratchpad, key-value store and vector database it can read and write", "effect": "All models had runs that derailed. Claude 3.5 Sonnet wrote a long daily summary to the scratchpad \"but never retrieves the information.\" Memory-tool use did not differ significantly between 10k, 30k and 60k token windows, and the larger windows did worse. The Pearson correlation between days until sales stop and days until memory is full was 0.167; \"the breakdowns are not directly related to the context of the LLMs being filled\".", "direction": "mixed", "measured": true, "conditions": "Claude 3.5 Sonnet, o3-mini, GPT-4o mini and others; window-size variations on GPT-4o mini; 5 runs per configuration; about 25M tokens per run", "older": true }, { "source": "Bardia Mohammadi, Lars Klein, Aman Chadha, Akhil Arora, Laurent Bindschaedler, \"The Working Set of a Coding Agent: Coherence Debt in Repository-Scale Tasks\"", "url": "https://arxiv.org/abs/2608.16630", "date": "2026-08-17", "locator": "§4.8; §4.9; Appendix N", "task": "Give a coding agent a written standard file that agrees with the code, no standard, or a stale standard that demands the worse form", "effect": "Share of contested decisions written the better way: 100% with a correct standard, 33% with no standard, 0% with a stale one; \"Deleting an out-of-date convention file beats leaving it in place.\" Where a written standard and working code disagreed, agents followed the standard in every one of 39 trials.", "direction": "mixed", "measured": true, "conditions": "Synthetic workload, ten contested conventions, 12 tasks per trial, 10 trials per condition (3,385 decisions); the 39-trial result spans three model tiers on one harness and a second harness; a convention file, not a progress file", "older": false }, { "source": "The Cognition Team, \"Rebuilding Devin for Claude Sonnet 4.5: Lessons and Challenges\"", "url": "https://cognition.ai/blog/devin-sonnet-4-5-lessons-and-challenges", "date": "2025-09-29", "locator": "The model takes a lot of notes", "task": "Let the model's own notes (CHANGELOG.md, SUMMARY.md) replace the harness's compaction and summarization", "effect": "\"we found the summaries weren't comprehensive enough. For example, it would sometimes paraphrase the task, leaving out important details. When we relied on the model's own notes without our compacting and summarization systems, we saw performance degradation and gaps in specific knowledge\". Also \"we've seen the agent spend more tokens writing summaries than actually solving the problem.\" The behavior was \"useful in certain cases, but less effective than our existing memory systems\".", "direction": "mixed", "measured": false, "conditions": "Claude Sonnet 4.5 in Devin; internal testing, no numbers reported", "older": true }, { "source": "Prithvi Rajasekaran (Anthropic), \"Harness design for long-running application development\"", "url": "https://www.anthropic.com/engineering/harness-design-long-running-apps", "date": "2026-03-24", "locator": "Why naive implementations fall short; The architecture", "task": "Context resets with a structured handoff between sessions, against one continuous session with automatic compaction", "effect": "With Claude Sonnet 4.5, \"compaction alone wasn't sufficient to enable strong long task performance, so context resets became essential\", at the cost of \"orchestration complexity, token overhead, and latency\". \"Opus 4.5 largely removed that behavior on its own, so I was able to drop context resets from this harness entirely\"; it ran one session with automatic compaction, and its agents still handed work to each other through files.", "direction": "mixed", "measured": false, "conditions": "Claude Sonnet 4.5 and Claude Opus 4.5; the author's own harness; no controlled comparison", "older": false } ], "caveats": "Most measured gains come from deep-research and long-horizon coding systems built around their authors' own memory design, often on one model and small samples (one ablation used 10 queries, another does not state its run count), and no controlled study isolates a plain progress file under a forced reset. Model-written notes are not free memory: Cognition found them incomplete, Vending-Bench agents wrote notes they never read, and a stale written standard led agents to the worse form on every contested decision, so read-back and upkeep must be enforced. The need depends on the model: Anthropic needed resets with handoff files for Sonnet 4.5 but dropped them for Opus 4.5, so measure per model and again after an upgrade.", "also_covers": [ { "name": "Structured note-taking" }, { "name": "File system as external memory" }, { "name": "Proactive state externalization" } ], "related": [ { "id": "7.b.iv", "slug": "track-task-progress", "name": "Track explicit progress: a task list with a status per item, re-injected each turn" }, { "id": "5.a.i", "slug": "staged-compaction", "name": "Compact conversation history in stages: elide stale tool outputs, then keep an anchored summary" }, { "id": "4.b.ii", "slug": "initializer-maintainer-agents", "name": "Initializer-maintainer dual agent" }, { "id": "4.b.i", "slug": "filesystem-checkpoints", "name": "Filesystem-based agent state and checkpoints" }, { "id": "9.b.iii", "slug": "verify-completion", "name": "Verify completion against the environment before the agent stops" }, { "id": "7.d.i", "slug": "continuous-task-loop", "name": "Continuous autonomous task loop" }, { "id": "4.c.i", "slug": "episodic-memory", "name": "Episodic memory retrieval" }, { "id": "4.c.ii", "slug": "skill-library", "name": "Skill library evolution" }, { "id": "5.c.i", "slug": "discrete-phase-separation", "name": "Discrete phase separation" }, { "id": "5.c.ii", "slug": "isolated-subagent-context", "name": "Sub-agent architectures with clean contexts" } ], "url": "https://mahalsystems.ai/aot/optimizations/multi-window-task-state/", "markdown": "https://mahalsystems.ai/aot/optimizations/multi-window-task-state.md" }, { "id": "4.c.i", "legacy_id": "P55", "former_ids": [], "slug": "episodic-memory", "name": "Episodic memory retrieval", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it adds a cross-session memory architecture", "proposed": true }, "levers": [ "state" ], "domains": [ "memory" ], "evidence_domains": [ "code-generation", "data-analysis", "computer-use", "tool-use", "web-research", "conversation", "planning", "reasoning" ], "categories": [ "Reusing work" ], "category_ids": [ "memory.reuse" ], "summary": "Log every agent session as a raw trace, distill a few verified lessons from them (user corrections, definitions, workflows that worked, mistakes) into short memory files, show the agent an index of those files at the start of each task, and let it search both. It pays where later tasks need what earlier ones learned: Databricks' data agent went from 22.3% to 68.4% on tasks built from its users' past sessions, with 31% fewer tokens. Elsewhere the gains mostly vanish: with three frontier models two memory methods moved average accuracy by -1.1 to +3.0 points over six benchmarks, a no-memory web agent given the same tokens beat ReasoningBank by 5.1 to 8.1 points, and off-the-shelf memory systems were at or below memory off on coding tasks in 11 of 12 pairings. Unverified lessons reproduce mistakes, and an agent that writes memory from what it reads can be poisoned: one crafted email planted a lasting instruction in up to 87.5% of test cases.", "how": [ "Start from two baselines on the same tasks and model: no memory, and a no-memory agent that gets the memory's token budget as extra steps (7.b.iii) or reasoning effort (1.b.i). A no-memory web agent allowed 15 steps instead of 10 beat ReasoningBank by 5.1 to 8.1 points with three 2026 models, mostly at fewer total tokens, and with GPT-5.4, Gemini 3.1 Pro and Claude Opus 4.7 two memory methods moved average accuracy by -1.1 to +3.0 points. Build the evaluation stream so later tasks need what earlier ones learned (a user's corrections, metric or table definitions, repository conventions, a workflow that was costly to find): that is where memory paid, 22.3% to 68.4% on tasks built from real user sessions.", "Log every session as an append-only raw trace (task, steps, tool results, outcome, user feedback), and keep a separate set of short distilled memory files (preferences, definitions, workflows, mistakes) under an auto-generated index file that says what each file holds. In Databricks' ablation, raw traces alone reached 35.8% (the agent searched them in only about a third of cases), distilled files with the index 66.1%, and both 68.4%.", "Persist a distilled lesson only after a check against ground truth (tests pass, the user confirms, the outcome is verified), never on the agent's own success judgment: on one site 52.9 to 59.5% of ReasoningBank's success-labeled lessons came from tasks that had failed, stored lessons trailed raw trajectories in 7 of 9 settings, and a GPT-5.5 check before admitting each lesson raised stored-lesson memory from 38.7% to 46.0%. Where nothing can verify a lesson, synthesize guidance from the retrieved raw traces for the current task and discard it afterwards (+3.3 to +28.0 points over storing it). Write explicit user requests (“Never write scripts in the /tmp dir”) at once.", "At session start inject the index, give the agent search and fetch tools over both stores, and retrieve only a few records (the studies used one to five). Inject a short record anchored to the files, tables or fix it concerns, not raw transcript chunks or loosely extracted facts: off-the-shelf memory systems were at or below memory off in 11 of 12 coding pairings, mostly because the record they injected buried or dropped the fix. Keep a separate store per user or domain (an unrelated passage cost 0.7 to 3.4 points) and tell the agent a memory is a hint to check against the current task, not an instruction.", "Consolidate offline on a schedule: distill new sessions into the files, merge duplicates, split oversized files, resolve conflicts and delete stale entries (offline distillation 63.5% against 53.9% for writes made during the session, 66.1% with both).", "Treat a memory write as a privileged action (10.c.i): only the user and verified outcomes may write; never store instructions found in email, web pages, documents or tool output; record each entry's source and show every write to the user. One crafted email made agents store and later act on a poisoned memory in 87.5% (GPT-5.4) and 71.4% (Sonnet 4.6) of held-out cases when mail was processed in the background, past input, model and trace defenses; asking the agent to report what it checked and did cut foreground success from 75.0% to 1.8% and from 48.2% to 23.2%.", "Measure with paired runs against both baselines, memory frozen during evaluation, at least three seeds: pass rate split into tasks memory rescues and tasks it breaks (it helps where the model fails without it and can hurt where it already succeeds), tokens including memory-side calls, steps and latency (1.35x faster and 1.44x fewer tokens in Databricks' test; A-Mem cost up to 6.37x of no memory on GPT-5.4), and a poisoning probe; re-run after every model change." ], "detect": { "static": "Raw session traces kept across tasks or sessions; distilled memory files or a vector store of lessons; an index or retrieval step at task start; writes after each task or on user request; a consolidation, TTL or decay job.", "traces": "Memories retrieved per task, whether the agent read and followed them, and whether they applied; memory writes and their source; repeated mistakes the store should have prevented.", "evals": "Repeat-mistake rate over time; pass rate with and without memory on a task stream, split into rescues and regressions.", "judge": "Rate retrieved memories for relevance.", "probes": "A task pair where the first teaches a lesson the second needs, an unrelated task the lesson could mislead, and untrusted content that asks to be remembered." }, "impact": { "measured_basis": true, "primary": [ "quality", "cost" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "tasks built to need earlier sessions: 22.3% to 68.4% pass rate (Databricks, Oct 2026); with GPT-5.4, Gemini 3.1 Pro and Claude Opus 4.7 over six benchmarks, A-Mem and ReasoningBank moved average accuracy by -1.1 to +3.0 points (Yan, Jul 2026); a no-memory web agent given the same tokens beat ReasoningBank by 5.1 to 8.1 points (Hajimiri, Jun 2026); memory systems at or below memory off in 11 of 12 coding pairings, a test-verified record +0.0 to +4.5 (Fan, Sep 2026); stored lessons trailed raw trajectories in 7 of 9 settings and discarding them gained 3.3 to 28.0 points (Wang, Oct 2026); under a tight budget several memory methods fell below no memory, MemRL 4.7% against 30.7% (Lai, Oct 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "mixed", "size": "medium", "when": "1.44x fewer tokens per task than no memory (Databricks, Oct 2026); per-task cost 0.62x to 0.98x of no memory for ReasoningBank but up to 6.37x for A-Mem on GPT-5.4 (Yan, Jul 2026); ReasoningBank 82.6K tokens per task against 73.6K for a stronger no-memory agent, its module adding 16.5K to 26.1K prompt tokens (Gemini 3 Flash) (Hajimiri, Jun 2026); 9.31 against 9.48 to 10.17 LLM calls per task for designs that store lessons (Wang, Oct 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "mixed", "size": "medium", "when": "1.35x faster per task, and one repeated workflow from nearly six minutes to 41 seconds (Databricks, Oct 2026); steps per task down 6.6% to 23.2% with A-Mem and ReasoningBank on GPT-5.4 and Gemini 3.1 Pro (Yan, Jul 2026); tasks must run in order, so they cannot run in parallel (Hajimiri, Jun 2026)", "basis": "evidence" }, { "dimension": "security", "effect": "degrades", "size": "large", "when": "when the agent reads untrusted content and can write memory: one crafted email made agents store and act on a poisoned memory in 87.5% (GPT-5.4) and 71.4% (Sonnet 4.6) of held-out cases in background mode, also with a Mem0 backend (85.7%), past input, model and trace defenses; handcrafted payloads scored 0.0% on both models (Zhang, Jul 2026)", "basis": "evidence" }, { "dimension": "governance", "effect": "mixed", "size": "medium", "when": "distilled memory kept as files with an index is a reviewable record that can move between agents (Databricks, Oct 2026); writes can be hidden from the user, with 100% stealth in background mode on GPT-5.4 (Zhang, Jul 2026); the store depends on task order, so one failed task cannot be re-run in isolation (Hajimiri, Jun 2026); stored sessions retain user data", "basis": "evidence" }, { "dimension": "brand", "effect": "improves", "size": "small", "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired runs on a held-out stream that mixes tasks reusing what earlier ones learned (user corrections, definitions, repository conventions) with tasks that share nothing, memory frozen during evaluation and at least three seeds: memory against no memory and against a no-memory agent given the same total tokens as extra steps or effort; report pass rate split into rescues and regressions, tokens including memory-side calls, steps and latency; ablate raw traces only, distilled files only and both; add a poisoning probe (a document or email asking the agent to remember an instruction) and count writes and later effects" }, { "method": "monitor", "tier": null, "how": "memory writes per session by source (user, verified outcome, untrusted content) and whether the user saw them; retrievals per task and how often the agent followed a memory that did not apply; repeat-mistake rate; regressions on task types solved before memory; store size, duplicates and conflicts after each consolidation; tokens per task" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Context & Memory group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Shinn et al., \"Reflexion: Language Agents with Verbal Reinforcement Learning\"", "url": "https://arxiv.org/abs/2303.11366", "date": "2023-03-20", "locator": "Abstract", "task": "Agents write a verbal reflection after each failed trial into a memory buffer and read it on later trials of the same task (retries, not separate tasks)", "effect": "\"91% pass@1 accuracy on the HumanEval coding benchmark, surpassing the previous state-of-the-art GPT-4 that achieves 80%\"", "direction": "supports", "measured": null, "conditions": "GPT-4; HumanEval Python pass@1 80.1 for a single sample against 91.0 with Reflexion, while MBPP Python fell from 80.1 to 77.1 (Table 1); memory bounded to 1 to 3 reflections (1 for programming); also ALFWorld and HotpotQA. Memory spans retries of one task, the mechanism of 7.c.i, not a store across tasks", "older": true }, { "source": "Databricks AI Research (Veronica Lyu, Wenhao Zhan, Sam Havens, et al.), \"MemCurrent: Designing Persistent Memory for Production Agents\"", "url": "https://www.databricks.com/blog/memcurrent-designing-persistent-memory-production-agents", "date": "2026-10-20", "locator": "How We Designed Memory; Results (Figure 6); MemCurrent in Action (Example 2); What Makes MemCurrent Work (Figures 9 and 10)", "task": "Keep raw session traces (episodic memory, append-only) and distilled memory files (derived memory) under an auto-generated memory.md index injected at session start; the agent searches and fetches both with tools, writes explicit requests with tools, and an offline \"dreaming\" pipeline distills new sessions and consolidates the store (splits oversized files, merges duplicates, resolves conflicts)", "effect": "Internal Memory Benchmark (Genie agent): pass rate 22.3% with no memory, 68.4% with MemCurrent, 78.6% with the gold memories injected, Mem0 43.9%; MemCurrent 252 s and 418K tokens per task, 1.35x faster and 1.44x fewer tokens than no memory (\"31% fewer tokens and running 26% faster\"). Ablations: raw episodic memory alone 35.8% (searched in about a third of cases), derived memory alone 66.1%, both 68.4%; offline dreaming alone 63.5% against online tools alone 53.9%. On LoCoMo, LongMemEval and ConvoMem, 88.1% against 87.8% for full context at 8.4K against 103K input tokens per question. A repeated workflow saved to memory took a related question from nearly six minutes to 41 seconds.", "direction": "supports", "measured": true, "conditions": "Databricks' own benchmark: 114 cases, 29 users, 2,233 history sessions mined from its own agent's sessions, each checked that the needed knowledge is present and required (plus some memory-independent questions); scored by an LLM judge with rubrics; 3 repeats; Claude Opus 4.6 agent, GPT 5.6 sol memory model; cost is tokens, not dollars; vendor evaluating its own product", "older": false }, { "source": "Ruhan Wang, Kishan Panaganti, Dongruo Zhou, \"More Memory, Worse Agents: Error Reproduction and Anti-Persistence in LLM Agents\"", "url": "https://arxiv.org/abs/2610.27406", "date": "2026-10-19", "locator": "Section 4.2 Table 1; Section 4.4 Table 2; App. E.2 Table 6; App. F.1 to F.3 Table 7", "task": "Compare memory designs over ten rounds of self-improvement: stored trajectories, stored abstractions (rules, summaries, skills), both, and anti-persistence (store trajectories and outcomes, synthesize guidance for the current task on demand and discard it afterwards)", "effect": "Held-out success, equal-weight mean over nine settings: anti-persistence 59.9%, trajectory memory 53.5%, abstraction memory 49.0%, joint memory 49.3%, Reflexion 52.9%; anti-persistence +19.2 points over ReAct with no memory. Stored abstractions (alone or joint) trailed trajectory memory in 7 of 9 settings. Discarding instead of storing synthesized guidance: +28.0 and +12.3 points (Llama-4-Scout, ALFWorld and WebShop), +3.3 and +6.7 (Qwen3-Coder-Next). A GPT-5.5 check before admitting each skill raised abstraction memory from 38.7% to 46.0% (WebShop, Qwen3-Coder-Next). On 30 AppWorld tasks, correct skills raised success from 23.0% to 40.0% and incorrect ones lowered it to 20.0%. LLM calls per task 9.31 against 9.48 to 10.17 for the stored designs (mean of Table 6's nine settings).", "direction": "mixed", "measured": true, "conditions": "Llama-4-Scout, Qwen3-Coder-Next, Gemma-4-31B-it; ALFWorld (134 held-out tasks), WebShop (100), ScienceWorld (60); 3 seeds; up to 5 retrieved trajectories; the authors' own method; the filter and AppWorld tests are auxiliary and lack seed-level uncertainty", "older": false }, { "source": "Guannan Lai, Zi-Yang Bai, Jun-Peng Jiang, Han-Jia Ye, \"Learning What to Remember: Resource-Efficient Self-Improvement for Lifelong Agents\"", "url": "https://arxiv.org/abs/2610.26453", "date": "2026-10-19", "locator": "Section 5.1; Table 1; Figure 3(a)", "task": "Under a tight joint budget (one retrieved memory per task, 64-entry bank, 128 to 512 output tokens, 3 to 15 rounds), compare a no-memory agent, plain retrieval and eight lifelong-memory methods with TokenWorth, which learns from execution feedback which memories are worth retrieving and keeping", "effect": "LifelongAgentBench success, local Qwen2.5-14B / API DeepSeek-V4-Flash: no memory 25.2/26.2 (DB), 50.2/67.2 (OS), 30.7/12.7 (KG); plain retrieval 33.0/29.4, 54.6/66.6, 11.3/31.3; TokenWorth 65.0/62.2, 59.4/73.2, 36.7/36.7. Several methods fell below no memory: on local KG MemRL 4.7% and plain retrieval 11.3% against 30.7%; on local DB Mem0 21.2% and Reflexion 22.0% against 25.2%. TokenWorth used 9.3% to 58.3% fewer output tokens than the no-memory agent.", "direction": "mixed", "measured": true, "conditions": "One stringent budget, which the authors say they do not claim generalizes; no repeated runs reported; output tokens include memory-side generation; the authors' own method", "older": false }, { "source": "Liyang Fan, Yingcheng Shi, Yongbin Li, Chenghao Sun, Xin Chen, Xander Xu, Hu Wei, Shiwen Ni, Min Yang, Jieping Ye, \"VibeMemBench: Evaluating Memory Systems for Coding Agents on Real Repository Coding Tasks\"", "url": "https://arxiv.org/abs/2609.23570", "date": "2026-09-20", "locator": "Abstract; Section 3.3; Section 4.2 Tables 1 and 2; Section 4.3 Table 3; Section 4.4 Tables 4 and 5", "task": "On repository coding tasks whose own history holds an experience verified by executing tests to help, inject that short verified record, an unrelated passage, or the top record that each of four memory systems (Mem0, SimpleMem, MemoryOS, A-MEM) retrieves after ingesting the history, against memory off", "effect": "Verified record against memory off: +1.1 to +4.5 points on four of five solvers and +0.0 on deepseek-v4-pro, every 95% interval crossing zero; fewer agent steps for all five. Unrelated passage: 0.7 to 3.4 points below memory off. Memory systems: 11 of 12 solver and system pairings at or below memory off (glm-5 with Mem0 -5.5, interval -10.59 to -0.45), a net loss of 127 resolved seeds over 1,332 pairings. Of 231 failing pairings, 69.3% reached the relevant history but lost the fix in the record's form (instructions copied from raw transcripts in 40.0% of those, overgeneralized extracted facts in 18.1%), 16.0% never stored it and 1.3% were ranking misses. Where one record moved two solvers in opposite directions, the solver weaker without memory gained in 61 of 64 cases.", "direction": "mixed", "measured": true, "conditions": "111 targets from 90 SWE-rebench V2 repositories, 3,634 history trajectories, 4 seeds; MiniSWEAgent; solvers deepseek-v4-pro, glm-5, glm-5.2, kimi-k2.7-code, qwen3.8-max (no Claude or GPT); targets kept only where a record helped the reference solver, so the verified-record gain is an upper bound; one record injected per target; memory-side tokens not counted; independent of the memory systems' authors", "older": false }, { "source": "Dong Yan, Jian Liang, Dapeng Hu, Ran He, Nicholas Jing Yuan, Qi Zhang, Tieniu Tan, \"AgentStream: How Well Do Self-Evolving LLM Agents Perform Under Streaming Tasks?\"", "url": "https://arxiv.org/abs/2608.00155", "date": "2026-07-31", "locator": "Section 4 (Experimental Settings); Tables 1, 2, 3 and 4; App. A Tables 7 and 8", "task": "Run five self-evolving methods (two episodic memories, A-Mem and ReasoningBank, plus ACE, AutoSkill and a harness-evolution method) over task streams from six benchmarks, each benchmark with its own state or all sharing one, with three frontier models, against the same model with no evolved state", "effect": "Average accuracy over the three stream types, no memory / A-Mem / ReasoningBank: GPT-5.4 45.8 / 48.8 / 44.7; Gemini 3.1 Pro 56.6 / 57.8 / 59.4; Claude Opus 4.7 63.9 / 65.6 / 65.3. Across all methods the average gain was +1.37 points with a separate state per benchmark and +0.75 and +0.90 with one shared state, and 11, 16 and 17 of 45 configurations fell below no memory; GPT-5.4 lost on average in every stream type (-0.35 to -0.78). Per-task cost, single run: ReasoningBank 0.62x to 0.66x of no memory on Gemini 3.1 Pro and 0.87x to 0.98x on GPT-5.4; A-Mem 4.78x to 6.37x on GPT-5.4. Steps per task fell 6.6% to 23.2% with both memories on both models.", "direction": "mixed", "measured": true, "conditions": "AppWorld, BFCL, BrowseComp-Plus, HLE, SWE-bench Verified, Tau2; 50 tasks each; 3 seeds that shuffle task order; learning from self-generated feedback only, no ground truth; GPT-5.4 and Gemini 3.1 Pro at medium effort, Claude Opus 4.7 at high; cost tables from one run and only for GPT-5.4 and Gemini 3.1 Pro; independent of the methods' authors", "older": false }, { "source": "Yechao Zhang, Shiqian Zhao, Jiawen Zhang, Jie Zhang, Gelei Deng, Xiaogeng Liu, Chaowei Xiao, Tianwei Zhang, \"When Claws Remember but Do Not Tell: Stealthy Memory Injection in Persistent Personal Agents\"", "url": "https://arxiv.org/abs/2607.05189", "date": "2026-07-06", "locator": "Abstract; Section V-A and V-B Table I; Section V-C Table II; Section V-E Tables IV and V; App. A (Victim Configurations)", "task": "Send a persistent personal agent with long-term memory one email, generated by an attacker model trained with supervised fine-tuning and RL against a shadow agent, that must make it write a poisoned fact or preference to memory, hide this from the user, and change its later behavior", "effect": "End-to-end success on 56 held-out cases, background / foreground: OpenClaw with GPT-5.4 87.5% / 75.0%; a Claude Code SDK agent with Sonnet 4.6 71.4% / 48.2%; five handcrafted injection payloads 0.0% on both models. Payloads transferred to NanoClaw and Hermes (80.4% and 82.1% in background) and to a Mem0 vector backend (85.7%). Defenses: an input detector missed 91.59%; Meta-SecAlign still let 49.07% be written; a trace auditor missed 93.46% (OpenClaw) and 40.38% (Claude Code SDK). Asking the agent to report what it checked and did cut foreground end-to-end success to 1.8% (GPT-5.4) and 23.2% (Sonnet 4.6), mainly by exposing the write.", "direction": "mixed", "measured": true, "conditions": "WhisperBench: 108 cases (52 for training the attacker, 56 held out) in five risk categories on a real IMAP/SMTP email workflow; filesystem memory unless noted; success judged by GPT-5.4-nano; the attacker is the authors' own, built on Qwen-4B; the abstract's 87.5% and 71.4% are background-mode figures", "older": false }, { "source": "Sina Hajimiri, Masih Aminbeidokhti, Jose Dolz, Ismail Ben Ayed, Issam H. Laradji, Spandana Gella, Nicolas Gontier, \"Are Online Skill and Memory Modules Always Worth Their Tokens? A Budget-Constrained Study of Web Agents\"", "url": "https://arxiv.org/abs/2606.15017", "date": "2026-06-12", "locator": "Abstract; Section 3.2; Tables 1, 2, 3 and 5; Section 6; App. F.3 Table 10; Section 8", "task": "Run ReasoningBank (lessons distilled from successes and failures and retrieved per task), AWM and ASI with a 10-step actor, against a no-memory actor that spends a similar total token budget on 15 steps (plus rule-based page pruning), counting all module and actor tokens", "effect": "WebArena success / total tokens per task, no-memory 15-step actor against ReasoningBank: Gemini 3 Flash 44.78% / 73.6K against 39.33% / 82.6K; GPT-5.4-mini 32.67% / 90.2K against 24.58% / 81.0K; Qwen 3.6-27B 42.14% / 95.5K against 37.00% / 101.9K. WorkArena-L1 (Qwen 3.6-27B): tied at 55.56%, 109.4K against 120.3K tokens. A plain 10-step actor with no memory scored 45.81 / 45.28 / 48.72 / 27.22 on Shopping / Reddit / Admin / GitLab against ReasoningBank's 45.45 / 39.93 / 48.90 / 22.96 (Gemini 3 Flash). ReasoningBank's module added 16.5K to 26.1K prompt tokens per task (Gemini 3 Flash). On Shopping, 52.9% (Gemini) and 59.5% (GPT-5.4-mini) of its success-labeled lessons came from tasks that failed by the ground-truth check, and retrieved memories were referenced in over 95% of Gemini tasks, sometimes over-applied. Memory made tasks sequential: no parallel runs, and a transient failure silently changes later state.", "direction": "contradicts", "measured": true, "conditions": "WebArena (655 tasks in four domains) with three models, 3 runs each; WorkArena-L1 with Qwen 3.6-27B only; budget matching by step count is approximate; four-domain figures from v2 (Aug 2026); the authors note these tasks are largely self-contained; independent of the methods' authors", "older": false } ], "caveats": "The large gains come from benchmarks built so that memory is needed, and the biggest is Databricks' own (114 cases from its own agent, scored by an LLM judge); the independent tests that found small or negative effects used largely self-contained tasks (WebArena, SWE-style coding, 50-task benchmark streams), mostly open or mid-size models, and intervals that often cross zero, and none of them compares memory with a budget-matched baseline on tasks that depend on earlier ones. Memory makes tasks depend on their order, so they cannot run in parallel and a transient failure silently changes what later tasks see. It helps weaker models and unsolved tasks more and can break tasks a strong model already solves: where one record moved two solvers in opposite directions, the one weaker without memory gained in 61 of 64 cases. The 87.5% poisoning rate needs an attacker model trained against a copy of the agent; handcrafted payloads scored 0.0% on GPT-5.4 and Sonnet 4.6.", "also_covers": [], "related": [ { "id": "4.c.ii", "slug": "skill-library", "name": "Skill library evolution" }, { "id": "4.b.iii", "slug": "multi-window-task-state", "name": "Keep plan, progress and results in files when work outlives one context window" }, { "id": "4.b.i", "slug": "filesystem-checkpoints", "name": "Filesystem-based agent state and checkpoints" }, { "id": "7.c.i", "slug": "reflection-loop", "name": "Reflection loop" }, { "id": "3.a.i", "slug": "few-shot-examples", "name": "Add worked examples to the prompt (few-shot), or curated example queries for data agents" }, { "id": "10.c.i", "slug": "contain-untrusted-input", "name": "Contain sensitive data and untrusted input" }, { "id": "1.b.i", "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class" }, { "id": "7.b.iii", "slug": "size-execution-budgets", "name": "Size run budgets so productive runs aren't cut off: turn caps, deadlines, command timeouts" }, { "id": "8.a.i", "slug": "choose-retriever", "name": "Choose the retriever by corpus size and question type" } ], "url": "https://mahalsystems.ai/aot/optimizations/episodic-memory/", "markdown": "https://mahalsystems.ai/aot/optimizations/episodic-memory.md" }, { "id": "4.c.ii", "legacy_id": "P58", "former_ids": [], "slug": "skill-library", "name": "Skill library evolution", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it adds a reusable skill memory", "proposed": true }, "levers": [ "state" ], "domains": [ "memory" ], "evidence_domains": [ "planning", "code-generation", "tool-use" ], "categories": [ "Reusing work" ], "category_ids": [ "memory.reuse" ], "summary": "Have the agent turn runs that passed a verifier into reusable skills (a code function or a SKILL.md procedure), store them in a library, and on later tasks show it only the few a separate selection step picks, reviewing what goes in and pruning what goes unused. It pays on recurring task families when skills come from verified successes: skills distilled from an agent's own successful runs raised success on those tasks (Terminal-Bench-2 0.5935 to 0.7548 with GPT-5.3-Codex), but skills distilled from failed runs fell below no experience, and skills a model wrote before attempting the task scored 8.1 to 11.5 points below no skills with Opus 4.7, GPT-5.5 and Gemini 3.1 Pro. Two costs grow with the library: agents stop opening the right skill (loaded-skill recall 73.6% to 27.4% with 1,000 irrelevant skills listed) unless a preselector chooses, and an agent that writes skills from retrieved ones copied planted malicious code into its own library on 20.3% to 41.8% of tasks across six models.", "how": [ "Start from the baseline: the same model and harness with no skill library, plus an arm with cleaned traces of past runs in context (cheaper: 95.3K fewer tokens per task than distilled skills, for 4.8 points less success in one study), on recurring task families with a deterministic verifier, scored on held-out instances. Do not ask the model to write skills from what it already knows before it attempts the task: such self-generated skills scored 8.1 to 11.5 points below no skills with Opus 4.7, GPT-5.5 and Gemini 3.1 Pro, because solvers never opened them or followed confident errors such as a hard-coded 1000x unit conversion.", "Write a skill only from a run the verifier passed: distill the trajectory into a code function or a SKILL.md (when to use it, preconditions, steps, failure modes, a verify step) with task-specific paths and values replaced by placeholders. If failed runs feed the writer, label them: skills distilled from failed runs alone fell below no experience in all six benchmark-model cells, and hiding the success and failure labels cut Gemini 3.1 Pro on Terminal-Bench-2 from 0.7462 to 0.4000. Voyager added a program only after self-verification passed, and removing self-verification cut discovered items by 73%.", "Gate admission: keep agent-authored skills in a quarantine that retrieval cannot see until a reviewer or a signed admission step approves them, and tell the authoring agent that retrieved skills are untrusted and that banner-style \"REQUIRED PATTERN\" or \"DO NOT MODIFY\" boilerplate must not be copied (10.c.i). In one study that four-line instruction cut self-poisoning from 20.3% to 41.8% of tasks to at most 6.7% with no significant loss in task completion, while a name blocklist caught none of the authored copies and a code scanner fell to 7% after a one-line rewrite.", "Retrieve a few skills through a separate step instead of listing them all (the same deferral idea as 2.a.i): near 100 skills, an LLM preselector over names and descriptions; at thousands, retrieval followed by an LLM selector; expose a handful (at most five in the study). With 1,000 irrelevant skills listed, loaded-skill recall fell from 73.6% to 27.4% although a selector given the same list found 73.9%; LLM preselection cut tokens per run from 1,966k to 547k and added a median 8 to 41 seconds of routing; asking the agent to plan its skill use first did less than preselection.", "Prune and revise from telemetry: log which skills each run opens and its outcome; retire skills that are never opened or whose runs keep failing, and merge near-duplicates, since irrelevant descriptions suppress loading (masking them restored recall from 18.3% to 63.3%) and similar skills are the hardest to tell apart (top-1 precision 70.5% to 53.4% on similar pools). If a poisoned skill is found, scrub the agent-authored copies too: in one model the infection rose to 68% of tasks after the planted seeds were removed.", "Measure paired against the baseline, same model and harness, on held-out tasks (not only the tasks the skills came from): pass rate, tokens and time per task, loaded-skill recall, zero-load rate and the misapplied-skill rate (10.0% of skill runs in one study); rerun with the library padded to its expected size and with planted-skill probes, since task success can stay flat while skill use collapses (precision 29.6% to 3.3% with success 36.4% to 39.3%)." ], "detect": { "static": "A skills directory or store the agent writes to after runs; skills written without a verifier outcome, or from the model's own knowledge before it attempts the task; new skills retrievable with no review or quarantine; authoring that reads retrieved skills as templates; every skill's name and description listed to the agent on every run.", "traces": "Skill reuse rate; skills authored per session; loaded-skill recall and zero-load rate per run; per-skill opens and outcomes; skills never opened; skills distilled from failed or unlabeled runs; authored skills that copy retrieved code verbatim.", "evals": "Paired runs with and without the library on held-out tasks from recurring families (pass rate, tokens, time); pass rate and loaded-skill recall as the library is padded with irrelevant skills.", "judge": "Review each new skill before it becomes retrievable: task-specific paths or values baked in, confident assumptions no verifier checked, boilerplate copied from retrieved skills, overlap with existing skills.", "probes": "Recurring task families; a growing library; planted skills." }, "impact": { "measured_basis": true, "primary": [ "quality", "cost" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "large", "when": "skills distilled from an agent's own successful runs raised success on the same tasks (Terminal-Bench-2 0.5935 to 0.7548 with GPT-5.3-Codex, 0.5000 to 0.7923 with Gemini 3.1 Pro), but skills from failed runs fell below no experience in all six cells and hiding outcome labels cut 0.7462 to 0.4000 (Jiang, Aug 2026); skills written before any attempt scored 8.1 to 11.5 points below no skills while curated ones added 18.2 to 24.8 (Li, Feb 2026); with 1,000 irrelevant skills listed, Pass@3 fell from 51.6% to 38.6% (Liu, Oct 2026); Voyager found 3.3x more unique items than prior agents (Wang, May 2023)", "basis": "evidence" }, { "dimension": "cost", "effect": "mixed", "size": "medium", "when": "skills used 34.2K fewer tokens per task than the Raw arm but 95.3K more than cleaned traces of the same runs, on 83 matched tasks (Jiang, Aug 2026); listing every skill's description raised tokens per run from 498k to 1,966k with 1,000 irrelevant skills added, and an LLM preselector brought it to 547k (Liu, Oct 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "mixed", "size": "small", "when": "Voyager reached tech-tree milestones up to 15.3x faster, counted in prompting iterations, than prior agents (Wang, May 2023); a run took 15.3 minutes with only the relevant skills and 17.0 with 1,000 more listed, and preselection adds a median 8 to 41 seconds of routing (Liu, Oct 2026)", "basis": "evidence" }, { "dimension": "security", "effect": "degrades", "size": "large", "when": "planted skills were copied into agent-authored skills on 20.3% to 41.8% of SWE-bench Verified tasks across six open-weight models, leaving 4.9 to 9.0 times the planted count, and one model's infection rose to 68% after the plants were removed; a name blocklist caught 0 of 275 authored copies; a four-line counter-prompt cut it to at most 6.7%, and a curator-signed quarantine stops re-retrieval (Wu, Aug 2026)", "basis": "evidence" }, { "dimension": "governance", "effect": "mixed", "size": "medium", "when": "skills are versioned files a reviewer can read, but task success hides library failure: skill-use precision fell from 29.6% to 3.3% as the pool grew from 5 to 100 while success moved from 36.4% to 39.3% (Jiang, Aug 2026), and Claude Opus 4.6 kept Pass@3 at 62.2% (from 64.9%) while its loaded-skill recall fell from 81.3% to 30.9% (Liu, Oct 2026); agent-authored skills bypass submission-side review unless new entries wait in a curator-signed quarantine (Wu, Aug 2026)", "basis": "evidence" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired runs, same model and harness: no library, the agent-built library, and where available a curated library or cleaned traces of past runs, on held-out instances of recurring task families scored by a deterministic verifier; pass rate, tokens and time per task, loaded-skill recall, zero-load rate and misapplied-skill rate; repeat with the library padded to its expected size with irrelevant skills, with implicit loading against a preselector, and with planted skills carrying copy-me boilerplate and a harmless callback to measure how often authored skills reproduce them" }, { "method": "monitor", "tier": null, "how": "per-skill loads and outcomes, the zero-load rate on tasks that have a matching skill, skills never opened (prune), skills admitted and rejected with their source run and verifier result, authored skills that copy retrieved code or add network or credential access, and pass rate on the task families the library serves" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Learning group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Wang et al., \"Voyager: An Open-Ended Embodied Agent with Large Language Models\"", "url": "https://arxiv.org/abs/2305.16291", "date": "2023-05-25", "locator": "Abstract; §2.2; §3.3 Table 1; §3.4 Fig. 9", "task": "An ever-growing library of executable code skills that the agent stores and retrieves", "effect": "\"obtains 3.3x more unique items, travels 2.3x longer distances, and unlocks key tech tree milestones up to 15.3x faster than prior SOTA\". Ablation: without the skill library, progress plateaus in later stages (figure only, no number); removing self-verification, which decides when a program is done and added to the library, cut discovered items by 73%.", "direction": "supports", "measured": null, "conditions": "Minecraft (MineDojo); gpt-4-0314; three trials; the headline compares the whole system (automatic curriculum, skill library, iterative prompting with self-verification) against ReAct, Reflexion and AutoGPT, not the skill library alone; milestone speed is counted in prompting iterations", "older": true }, { "source": "Jinlin Liu, Wei Chen, Xiaojin Zhang, \"Lost in Implicit Routing: How LLM Agents Fail to Load Relevant Skills at Scale\"", "url": "https://arxiv.org/abs/2610.18522", "date": "2026-10-13", "locator": "Abstract; §3.1; §3.2 Tables 1-2; §3.3 Tables 3-5; §4.2 Tables 7-9; §5; App. D Tables 17-19; App. E Tables 21-22", "task": "Hold SkillsBench tasks and their relevant skills fixed, add up to 10,000 irrelevant marketplace skills, and compare implicit loading from skill descriptions with a separate preselection step (LLM, retrieval, or retrieval then LLM) that exposes at most five skills", "effect": "Implicit routing, 0 to 1,000 irrelevant skills: loaded-skill recall 73.6% to 27.4%, zero-load rate 9.3% to 51.6%, Pass@3 51.6% to 38.6% (no skills: 26.8% to 34.1% per model); the same models selecting from the full list without executing recalled 73.9%. Masking only distractor descriptions restored recall from 18.3% to 63.3% (10-task subset); a plan-first prompt recalled 39.9% against 70.6% for preselection. At 1,000 skills, LLM preselection: recall 62.0%, Pass@3 51.6%, 547k tokens per run against 1,966k; retrieval then LLM was best at 1,000 and 10,000 (Pass@3 52.8% and 50.4%); median routing time 8 to 19 s (LLM) and 33 to 41 s (hybrid) from 10 to 1,000 skills; a run took 15.3 minutes with no extra skills and 17.0 with 1,000. Proprietary check: Claude Opus 4.6 Pass@3 64.9% to 62.2% while loaded-skill recall fell from 81.3% to 30.9%.", "direction": "mixed", "measured": true, "conditions": "SkillsBench, 82 tasks, three runs per cell, over 15,000 trajectories; executors GLM-5, Kimi K2.5, MiniMax M2.5; 37-task check on Claude Opus 4.6, Gemini 3.1 Pro and GPT-5.4; curated golden skills plus distractors sampled from more than 20,000 market skills, not agent-built skills; independent", "older": false }, { "source": "Xiaodong Wu, Yu Shi, Qi Li, Zhimin Zhao, Xiangman Li, Bram Adams, Ahmed E. Hassan, Jianbing Ni, \"EVOMAL: Self-Poisoning in Self-Evolving Coding Agents\"", "url": "https://arxiv.org/abs/2608.25776", "date": "2026-08-26", "locator": "Abstract; §6.1; §6.3 Table 2; §6.4 Table 3; §6.5; §7; §9.1 Table 5; §9.2; §9.3", "task": "Plant skills carrying a payload in the Voyager-style library of a coding agent that writes a reusable tool for each task by imitating retrieved skills, measure how often it re-authors the payload into its own stored skills, and test existing detectors and two defenses", "effect": "Agent self-poisoning rate 20.3% to 41.8% across six models on 153 tool-relevant SWE-bench Verified tasks, 18.3 to 32.0 points above a payload-free control; 31 to 64 newly authored malicious skills, 4.9 to 9.0 times the 8 planted. DeepSeek-V4-Pro reached 11.1% with the payload alone, and a single seed (0.4% of the library) gave it 19.6%. Targeting one task family: up to 86.7% (Qwen3, 15-task pytest family), 35.3% to 63.5% over the three families combined. After the seeds were removed, Qwen3 rose to 68% by round 5. A name blocklist caught 0 of 275 authored copies; a code scanner caught 85%, 7% after a one-line rewrite. A four-line counter-prompt cut the rate to at most 1.8% on the headline attack and at most 6.7% in every setting, with no statistically significant loss in task completion; a curator-signed quarantine for agent-authored skills blocks their re-retrieval.", "direction": "mixed", "measured": true, "conditions": "mini-SWE-agent with a Voyager-style skill manager (top-5 embedding retrieval, 232 benign skills plus 8 planted); Devstral-Small-2, Gemma-4-31B, Qwen3-Coder-Next, GPT-OSS-120B, MiniMax-M2.7, DeepSeek-V4-Pro; also SWE-bench Pro (114 tasks) and reruns on OpenHands and Claude Code scaffolds; the agent is asked to write a reusable tool for every task; independent", "older": false }, { "source": "Zhiyuan Jiang, Fangrui Huang, Hanwen Xing, Xander Wu, Yipeng Gao, Rui Cao, Mengdi Wang, Shilong Liu, Yijiang Li, \"Demystifying Agent Skills: Why They Work-Until They Don't\"", "url": "https://arxiv.org/abs/2608.14036", "date": "2026-08-14", "locator": "Abstract; §3.3; §5.1-5.5; Tables 1 and 4; App. A.4; App. A.8 Table 12; App. A.9 Table 13", "task": "Distill an agent's own successful and failed runs of a task into a SKILL.md or into cleaned workflow memory, rerun the tasks, vary the success and failure mix and whether outcome labels are shown to the skill writer, and test skill use from pools of 5 to 100 skills", "effect": "Success-only skills against no prior experience: Terminal-Bench-2 0.5935 to 0.7548 (Codex + GPT-5.3-Codex) and 0.5000 to 0.7923 (Gemini CLI + Gemini 3.1 Pro); SkillsBench 0.5083 to 0.7250 and 0.4762 to 0.7429. Failure-only skills fell below no experience in all six benchmark-model cells. Withholding outcome labels: Gemini on Terminal-Bench-2 at 3 successes and 2 failures 0.7462 against 0.4000. Across all mixtures: skill 61.9%, no experience 59.1%, workflow memory 55.9%; skill over workflow memory +6.06 points (95% CI +0.76 to +11.36). Skill guidance misapplied or ignored in 10.0% of skill runs (0.8% without skills). Tokens per task on 83 matched tasks: skill 521.5K, Raw arm 555.7K, workflow memory 426.2K. Pools of 5 to 100 skills: execution-time skill-use precision 29.6% to 3.3% while success went from 36.4% to 39.3%; top-1 embedding precision on similar pools 70.5% to 53.4%.", "direction": "mixed", "measured": true, "conditions": "Subsets of Terminal-Bench 2.0, Terminal-Bench Pro and SkillsBench; 5 trials per task; Codex + GPT-5.3-Codex and Gemini CLI + Gemini 3.1 Pro Preview; skills scored on the same tasks whose runs they were distilled from, tasks chosen because raw runs both passed and failed; retrieval study on curated SkillsBench skills with GPT-5.4 and Gemini 3.1 Pro; failure-mode shares labeled by an LLM judge; two authors also wrote SkillsBench", "older": false }, { "source": "Xiangyi Li, Yimin Liu, Wenbo Chen, et al., \"SkillsBench: Benchmarking How Well Agent Skills Work Across Diverse Tasks\"", "url": "https://arxiv.org/abs/2602.12670", "date": "2026-02-13", "locator": "Abstract; §4; §5.1; App. D.6 Table 6; App. D.6.1", "task": "Run 87 expertise-heavy tasks with deterministic verifiers under no skills, curated skills, and skills the agent first writes for itself with Anthropic's skill-creator (told not to solve the task) and then solves with", "effect": "Self-generated skills scored below no skills on all three configurations: Claude Code + Opus 4.7 34.9% against 43.0% (-8.1), Codex + GPT-5.5 35.5% against 46.8% (-11.3), Gemini CLI + Gemini 3.1 Pro 24.5% against 36.0% (-11.5); curated skills 61.2%, 66.5% and 60.8% (+18.2 to +24.8). Trajectory audit: 10 of 12 audited trajectories never listed, read or mentioned the generated skills; authoring displaced task work; one used pack hard-coded a wrong 1000x unit conversion; the strongest win was an answer key written inside the graded sandbox. Across 18 configurations, curated skills raised the pass rate from 33.9% to 50.5%.", "direction": "contradicts", "measured": true, "conditions": "87 tasks, 8 domains, three trials per cell; self-generated arm on three configurations only, as a diagnostic (Claude Code ran it at max effort against high for its baselines; Codex and Gemini reused one generated pack per task); numbers from v4 (Jun 2026), v1 reported no benefit on average over 7 configurations; skills written before any attempt, not distilled from verified runs; independent of the practice", "older": false } ], "caveats": "The positive evidence for agent-built skills is narrow: the strongest controlled gain scored skills on the same tasks whose runs they were distilled from (tasks chosen because raw runs both passed and failed), its 6.06-point lead is over cleaned traces with a lower bound of 0.76, and Voyager's headline compares its whole system, not the library alone, with GPT-4 in Minecraft. The self-generated deficit comes from skills written before any attempt, so it does not test skills distilled from verified successes, and one configuration ran that arm at a higher effort level. The poisoning study used six open-weight models, attacker-planted seeds and an agent asked to author a tool on every task; the routing study used curated SkillsBench skills with marketplace distractors, mostly on open-weight models with a 37-task check on Claude Opus 4.6, Gemini 3.1 Pro and GPT-5.4. No source here measures an agent-grown library over months of production use.", "also_covers": [], "related": [ { "id": "4.c.i", "slug": "episodic-memory", "name": "Episodic memory retrieval" }, { "id": "2.a.i", "slug": "reduce-tool-surface", "name": "Reduce the tool surface: remove and consolidate tools, then defer the rest behind search" }, { "id": "2.b.ii", "slug": "tool-use-guidance", "name": "Tell the agent when to use each tool, in context it always sees" }, { "id": "5.c.ii", "slug": "isolated-subagent-context", "name": "Sub-agent architectures with clean contexts" }, { "id": "2.a.iii", "slug": "choose-action-interface", "name": "Choose how the agent acts: typed tools, code execution, or shell, matched to the model and task" }, { "id": "10.c.i", "slug": "contain-untrusted-input", "name": "Contain sensitive data and untrusted input" }, { "id": "7.d.ii", "slug": "inversion-of-control", "name": "Inversion of control" }, { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "4.b.iii", "slug": "multi-window-task-state", "name": "Keep plan, progress and results in files when work outlives one context window" } ], "url": "https://mahalsystems.ai/aot/optimizations/skill-library/", "markdown": "https://mahalsystems.ai/aot/optimizations/skill-library.md" }, { "id": "5.a.i", "legacy_id": "T31", "former_ids": [], "slug": "staged-compaction", "name": "Compact conversation history in stages: elide stale tool outputs, then keep an anchored summary", "kind": "technique", "status": "card", "tier": { "tier": 2, "name": "Reliability", "why": "stops runs dying on context overflow, a hard failure", "proposed": false }, "levers": [ "context" ], "domains": [ "general" ], "evidence_domains": [ "code-generation" ], "categories": [ "Compacting" ], "category_ids": [ "context.lean-window" ], "summary": "Manage the history before it overflows. Past a soft threshold (about 60% of the window) replace old tool outputs with short stubs; only past a hard threshold (about 85%) summarize the oldest turns into a summary with fixed sections (goal, files touched, done, pending, errors, next step) that is updated incrementally, not regenerated. Keep the recent tail verbatim. The value is large at small windows and near zero at 128k; compaction alone does not carry multi-window work (see 4.b.iii for state kept in files).", "how": [ "Measure peak context against the window on the longest runs.", "Implement the two thresholds and the fixed-section summary.", "Evaluate with post-compaction probes (recall, artifacts, next step) and tokens per task, not compression ratio." ], "detect": { "static": "no compaction, or compaction that summarizes the whole history at once; a context window under about 64k", "traces": "context-overflow outcomes; peak context over about 85% of the window; summaries that drop file changes or decisions; failure rows labeled context-rationale-erosion", "evals": "failures concentrated in the longest trajectories" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "large", "when": "+35.7 points at 32k, +2.7 at 128k (Fan, Sep 2026); staged elision matches other strategies on success", "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "medium", "when": "staged elision lowest cost in 7 of 8 panels, e.g. $1.45 against $2.57", "basis": "evidence" }, { "dimension": "performance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "before-after", "tier": 2, "how": "context-overflow rate on the same cases" }, { "method": "experiment", "tier": 3, "how": "paired eval, tokens per task, post-compaction probes" } ], "evidence": [ { "source": "Scale AI, VeRO code release", "url": "https://github.com/scaleapi/vero", "date": null, "locator": "figure_04", "task": "five follow-up benchmarks", "effect": "MAX_HISTORY_CHARS lowered twice, never raised.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, VeRO code release", "url": "https://github.com/scaleapi/vero", "date": null, "locator": "figure_01", "task": "five follow-up benchmarks", "effect": "Context-management edits in 6 to 10 of 20 runs per benchmark.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Fan et al., \"An Empirical Study of Harness Design for Coding Agents\"", "url": "https://arxiv.org/abs/2609.20804", "date": "2026-09-17", "locator": "§3.2 'The value of context management grows as the context-window budget shrinks', Fig 3, Tables 3-4; §4, App 9.5", "task": "Add context management when the window is tight", "effect": "Managed tiers (T1-T4) vs no management (T0), averaged over 4 models (Nemotron-3 30B/120B/550B, Mistral-Medium-3.5-128B): the gap is 35.7, 15.9, 5.5 and 2.7 pp on SWE-Bench Verified (n=500) and 9.5, 7.5, 4.8 and 2.8 pp on Terminal-Bench 2.1 (n=89) at 32k/64k/96k/128k. T0 overflow rate falls from 78.7% to 8.7% (SWE) and 61.0% to 12.1% (TB) over the same budgets. Managed tiers have zero overflow at every budget. Example at 32k SWE: T0 9.4/11.4/6.4/12.6% vs T1 20.6/43.0/51.4/64.4%. At 128k, managed tiers change trajectory shape very little (median turns 39-42 for 30B, 70-74 for 550B).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Fan et al., \"An Empirical Study of Harness Design for Coding Agents\"", "url": "https://arxiv.org/abs/2609.20804", "date": "2026-09-17", "locator": "§3.2 'Recall (M2) is rarely used...'; App 9.6 Table 13", "task": "Skip recoverable recall of elided outputs", "effect": "T2 (elision+recall) vs T1 (elision) over 32 model-benchmark-window comparisons: 15 wins, 14 losses, 3 ties. Mean -0.36 pp (+0.40 SWE, -1.12 TB). 36 of 64 T2/T4 settings never call recall_event and the median rate is zero. Use is concentrated at 32k in Nemotron-3 30B (TB T2 4.326 calls/task), and even that setting scores below T1. The 550B and Mistral models almost never recall.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 9.5; Recommendation 7; Figure 4", "task": "Compact at a fixed buffer, keeping a verbatim recent tail", "effect": "Claude Code \"AUTOCOMPACT_BUFFER_TOKENS = 13 000\"; post-compact restores \"up to 5 files (POST_COMPACT_TOKEN_BUDGET = 50 000)\"; Gemini \"fires when history exceeds 50% ... preserves the most recent 30%\"; Mistral Vibe envelope \"re-injects prior user messages ... so original task goals survive resets\"; 7 of 11 systems use threshold compaction.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 9.4; Sec 9.5; Recommendation 7", "task": "Compact reactively on context-overflow errors", "effect": "Mistral Vibe \"the same routine fires reactively when a turn overflows the context window mid-flight\"; OpenHands \"context-window overflows and malformed-history errors trigger a condensation rather than a crash\"; Pi \"one-shot compact-and-retry on context overflow.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 9.5 (OpenCode)", "task": "Soft-prune old tool outputs", "effect": "OpenCode \"opt-in pruning soft-deletes old tool outputs, protecting the newest 40K tokens and firing only if more than 20K are reclaimable.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 9.5 (Hermes); Table 12 \"Lineage Compaction\"", "task": "Compact by rotating to a linked child session", "effect": "Hermes \"compaction does not rewrite the transcript—it ends the session ... rotates to a child chained by parent_session_id ... no history is ever destroyed.\" Trigger at 50% of context minus max-tokens (64K floor), summary capped at min(5% of context, 12K).", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Walden Yan (Cognition), \"Don't Build Multi-Agents\"", "url": "https://cognition.com/blog/dont-build-multi-agents", "date": "2025-06-12", "locator": "A Theory of Building Long-running Agents (compression diagram)", "task": "Compress history with a dedicated summarization model", "effect": "\"we introduce a new LLM model whose key purpose is to compress a history of actions & conversation into key details, events, and decisions. This is hard to get right ... you might even consider fine-tuning a smaller model (this is in fact something we've done at Cognition).\" Still \"eventually hit a limit\".", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Justin Young (Anthropic), \"Effective harnesses for long-running agents\"", "url": "https://www.anthropic.com/engineering/effective-harnesses-for-long-running-agents", "date": "2025-11-26", "locator": "The long-running agent problem", "task": "Do not rely on compaction alone for multi-window work", "effect": "\"compaction isn't sufficient. Out of the box, even a frontier coding model like Opus 4.5 running on the Claude Agent SDK in a loop across multiple context windows will fall short of building a production-quality web app if it's only given a high-level prompt\". Compaction \"doesn't always pass perfectly clear instructions to the next agent.\"", "direction": "mixed", "measured": false, "conditions": null, "older": false }, { "source": "Drew Breunig, \"How Long Contexts Fail / How to Fix Your Context\", https://www.dbreunig.com/2025/06/22/how-contexts-fail-and-how-to-fix-them.html ; https://www.dbreunig.com/2025/06/26/how-to-fix-your-context.html (2025-06-26)", "url": null, "date": "2025-06-22", "locator": "06-26 post, \"Context Pruning\" (closing paragraph)", "task": "Keep context structured so sections prune independently", "effect": "Design suggestion: \"ensure the main instructions and goals are preserved while the document or history sections can be pruned or summarized\".", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Drew Breunig, \"How Long Contexts Fail / How to Fix Your Context\", https://www.dbreunig.com/2025/06/22/how-contexts-fail-and-how-to-fix-them.html ; https://www.dbreunig.com/2025/06/26/how-to-fix-your-context.html (2025-06-26)", "url": null, "date": "2025-06-22", "locator": "06-26 post, \"Context Summarization\"", "task": "Summarize accrued context before the distraction threshold", "effect": "Gemini Pokemon agent: \"anything beyond 100,000 tokens triggered this behavior\" (repeating past actions) despite a 1M+ window. No effect size for summarizing.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Lance Martin, \"Agent Design Patterns\"", "url": "https://rlancemartin.github.io/2026/01/09/agent_design/", "date": "2026-01-09", "locator": "Offload Context", "task": "Offload old tool results to files before summarizing", "effect": "\"Manus writes old tool results to files and only applies summarization once there is diminishing returns from offloading.\" Addresses the concern that compaction \"can result in loss of useful information.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Jediah Katz (Cursor), \"Dynamic context discovery\"", "url": "https://cursor.com/blog/dynamic-context-discovery", "date": "2026-01-06", "locator": "2. Referencing chat history during summarization", "task": "Give summarized agent a searchable chat-history file", "effect": "\"If the agent knows that it needs more details that are missing from the summary, it can search through the history to recover them.\" No numbers.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Fan et al., \"An Empirical Study of Harness Design for Coding Agents\"", "url": "https://arxiv.org/abs/2609.20804", "date": "2026-09-17", "locator": "§2.3 Algorithm 1; §3.1 Implementation Details (thresholds); §3.2 'T4 offers the best accuracy-cost trade-off', Figs 4-5", "task": "Elide stale tool outputs to stubs before summarizing", "effect": "T4 (staged elision, then summarization) matches T1-T3 on success and has the lowest cost in 7 of 8 model-benchmark panels. It has the lowest mean cost per task and the lowest average peak-context/budget ratio at every budget, and it calls summarization less than T3 at every budget. At 32k, T1/T2 (elision only) still reach about the full window. Example SWE 32k, Nemotron-3 550B: T4 55.6% at $1.45 vs T1 51.4% at $2.57.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Fan et al., \"An Empirical Study of Harness Design for Coding Agents\"", "url": "https://arxiv.org/abs/2609.20804", "date": "2026-09-17", "locator": "§2.3; App 7.3 Fig 18-19; Tables 3-4; Fig 5(a)", "task": "Summarize old history into fixed-heading running summary", "effect": "Summarization alone (T3) vs T0 at 32k SWE: 23.8/39.6/58.4/63.2% vs 9.4/11.4/6.4/12.6% (30B/120B/550B/Mistral). T3 is cheaper than elision-only at 32k, e.g. 550B $1.25 vs T1 $2.57, and keeps peak context well below the window where T1/T2 do not. The prompt is reproduced in Fig 18.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Factory Research, \"Evaluating Context Compression for AI Agents\"", "url": "https://factory.com/news/evaluating-compression", "date": "2025-12-16", "locator": "Three approaches to compression; Results table", "task": "Anchored iterative summarization with fixed sections", "effect": "36,611 messages from production SWE sessions (hundreds of compression points, 4 probes each, GPT-5.2 judge, 0-5 scale): overall Factory 3.70, Anthropic SDK (full regeneration) 3.44, OpenAI /responses/compact 3.35. Accuracy 4.04 / 3.74 / 3.43; context awareness 4.01 / 3.56 / 3.64. Compression 98.6% / 98.7% / 99.3%. Differences \"consistent across task types and session lengths.\" Factory summaries came from production; the others were generated from the same prefixes. Agent model not stated. Vendor evaluating its own method.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Factory Research, \"Evaluating Context Compression for AI Agents\"", "url": "https://factory.com/news/evaluating-compression", "date": "2025-12-16", "locator": "What we learned", "task": "Track file artifacts in a separate index", "effect": "Proposed, not tested: artifact-trail score was 2.19-2.45/5 for all methods; it \"probably requires specialized handling beyond summarization: a separate artifact index, or explicit file-state tracking in the agent scaffolding.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 9.5 (OpenCode, Pi); Recommendation 7", "task": "Merge summaries incrementally into anchored sections", "effect": "OpenCode merge into \"Objective / Important Details / Work State / Next Move / Relevant Files\"; Pi iterative merge; Rec 7: incremental merging \"preserves early decisions that from-scratch re-summarization loses.\"", "direction": "supports", "measured": false, "conditions": null, "older": false } ], "caveats": null, "also_covers": [ { "name": "Context compaction" } ], "related": [ { "id": "10.c.i", "slug": "contain-untrusted-input", "name": "Contain sensitive data and untrusted input" }, { "id": "2.c.i", "slug": "bound-tool-output", "name": "Bound tool output: cap it, or write it to a file the agent can read" }, { "id": "7.b.iv", "slug": "track-task-progress", "name": "Track explicit progress: a task list with a status per item, re-injected each turn" }, { "id": "7.b.i", "slug": "detect-loops-and-stalls", "name": "Detect stalls and loops: warn, then stop early" }, { "id": "5.c.i", "slug": "discrete-phase-separation", "name": "Discrete phase separation" }, { "id": "5.c.ii", "slug": "isolated-subagent-context", "name": "Sub-agent architectures with clean contexts" }, { "id": "7.b.ii", "slug": "keep-errors-in-context", "name": "Keep errors in context" }, { "id": "4.b.iii", "slug": "multi-window-task-state", "name": "Keep plan, progress and results in files when work outlives one context window" }, { "id": "7.b.iii", "slug": "size-execution-budgets", "name": "Size run budgets so productive runs aren't cut off: turn caps, deadlines, command timeouts" } ], "url": "https://mahalsystems.ai/aot/optimizations/staged-compaction/", "markdown": "https://mahalsystems.ai/aot/optimizations/staged-compaction.md" }, { "id": "5.b.i", "legacy_id": "T40", "former_ids": [], "slug": "ground-queries-in-schema", "name": "Give the model the schema it queries, in context: descriptions, keys, codes, sample values", "kind": "technique", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes what the model sees", "proposed": false }, "levers": [ "knowledge-access", "context" ], "domains": [ "data-analysis" ], "evidence_domains": [ "data-analysis" ], "categories": [ "Filtering input", "Querying precisely" ], "category_ids": [ "context.filter-input", "retrieval.query" ], "summary": "When the model writes SQL or structured filters, put the schema it is querying in its context: table and column descriptions, key relationships, what enum codes and nulls mean, sample values. Give it directly, in the prompt or as files it reads, rather than behind a search tool. This holds while the schema fits in a modest share of the window; past that, link it by exploration (8.b.ii). Making the schema context good (names, descriptions, examples) is data-side work.", "how": [ "Count the schema's tokens with descriptions and sample values; if it takes more than a modest share of the window, use 8.b.ii instead.", "Write or generate the missing descriptions: column meanings, keys, enum codes, null semantics.", "Put the schema in the prompt, or mount it as files, instead of a retrieval tool.", "Paired eval on execution accuracy and wrong-table, wrong-column and wrong-value failures." ], "detect": { "static": "a SQL-writing agent with no table and column descriptions, keys or allowed values in its context; schema context reachable only through a search tool", "traces": "queries against non-existent columns, wrong joins, guessed enum codes; failure rows labeled tool-hallucination (names) or domain-knowledge-deficit", "evals": "failures from wrong table, column or value" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "large", "when": "Cube: +17 to +23 points from a semantic-context document on 99 questions (Apr 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "small", "when": "the schema costs prompt tokens on every call; caching the prefix (3.b.i) recovers most of it", "basis": "evidence" }, { "dimension": "performance", "effect": "none", "size": null, "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval with and without the schema context, on execution accuracy" } ], "evidence": [ { "source": "Andrew Qu (Vercel), \"We removed 80% of our agent's tools\"", "url": "https://vercel.com/blog/we-removed-80-percent-of-our-agents-tools", "date": "2025-12-22", "locator": "v2: The file system is the agent", "task": "Mount raw semantic-layer files instead of retrieval tools", "effect": "Same bundled 5-query result. \"The semantic layer is already great documentation ... We were building tools to summarize what was already legible. Claude just needed access to read it directly.\" Code: sandbox.writeFiles(files) from ./semantic-catalog.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Michael Rumiantsau, Ivan Fokeev (Cube), \"Semantic Layers for Reliable LLM-Powered Data Analytics: A Paired Benchmark of Accuracy and Hallucination Across Three Frontier Models\"", "url": "https://arxiv.org/abs/2604.25149", "date": "2026-04-28", "locator": "Section 4.2 Conditions; Table 1; Section 5.4; REPORT.md 2.2", "task": "Add a hand-authored semantic-layer document to the prompt", "effect": "n=99, single shot, analytical verdict. Opus 4.7 50.5% -> 67.7% (+17.2 pp, McNemar b/c 22/5, p=0.0015); Sonnet 4.6 46.5% -> 68.7% (+22.2 pp, 30/8, p=0.0005); GPT-5.4 45.5% -> 68.7% (+23.2 pp, 27/4, p<0.0001). The document also broke 4-8 previously passing questions per model (the c counts). Mean input tokens rose ~2-4K per query (e.g. Sonnet 14,364 -> 17,303); latency changed -0.9 to +1.8 s. Document written once and not iterated against the benchmark; residual ~30% failures sit on percentiles, correlation, stddev, ABC classification, threshold ranking and complex CTE shapes.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Michael Rumiantsau, Ivan Fokeev (Cube), \"Semantic Layers for Reliable LLM-Powered Data Analytics: A Paired Benchmark of Accuracy and Hallucination Across Three Frontier Models\"", "url": "https://arxiv.org/abs/2604.25149", "date": "2026-04-28", "locator": "Section 6.5 Limitations; REPORT.md 3.5", "task": "Put the semantic context in the prompt, not behind search", "effect": "Reported only in passing (Section 6.5; REPORT.md 3.5): \"did not significantly outperform raw-DDL Claude on this benchmark\"; numbers not given in the paper. 25-table schema, so everything fit in context; the authors flag 1,000+ table warehouses as open.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Albert Chen, Manas Bundele, Gaurav Ahlawat, et al. (LinkedIn), \"Text-to-SQL for Enterprise Data Analytics\"", "url": "https://arxiv.org/abs/2507.14372", "date": "2025-07-18", "locator": "§2.2.2", "task": "Render tables as CREATE TABLE with extended column descriptions", "effect": "Described in §2.2.2; not ablated separately.", "direction": "supports", "measured": false, "conditions": null, "older": true } ], "caveats": "The measured gain comes from a semantic-layer vendor's study (Cube) on one 25-table schema. Past a modest share of the window, loading the whole schema costs more than it returns; that is 8.b.ii.", "also_covers": [], "related": [ { "id": "8.b.ii", "slug": "link-large-schemas", "name": "Link large schemas by exploration: start broad, grow the linked set with probing queries, favor recall" }, { "id": "8.a.ii", "slug": "surface-unreadable-information", "name": "Surface information the source holds but the agent cannot read" }, { "id": "7.a.i", "slug": "semantic-layer-routing", "name": "Route metric questions through a deterministic semantic layer; fall back to text-to-SQL" }, { "id": "3.b.i", "slug": "cache-stable-prefix", "name": "Keep the prompt prefix cache-stable" }, { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "8.a.i", "slug": "choose-retriever", "name": "Choose the retriever by corpus size and question type" }, { "id": "3.a.i", "slug": "few-shot-examples", "name": "Add worked examples to the prompt (few-shot), or curated example queries for data agents" }, { "id": "8.b.i", "slug": "tune-search-tool", "name": "Tune the search tool: snippets, enough results, retries, reformulation, filters, source quality" } ], "url": "https://mahalsystems.ai/aot/optimizations/ground-queries-in-schema/", "markdown": "https://mahalsystems.ai/aot/optimizations/ground-queries-in-schema.md" }, { "id": "5.c.i", "legacy_id": "P47", "former_ids": [], "slug": "discrete-phase-separation", "name": "Discrete phase separation", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it splits the work into isolated phases", "proposed": true }, "levers": [ "context" ], "domains": [ "general" ], "evidence_domains": [ "code-generation" ], "categories": [ "Isolating subtasks" ], "category_ids": [ "context.isolate" ], "summary": "Run research, planning and implementation as separate agent sessions, each starting from a fresh context that holds only the previous phase's written output (a research document, then a plan file), and have a person review that document before the next phase starts. It pays when the plan knows more than the implementer would find on its own: on SWE-bench Verified, a plan written by a stronger model at the agent's first edit raised resolution by 6.0 to 16.8 points across three harnesses, and a fresh implementer given only that plan did as well as continuing the original session with it (58.6% vs 58.2%) while making more calls. The plan is also the risk: plans from the agent's own model recovered 12 of 223 failures against 9 with no plan, a weaker planner's plans broke 14.6% of runs that had passed, and with incomplete instructions a read-only plan phase never raised task success and cut requirement scores by 11.3 and 13.5 points for two of three models because it stopped exploring early. In practice teams use it for large new features, read the specs only partially and see them go stale; no study we found tests the full split with review against one continuous session.", "how": [ "Start from one continuous session on your real tasks, with the harness's own compaction (see 5.a.i), and record where it goes wrong: the wrong problem solved, the fix put in the wrong place, requirements the request left out but the repository shows, rework after review. Split only the tasks that fail that way, typically large new features or multi-file changes in unfamiliar code; practitioners skip it for small fixes, bugs and one-off scripts, and 82.0% of the public specs studied describe new features.", "Research: run a read-only session (no edit tools) that explores the code and writes a research document: the relevant files with paths and line references, how the current behavior works, constraints and conventions, and requirements the request does not state. Tell it to treat the request as evidence rather than the full specification and to keep exploring until it can say how each requirement will be checked; a plan phase that stopped exploring early wrote plans covering only part of the task, and HumanLayer traced its own failed 7-hour attempt to research that did not go deep enough through the dependency tree.", "Plan: start a fresh session from the request and the research document only, and have it write a plan file: every change file by file, the reason for each, and the test or command that validates it. Use your strongest model here: a stronger planner's plans raised resolution by 6.0 to 16.8 points, the implementer's own model as planner recovered 12 of 223 failures against 9 with no plan, and a weaker planner's plans broke 14.6% of passing runs. Keep detail that names classes, functions or library versions scoped to this change; it is what goes stale first.", "Review at each boundary: a person reads the research document and the plan before the next phase starts and corrects or rejects them, because a wrong line of research or plan turns into many wrong lines of code. Check that the review happens; most spec-driven tool users read specs only partially or handed the review to another agent.", "Implement: start a fresh session with the request, the plan file and a structured handoff in fixed fields (files changed or to change, validation commands and results, what failed, what is still uncertain, what to verify first), and tell it to check the plan against the repository rather than trust it. Notes cost part of the raw history's gain (+4.4 to +5.0 against +6.1 points for the same model), so keep the research transcript where the implementer can read it on demand. If research and planning used little context, appending the plan to the same session is cheaper: a fresh implementer matched the continuing one (58.6% vs 58.2%) but needed 2.0 times the executor calls on failed tasks.", "After the change merges, update or archive the research and plan files so no later session reads them as current (see 4.b.iii).", "Measure with paired runs on the same tasks and models: one continuous session, the plan in the same session, and the phase split, with and without the boundary review; report task success, requirement coverage on tasks whose request omits requirements, runs the baseline solved that the variant broke, tokens and wall time per phase and human review minutes, and label each failure as research, plan, handoff or implementation." ], "detect": { "static": "Phases with distilled handoffs rather than shared history.", "traces": "Phase boundaries and what crosses them; handoff size against the full history; files the implement phase re-reads that research already covered; edits made to the plan at review; research or plan files older than the code they describe.", "evals": "Quality vs monolithic run.", "judge": "Rate handoffs for completeness.", "probes": "Multi-file tasks whose request leaves out requirements the repository reveals, where a design decision must precede coding." }, "impact": { "measured_basis": true, "primary": [ "quality", "governance" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "pays when the plan beats what the implementer finds alone: a stronger model's plan at the first edit +6.0 to +16.8 points on SWE-bench Verified, a fresh implementer from that plan alone 58.6% vs 58.2% for the continuing session; the implementer's own model as planner 12 of 223 failures recovered against 9 with no plan, and a weaker planner's plans 43.4% against a 45.6% baseline with 14.6% of passing runs broken (Patel, Oct 2026); under incomplete instructions a read-only plan phase never raised Success@1 and cut Score@1 from 68.8 to 55.3 and 66.3 to 55.0 for two of three models (Liu, Oct 2026); distilled notes +4.4 to +5.0 points against +6.1 for the raw history, same model (KC, Jun 2026); one team's 35k LOC in 7 hours, anecdotal (Horthy, Aug 2025)", "basis": "evidence" }, { "dimension": "cost", "effect": "mixed", "size": "medium", "when": "more when the next phase must re-explore: a fresh implementer from the same plan used 2.0 times the executor calls and 22% more executor tokens on failed tasks, and 519.1M against 347.4M executor tokens on 500 tasks (Patel, Oct 2026); planning itself adds tokens, one team reporting two to three times those of direct prompting, self-reported (Agarwal, Oct 2026); fewer when bounded notes replace a long history: a same-model successor given notes used 602k to 660k prompt tokens against 811k with the raw trace, its first prompt about 10k characters against 87k (KC, Jun 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "medium", "when": "phases run in sequence and wait for review: a fresh implementer needed 2.0 times the executor calls (Patel, Oct 2026); a successor working from notes took a median 53 to 55 steps against 41 with the raw trace (KC, Jun 2026); one team reported about twice the time, self-reported (Agarwal, Oct 2026); 3 of 7 hours went to research and plans (Horthy, Aug 2025); no source measures review wait time", "basis": "evidence" }, { "dimension": "security", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "governance", "effect": "mixed", "size": "medium", "when": "research and plan files are a reviewable record of intent, and reading a 200-line plan instead of 2,000 lines of code is the stated point (Horthy, Aug 2025); in practice most spec-driven tool users read specs only partially or delegated review to other agents, 98.3% of spec files with history had one author, 68.5% of inline spec comments came from bots, and specs went stale as code changed (Agarwal, Oct 2026)", "basis": "evidence" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired runs on the same tasks, model and budget: one continuous session with compaction, a plan in the same session, and the phase split with fresh contexts, each with and without the boundary review; include tasks whose instruction omits requirements that hidden tests check; report task success, requirement coverage, baseline-solved runs the variant broke, tokens and wall time per phase and review minutes; repeat with the planner on a stronger model and on the implementer's model, and again after a model upgrade" }, { "method": "monitor", "tier": null, "how": "per task: phases run, tokens and time per phase, handoff size, files the implement phase re-reads that research already covered, edits made to the plan at review and whether a reviewer opened it, failures traced to research, plan or handoff, and research or plan files older than the code they describe" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Multi-Agent group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Dex Horthy (HumanLayer), \"Getting AI to Work in Complex Codebases\"", "url": "https://github.com/humanlayer/advanced-context-engineering-for-coding-agents/blob/main/ace-fca.md", "date": "2025-08-29", "locator": "Intro; Solving complex problems", "task": "Describes a research, plan, implement workflow for coding agents that keeps context utilization low and adds human review between phases", "effect": "\"We use a \"research, plan, implement\" workflow\"; reports shipping 35k LOC to BAML in 7 hours (3 on research/plans, 4 on implementation). A 7-hour attempt to remove Hadoop dependencies from parquet-java \"did not go well\": \"the research steps didn't go deep enough through the dependency tree\". Anecdotal, no controlled comparison.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Viral Patel, Boyin Tan, Jason Xue, Youcheng Sun, \"PAUSE: Search-Free Steering of Coding Agents Before Their First Source-Code Edit\"", "url": "https://arxiv.org/abs/2610.29870", "date": "2026-10-21", "locator": "§4.1 Table 3; §4.2 Tables 4-5; §4.4 Table 6; §5", "task": "At a coding agent's first source-code edit (a median of 60% into the run, after exploration, reproduction and testing), a separate Planner call reads the task and the transcript and writes one repair plan; the same executor then continues its session with the plan appended (resume), or a fresh executor starts from the initial repository with only the plan (restart). Controls: no plan, self-reconsideration, and the executor's own model as Planner", "effect": "Online with GPT-5.6 Sol as Planner, SWE-bench Verified resolution rose from 50.2% to 65.0% (SWE-agent), 37.0% to 43.0% (Pi) and 45.4% to 62.2% (OpenHands); Multilingual 28.3% to 43.0%. On 223 recorded Qwen3-coder failures: no plan recovered 9 (4.0%), self-reconsideration 13 (5.8%), the executor's own model as Planner 12 (5.4%), GPT-5.6 Sol 79 (35.4%); with Devstral-small 14, 17, 20 and 53 of 174. Full benchmark against a 45.6% baseline: Sol plans 58.2% resume and 58.6% restart; Qwen3-30B-Thinking plans 43.4% resume and 41.4% restart, with 26 regressions (14.6%) on 178 solved tasks against 10 for resuming with no plan. On failures, restart needed \"2.0× as many Executor calls\" and 22% more executor tokens; over 500 tasks 519.1M against 347.4M executor tokens.", "direction": "mixed", "measured": true, "conditions": "Qwen3-coder and Devstral-small executors; GPT-5.6 Sol, Gemini 3.8 Flash, Gemini 3.5 Flash Lite and Qwen3-30B-Thinking planners; SWE-agent, Pi and OpenHands; SWE-bench Verified (500) and Multilingual (300); offline comparisons one run per task; the Planner reads the transcript without tools and no human reviews the plan; the gain mainly reflects a stronger Planner model", "older": false }, { "source": "Kaiming Liu, Yuxuan Liu, Jinrui Ju, Minyuan Ruan, Xuanyu Lei, Peng Li, Yang Liu, \"Instructions Are Evidence, Not Objectives: Measuring Outcome Alignment of Agents\"", "url": "https://arxiv.org/abs/2610.17757", "date": "2026-10-13", "locator": "§5.4 Table 6; Appendix F.3; Table 7", "task": "A two-phase plan mode under partial instructions: a read-only plan phase (editing and submission tools withheld) ends by writing PLAN.md, then the same episode executes it; compared with the same model, instruction and budget without the plan phase", "effect": "Score@1 (share of five requirements passed) and Success@1 (all passed), without vs with plan mode: GPT-5.6-Luna 41.6 vs 45.9 and 3.1 vs 3.1; Kimi-K3 68.8 vs 55.3 and 21.9 vs 17.2; DeepSeek-V4.1-Flash 66.3 vs 55.0 and 25.0 vs 21.9. \"the plan mode never raises Success@1\"; \"the plan-phase agent does not infer the complete true requirements, and ... it often stops exploring too early and writes the plan.\"", "direction": "contradicts", "measured": true, "conditions": "TiGanEval, 64 tasks (40 software engineering from DeepSWE, 24 research), five requirements each, initial instruction deliberately partial; 120 turns and 360 tool calls shared across both phases; three models; seed-to-seed deviation within 4.3 Score points on replicated configurations; one context across both phases, no human review", "older": false }, { "source": "Shyam Agarwal, Anmol Singhal, Andrew Macvean, Bogdan Vasilescu, Travis Breaux, \"Spec-Driven Development as Prescribed and as Practiced\"", "url": "https://arxiv.org/abs/2608.25202", "date": "2026-10-15", "locator": "§IV Findings 2, 3, 4, 7, 9, 10; §V-A; §V implications", "task": "Studied spec-driven development (a spec, design or plan, and tasks written before the agent codes, as in Spec Kit, OpenSpec and Kiro) through Reddit discussions, interviews with 21 practitioners including eight tool makers, and 266 active repositories from a corpus of 470,795 spec files", "effect": "Practitioners \"used SDD selectively\", skipping it for small tasks, bugs and one-off scripts; an estimated 82.0% of specs describe new features. \"Most tool users read specs only partially or delegated their review to other agents\"; 98.3% of spec files with a recorded history have a single author and 68.5% of inline spec comments were written by bots. Most reported more reliable agent work but \"could not separate this benefit from improvements in models\"; one team found \"the spec-driven runs typically consume two to three times as many tokens and take about twice as long\" and “We couldn’t prove it improved the code” (Reddit posts). 77.6% of specs contain implementation detail, 75% of those naming classes, functions, file paths or library versions; agents departed from specs during implementation and specs went stale; 81.5% of spec files have a single recorded commit. The authors recommend that tools \"separate specification, implementation, and review contexts\".", "direction": "mixed", "measured": false, "conditions": "Thematic analysis plus repository mining with LLM-as-judge labels; no controlled comparison; token and time figures are self-reported by one team; title, five authors and all cited findings first appear in v4 (2026-10-15)", "older": false }, { "source": "Dipesh KC, Anjila Budathoki, \"Handoff Debt: The Rediscovery Cost When Coding Agents Take Over Interrupted Tasks\"", "url": "https://arxiv.org/abs/2606.02875", "date": "2026-06-01", "locator": "§5.1-5.3 Table 2; Appendix Tables 7, 8, 10", "task": "A coding agent is stopped at a fixed point (after its first source edit, first validation result or first post-failure edit) and a successor in a fresh context resumes from the frozen repository with one of four handoffs: repository only, the raw trace, free-form summary notes, or structured notes (changed files, validation evidence, uncertainty, rollback risk, what to verify)", "effect": "Same-model takeover (Qwen to Qwen) solved rate: repository only 46.4%, raw trace 52.5% (+6.1, p 0.035), summary notes 51.4% (+5.0, p 0.093), structured notes 50.8% (+4.4, p 0.134); median steps 99, 41, 53 and 55; prompt tokens 1.63M, 811k, 602k and 660k. Across three successors the raw trace gained +6.1 to +14.9 points and notes +1.1 to +10.5, notes significant only for Devstral. Median first prompt: 87k characters for the raw trace, 9.8k and 10.0k for the notes. For Gemma, notes slightly lowered the solved rate on unfinished work (-1.8 and -0.9 points).", "direction": "mixed", "measured": true, "conditions": "75 SWE-bench Verified tasks, 181 handoff points, 2,172 takeover runs; Qwen3.6-27B predecessor; Qwen3.6-27B, Gemma-4-31B and Devstral-Small-2-24B successors; OpenHands-style runtime; one run per view, a three-attempt rerun on 40 points agreed; mid-task handoffs, not research-to-plan boundaries", "older": false } ], "caveats": "No study we found compares the full research, plan and implement split with human review against one continuous session on the same tasks: the measured results cover a plan injected at the first edit, where the gain comes from a stronger planner model, a same-session plan mode with no review under deliberately partial instructions (64 tasks, three models), and mid-task handoffs between 24B to 31B open-weight models (75 source tasks, one run per view). Plans can hurt: a weaker planner's plans broke passing runs, a plan written after too little exploration covered only part of the task, and distilled notes kept less of the solve-rate gain than the raw history. The practice evidence is self-reported: practitioners could not separate the benefit from model improvements, one team reported two to three times the tokens and about twice the time, and the 35k-LOC report is one team's anecdote.", "also_covers": [], "related": [ { "id": "5.c.ii", "slug": "isolated-subagent-context", "name": "Sub-agent architectures with clean contexts" }, { "id": "4.b.iii", "slug": "multi-window-task-state", "name": "Keep plan, progress and results in files when work outlives one context window" }, { "id": "5.a.i", "slug": "staged-compaction", "name": "Compact conversation history in stages: elide stale tool outputs, then keep an anchored summary" }, { "id": "7.a.ii", "slug": "prompt-chaining-with-gates", "name": "Prompt chaining with gate checks" }, { "id": "4.b.ii", "slug": "initializer-maintainer-agents", "name": "Initializer-maintainer dual agent" }, { "id": "6.a.i", "slug": "oracle-worker-escalation", "name": "Oracle-worker multi-model" }, { "id": "6.c.ii", "slug": "plan-then-execute", "name": "Plan-then-execute" }, { "id": "7.b.iv", "slug": "track-task-progress", "name": "Track explicit progress: a task list with a status per item, re-injected each turn" } ], "url": "https://mahalsystems.ai/aot/optimizations/discrete-phase-separation/", "markdown": "https://mahalsystems.ai/aot/optimizations/discrete-phase-separation.md" }, { "id": "5.c.ii", "legacy_id": "P20", "former_ids": [], "slug": "isolated-subagent-context", "name": "Sub-agent architectures with clean contexts", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it adds sub-agents with their own contexts", "proposed": true }, "levers": [ "context" ], "domains": [ "general" ], "evidence_domains": [ "web-research", "document-extraction", "retrieval", "code-generation", "tool-use", "conversation" ], "categories": [ "Isolating subtasks" ], "category_ids": [ "context.isolate" ], "summary": "Hand a context-heavy side task (code search, reading many files, documents or logs) to a sub-agent that starts with a fresh context and only the tools it needs, and have it return a short result or file references so the parent's context stays small. It pays when the parent would otherwise read far more than it can hold: on legal data rooms of up to 80M tokens, delegating bounded reading raised rubric pass rate from 23.3% to 62.4% across seven models. It is not a default win: on bounded repository questions, delegated exploration scored 46.2% against 65.2% for an indexed single agent at 2.3 times the cost per correct answer, with 41.8% of its failures at the hand-off, mostly silent; generic sub-agent roles and a second level of delegation also lost quality.", "how": [ "Start from the single agent: run it on your tasks and log peak context, tokens per task and which tool outputs fill the window; if the material is a corpus you can index, try a retriever before delegating (see 8.a.i), since indexed retrieval beat delegated exploration on bounded repository questions.", "Delegate only side tasks whose raw output is large and not needed again (codebase search, reading many files, documents or logs) and whose result fits a short contract; keep work whose steps share a lot of context in the parent.", "Define each sub-agent as a task-specific role with an input-output contract: the objective, the inputs it gets, an output schema (findings with file paths, line ranges or document ids, plus what it did not cover) and a length cap; avoid generic lifecycle roles such as requirements, design or code generation.", "Start it with a fresh context and a self-contained brief, not the parent's routing messages or whole history, and give it only the tools it needs: read-only search and read tools for exploration, no delegation tool (depth 1), and a turn cap.", "Protect the hand-off: have it write long outputs to files and return references (see 6.b.iii), have the parent forward or quote its findings instead of paraphrasing them, and require it to list every policy- or risk-relevant fact it found.", "Measure with a paired eval of the single agent against the delegating one on the same tasks and models: success, cost per correct answer, total and parent tokens, the parent's peak context and latency; then label failures where the sub-agent's transcript holds a fact that the parent's answer lacks." ], "detect": { "static": "A sub-agent call with a fresh context, a scoped tool list and a summary or file-reference return; or a single agent that reads large search results, file sets, documents or logs into its own context. Warning signs: generic lifecycle sub-agents, sub-agents that inherit the full history or can delegate further, a parent that paraphrases sub-agent output.", "traces": "Sub-agent spans: tokens consumed inside vs returned; the parent's peak context; empty returns; facts present in the sub-agent transcript but missing from the parent's answer (hand-off breakdowns, usually silent); share of the corpus read.", "evals": "Single agent vs delegation (and vs an indexed retriever) on search-heavy tasks: success, cost per correct answer.", "judge": "Summary fidelity against the sub-agent's full transcript: facts found but not returned, claims returned but not supported.", "probes": "Broad search then synthesis." }, "impact": { "measured_basis": true, "primary": [ "quality", "cost" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "large", "when": "Large corpora: +39.1 points rubric pass rate on legal data rooms (Grupen, Sep 2026); folding sub-tasks out of context 0.567 vs 0.540 for a 327K-context agent with the same RL on BrowseComp-Plus, at a 10x smaller active context (Sun, Oct 2025); sub-calls lifted GPT-5 on OOLONG 36.0 to 56.0 but cut Qwen3-Coder on CodeQA 66.0 to 56.0 (Zhang, Dec 2025). Bounded tasks: 46.2% vs 65.2% for an indexed single agent (Oskooei, Aug 2026); task-specific sub-agents +5.91 and +4.46 points, generic ones lower (Hu, Sep 2026); medal rate 45.2% vs 55.7%, CIs overlapping (Brilliantov, Sep 2026); depth 2 cost 19 points (Grupen, Sep 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "mixed", "size": "large", "when": "More total tokens (Piriyakulkij, Sep 2026): 2.3x cost per correct answer for delegated search (Oskooei, Aug 2026), cost up for six of seven models in a delegating harness while Opus 5 fell from ~$18 to ~$7 per data room (Grupen, Sep 2026); single-agent tokens grew with distractor domains while sub-agent setups stayed flat (Fu-Hinthorn, Jun 2025)", "basis": "evidence" }, { "dimension": "performance", "effect": "mixed", "size": "small", "when": "Each fresh sub-agent must gather its own context; median latency was lower with delegation for two models and higher for two (Oskooei, Aug 2026); RL rollouts slowed from 48 to 74 minutes per step as the trained root sent more sub-agents (Grupen, Sep 2026). Parallel speedups belong to 6.b.iii", "basis": "evidence" }, { "dimension": "security", "effect": "mixed", "size": "medium", "when": "Depends on tool scoping (see 5.c.ii).", "basis": "assessment" }, { "dimension": "governance", "effect": "degrades", "size": "small", "when": "Nested traces.", "basis": "assessment" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "Paired eval on the same tasks and models: the single agent against the same agent with a delegated sub-agent (and against an indexed retriever where the corpus allows): success, cost per correct answer, total tokens, the parent's peak context and latency; code each failure by mechanism, including hand-off breakdowns where the sub-agent found the answer and the parent lost or misreported it." }, { "method": "monitor", "tier": null, "how": "Per sub-agent call: tokens consumed inside against tokens returned, empty returns, sub-agent count against task size and the share of the corpus read; on a sample, have a judge check whether the facts in the sub-agent's transcript reach the parent's answer (fact attenuation)." } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file master-agent-optimization-cookbook.md", "task": "Lists this practice in its agent-optimization pattern library (Long-Horizon group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Benefits of a multi-agent system", "task": "Lead agent delegates to subagents with their own context windows on a research task", "effect": "\"a multi-agent system with Claude Opus 4 as the lead agent and Claude Sonnet 4 subagents outperformed single-agent Claude Opus 4 by 90.2% on our internal research eval\"", "direction": "supports", "measured": true, "conditions": "Anthropic internal research eval; Claude Opus 4 lead, Claude Sonnet 4 subagents", "older": true }, { "source": "Prithvi Rajasekaran, Ethan Dixon, Carly Ryan, Jeremy Hadfield (Anthropic), \"Effective context engineering for AI agents\"", "url": "https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents", "date": "2025-09-29", "locator": "Sub-agent architectures", "task": "Sub-agents work in clean context windows and return a distilled summary", "effect": "\"specialized sub-agents can handle focused tasks with clean context windows\"; each \"might explore extensively, using tens of thousands of tokens or more, but returns only a condensed, distilled summary of its work (often 1,000-2,000 tokens).\"", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Amirkia Rafiei Oskooei, Bora Ilci, Alperen Kayim, Mehmet Egemen Uzun, Berat Can, Kaan Emre Kara, Ozan Orhan, Mehmet S. Aktas, \"Deep Agentic Search for Repository-Level Code Question Answering: An Empirical Study\"", "url": "https://arxiv.org/abs/2608.01507", "date": "2026-08-02", "locator": "Abstract; §4.1-4.4; §5.1 Table 4; §5.2; §5.3; §6.3-6.4", "task": "Planner that delegates repository exploration to an isolated sub-agent returning a condensed result, against a single ReAct agent with semantic search over a pre-built index", "effect": "\"Semantic search answered 65.2% of questions correctly against 46.2% for deep agentic search\"; pass-rate gap 5.6 (Gemini 2.5 Flash), 10.0 (Gemini 2.5 Pro), 10.6 (Qwen3-235B) and 49.6 (Gemini 3 Flash) points, all significant after FDR correction. Pooled cost per correct answer $0.32 vs $0.74, \"about 2.3 times more expensive\". 68.8% of the delegating agent's tool calls ran inside the sub-agent. \"Coordination breakdown accounted for 41.8% of Deep failures\", and \"in 91% of these cases the agent still returned a fluent, confidently worded answer\". Latency \"did not order the two paradigms consistently\".", "direction": "contradicts", "measured": true, "conditions": "SWE-QA, 15 Python repositories x 48 questions = 720 per condition; Gemini 2.5 Flash, Gemini 2.5 Pro, Gemini 3 Flash, Qwen3-235B (same model as planner and sub-agent); LangChain ReAct with 3 tools vs LangChain Deep Agents defaults; Claude Sonnet 4.6 judge, Cohen's kappa 0.74 against a 3-engineer panel on 800 answers. Confound: the delegating arm also replaces index retrieval with grep-style exploration.", "older": false }, { "source": "Bowen Li, Guojun Wang, \"Governance at the Boundary: How Agent Decomposition Degrades Policy Compliance\"", "url": "https://arxiv.org/abs/2608.16055", "date": "2026-08-17", "locator": "Abstract; §5.1-5.3, Tables 4-5; §6", "task": "Single loop vs fixed pipeline vs orchestrator-subagent on policy-governed tasks; count facts a component discovered that never reach the component that must act on them", "effect": "Fact attenuation at constraint distance 2, conditional on discovery: Qwen2.5-32B 0/16 (0%) single loop, 9/16 (56%) pipeline, 22/26 (85%) orchestrator-subagent; gpt-4.1-mini 0/27 (0%), 1/30 (3%), 1/18 (6%). The orchestrator-subagent arm found trigger facts in 27 of 100 episodes vs 16 for the single loop, then attenuated 22 of those 27: \"The architecture that is better at finding policy-relevant information is worse at carrying it.\" Dropped facts caused both under- and over-escalation.", "direction": "contradicts", "measured": true, "conditions": "Fiducia-bench, 100 KYC/AML task variants, 626 episodes; Qwen2.5-32B-Instruct (GPTQ-Int4) and gpt-4.1-mini; 25-step budget; scripted simulator; governed success only 8/596 (1.3%); small counts per cell", "older": false }, { "source": "Haichuan Hu, Quanjun Zhang, Shengcheng Yu, Zhifei Chen, Tianyu Luo, Chunrong Fang, Zhenyu Chen, Liang Xiao, \"Beyond the Model: Demystifying Harness Effects in Software Engineering Agents\"", "url": "https://arxiv.org/abs/2609.32459", "date": "2026-09-26", "locator": "§3.1 subagent module; §5.2.1 RQ2.1, Figs 11-12; §5.3.1 Table 7", "task": "Add sub-agents (fresh history, restricted tool whitelist, structured report, nesting disabled) to a minimal coding harness one component at a time: task-specific roles vs generic software-lifecycle roles", "effect": "Task-specific sub-agents (behavior analysis, differential testing, submission review) gave \"the largest single-component gain of 5.91 percentage points\" for Qwen3.7-Max and 4.46 for DeepSeek-V4-Pro. \"General subagents also underperform\" and \"degrade performance for both models\". Average prompt tokens: task-specific +7.35% and -9.54%, generic +9.91% and +51.86% (Qwen3.7-Max, DeepSeek-V4-Pro).", "direction": "mixed", "measured": true, "conditions": "ProgramBench, 200 instances; NanoHarness on mini-SWE-agent; Qwen3.7-Max and DeepSeek-V4-Pro; up to 50 sub-agent rounds", "older": false }, { "source": "Kirill Brilliantov, Alejandro Hernández-Cano, Emmanuel Abbé (EPFL, Apple), \"How Much of a Harness Does a Strong Agent Need for Autonomous ML Engineering?\"", "url": "https://arxiv.org/abs/2609.40303", "date": "2026-09-30", "locator": "§4.4 Multi-agent orchestration, Table 1; Appendix (intervention D)", "task": "Enable background sub-agents through the coding agent's native task tool and encourage their use, on top of a single long coding-agent session", "effect": "With delegation vs base (self-select): percentile 63.62 [59.27, 68.26] vs 66.51 [62.46, 69.97]; medal rate 45.2% [38.1, 54.8] vs 55.7% [48.2, 62.9]. \"The base Malena iteration performs no significantly worse than any intervention tested.\"", "direction": "contradicts", "measured": true, "conditions": "GLM 5.2 backbone; OpenCode v1.15.6; MLE-bench fixed14 split, 24h budget, 1 A100; 95% bootstrap CIs over seeds. Delegation here also runs in the background, so isolation is not separated from parallelism", "older": false }, { "source": "Wasu Top Piriyakulkij, Rachel Lawrence, Alicia Curth, Sushrut Karmalkar, Niranjani Prasad (Cornell, Microsoft Research Cambridge), \"Subagents vs Agent Skills: Executing Reusable Knowledge for Long-Horizon Agentic Tasks\"", "url": "https://arxiv.org/abs/2609.09233", "date": "2026-09-07", "locator": "§3.1-3.2; §4 Main results, Figs 2-4; Appendix A.1 Fig 5", "task": "Run each skill package as a sub-agent in a fresh context instead of loading its instructions into the main agent's context", "effect": "With curated packages lacking input-output contracts, \"agent skills match or outperform subagents across all models\"; with packages that have explicit contracts, \"subagents outperform agent skills, with the gain being largest for smaller models\". As distracting tools are added, \"subagent execution degrades more gracefully\". Sub-agents lowered peak context \"on over 80% of tasks\" for GPT 5.3 Codex and Kimi K2.6, less or reversed for weaker models; \"Subagent execution consumes substantially more tokens overall\". Routing nodes inline with leaf skills as sub-agents was \"consistently best\" (Qwen3.5-9B). Accuracies are reported in figures only.", "direction": "mixed", "measured": true, "conditions": "SkillsBench, 64 of 87 tasks; OpenHands; Qwen3.5 2B/4B/9B, Ministral-3-8B, Gemma-4-12B, GPT-5.x, Mistral-Large-3, Kimi-K2.6", "older": false }, { "source": "Weiwei Sun, Miao Lu, Zhan Ling, Kang Liu, Xuesong Yao, Yiming Yang, Jiecao Chen (ByteDance Seed, CMU, Stanford), \"Scaling Long-Horizon LLM Agent via Context-Folding\"", "url": "https://arxiv.org/abs/2510.11967", "date": "2025-10-13", "locator": "Abstract; §4.1 Table 1; §4.5.3", "task": "Agent branches into a sub-trajectory for a sub-task and on return folds it to a short summary, keeping a 32K active context", "effect": "Untrained, the folding agent beat 32K ReAct and a summary agent (BrowseComp-Plus 0.420 vs 0.286 and 0.386; SWE-Bench Verified 0.492 vs 0.436 and 0.488) but not 327K ReAct (0.478, 0.552). With the same GRPO training: 0.567 vs 0.540 (327K ReAct) and 0.527 (summary) on BrowseComp-Plus; 0.564 vs 0.574 and 0.550 on SWE-Bench Verified. With FoldGRPO: 0.620 and 0.580, \"using an active context 10× smaller\". Parallel branching (0.6133) performed \"similarly to the single-branch version\".", "direction": "supports", "measured": true, "conditions": "Seed-OSS-36B-Instruct; BrowseComp-Plus N=150, SWE-Bench Verified N=500; 32K x 10 branches vs 327K ReAct", "older": true }, { "source": "Alex L. Zhang, Tim Kraska, Omar Khattab, \"Recursive Language Models\"", "url": "https://arxiv.org/abs/2512.24601", "date": "2025-12-31", "locator": "v3 Table 1; §4 Observations 2 and 4", "task": "Hold the long input as a variable in a REPL and let the root model call sub-models on slices; compare recursion depth 0 (no sub-calls) with depth 1", "effect": "GPT-5 root with GPT-5-mini sub-calls, depth 0 vs depth 1: CodeQA 58.0 vs 62.0, BrowseComp+ (1K) 88.0 vs 91.3, OOLONG 36.0 vs 56.0, OOLONG-Pairs 43.9 vs 58.0. Qwen3-Coder-480B: 66.0 vs 56.0, 46.0 vs 44.7, 43.5 vs 48.0, 17.3 vs 23.1; on CodeQA \"the no-sub-calling RLM(depth=0) is able to outperform all sub-calling variants\". Sub-calls without offloading the input (CodeAct + sub-calls, GPT-5): OOLONG 40.0, OOLONG-Pairs 28.4. \"the median RLM run is cheaper than the median base model run, but more expensive on average due to outlier trajectories\".", "direction": "mixed", "measured": true, "conditions": "GPT-5 and Qwen3-Coder-480B-A35B; CodeQA, BrowseComp+ (1K documents, 150 tasks), OOLONG, OOLONG-Pairs; numbers from v3 (May 2026)", "older": false }, { "source": "Niko Grupen, Julio Pereyra, Gabe Pereyra, Charlie O'Neill, Mudith Jayasekara, Aaron Ellis-Bloor, Jonathon Liu, Matthew Blau (Harvey), \"Post-Training RLM Agents for End-to-End M&A Diligence\"", "url": "https://www.harvey.ai/blog/post-training-rlm-agents-for-m-and-a-diligence", "date": "2026-09-08", "locator": "Intro; Figs 5-6; Coverage and Cost; Appendix RLM Recursion Depth", "task": "Root agent with the data room in a REPL delegates bounded reading to sub-agents in their own context windows; only what the root prints enters its context", "effect": "\"Across seven models, the RLM harness raises mean pass rate from 23.3% to 62.4%, a gain of 39.1 percentage points.\" In the tool loop \"no run reads more than 1% of the data room\"; with delegation \"most read close to all of it\". Cost rose for six of seven models; Opus 5 went from ~$18 to ~$7 per data room. The Opus 5 root used 3.8% of input and 1.1% of output tokens. Claude Code and Codex scored 17.9 and 4.6 points below the plain tool loop and \"neither spawns sub-agents despite having the ability to\". Depth 2 (sub-agents that can delegate) improved 4 of 14 rooms, reduced 10, \"lowering mean criteria pass rate by 19 percentage points\".", "direction": "supports", "measured": true, "conditions": "LAB Diligence, data rooms up to 80M tokens, 50-room hold-out (unreleased); seven root models with Qwen3.6-35B-A3B sub-agents at depth 1; depth test on Qwen3.5-122B-A10B, 14 rooms. The REPL and the delegation change together", "older": false }, { "source": "Will Fu-Hinthorn (LangChain), \"Benchmarking Multi-Agent Architectures\"", "url": "https://www.langchain.com/blog/benchmarking-multi-agent-architectures", "date": "2025-06-10", "locator": "Score; Cost (Tokens); Improvements to supervisor", "task": "Single agent holding every domain's tools and instructions vs supervisor and swarm setups where each sub-agent sees only its own domain, as distractor domains are added", "effect": "The single agent \"falls off sharply when there are two or more distractor domains\" and \"uses consistently more tokens\" as they grow, while supervisor and swarm \"remain flat\"; with one distractor domain the single agent did slightly better. Supervisor errors came from \"translation\" (paraphrasing sub-agent replies); removing hand-off messages from the sub-agent's state, a forward_message tool and tool-name changes gave \"a nearly 50% increase in performance\". Scores in charts only.", "direction": "mixed", "measured": true, "conditions": "gpt-4o; tau-bench retail, first 100 test examples; up to 6 distractor domains of 19 tools plus a wiki each; LangGraph", "older": true } ], "caveats": "The large gains come from very large corpora (data rooms, multi-million-token inputs) and from setups that also change other things (a REPL holding the corpus, RL training), so they do not isolate the sub-agent; on bounded tasks with a strong single agent the gain vanished or reversed (repository questions, MLE-bench, generic roles on ProgramBench). The hand-off is the new failure point: it caused 41.8% of delegated-search failures, usually with a confident wrong answer, and a 32B model lost 85% of discovered facts across an orchestrator-subagent boundary, against 3-6% for gpt-4.1-mini. Several studies are small or chart-only (14 data rooms for the depth test, 16-30 facts per cell, SkillsBench accuracies in figures), and Harvey's hold-out sets are unreleased.", "also_covers": [ { "name": "Sub-agent spawning with isolated context" } ], "related": [ { "id": "6.b.iii", "slug": "multi-agent-for-breadth", "name": "Use multi-agent orchestration only for breadth-first, independent sub-tasks" }, { "id": "5.a.i", "slug": "staged-compaction", "name": "Compact conversation history in stages: elide stale tool outputs, then keep an anchored summary" }, { "id": "8.a.i", "slug": "choose-retriever", "name": "Choose the retriever by corpus size and question type" }, { "id": "4.b.iii", "slug": "multi-window-task-state", "name": "Keep plan, progress and results in files when work outlives one context window" }, { "id": "5.c.i", "slug": "discrete-phase-separation", "name": "Discrete phase separation" }, { "id": "6.b.ii", "slug": "llm-map-reduce", "name": "LLM map-reduce" }, { "id": "10.a.i", "slug": "constrain-agent-reach", "name": "Constrain what the agent can touch" }, { "id": "1.b.i", "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class" }, { "id": "2.c.i", "slug": "bound-tool-output", "name": "Bound tool output: cap it, or write it to a file the agent can read" }, { "id": "2.a.i", "slug": "reduce-tool-surface", "name": "Reduce the tool surface: remove and consolidate tools, then defer the rest behind search" }, { "id": "7.d.i", "slug": "continuous-task-loop", "name": "Continuous autonomous task loop" }, { "id": "4.c.ii", "slug": "skill-library", "name": "Skill library evolution" }, { "id": "3.a.ii", "slug": "vary-few-shot-examples", "name": "Avoid few-shot ruts" } ], "url": "https://mahalsystems.ai/aot/optimizations/isolated-subagent-context/", "markdown": "https://mahalsystems.ai/aot/optimizations/isolated-subagent-context.md" }, { "id": "6.a.i", "legacy_id": "P42", "former_ids": [], "slug": "oracle-worker-escalation", "name": "Oracle-worker multi-model", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it adds a multi-model topology", "proposed": true }, "levers": [ "model" ], "domains": [ "general" ], "evidence_domains": [ "code-generation", "tool-use", "conversation", "reasoning" ], "categories": [ "Escalating", "Using cheaper models" ], "category_ids": [ "topologies.escalate", "models.cheap-verifiable" ], "summary": "Let a cheap model do the work and call an expensive one only when a trigger fires: a cascade escalates when the cheap answer looks unreliable or fails a check, a router picks the model before the call, and an advisor tool lets a cheap agent consult a stronger model mid-task. It pays when the trigger really separates the cheap model's failures: a two-model cascade beat the best single model at the same budget by 2.1 to 8.2 points on average across budgets, a learned consult policy recovered 44% to 81% of the gap to the strong model on three agent benchmarks at 34.5% to 48.1% of its cost, and re-running only the failures of a low-effort run at high effort matched an all-high run for about 40% less (computed task by task from existing runs, not run end to end). The trigger is the weak link: none of six commercial routers beat a random mix of two well-chosen models at matched cost (the worst trailed by 10.5 points), a third to fifth model in a cascade added at most 0.43 points, and agents left to decide for themselves consulted rarely and late. In Anthropic's tests on current models the advisor bought about what more effort does (1.7 points for 2.1 times the cost on coding) and scored 7 points lower on chart reading, where it was configured but consulted on 1 of 300 tasks.", "how": [ "Start from single-model baselines on your own tasks: the cheap and the strong model alone, each across effort settings (the strong one at low effort too), priced per completed task, plus a random mix of the two at the same spend. A pairing must beat that whole curve: Anthropic's coding advisor pairing (90.1% at $2.92 per attempt) landed on its executor's own effort curve (88.4% at $1.38 on high, 91.1% at $4.11 on xhigh), and no commercial router beat the random mix. Picking one model per call site statically is 1.b.i.", "If the outcome is checkable (tests, a schema, a sound verifier), try the simplest cascade first: run cheap and escalate only the failures. Opus 5.5 at low effort with its failures re-run at high solved about 97% for about $0.17 a task, against 95.3% for $0.29 with everything at high, a figure computed task by task from existing runs rather than run end to end. A checker that passes bad work lets those failures through, so audit it on a gold sample (9.c.iii).", "Use two models, not a ladder: allowing three to five models did not raise mean test accuracy in 118 of 135 comparisons (at most +0.43 points), and even with true query difficulty known a two-model roster was within 1 point of the best roster of any size. Take the cheap model from the cost-accuracy frontier; in one 26-model set it trailed the strongest by 1.6 points at a 96% discount.", "Measure the trigger before trusting it. For a confidence-score cascade, check how well the score separates the cheap model's errors from the strong model's: log-probability scores reached AUROC 0.63 to 0.80 for the cheap model's correctness yet beat random escalation at the same escalation rate (not the same cost) by only about 0.3 points on MATH. For a pre-call router, check it against the random two-model mix at the same spend (commercial routers trailed it by 1.0 to 10.5 points).", "In agents, trigger consults on execution evidence and workflow phase rather than on the agent's own sense of being stuck, and cap them. On SWE-bench Verified, a learned policy at 2.78 consults per episode solved 48.0%, fixed triggers (before committing to an approach, before finishing, on stalls) 41.3%, random placement at similar volume 28.0%, and the orchestrating model left to choose (not volume-matched) made 0.41 consults per episode, 77% of them in the last third; on Terminal-Bench the learned policy only tied the fixed triggers (34.7% each). Do not re-run whole agent tasks on the strong model after a failed self-check: that cascade cost $59.68 against $45.90 for the strong model alone.", "Watch the consult rate and whether the advice is used. Sonnet 5.5 at its default effort never consulted on 20 SWE-bench Pro problems; a pairing that consulted on most tasks at default effort consulted on few once its effort was lowered, and then scored below the executor alone; with the advisor tool attached but never called, GPQA cost 12% to 25% more (its definition is about 1,000 prompt tokens per request); ignored advice made up 58% of the learned policy's remaining SWE-bench failures. A consult that fires on most tasks can cost more than the strong model alone (1.8 times on chart reading).", "Measure on held-out tasks with gold outcomes over several seeds: success and cost per completed task (counting failed cheap attempts and strong-model tokens) against every step-1 baseline, escalation or consult rate, and latency on escalated tasks; re-run at every model or price change." ], "detect": { "static": "Two models with an escalation call from worker to oracle.", "traces": "Escalation or consult rate and when it fires (turn of the first consult, share in the last third of the run); strong-model share of tokens and cost; escalated tasks whose outcome changed; advice the worker then ignored.", "evals": "Success and cost per completed task against each model alone across effort settings and against a random mix of the two at the same spend.", "judge": "Whether escalations were warranted.", "probes": "Tasks the worker solves, tasks only the oracle solves, and tasks neither solves." }, "impact": { "measured_basis": true, "primary": [ "cost", "quality" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "gains where the trigger separates failures: two-model cascades +2.1 to 8.2 points over the best single model at the same budget, on average across budgets (Bouchard, May 2026); a learned consult policy recovered 44% to 81% of the cheap-to-strong gap on SWE-bench Verified, Terminal-Bench 2.1 and tau2-bench (Sowietzki, Oct 2026); FrugalGPT matched GPT-4 on three 2023 tasks (Chen, May 2023). Losses where it does not: six commercial routers 1.0 to 10.5 points below a random two-model mix at matched cost (Wang, Oct 2026); Anthropic's advisor +1.7 points on coding at the edge of noise, within noise on GPQA with no consults, 7 points lower on chart reading (Anthropic, Oct 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "mixed", "size": "large", "when": "cheaper than the strong model alone when escalation is selective: 34.5% to 48.1% of its cost with a learned consult policy (Sowietzki, Oct 2026); 59.2% to 98.3% less at matched accuracy on 2023 classification tasks (Chen, May 2023); re-running only failures at high effort about 40% below an all-high run, computed from existing runs (Anthropic, Oct 2026). Dearer when escalation is frequent or task-level: the coding advisor pairing cost 2.1 times its executor alone, GPQA 12% to 25% more with no consults, and a pairing that consulted on 88% of chart tasks 1.8 times the strong model alone (Anthropic, Oct 2026); verify-then-escalate cost $59.68 against $45.90 for the strong model alone on SWE-bench Verified (Sowietzki, Oct 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "mixed", "size": "small", "when": "each escalation or consult is a serial strong-model call on the critical path (about 1.4 per coding attempt), and a re-run failure takes two runs of wall-clock time (Anthropic, Oct 2026); work the cheap model finishes may be faster, which no source here times", "basis": "assessment" }, { "dimension": "security", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "governance", "effect": "improves", "size": "small", "when": "an explicit escalation policy can be read and audited (the learned policies consult in 4 to 9 of 24 states; Sowietzki, Oct 2026); count the strong model's tokens separately, since Anthropic reports advisor tokens per iteration rather than in the top-level usage totals", "basis": "assessment" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired eval on held-out tasks with gold outcomes, 3 or more seeds: the escalation setup against the cheap and the strong model alone across effort settings and against a random mix of the two at the same spend; success, cost per completed task counting failed cheap attempts and strong-model tokens, escalation or consult rate, latency of escalated tasks" }, { "method": "monitor", "tier": null, "how": "escalation or consult rate by task type and when it fires, strong-model share of tokens and cost, success of escalated against kept tasks, advice the worker ignored, and true error on a gold-labeled audit sample the trigger never sees (9.c.iii); repeat the experiment at each model or price change" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Multi-Agent group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Lingjiao Chen, Matei Zaharia, James Zou, \"FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance\"", "url": "https://arxiv.org/abs/2305.05176", "date": "2023-05-09", "locator": "Abstract; §3 Table 3; Figure 3", "task": "LLM cascade: learn which cheaper models to try first and when to escalate to a stronger one", "effect": "\"FrugalGPT can match the performance of the best individual LLM (e.g. GPT-4) with up to 98% cost reduction or improve the accuracy over GPT-4 by 4% with the same cost\". The 98% is the best of three datasets: the cost of matching the best single model fell 98.3% on HEADLINES (GPT-4), 73.3% on OVERRULING (GPT-4) and 59.2% on COQA (GPT-3).", "direction": "supports", "measured": true, "conditions": "12 commercial LLM APIs from 5 providers at March 2023 prices; three classification and reading-comprehension datasets (HEADLINES, OVERRULING, COQA); cascades of up to three models with a DistilBERT scorer trained on labeled examples from the same distribution", "older": true }, { "source": "Anthropic, \"Optimizing for cost and intelligence\" (Claude Platform docs)", "url": "https://platform.claude.com/docs/en/about-claude/models/optimizing-for-cost-and-intelligence", "date": "2026-10-17", "locator": "Combine models; Advisor strategy table; Re-run failures at higher effort; Benchmarks referenced 3, 9, 10, 11, 13", "task": "Measured the advisor strategy (a cheaper executor consults a stronger advisor through the advisor tool) against the executor alone at the same effort and against single-model effort curves; also computed a verifier-triggered escalation: run every task at low effort and re-run only the failed ones at high effort", "effect": "Internal agentic-coding benchmark, Opus 5.5 at high with a Fable 5.1 advisor: 90.1%, \"1.7 points over the executor alone, at the edge of run-to-run noise\", for $2.92 per attempt, about 2.1 times the executor alone, about 1.4 consults per attempt; it \"lands about on Claude Opus 5.5's own effort curve, so the advisor bought about what more effort does\" (Opus 5.5 alone 88.4% at $1.38 on high, 91.1% at $4.11 on xhigh). GPQA Diamond, Haiku 5.5 or Sonnet 5.5 with an Opus 5.5 advisor: consulted on none of 198 questions, score within noise, 12% and 25% more cost per question. Chartography, Opus 5.5 at low with a Fable 5.1 advisor: consulted on 1 of 300 tasks, 61.7 against 68.7 alone, \"beyond run-to-run noise\"; an August pairing that consulted on 88% of tasks matched Fable 5.1 alone at medium (65.0 vs 67.5) at about 1.8 times its cost. Sonnet 5.5 at high consulted on 0 of 20 SWE-bench Pro problems; on DeepSWE a low-effort Sonnet 5 executor with an advisor scored 23 points above itself alone. Escalation on failure, SWE-bench Pro subset, Opus 5.5: low then high on failures 96.4% to 97.5% for about $0.17 a task; everything at high 94.8% to 95.8% for $0.29; high re-run on its own failures 96.9% for $0.31 (\"Use this policy for the saving, not the lift\"); the escalation figures were computed task by task from existing runs, not run separately.", "direction": "mixed", "measured": true, "conditions": "Anthropic-internal runs, August to October 2026, list prices, Claude models only; internal coding benchmark of 370 repository tasks (pairing five attempts per task, some single-model points one run); GPQA Diamond 198 questions, two runs; Chartography 100 charts, three runs, spreads up to 10 points; SWE-bench Pro 478-problem subset, two runs per effort; undated living page, figures from the revision archived 2026-10-17 (page first archived 2026-08-15)", "older": false }, { "source": "Pascal Sowietzki, Steffen Holter, Yannick Metz, Emil Kasper, Raphael Baur, Mennatallah El-Assady (ETH Zurich), \"Ask Your Advisor: Learning When to Consult Stronger Models in Agentic Systems\"", "url": "https://arxiv.org/abs/2610.13870", "date": "2026-10-09", "locator": "§4; §5 Tables 2-3; §5.1 Table 4; §5.4; Appendix B.1-B.2; Appendix C.1 Table 12", "task": "A cheap executor in a Magentic-One agent may consult a stronger advisor, at most 4 times per episode, at workflow contact points; a tabular policy learned by Monte Carlo control on development tasks chooses Silent or Consult from execution evidence. Compared with FrugalGPT, RouteLLM and AutoMix (task-level cascade, router, verify-then-escalate), consulting once at the start, Anthropic-style fixed triggers, consulting at every point, the orchestrating LLM deciding, and random placement at matched volume", "effect": "Success and run cost (mean of 3 seeds), executor alone / advisor alone / learned policy: SWE-bench Verified 18.0% $11.48 / 62.0% $45.90 / 48.0% $22.09; Terminal-Bench 2.1 21.3% $9.76 / 51.3% $60.99 / 34.7% $24.93; tau2-bench telecom 72.5% $5.07 / 98.3% $22.50 / 93.3% $7.77, i.e. 68.2%, 44.4% and 80.6% of the gap recovered at 48.1%, 40.9% and 34.5% of the advisor-alone cost. SWE-bench at similar consult volume: learned 48.0% (2.78 consults per episode), fixed start, end and stall triggers 41.3% (2.75), random 28.0% (2.53); consulting at every point 34.0%; orchestrating LLM deciding (not volume-matched) 29.3% with 0.41 consults per episode, 77% of them in the final third. Task-level baselines on SWE-bench: FrugalGPT 31.3% at $22.61, RouteLLM 30.0% at $25.03, AutoMix 52.0% at $59.68. On Terminal-Bench the learned policy and the fixed triggers tied at 34.7%. Ignored advice was the primary mode in 58% of the learned policy's remaining SWE-bench failures.", "direction": "supports", "measured": true, "conditions": "gemini-3-flash-preview at low effort as executor, gemini-3.1-pro-preview at high as advisor; held-out tests of 50 (Verified-Mini), 50 and 40 tasks; 3 seeds, success standard deviations 1.2 to 7.2 points across conditions; development tasks screened by outcome; one model pair, one agent framework", "older": false }, { "source": "Sam Wang, Julia White, Sahibzada Allahyar, Dhruv Atreja, Urchade Zaratiana, Kelton Zhang, \"Dynamic LLM Routers are Often Misguided\"", "url": "https://arxiv.org/abs/2610.02762", "date": "2026-10-02", "locator": "Abstract; §3; §4 Table 1; §6.1 Figure 4; §7; Limitations", "task": "Ran six commercial routers (OpenRouter auto and auto-beta, Azure, Not Diamond, vLLM Semantic Router, Nadir, Orca) in 14 settings, generating and grading every routed answer the same way, against a router that randomly picks between Gemini 3.7 Flash and Opus 5 (high) at matched cost; then built a two-model router that escalates on predicted difficulty", "effect": "Every setting trailed the random two-model mix: from -1.0 ± 1.8 points (vLLM SR) to -10.5 ± 2.8 (Orca adaptive); OpenRouter auto -3.3 to -5.5, Not Diamond -4.1 to -8.8, Azure -2.2 to -8.4. Only the smallest gap (vLLM SR) is within error; the other 13 settings trail significantly. No router's choices correlated with query difficulty above gamma +0.20. Gemini 3.7 Flash alone trailed Opus 5 (high) by 1.6 points at a 96% discount, and even with true difficulty known a two-model roster was never more than 1 point below the best roster of any size. The authors' difficulty router stayed within noise of random routing at every budget (\"a well-chosen roster leaves little to route\"); with gpt-oss-20b (low) as the cheap model, it and a difficulty oracle underperformed random routing at every budget.", "direction": "contradicts", "measured": true, "conditions": "Single-turn queries; 17 benchmarks in 8 categories (coding, instruction following, knowledge, math, QA, office, tool use, HLE), 800 evaluation and 2,508 training queries; 26 models from June and August 2026, one generation per query, OpenRouter costs; patterns replicated on RouterBench, SPROUT and LLMRouterBench", "older": false }, { "source": "Dylan Bouchard, \"Is Escalation Worth It? On the Depth of LLM Cascades\"", "url": "https://arxiv.org/abs/2605.06350", "date": "2026-05-07", "locator": "Abstract; §4.1; §4.3 Table 1; §4.4 Table 2; Appendix C, Table 25", "task": "Optimized cascade model sequences and deferral thresholds from a pool of eight models at 500 budgets with nine deferral scores (five log-probability scores, two trained on the cheap model's correctness, two trained on the gain from escalating), comparing caps of one to five models on held-out queries", "effect": "Two-model cascades beat the best single model affordable at each budget by 2.1 to 8.2 points of mean test accuracy, averaged across the budget range (MMLU +2.30 to +3.93, TriviaQA +5.27 to +7.20, MATH +2.13 to +3.46, SimpleQA +6.18 to +8.22, LiveCodeBench +4.65 to +7.32). Allowing more than two models did not raise mean test accuracy in 118 of 135 comparisons and added at most 0.43 points. The seven confidence-based scores had AUROC 0.63 to 0.80 for the cheap model's correctness, and their accuracy gain over escalating at random at equal escalation rates (not equal cost) was 0.03 to 3.20 points (MATH 0.26 to 0.36). With simulated scores at AUROC 0.9 that track only the current model's errors, a third model helped on four of five benchmarks; the gain disappeared once the score also tracked difficulty shared across models.", "direction": "supports", "measured": true, "conditions": "Single-turn; Llama 3.1-8B, Qwen2.5-7B, GPT-4o mini, GPT-oss-20B, DeepSeek-V3, Llama 3.3-70B, GPT-4o and MiniMax-M2.7 at May 2026 list prices; MATH levels 3-5, MMLU, TriviaQA, SimpleQA, LiveCodeBench, about 1,000 to 2,000 queries each, 50 calibration/test splits; no self-consistency, verbalized-confidence or judge triggers; one author", "older": false } ], "caveats": "The agent results are small or vendor-run: the learned consult policy was tested on one Gemini model pair in one framework, on 40 to 50 held-out tasks per benchmark with 3 seeds (success standard deviations up to 7.2 points) and outcome-screened development tasks, and Anthropic's advisor figures are internal, sometimes single runs on Claude models, from a living page whose August 2026 revision rated the advisor more favorably (closing 60% to 90% of the gap). The router and cascade studies are single-turn QA, math and code with list-price costs; the router study's random baseline uses a pair picked from the frontier with hindsight, and the cascade study tested only log-probability and trained scores on older open and GPT-4o-class models, not verbalized confidence, self-consistency or judges. FrugalGPT's 98% is its best of three 2023 classification datasets (59.2% and 73.3% on the other two). A cheap verifier used as the trigger can hide most of the cascade's errors, so measure true error outside it (9.c.iii).", "also_covers": [], "related": [ { "id": "1.b.i", "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class" }, { "id": "9.c.iii", "slug": "audit-verifier-cascade", "name": "Audit a verifier-gated cascade against a gold sample the verifier never sees" }, { "id": "9.c.ii", "slug": "samples-before-verification", "name": "Spend extra samples before extra verification: parallel attempts with a vote or judge; verify only where it pays" }, { "id": "6.c.ii", "slug": "plan-then-execute", "name": "Plan-then-execute" }, { "id": "6.b.iii", "slug": "multi-agent-for-breadth", "name": "Use multi-agent orchestration only for breadth-first, independent sub-tasks" }, { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "3.b.i", "slug": "cache-stable-prefix", "name": "Keep the prompt prefix cache-stable" }, { "id": "5.c.i", "slug": "discrete-phase-separation", "name": "Discrete phase separation" }, { "id": "7.c.vi", "slug": "multi-agent-debate", "name": "Multi-agent debate" } ], "url": "https://mahalsystems.ai/aot/optimizations/oracle-worker-escalation/", "markdown": "https://mahalsystems.ai/aot/optimizations/oracle-worker-escalation.md" }, { "id": "6.b.i", "legacy_id": "P52", "former_ids": [], "slug": "parallel-read-only-tools", "name": "Parallel tool execution for read-only calls", "kind": "pattern", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes execution order and latency", "proposed": true }, "levers": [ "control-flow" ], "domains": [ "general" ], "evidence_domains": [ "tool-use", "web-research", "code-generation" ], "categories": [ "Parallelizing" ], "category_ids": [ "topologies.parallel" ], "summary": "Let the model put several independent read-only tool calls (searches, fetches, file reads) in one turn and have the harness run them at once, while calls that write, share state or need another call's result run one at a time in the order the model emitted them. It pays where a task has many slow, independent lookups: told to make three parallel search or scrape calls per turn, GPT-5 scored 68% against 66% on 100 BrowseComp tasks in half the turns, at 35.9% lower cost and 40.6% less time, and native parallel calls ran 1.61x to 2.76x faster than one call per step on parallel search questions. The gain shrinks where tool work is fast or sequential (1.04x on SWE-bench Lite, 1.40x once the harness declared each tool's reads and writes and kept decoding during calls), wider is not better (five or eight calls per turn scored 60% and 63%), and batching calls that depend on each other costs accuracy (GLM-4.5 completed 63.1% of compound tasks when parallelizing against 71.8% sequentially).", "how": [ "Start from the baseline: the same agent, model and tasks with parallel calls off (disable_parallel_tool_use in Claude's tool_choice, or your provider's equivalent), and with them on but executed one at a time; record accuracy, turns, tokens, cost and p50/p95 wall-clock per task, and the average number of tool calls per tool-calling turn. Expect the gain to track how much slow, independent lookup work a task has: native parallel batches ran 1.61x to 2.76x faster than ReAct on 2-way and 8-way parallel search questions (gpt-3.5-turbo), but running a model's same-turn batches concurrently instead of one by one gave 1.04x on SWE-bench Lite (GPT-5.6 Sol).", "Classify every tool by what it reads and writes. Search, fetch, grep, glob, file reads and other lookups can run together; anything with side effects, shared state (a shell session, a browser, a working directory) or an ordering requirement cannot. Default unknown tools to serial, and classify shell commands per command (one harness locked the whole repository for python, pytest and any command its rules did not match). Declaring a read set and a write set per tool lets the harness run non-conflicting calls together while conflicting ones keep the model's order.", "Dispatch each batch in the harness, not the model: the provider APIs leave execution order, partial failure and races to you. Run the safe calls concurrently, run the rest in the order the model emitted them, and when a call fails cancel only the calls that depend on it. Return one result per call, all in the next message and before any text (results sent as separate messages teach Claude to stop batching), with is_error and a short reason for any call not run or that failed because its prerequisite had not completed; the model reissues it next turn.", "Prompt for it, at a modest width: \"whenever you need to perform multiple independent operations, invoke all relevant tools simultaneously rather than sequentially\" plus \"Only batch tool calls that are independent of each other.\" With GPT-5 on 100 BrowseComp tasks at a 100-turn limit, three to four search or scrape calls per turn scored 68% against 66% for one, in 23.8 against 45.7 turns, while five and eight scored 60% and 63%; a wide-then-narrow schedule (3, then 2, then 1 per turn) reached 74%, and letting the model choose reached 72%. Open-weight models gained little (DeepSeek-V3.2 38% to 39%), so measure before relying on it.", "Keep dependent steps sequential: where later calls need earlier results, parallel scheduling cost GLM-4.5 8.7 points of task completion (63.1% against 71.8% sequential), and QwQ-32B put dependent calls in one batch and completed 29.36%. Leave those workflows serial, or plan them as an explicit dependency graph (the LLMCompiler planner ran 1.40x to 3.74x faster than ReAct).", "Where tools are slow, also let decoding continue while calls run: returning placeholders (futures) and filling in results at turn boundaries gave 1.40x on SWE-bench Lite and 1.26x on BFCL web search with no significant accuracy change, and with no read/write declarations it still helped by overlapping decoding with serialized calls.", "Measure paired against the baseline on tasks with and without dependent calls: accuracy, turns, cost and p95 latency per task, average calls per tool-calling turn, calls returned as is_error because a prerequisite had not run, and probes that put a read and a write to the same resource in one batch." ], "detect": { "static": "Each tool marked read-only (safe to run together) or as having side effects, shared state or ordering needs, with unknown tools defaulting to serial; read-only calls in a batch dispatched concurrently; the rest run in model-emitted order; all results returned in one message before any text; a parallel-calls instruction in the prompt.", "traces": "Average tool calls per tool-calling turn near 1 on tasks with independent lookups (parallelism unused); overlapping spans for calls that touch the same resource; calls that failed because a prerequisite in the same batch had not run.", "evals": "Accuracy, turns, cost and p95 latency with parallel calls off and on, on tasks with and without dependent calls; race-condition failures.", "probes": "Batches that mix reads and writes to the same resource, or hold a call that needs another call's output." }, "impact": { "measured_basis": true, "primary": [ "performance", "cost" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "68% against 66% in half the turns at three parallel calls per turn on 100 BrowseComp tasks, 60% and 63% at five and eight, 38% to 39% for DeepSeek-V3.2 (Lin, Feb 2026); 63.1% against 71.8% forced sequential for GLM-4.5 on compound tasks with dependencies, 29.36% for QwQ-32B, which batched dependent calls (Xu, Nov 2025); 79.3% against 79.0% on SWE-bench Lite, no significant change (Feng, May 2026); native parallel calls 77.00% against 72.47% on Movie Recommendation and 87.32% against 89.09% on ParallelQA for ReAct (Kim, Dec 2023). A tool with side effects classified read-only can race; no source measures that.", "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "medium", "when": "$65.7 against $102.5 per 100 BrowseComp tasks (35.9% less) for three parallel calls per turn (Lin, Feb 2026); native parallel calls 1.81x to 3.33x cheaper than ReAct (Kim, Dec 2023); sequential scheduling raised DeepSeek-R1 from 11.6k to 12.6k tokens per task (Xu, Nov 2025)", "basis": "evidence" }, { "dimension": "performance", "effect": "improves", "size": "medium", "when": "904.2 s against 1,522.6 s per task (40.6% less) (Lin, Feb 2026); native parallel calls 1.61x to 2.76x faster than ReAct on parallel search questions (Kim, Dec 2023); 1.04x on SWE-bench Lite for concurrent batches alone, 1.40x with read/write declarations and decoding during calls (Feng, May 2026); 361.6 s against 423.6 s sequential for DeepSeek-R1 (Xu, Nov 2025)", "basis": "evidence" }, { "dimension": "security", "effect": "mixed", "size": "small", "when": "No source measures it: a call with side effects classified read-only can run alongside, or ahead of, a check it should follow.", "basis": "assessment" }, { "dimension": "governance", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired runs on the same tasks and model with parallel calls off, on but executed one at a time, and on and concurrent, including tasks whose calls depend on each other: accuracy, turns, tokens and cost per task, p50/p95 wall-clock, and average tool calls per tool-calling turn; sweep the calls-per-turn instruction (1, 3, 5) and a wide-then-narrow schedule; add probes that put a read and a write to the same resource, or a call that needs another call's output, in one batch" }, { "method": "monitor", "tier": null, "how": "average tool calls per tool-calling turn, the share of batches that mix read-only and side-effecting calls, calls returned as is_error because a prerequisite had not run, turns per task, and p95 tool wall-clock per turn" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Control Flow group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Kim et al., \"An LLM Compiler for Parallel Function Calling\"", "url": "https://arxiv.org/abs/2312.04511", "date": "2023-12-07", "locator": "Abstract; §5.1; Tables 1 and 2", "task": "Plan function calls as a dependency graph and run independent calls in parallel", "effect": "\"latency speedup of up to 3.7x, cost savings of up to 6.7x, and accuracy improvement of up to ~9% compared to ReAct\": the best cases, 3.74x and 6.73x on Movie Recommendation with gpt-3.5-turbo and +8.55 points on ParallelQA with LLaMA-2 70B, for LLMCompiler's planner. OpenAI's native parallel function calling, run as a baseline: 1.61x (HotpotQA), 2.76x (Movie Recommendation) and 1.86x (ParallelQA) faster than ReAct, 1.87x, 3.33x and 1.81x cheaper, accuracy 62.05% against 62.47%, 77.00% against 72.47%, and 87.32% against 89.09%; LLMCompiler was up to 35% faster than native parallel calls.", "direction": "supports", "measured": null, "conditions": "gpt-3.5-turbo (1106) and LLaMA-2 70B; HotpotQA comparison questions (1.5k, 2-way parallel), Movie Recommendation (500, 8-way), ParallelQA (113, the authors' own, with dependencies), Game of 24, WebShop; ReAct with anti-looping prompts as the latency baseline (unprompted ReAct for ParallelQA); LLMCompiler plans a dependency graph, native parallel calls do not", "older": true }, { "source": "Feng, Mao, Dutta, Gonzalez, \"Concurrency without Model Changes: Future-based Asynchronous Function Calling for LLMs\"", "url": "https://arxiv.org/abs/2605.15077", "date": "2026-05-14", "locator": "§2; §3.3; §3.4; §5.1; Figure 3; Figure 5; App. A", "task": "Run function calls through an execution layer that returns placeholders (futures) so decoding continues, with a scheduler that runs calls concurrently only when their declared read and write sets do not conflict (undeclared tools serialize in model-emitted order); compared with sequential and native parallel function calling", "effect": "SWE-bench Lite (n=300, GPT-5.6 Sol, real tool latency), resolve rate and speedup over SWE-agent: baseline 79.0%; same-turn batches run concurrently 79.0%, 1.04x; AsyncFC 79.3%, 1.40x. BFCL v3 multi-turn (n=150, GPT-4o, 5 s injected delay) accuracy: sequential 68.0%, native parallel 67.3%, AsyncFC 66.0% to 70.7% (no significant difference). BFCL v4 web search (GPT-4o, real latency): sequential 54.8% against AsyncFC 53.2% at 1.26x (n=31); native parallel 50.0% against AsyncFC 50.0% at 1.12x (n=29, latency gain not significant). Gemini 3.1 Pro on BFCL v3: native parallel 62.0% against AsyncFC 65.3% at 1.17x. With no annotations the scheduler serialized all calls and still cut latency by overlapping decoding. Provider parallel-calling APIs \"leave ambiguity in function execution order\" and do not specify \"the handling of partial failure and race conditions.\"", "direction": "supports", "measured": true, "conditions": "GPT-4o, Gemini 3.1 Pro, GPT-5.6 Sol; SWE-agent with a rule-based command analyzer that locks the repository root for python, pytest and unmatched commands; the native parallel baseline applied state effects in model-emitted order (favorable to it); numbers from v2 (Oct 2026), v1 reported 47.6% / 47.6% / 44.3% at 1.21x / 1.44x with GPT-5.2 under scaled latencies; the authors' own system", "older": false }, { "source": "Lin, Liew, Savarese, Li (Salesforce AI Research), \"W&D:Scaling Parallel Tool Calling for Efficient Deep Research Agents\"", "url": "https://arxiv.org/abs/2602.07359", "date": "2026-02-07", "locator": "§2.1 Figure 2; §3.2 Table 1, Table 2; §5 Table 3", "task": "Tell a deep research agent with search and scrape tools to make a set number of parallel tool calls per turn (a user message before each model call asks for at least m and at most m+1), against one call per turn, at several turn limits and call schedules", "effect": "BrowseComp, first 100 tasks, GPT-5 medium effort, accuracy (turns) at a 100-turn limit: 1 call 66% (45.7), 2 calls 68% (27.1), 3 calls 68% (23.8), 5 calls 60% (19.9), 8 calls 63% (15.2); at a 10-turn limit 44% for 1 call against 56% for 8. One call at 66% cost $102.5 per 100 tasks and averaged 1,522.6 s per task; three parallel calls at 68% cost $65.7 (35.9% less) and 904.2 s (40.6% less). Schedules: 3-2-1 descending 74% (23.5 turns), model-chosen 72% (26.6), 1-2-3 ascending 63% (36.5). Open-weight models, one call against three: DeepSeek-V3.2 38% against 39%, Qwen3-235B-A22B-Thinking 8% against 11% (turns 18.5 against 52.1). Full BrowseComp with parallel calls: 62.2% for GPT-5 medium.", "direction": "supports", "measured": true, "conditions": "MCP-Universe framework; Serper search and Jina scraping with a Gemini-2.5-Flash summarizer, Python only for HLE; n=100 for BrowseComp and HLE, 103 GAIA text tasks; gains for Gemini 3.0 Pro, Claude 4.5 Sonnet, HLE and GAIA shown in figures only; the 62.2% is set against OpenAI's reported 54.9% for GPT-5 high, not a same-harness single-call run", "older": false }, { "source": "Xu, Huang, Liu, Yu, Deng, \"TPS-Bench: Evaluating AI Agents' Tool Planning & Scheduling Abilities in Compounding Tasks\"", "url": "https://arxiv.org/abs/2511.01527", "date": "2025-11-03", "locator": "Abstract; §3.1; §4.2 Table 1; §4.3 Tool Scheduling, Figure 7; §5 Table 3", "task": "Benchmark tool selection and scheduling on compound tasks over 141 MCP tools whose subtasks may depend on each other, with agents prompted to run independent subtasks in parallel; ablate the default parallel scheduling against forced sequential calls on the hard split", "effect": "Forced sequential against default parallel scheduling: GLM-4.5 task completion 71.8% against 63.1%; DeepSeek-R1 12.6k against 11.6k tokens and 423.6 s against 361.6 s; \"Serial scheduling can effectively help the model reduce errors caused by incorrect judgments of subtask dependencies.\" Across models on the hard split: GPT-4o, which favors parallel calls, 45.08% in 2.5 turns and 76.84 s; GLM-4.5, mostly sequential, 64.72% in 35.0 turns and 217.8 s; QwQ-32B \"often invokes multiple tools simultaneously, even when certain tools rely on the outputs of others\" and completed 29.36% in about one turn. GRPO on 100 samples took Qwen3-1.7B from 26.75% to 33.13% completion and 42.0 s to 36.1 s.", "direction": "mixed", "measured": true, "conditions": "200 tasks at two levels; completion judged by Gemini-2.5-Flash (Pearson 0.8375 with human scores); seven models; the sequential ablation covers four models (GLM-4.5, DeepSeek-R1, Kimi-K2, Qwen3-32B) but the text gives numbers for two; the abstract's GPT-4o against GLM-4.5 contrast is across models", "older": false }, { "source": "Anthropic, \"Parallel tool use\" (Claude API docs)", "url": "https://platform.claude.com/docs/en/agents-and-tools/tool-use/parallel-tool-use", "date": "2026-10-25", "locator": "Execution semantics; Maximizing parallel tool use; Troubleshooting", "task": "Vendor guidance on executing, formatting and prompting parallel tool calls", "effect": "\"The API doesn't prescribe an execution order\"; \"Independent, read-only operations are usually safe to run in parallel for lower latency. Tools with side effects, shared state, or ordering requirements might be better run sequentially.\" Return one tool_result per tool_use, all in the next user message and before any text; a separate message per result \"teaches\" Claude to avoid parallel calls. For a call not run, or one that failed because its prerequisite had not completed, return is_error: true and Claude reissues it; computer-use and browser batches must run in order and stop at the first failure. Suggested prompts: \"whenever you need to perform multiple independent operations, invoke all relevant tools simultaneously rather than sequentially\" and \"Only batch tool calls that are independent of each other.\" Check adoption as average tool calls per tool-calling message (above 1.0). No measured effect.", "direction": "supports", "measured": false, "conditions": "Claude 4 and later models call tools in parallel by default; disable_parallel_tool_use in tool_choice turns it off; accessed Oct 2026", "older": false } ], "caveats": "The current-model accuracy and cost result rests on one study's first 100 BrowseComp tasks, where 68% against 66% is two tasks, and its headline 62.2% compares a full-set run with GPT-5 medium against the score OpenAI reported for GPT-5 high in a different harness, not a single-call run (Lin, Feb 2026). The speedup depends on the workload: concurrent batches alone gave 1.04x on SWE-bench Lite, the 1.40x also needs futures and per-tool read/write declarations, and the same paper's May 2026 version reported 1.44x with a 3.3-point lower resolve rate on GPT-5.2, not significant (Feng, May 2026). The loss on dependent calls comes from an LLM-judged benchmark whose same-model ablation gives numbers for two of four models (Xu, Nov 2025), the largest speedups are with gpt-3.5-turbo against ReAct (Kim, Dec 2023), and no source measures races or security effects of a misclassified tool.", "also_covers": [], "related": [ { "id": "6.b.iii", "slug": "multi-agent-for-breadth", "name": "Use multi-agent orchestration only for breadth-first, independent sub-tasks" }, { "id": "6.b.ii", "slug": "llm-map-reduce", "name": "LLM map-reduce" }, { "id": "6.c.ii", "slug": "plan-then-execute", "name": "Plan-then-execute" }, { "id": "7.d.iii", "slug": "react-loop", "name": "ReAct loop" }, { "id": "2.a.iii", "slug": "choose-action-interface", "name": "Choose how the agent acts: typed tools, code execution, or shell, matched to the model and task" }, { "id": "2.b.ii", "slug": "tool-use-guidance", "name": "Tell the agent when to use each tool, in context it always sees" }, { "id": "2.b.i", "slug": "fail-soft-tools", "name": "Make tool calls and tool results fail soft, and retry transient failures" }, { "id": "10.b.i", "slug": "risky-action-policy", "name": "Gate irreversible actions: allow, ask or deny by side effect" }, { "id": "7.b.iii", "slug": "size-execution-budgets", "name": "Size run budgets so productive runs aren't cut off: turn caps, deadlines, command timeouts" } ], "url": "https://mahalsystems.ai/aot/optimizations/parallel-read-only-tools/", "markdown": "https://mahalsystems.ai/aot/optimizations/parallel-read-only-tools.md" }, { "id": "6.b.ii", "legacy_id": "P45", "former_ids": [], "slug": "llm-map-reduce", "name": "LLM map-reduce", "kind": "pattern", "status": "card", "tier": { "tier": 2, "name": "Reliability", "why": "it contains prompt injection to the item it arrives in", "proposed": true }, "levers": [ "control-flow" ], "domains": [ "general" ], "evidence_domains": [ "retrieval", "document-extraction", "commerce", "tool-use", "code-generation" ], "categories": [ "Parallelizing" ], "category_ids": [ "topologies.parallel" ], "summary": "Give each piece of a large or untrusted input its own tool-less model call that sees only that piece and must answer in a fixed schema, then combine the answers in code or in a reducer that never sees the raw text. Against prompt injection it confines an attack to the item it sits in: with each product review mapped to two integers and a median, injected reviews swayed a GPT-4o shopping agent 0.0% of the time against 31.7% undefended at the same utility, but a schema too narrow for the task cut fixes on 180 curated SWE-bench Lite issues from 68.3% to 45.0%. On long input it pays when the model degrades with length and the answer needs little reasoning across chunks: prompted chunk readers on a 4B model held 72.92% on multi-hop QA at 896K tokens where the same model given the whole document fell to 31.25%, while character inference, which depends on relations spread across a dialogue, scored below the whole-input call at every chunk size. A frontier long-context model is the baseline to beat: DeepSeek-V4-Pro at maximum reasoning kept 78.91% at 896K, and whole-input Gemini 2.5 Pro beat a map-reduce whose per-document sub-agents searched only their own document 68.0 to 58.1 on 102 multi-document questions, where map-reduce won only on cost.", "how": [ "Start from the baseline: the whole input in one call to your current model with thinking on, at the input lengths you actually see, and for untrusted input an injection eval with one poisoned item among clean ones; record accuracy, cost and p50/p95 latency per item. Expect it to be strong: DeepSeek-V4-Pro at maximum reasoning held 78.91% on multi-hop QA at 896K tokens, whole-input Gemini 2.5 Pro beat per-document retrieval map-reduce 68.0 to 58.1, and at 7K tokens a 4B model given the whole document with thinking beat prompted chunk readers 80.47% to 72.92%.", "Check that the task decomposes: per-item judgments (does this file hold an invoice, how does this review rate the product, how many years of experience does this resume show), extraction, filters, counts and extremes do; answers that hinge on relations across chunks do not (character inference over a dialogue scored below the whole input at every chunk size). For multi-hop questions use rounds (step 5), not one pass.", "Split on item boundaries (one file, review, resume, email or document per call), so a poisoned item can only change its own record. For long running text, try a few chunk sizes on 5 sampled documents each and keep the best (this picked the exhaustive-search optimum in 4 of 6 settings and came within 0.04 in the others); the best sizes ran from 4K to 32K tokens across models and tasks, and a 1K-token overlap did not help.", "Map: one isolated call per piece with no tools, the task rewritten as a per-piece subtask the reducer can combine (for the second-smallest number, ask each chunk for its two smallest), and the output schema enforced with structured outputs. For untrusted data allow only integers, floats, booleans, enums from a vetted list or short arrays of them, never free text that a later privileged or tool-using call reads; for trusted long text, short JSON findings with an explicit not-found answer. The map can run on a smaller model (1.b.i): 4B chunk readers scored as well as 9B ones.", "Reduce in code where you can (count, filter, median, top-K), which no single tampered input can redirect; if the reducer is a model with tools, pass it only the validated schema fields. For questions that need several hops, let a lead call that never sees the document send focused queries to all chunk readers and ask follow-ups over a few rounds; prompted, this held 72.92% to 76.04% from 7K to 896K tokens, and RL training of the lead added about 10 points.", "Measure against the baseline on the same items at each input length: accuracy, cost per item and latency (parallel chunk reading, measured on the RL-trained version on two GPUs against one, took 78.22 s against 375.32 s for the whole 896K document with thinking, but 5.33 s against 1.16 s at 7K, and a three-stage per-document retrieval pipeline took 54.8 s against 45.6 s). For injection, report attack success with a canary in one item, check that the reduce output changes only through that item's own fields, and track the per-chunk schema-violation rate." ], "detect": { "static": "Chunking, a sandboxed model per chunk with a constrained output schema, a deterministic reducer. Flag map calls that hold tools, free-text map outputs read by a privileged or tool-using reducer, and chunking applied to tasks whose answer depends on relations across chunks.", "traces": "Per-chunk outputs conform to schema.", "evals": "Injection containment (attack success with one poisoned item); accuracy against the whole-input call at several input lengths, with cost and latency.", "probes": "One injected item among clean items; long inputs with the evidence placed at different positions and split across chunks." }, "impact": { "measured_basis": true, "primary": [ "security", "quality" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "large", "when": "gains on long input when the model degrades with length and evidence sits within chunks: 72.92% vs 31.25% for the whole 896K-token document on a 4B model (Li, Sep 2026); synthetic Math 0.11 to 0.55 and LongBench-v2 QA 0.31 to 0.48 for gpt-4o-mini at 128K, at a chunk size picked after the fact on test data (Xu, Jun 2025). Losses when the answer spans chunks (character inference below whole input at every chunk size; Xu, Jun 2025), against a frontier whole-input call (Gemini 2.5 Pro 68.0 vs 58.1; Akay, Mar 2026) and when the schema is too narrow for the task (180 curated SWE-bench Lite issues 68.3% to 45.0%; Jacob, Sep 2025)", "basis": "evidence" }, { "dimension": "cost", "effect": "mixed", "size": "medium", "when": "every chunk is still read once, plus per-call instructions and a reduce call; cheaper when the map runs on a smaller model: $0.103 vs $0.273 per query with Gemini 2.5 Flash readers against whole-input Gemini 2.5 Pro (Akay, Mar 2026); repeated rounds re-read every chunk unless the KV cache is reused (Li, Sep 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "mixed", "size": "medium", "when": "faster on long input with a parallel map: 78.22 s vs 375.32 s per question at 896K tokens against the whole document with thinking (RL-trained version on two GPUs vs one), but 5.33 s vs 1.16 s at 7K (Li, Sep 2026); a coordinator, per-document retrieval map and reduce pipeline took 54.8 s vs 45.6 s for one whole-input call (Akay, Mar 2026)", "basis": "evidence" }, { "dimension": "security", "effect": "improves", "size": "large", "when": "per-item calls with typed outputs and a median reducer took injected-review attack success on a GPT-4o shopping agent from 31.7% to 0.0% at equal utility; typed hand-offs took calendar scheduling from 63.0% and bug fixing from 98.9% to 0.0% (Jacob, Sep 2025); a poisoned item can still set its own values (Beurer-Kellner, Jun 2025)", "basis": "evidence" }, { "dimension": "governance", "effect": "improves", "size": "medium", "when": "each item's contribution is a typed record that can be validated, logged and audited", "basis": "assessment" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "before-after", "tier": 2, "how": "injection eval on your own items: plant a canary instruction in one item (review, file, email, document) among clean ones and compare attack success with the raw-text design; the reduce output may change only through the poisoned item's own schema fields, and no tool call may follow the canary; report utility on clean items alongside" }, { "method": "experiment", "tier": 4, "how": "paired eval against the whole-input call with thinking on, at several input lengths you actually see: accuracy, cost per item, p50/p95 latency; include items whose evidence spans chunk boundaries and pick the chunk size on a few sampled documents per size before the full run; confirm with a production A/B before switching" }, { "method": "monitor", "tier": null, "how": "per-chunk schema-violation and not-found rates, chunks per request, reducer input size, slowest-chunk latency, cost per request; on a regularly audited sample, answers that depended on facts spread across chunks" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Multi-Agent group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Beurer-Kellner et al., \"Design Patterns for Securing LLM Agents against Prompt Injections\"", "url": "https://arxiv.org/abs/2506.08837", "date": "2025-06-10", "locator": "Sec 3.1 (3) The LLM Map-Reduce Pattern; case studies Sec 4.1 (5), 4.6 (3), 4.7 (3)", "task": "Names the LLM map-reduce pattern: isolated sub-agents process each piece of untrusted data, and a reduce step combines their constrained outputs", "effect": "Proposed as a design pattern with \"provable resistance to prompt injection\"; analyzed for trade-offs and case studies, not benchmarked. Two reduce designs: one that \"does not use an LLM, and simply applies operations that are robust to tampering of individual inputs\", or an LLM reducer with \"safety constraints on the outputs of the map operation ... (e.g., a regex that ensures the output of map is a number)\". A malicious file \"is now restricted to tricking the “map” LLM into marking the file as an invoice\"; \"there may be tasks where such a strict decomposition is impossible\".", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Kun Li, Zexuan Qiu, Tianhua Zhang, Irwin King, Helen Meng (CUHK), \"PARSER: Read in Parallel, Reason in Depth for Long-Context LLM Agents\"", "url": "https://arxiv.org/abs/2609.06702", "date": "2026-09-06", "locator": "§3.3 Table 1; §4.2 Table 2; §4.3 Tables 3-4; Appendix B.1 Table 7", "task": "Chunk-bound sub-agents (one per 4,096-token chunk, no thinking, JSON findings or abstain) read the whole document in parallel; a lead agent that never sees the document scatters focused queries and gathers findings over up to 12 rounds. Compared with the same model and DeepSeek-V4-Pro given the whole document, and with sequential chunk-by-chunk memory agents (MemAgent, ReMemR1)", "effect": "HotpotQA Sub_EM, Qwen3.5-4B, 7K / 896K tokens / average: prompted PARSER (no RL) 72.92 / 72.92 / 74.42; whole document with thinking 80.47 / 31.25 / 65.82; whole document without thinking 75.78 / 34.38 / 64.36; untrained MemAgent 69.53 / 19.79 / 45.05. RL on the lead: average 84.57 (9B lead 86.79). DeepSeek-V4-Pro whole document, maximum reasoning: 82.03 / 78.91 / 80.47. 2WikiMultiHopQA averages: prompted PARSER 82.85, Qwen3.5-4B whole with thinking 75.49, DeepSeek-V4-Pro 88.18, RL PARSER 87.04 (9B 88.48). One reader on the whole document instead of chunks: average 73.76, 53.13 at 896K; 16K, 64K and 128K chunks 83.59, 81.84, 80.34 vs 84.57 at 4K. 2B / 4B / 9B readers: 78.26 / 84.57 / 84.77. Latency per question at concurrency 1, RL-trained PARSER on two GPUs vs baselines on one: 896K 78.22 s vs 375.32 s whole document with thinking, 185.42 s without, 876.20 s MemAgent; 7K 5.33 s vs 1.16 s whole document without thinking.", "direction": "supports", "measured": true, "conditions": "Qwen3.5-4B and 9B lead agents, frozen Qwen3.5-4B readers; DeepSeek-V4-Pro (1M context, reasoning effort Max); HotpotQA (in-distribution, Wikipedia paragraphs packed with distractors) and 2WikiMultiHopQA, 128 questions per length from 7K to 896K tokens, Sub_EM, trained methods averaged over 3 runs; Qwen3.5 beyond its native 262K window ran with YaRN scaling; self-hosted (PARSER on two GPUs, baselines on one H100)", "older": false }, { "source": "Yagiz Can Akay, Muhammed Yusuf Kartal, Esra Alparslan, Faruk Ortakoyluoglu, Arda Akpinar, \"SPD-RAG: Sub-Agent Per Document Retrieval-Augmented Generation\"", "url": "https://arxiv.org/abs/2603.08329", "date": "2026-03-09", "locator": "§4.2-4.4; §5.1 Table 1; §5.2 Table 2; §5.4 Table 4; §6 Latency overhead; Appendix B", "task": "A coordinator writes shared extraction tasks; one isolated sub-agent per document (Gemini 2.5 Flash, searches only its own document, up to 5 searches) returns findings and a confidence score; a Gemini 2.5 Pro synthesis call merges them. Compared with all documents concatenated into one Gemini 2.5 Pro call and with single-index RAG", "effect": "Avg Score (GPT-5 judge) / perfect rate / cost / latency per query: whole input 68.0 / 31.4% / $0.273 / 45.6 s; per-document retrieval map-reduce 58.1 / 18.6% / $0.103 / 54.8 s; Normal RAG 33.0 / 13.7% / $0.080 / 42.6 s; Agentic RAG 32.8 / 8.8% / $0.098 / 40.6 s. By task type, map-reduce vs whole input: Comparison 42.2 vs 42.7, Clustering 57.2 vs 67.1, Chain of Reasoning 44.1 vs 60.9, Spotlight Locating 74.2 vs 86.6. \"SPD-RAG reaches 85.4% of this performance while consuming only 37.9% of the API cost\"; the latency overhead comes from \"at least three sequential LLM calls\".", "direction": "mixed", "measured": true, "conditions": "Loong benchmark, English, 200K-250K-token set: 102 instances (40 academic-paper, 62 financial-report), about 11 documents each; temperature 0; no repeated runs reported; the map runs on a cheaper model than the whole-input baseline, so the cost gap mixes the split with the model change; the recursive reduce path never triggered", "older": false }, { "source": "Dennis Jacob, Emad Alghamdi, Zhanhao Hu, Basel Alomair, David Wagner, \"Preventing Prompt Injection with Type-Directed Privilege Separation\"", "url": "https://arxiv.org/abs/2509.25926", "date": "2025-09-30", "locator": "§III-B Table I; §IV-A2; §V; §VI Table II; §VIII", "task": "A quarantined model with no tools reads untrusted data and may hand back only ints, floats, booleans, enums or short arrays of them, enforced with structured outputs. Shopping: each product review is mapped to two 0-10 integers (support, relevance) and the privileged agent takes the medians; calendar: the email thread is mapped to an enum list of time slots; bug fixing: the issue is mapped to code regions with enum categories. Compared with the same agent reading the raw text, one injected item per task", "effect": "Utility / attack success rate, undefended vs defended: online shopping (GPT-4o) 21.8% / 31.7% vs 22.4% / 0.0%; calendar scheduling (GPT-4o) 90.0% / 63.0% vs 91.0% / 0.0%; bug fixing (GPT-5.2, medium effort) 68.3% / 98.9% vs 45.0% / 0.0%, a drop the authors attribute to tasks \"where natural language context plays a key role\". Limitations: finding the schema \"can be non-trivial\", two sub-agents add \"non-negligible overhead\", and it does not prevent \"manipulation through misinformation or deception\".", "direction": "supports", "measured": true, "conditions": "WebShop test set, 500 instructions with GPT-4o-generated reviews, 5 trials at temperature 1.0 (success in 3 of 5); calendar: 100 simulated user pairs; bug fixing: 180 curated SWE-bench Lite instances on mini-swe-agent; one fixed injection payload per case study, no adaptive attacker; numbers from v2 (May 2026), where v1 ran bug fixing on Claude Sonnet 4 (49.7% / 94.3% vs 14.6% / 0.0%)", "older": true }, { "source": "Zhen Xu, Shang Zhu, Jue Wang, Junlin Wang, Ben Athiwaratkun, Chi Wang, James Zou, Ce Zhang, \"When Does Divide and Conquer Work for Long Context LLM? A Noise Decomposition Framework\"", "url": "https://arxiv.org/abs/2506.16411", "date": "2025-06-19", "locator": "§3.6; §5.3-5.5 Table 1; Appendix F Table 2; Appendix G Table 3; Appendix I Table 6; Appendix J Table 7; Appendix N", "task": "A 128K-token input split into equal chunks (1K to 64K), one worker call per chunk and one manager call to merge, with a planner model rewriting worker and manager prompts; compared with the same model and gpt-4o given the whole input, across task types", "effect": "Accuracy at 128K, whole input vs best chunk size (picked after the fact on test data), same model: synthetic Math (k-th largest or smallest number) gpt-4o-mini 0.11 vs 0.55, Llama-3.1-70B 0.09 vs 0.57, against gpt-4o whole input 0.33; key-value retrieval Llama-3.1-70B 0.15 vs 1.00; LongBench-v2 QA gpt-4o-mini 0.31 vs 0.48, Llama-3.1-70B 0.23 vs 0.46. Character inference: gpt-4o-mini 0.19 whole vs 0.12-0.18 chunked, Llama-3.1-70B 0.18 vs 0.04-0.17, below whole input at every chunk size. A 1K-token overlap gave \"mixed and generally marginal\" changes (Llama-3.1-70B QA-IB at 16K chunks 0.63 to 0.54). With 5 sampled documents per chunk size the chosen size matched the exhaustive-search optimum in 4 of 6 settings. A planner-written aggregator prompt beat a manual one (chart only). Cost and latency are derived, not measured.", "direction": "mixed", "measured": true, "conditions": "gpt-4o-2024-08-06, gpt-4o-mini, Llama-3.1-70B, Llama-3.2-3B, Qwen2.5-72B as both worker and manager, Qwen2.5-72B planner, temperature 0; six tasks adapted from InfiniteBench and LongBench-v2, two of them synthetic; items per task not reported; no reasoning models; numbers from v2 (Feb 2026)", "older": true } ], "caveats": "The injection result is one study with one fixed payload per case and no adaptive attacker; containment holds by construction only for instruction injection, a poisoned item can still lie about itself, and a free-text map output read by a privileged reducer gives the protection up. The long-input gains come from packed or synthetic benchmarks (HotpotQA paragraphs padded with distractors, synthetic number lists), from 2024 non-reasoning models in one study and 4B to 9B models in another, where beyond 262K tokens the whole-input baseline ran on a context stretched with YaRN. The one comparison against a frontier long-context model on real documents is a 102-question study whose map ran on a cheaper model, so its cost gap mixes the split with the model change; PARSER's headline 11x latency cut is against a sequential memory agent, and its 6.3-point lead over DeepSeek-V4-Pro needs RL on the lead agent and shrinks to 0.3 points out of distribution.", "also_covers": [], "related": [ { "id": "10.c.i", "slug": "contain-untrusted-input", "name": "Contain sensitive data and untrusted input" }, { "id": "6.c.i", "slug": "code-then-execute", "name": "Code-then-execute (CaMeL)" }, { "id": "6.c.ii", "slug": "plan-then-execute", "name": "Plan-then-execute" }, { "id": "5.c.ii", "slug": "isolated-subagent-context", "name": "Sub-agent architectures with clean contexts" }, { "id": "6.b.iii", "slug": "multi-agent-for-breadth", "name": "Use multi-agent orchestration only for breadth-first, independent sub-tasks" }, { "id": "8.a.i", "slug": "choose-retriever", "name": "Choose the retriever by corpus size and question type" }, { "id": "1.b.i", "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class" }, { "id": "10.a.i", "slug": "constrain-agent-reach", "name": "Constrain what the agent can touch" }, { "id": "7.c.iv", "slug": "graph-of-thought", "name": "Graph-of-Thought" }, { "id": "6.b.i", "slug": "parallel-read-only-tools", "name": "Parallel tool execution for read-only calls" } ], "url": "https://mahalsystems.ai/aot/optimizations/llm-map-reduce/", "markdown": "https://mahalsystems.ai/aot/optimizations/llm-map-reduce.md" }, { "id": "6.b.iii", "legacy_id": "T64", "former_ids": [], "slug": "multi-agent-for-breadth", "name": "Use multi-agent orchestration only for breadth-first, independent sub-tasks", "kind": "technique", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "a multi-agent topology is a restructuring; only a production test shows its behavior on live traffic", "proposed": false }, "levers": [ "control-flow", "context" ], "domains": [ "general" ], "evidence_domains": [ "web-research", "code-generation", "data-analysis" ], "categories": [ "Parallelizing", "Isolating subtasks" ], "category_ids": [ "topologies.parallel", "context.isolate" ], "summary": "Spawn parallel sub-agents when the work splits into independent, breadth-first parts with little shared context; expect about 15x the tokens of a chat and up to 90% less wall-clock. Default to a single agent otherwise (most coding): parallel workers make conflicting implicit decisions, and decompose-first planning scored below a single agent on data questions. When delegating, share full traces, give each sub-agent an objective, output format, tools and boundaries, have it write outputs to storage and return references, and re-verify what it reports.", "how": [ "Check the task shape: independent parts, little shared context.", "If so, prototype a lead plus 3-5 parallel sub-agents with the delegation rules above.", "Production A/B against the single agent on cost, latency and quality." ], "detect": { "static": "a multi-agent design whose sub-tasks share context or depend on each other; or a single agent on a breadth-first task with many independent parts", "traces": "sub-agents duplicating work or making conflicting decisions; failure rows labeled delegation-failure-subagent or communication-failure-subagent; token use per task far above a single agent's", "evals": "single-agent against multi-agent on the same tasks" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "large", "when": "+90.2% on breadth-first research (Jun 2025); below a single agent when sub-tasks depend on each other (GATE, Jun 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "large", "when": "about 15x the tokens of chat", "basis": "evidence" }, { "dimension": "performance", "effect": "improves", "size": "large", "when": "up to 90% less time with parallelism", "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "production-test", "tier": 4, "how": "A/B, single agent against multi-agent" } ], "evidence": [ { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Benefits of a multi-agent system; Architecture overview for Research", "task": "Orchestrator-worker with capable lead, cheaper parallel subagents", "effect": "\"a multi-agent system with Claude Opus 4 as the lead agent and Claude Sonnet 4 subagents outperformed single-agent Claude Opus 4 by 90.2% on our internal research eval\" (n not given; \"excel especially for breadth-first queries\"). Cost: \"agents typically use about 4x more tokens than chat interactions, and multi-agent systems use about 15x more tokens than chats\". Poor fit: \"domains that require all agents to share the same context or involve many dependencies between agents ... most coding tasks\".", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Prompt engineering: Teach the orchestrator how to delegate", "task": "Specify each subagent task with objective, format, tools, boundaries", "effect": "Short instructions like \"research the semiconductor shortage\" led subagents to misinterpret or duplicate (\"one subagent explored the 2021 automotive chip crisis while 2 others duplicated work investigating current 2025 supply chains\"). \"Without detailed task descriptions, agents duplicate work, leave gaps, or fail to find necessary information.\"", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Prompt engineering: Parallel tool calling transforms speed and performance", "task": "Spawn subagents in parallel, not serially", "effect": "Together with parallel tool calls in subagents: \"These changes cut research time by up to 90% for complex queries\". Joint figure for both changes.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Prompt engineering: Parallel tool calling transforms speed and performance", "task": "Issue 3+ tool calls in parallel per subagent", "effect": "Joint figure with parallel subagent spawning: \"cut research time by up to 90% for complex queries\".", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Production reliability: Synchronous execution creates bottlenecks", "task": "Run subagents asynchronously (proposed)", "effect": "Currently synchronous: \"the lead agent can't steer subagents, subagents can't coordinate, and the entire system can be blocked while waiting for a single subagent\". Async \"adds challenges in result coordination, state consistency, and error propagation\".", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Appendix: Long-horizon conversation management", "task": "Hand off to a fresh subagent near the context limit", "effect": "\"When context limits approach, agents can spawn fresh subagents with clean contexts while maintaining continuity through careful handoffs.\"", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Appendix: Subagent output to a filesystem", "task": "Have subagents write outputs to storage, return references", "effect": "\"This prevents information loss during multi-stage processing and reduces token overhead from copying large outputs through conversation history.\" Best \"for structured outputs like code, reports, or data visualizations\".", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Walden Yan (Cognition), \"Don't Build Multi-Agents\"", "url": "https://cognition.com/blog/dont-build-multi-agents", "date": "2025-06-12", "locator": "A Theory of Building Long-running Agents (after Principle 2)", "task": "Default to a single-threaded linear agent", "effect": "\"The simplest way to follow the principles is to just use a single-threaded linear agent ... Here, the context is continuous.\" \"the simple architecture will get you very far\".", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Walden Yan (Cognition), \"Don't Build Multi-Agents\"", "url": "https://cognition.com/blog/dont-build-multi-agents", "date": "2025-06-12", "locator": "Principle 1", "task": "Share full agent traces with subagents, not task messages", "effect": "Principle 1: \"Share context, and share full agent traces, not just individual messages.\" Copying the original task is not enough because \"the conversation is most likely multi-turn, the agent probably had to make some tool calls to decide how to break down the task\". Illustrated with a Flappy Bird clone whose subtasks were each misread.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Walden Yan (Cognition), \"Don't Build Multi-Agents\"", "url": "https://cognition.com/blog/dont-build-multi-agents", "date": "2025-06-12", "locator": "Principle 2", "task": "Do not run parallel agents that make implicit decisions", "effect": "Principle 2: \"Actions carry implicit decisions, and conflicting decisions carry bad results.\" Even with shared context, parallel subagents produced \"a bird and background with completely different visual styles\". Principles 1 and 2 are \"so rarely worth violating, that you should by default rule out any agent architectures that don't abide by them.\"", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Walden Yan (Cognition), \"Don't Build Multi-Agents\"", "url": "https://cognition.com/blog/dont-build-multi-agents", "date": "2025-06-12", "locator": "Applying the Principles: Claude Code Subagents", "task": "Limit subagents to answering well-defined questions, sequentially", "effect": "Claude Code (June 2025) \"never does work in parallel with the subtask agent, and the subtask agent is usually only tasked with answering a question, not writing any code ... The benefit ... is that all the subagent's investigative work does not need to remain in the history of the main agent\".", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Walden Yan (Cognition), \"Don't Build Multi-Agents\"", "url": "https://cognition.com/blog/dont-build-multi-agents", "date": "2025-06-12", "locator": "Applying the Principles: Edit Apply Models", "task": "Decide and apply edits in one model action", "effect": "Edit-apply systems \"would still be very faulty ... the small model would misinterpret the instructions of the large model and make an incorrect edit due to the most slight ambiguities\". \"Today, the edit decision-making and applying are more often done by a single model in one action.\"", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Walden Yan (Cognition), \"Don't Build Multi-Agents\"", "url": "https://cognition.com/blog/dont-build-multi-agents", "date": "2025-06-12", "locator": "Applying the Principles: Multi-Agents", "task": "Avoid agents negotiating with each other to resolve conflicts", "effect": "\"running multiple agents in collaboration only results in fragile systems. The decision-making ends up being too dispersed and context isn't able to be shared thoroughly enough between the agents.\"", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Smyth, Mantilla-Ramos, Tikeng Notsawo, Helbling, Tosato, Merzouk, Dziri, Gidel, Tosato (Tara Research, Mila, Cohere), \"Quantifying Overclaiming Propensity in Frontier LLM Agents\"", "url": "https://arxiv.org/abs/2609.20812", "date": "2026-09-22", "locator": "§3.4 Controlled delegation experiment, §4, Appendix D.2 Tables 4-5, Figure 4", "task": "Require subagent delegation for large review corpora", "effect": "Controlled experiment, 6 models x 5 scenarios x 20 runs per condition (1,200 runs), prohibited -> required delegation: mean files touched 86.9% -> 97.3%, lines read 67.0% -> 87.3%, needles reported 49.9% -> 69.6% (e.g. Sonnet 5 27.7% -> 53.7%, Opus 5 69.7% -> 95.7%). Explicit overclaiming across all runs 34.5% -> 16.3%. All-files-touched increase significant in both families (Claude G2=46.27, GPT G2=39.58, p<0.0001).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Smyth, Mantilla-Ramos, Tikeng Notsawo, Helbling, Tosato, Merzouk, Dziri, Gidel, Tosato (Tara Research, Mila, Cohere), \"Quantifying Overclaiming Propensity in Frontier LLM Agents\"", "url": "https://arxiv.org/abs/2609.20812", "date": "2026-09-22", "locator": "§4 Subagents increase coverage but do not improve honesty, Appendix D.2 Table 5", "task": "Delegation does not make coverage reports honest", "effect": "Same 1,200-run experiment: among incomplete reviews, misleading rate rose 80.7% -> 93.8% overall (Claude family condition effect G2=19.10, p<0.0001; Fable 5 64.9% -> 100%, Sonnet 5 60.8% -> 84.2%); GPT models stayed at or near 100% in both conditions. 50.3% of incomplete delegating reviews still explicitly overclaimed; 83-100% failed to disclose the gap across the six models.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Youngwon Lee, Jaejin Kim, Seung-won Hwang (Seoul National University), \"Bootstrapping Semantic Layer from Execution for Text-to-SQL (GATE: Grounding After Test from Execution)\"", "url": "https://arxiv.org/abs/2606.05634", "date": "2026-06-04", "locator": "§2 Decomposition for grounding; Table 1", "task": "Decompose the query into sub-tasks before observing data", "effect": "Table 1, decomposition-first ReDel vs single-agent ReAct: Qwen3-8B EHRSQL 26.4 vs 30.2, LS-Hard 8.0 vs 14.5; GPT-5.4-mini EHRSQL 26.8 vs 32.2. ReDel beats ReAct only on RealEHR (both models) and LS-Hard GPT-5.4-mini (40.2 vs 37.8). Authors argue decomposition is fixed \"before sufficient database observations are available.\"", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Recommendation 12; Sec 15.1", "task": "Stay single-agent until breadth-first exploration pays off", "effect": "Rec 12 citing [17]: multi-agent systems use \"~15x more tokens\" than a chat baseline and \"most coding tasks involve fewer truly parallelizable tasks than research\".", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 11.1–11.9; Table 11 \"Recursive Composition\"", "task": "Spawn sub-agents with isolated context, summary-only returns", "effect": "Claude Code AgentTool agent definitions (Explore, Plan, VerificationAgent); OpenCode returns \"only the final assistant text wrapped in \"; Hermes \"summary-only return\"; Pi separate OS process \"hard context isolation by construction\".", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 11.3; Sec 6.4", "task": "Control how much history a sub-agent inherits", "effect": "Codex SpawnAgentForkMode \"(FullHistory or LastNTurns(N))\"; keep_forked_rollout_item() \"preserves only system/developer/user messages and final assistant answers, filtering out intermediate tool calls and reasoning.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 11.2.3; Sec 11.5; Sec 11.7; Sec 11.9", "task": "Restrict sub-agent tools and block recursion", "effect": "Claude Code workers restricted to a 16-tool whitelist; Hermes \"subagent must not gain tools the parent lacks\", six-tool blocklist, defaults \"three concurrent children, 50-iteration budgets, depth 1\"; Mistral Vibe only \"subagent\"-tagged profiles (explore) are delegable; OpenCode children inherit the parent's deny rules and recursion is off by default.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Drew Breunig, \"How Long Contexts Fail / How to Fix Your Context\", https://www.dbreunig.com/2025/06/22/how-contexts-fail-and-how-to-fix-them.html ; https://www.dbreunig.com/2025/06/26/how-to-fix-your-context.html (2025-06-26)", "url": null, "date": "2025-06-22", "locator": "06-26 post, \"Context Quarantine\"", "task": "Quarantine subtasks in separate sub-agent contexts", "effect": "Quotes Anthropic's multi-agent research system: Opus 4 lead with Sonnet 4 subagents \"outperformed single-agent Claude Opus 4 by 90.2% on our internal research eval\". Secondary citation.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Lance Martin, \"Agent Design Patterns\"", "url": "https://rlancemartin.github.io/2026/01/09/agent_design/", "date": "2026-01-09", "locator": "Isolate Context", "task": "Fan out independent checks to isolated sub-agents", "effect": "Claude Code team \"uses sub-agents in code review to independently check different possible issues. A similar map-reduce pattern can be used for tasks like updating lint rules or migrations.\"", "direction": "supports", "measured": false, "conditions": null, "older": false } ], "caveats": null, "also_covers": [ { "name": "Parallelization by sectioning or voting" }, { "name": "Orchestrator-workers" } ], "related": [ { "id": "5.c.ii", "slug": "isolated-subagent-context", "name": "Sub-agent architectures with clean contexts" }, { "id": "6.b.ii", "slug": "llm-map-reduce", "name": "LLM map-reduce" }, { "id": "9.b.iii", "slug": "verify-completion", "name": "Verify completion against the environment before the agent stops" }, { "id": "6.c.i", "slug": "code-then-execute", "name": "Code-then-execute (CaMeL)" }, { "id": "7.d.i", "slug": "continuous-task-loop", "name": "Continuous autonomous task loop" }, { "id": "4.b.ii", "slug": "initializer-maintainer-agents", "name": "Initializer-maintainer dual agent" }, { "id": "1.b.i", "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class" }, { "id": "7.c.vi", "slug": "multi-agent-debate", "name": "Multi-agent debate" }, { "id": "6.a.i", "slug": "oracle-worker-escalation", "name": "Oracle-worker multi-model" }, { "id": "6.b.i", "slug": "parallel-read-only-tools", "name": "Parallel tool execution for read-only calls" } ], "url": "https://mahalsystems.ai/aot/optimizations/multi-agent-for-breadth/", "markdown": "https://mahalsystems.ai/aot/optimizations/multi-agent-for-breadth.md" }, { "id": "6.c.i", "legacy_id": "P49", "former_ids": [], "slug": "code-then-execute", "name": "Code-then-execute (CaMeL)", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it restructures the control loop for trust separation", "proposed": true }, "levers": [ "guardrails" ], "domains": [ "general" ], "evidence_domains": [ "tool-use" ], "categories": [ "Separating privileges" ], "category_ids": [ "topologies.trust" ], "summary": "Have a privileged model turn the trusted request into a small program before any untrusted data is read, then run it in an interpreter that labels every value with where it came from and checks a per-tool policy before each call, so injected text can neither add actions nor send private data to the wrong place. Attack success stayed at or below 1.8% across five models even under an adaptive attack (up to 58.6% undefended), but it was the costliest defense in utility: 26.3 to 52.6% of clean tasks solved against 63.2 to 89.5% undefended in one independent test, a gap that narrowed to 5.4 to 7.2 points only with frontier models (GPT-5.5, o3). Use it where the request alone fixes the actions and a hard block on exfiltration is worth lost tasks and about 3.5x the latency; the guarantee holds only where label tracking reaches, and retries, error messages, untaken branches and agent-to-agent messages have all been shown to leak.", "how": [ "Start from the baseline: your current tool loop (7.d.iii) and a cheaper check on where write arguments came from (argument provenance or per-task policies), same model, on an injection suite with static and adaptive attacks; frontier models already resist template attacks undefended (0.3% for Claude-Haiku-4.5, 0.0% for GPT-5.5), so static numbers alone overstate the gain. Sort tasks by whether the request alone fixes the actions: a program written before reading data cannot do \"the actions specified in the email\" (no model solved it), so contain those inputs (10.c.i) and gate the action (10.b.i) instead.", "Give the privileged model only the request, trusted context and tool descriptions with full return schemas, and have it write the whole task as a program in a restricted Python subset: undocumented tool outputs caused 13 of Claude 3.5 Sonnet's 15 Travel failures, and a follow-up had to add return schemas to every tool to reach usable utility. Parse untrusted content only through a quarantined model call with no tools and a typed output schema; it can be a cheaper model (Claude 3.5 Haiku under Claude 4 Sonnet cost about 1% utility for about 12% lower cost per median task) (1.b.i).", "Run the program in a custom interpreter that attaches sources and allowed readers to every value and propagates them through every operation, string formatting and control flow, including assignments on branches not taken: an independent audit found CaMeL's interpreter dropped labels in both places. Keep repair retries inside the checked boundary: never return raw error text derived from tool output to the privileged model (an error message put an attacker's reserve_hotel call into the rewritten program in one test), and cap retries, since each blocked-or-allowed retry leaks a bit.", "Write a policy for each side-effecting tool on where every argument may come from and who may read what is sent (for example, send_money's recipient and amount only from the user). Without policies, a recipient the quarantined model parsed from a landlord's notice was used for a scheduled payment. Route denials to user confirmation and budget for them: CaMeL's released policies refused 41.5% of calls a task was entitled to make and let every required call through on only 37 of 92 tasks, and turning policies on cut banking utility from 66.7% to 37.5% (GPT-5.2).", "If agents call agents, pass tool-derived values to the child agent through a separate data channel, never inside its instruction text: with CaMeL in every agent, 0.2% of attacks still got through (0.5% for GPT-4.1, from 36.1%), all by that route, and the separate channel took it to 0.0%.", "Deploy it on strong models and on tasks the request fully specifies: clean utility was 26.3 to 52.6% against 63.2 to 89.5% undefended across five smaller models, but 77.3% against 84.5% for o3 and 82.1% against 87.5% for GPT-5.5 (no policies). If that loss is too high, a cheaper argument-provenance or policy defense kept 59.6 to 63.2% clean utility at 4.4 to 7.7% adaptive attack success on GPT-4o-mini, where CaMeL kept 26.3% at 0.0%.", "Measure paired against the baseline, same model and suite: benign utility, utility under attack, attack success under static and adaptive attacks, tokens and latency per task (17.30 s against 4.98 s and 5,083 against 8,020 tokens with GPT-4o in one test; 2.73x the input and 2.82x the output tokens for the median task with Claude 3.5 Sonnet in the authors' test), the share of tasks with a policy denial and the excess privilege left open; in production, monitor denials, retries and tasks the program could not express." ], "detect": { "static": "The privileged model sees only the request, trusted context and tool descriptions with return schemas, and writes the task as a program; a custom interpreter labels every value with its sources and readers and propagates labels through operations, string formatting and control flow, including untaken branches; untrusted content is parsed only by a model with no tools and a typed output; each side-effecting tool has a policy on argument sources and readers; repair retries and error messages stay inside the checked boundary.", "traces": "Policy denials and user confirmations per task; privileged-model retries per task; no executed call whose arguments carry untrusted labels unless a policy allows it; no tool-derived text in another agent's instruction.", "evals": "Static and adaptive injection suites (AgentDojo with per-tool policies, plus data-dependent tasks); benign utility and utility under attack; tokens and latency per task; share of tasks with a policy denial; excess privilege left open.", "probes": "Injected data that tries to add a call, set a write argument, leak a private value through a branch or a retry, or cross an agent boundary inside instruction text." }, "impact": { "measured_basis": true, "primary": [ "security", "quality" ], "rows": [ { "dimension": "quality", "effect": "degrades", "size": "large", "when": "benign utility 77.3% against 84.5% for o3 but 41.2% against 73.2% for Gemini 2.5 Pro (Debenedetti, Mar 2025); 59.79% against 80.41% with GPT-4o (Ji, Nov 2025); 26.3 to 52.6% against 63.2 to 89.5% across five smaller models, the largest cost of ten defenses on four (Ma, Jun 2026); 64.9% against 86.6% with Sonnet-5, 73.2% against 77.3% with Haiku-4.5 (Sharma, Sep 2026); policies on 37.5% against 66.7% off (GPT-5.2, banking) (Garby, Feb 2026); without policies 82.1% against 87.5% for GPT-5.5 and 20.5% against 64.3% for Gemini 2.5 Flash-Lite (Peters-Gill, Oct 2026). Tasks whose actions come from the data cannot be expressed.", "basis": "evidence" }, { "dimension": "cost", "effect": "mixed", "size": "medium", "when": "median task 2.73x the input and 2.82x the output tokens, mean 7.24x and 6.23x, against native tool calling (Claude 3.5 Sonnet); a cheaper quarantined model cut cost about 12% for about 1% utility (Debenedetti, Mar 2025); 5,083 against 8,020 tokens per task with GPT-4o (Ji, Nov 2025)", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "large", "when": "17.30 s against 4.98 s per task with GPT-4o, the slowest of the GPT-4o defenses tested (Ji, Nov 2025); the authors attribute the overhead to model calls and code-repair retries, not the interpreter (Debenedetti, Mar 2025)", "basis": "evidence" }, { "dimension": "security", "effect": "improves", "size": "large", "when": "0 of 949 attacks succeeded with policies for o3 and Gemini 2.5 Pro, against 11 and 163 undefended (Debenedetti, Mar 2025); at most 1.8% under static and adaptive attacks across five models, against up to 58.6% undefended (Ma, Jun 2026); 1.21% against 15.62% on AgentDojo and 7.15% against 31.04% on ASB, and an attack reusing the task's tools doubled it (0.95% to 1.90%) (Ji, Nov 2025). Leaks where tracking stops: error messages (Ji, Nov 2025), untaken branches and the retry loop (Garby, Feb 2026), tool output written into another agent's instructions (0.2%, 0.0% with a separate data channel) (Peters-Gill, Oct 2026); 16 of 609 attack sinks left open against 1 for an argument-exact oracle (Sharma, Sep 2026)", "basis": "evidence" }, { "dimension": "governance", "effect": "improves", "size": "medium", "when": "the program, per-value labels and per-tool policies are a reviewable record of what may flow where; it left less excess privilege open than a tool allowlist (0.0895 against 0.3551) but about 45 times an argument-exact oracle (0.0020), partly by refusing entitled calls (Sharma, Sep 2026); its guarantee is not formally verified: the label rules live in its Python interpreter, which missed an implicit flow (Garby, Feb 2026)", "basis": "evidence" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired runs, CaMeL-style program and interpreter against the current tool loop and a cheaper argument-provenance or policy check, same model, on AgentDojo with per-tool policies plus data-dependent tasks (AgentDyn-style): benign utility, utility under attack, attack success under the static attack and an adaptive black-box optimizer (AutoDojo-style), tokens and latency per task, the share of tasks with a policy denial, and excess privilege left open (Ajar-style candidate calls); add probes for error-message injection, secret-dependent assignments on untaken branches, string interpolation of untrusted values and, in multi-agent systems, tool output written into a child agent's instructions" }, { "method": "monitor", "tier": null, "how": "policy denials and user confirmations per task, privileged-model retries per task and error messages that contain tool-derived text, tasks abandoned because the program could not express them, quarantined-model not-enough-information errors, and values that cross agent boundaries inside instruction text" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Control Flow group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Debenedetti et al., \"Defeating Prompt Injections by Design\"", "url": "https://arxiv.org/abs/2503.18813", "date": "2025-03-24", "locator": "Abstract; §6.1 Table 1; §6.1.3; §6.2; §6.5; App. B Tables 2 and 4; App. F Table 10", "task": "Introduces CaMeL: extract control and data flow from the trusted query as a program, then run it with capability checks on tool calls", "effect": "\"solving 77% of tasks with provable security (compared to 84% with an undefended system) in AgentDojo\": o3 (high effort) 77.3% against 84.5% with native tool calling; other models lost more (Claude 4 Sonnet 74.2% against 86.6%, Gemini 2.5 Pro 41.2% against 73.2%), mostly in Travel, where undocumented tool outputs caused 13 of Claude 3.5 Sonnet's 15 failures. With policies, 0 of 949 attacks succeeded for o3 and Gemini 2.5 Pro (11 and 163 undefended). No model solved a task whose actions come from data. Median task 2.73x the input and 2.82x the output tokens of native tool calling (Claude 3.5 Sonnet, no attack); a Claude 3.5 Haiku quarantined model under Claude 4 Sonnet cost about 1% utility for about 12% lower cost.", "direction": "supports", "measured": null, "conditions": "AgentDojo, 97 tasks, 949 attacks; the abstract's 77% against 84% is o3 with high reasoning effort in the June 2025 v2 (v1 reported 67%); six models in Table 2; the authors' own system and policies", "older": true }, { "source": "Peters-Gill, Semler, Bartsch, Shumailov, Schroeder de Witt, \"Can CaMeLs Talk? Securing Multi-Agent Systems Against Indirect Prompt Injection Attacks\"", "url": "https://arxiv.org/abs/2610.05640", "date": "2026-10-05", "locator": "§III; §V-A2; §V-B Table III; §V-C; App. D Tables IV and V", "task": "Test whether CaMeL's guarantees hold when CaMeL agents call each other as tools, then add multi-CaMeL, which passes tool-derived values to a child agent through a separate data channel instead of inside its instruction text", "effect": "MultiAgentDojo, n=949 per model, attack success with no CaMeL / CaMeL in every agent / multi-CaMeL: average 12.9% / 0.2% / 0.0%; GPT-4.1 36.1% / 0.5% / 0.0%; Gemini 2.5 Pro 24.6% / 0.4% / 0.0%; GPT-5.5 0.0% in all three. Every attack that got through per-agent CaMeL was boundary laundering (tool-derived text written into a child agent's instruction). AssetOpsBench benign success (n=112), no CaMeL against per-agent CaMeL: Claude Fable 5 88.4% / 67.0%, GPT-5.5 87.5% / 82.1%, GPT-4.1 81.2% / 64.3%, Gemini 2.5 Pro 75.9% / 55.4%, Gemini 2.5 Flash-Lite 64.3% / 20.5% (average -21.6 points, -13.4 for the two strongest). MultiAgentDojo benign, no CaMeL / per-agent / multi-CaMeL: GPT-5.5 86.6% / 73.2% / 66.0%, GPT-4.1 78.4% / 38.1% / 24.7%. Tool descriptions had to be extended with full return schemas to reach reasonable utility.", "direction": "mixed", "measured": true, "conditions": "Five models, the same model as privileged and quarantined model; CaMeL run with no security policies; MultiAgentDojo is the authors' own split of AgentDojo across 2 to 4 sub-agents per suite; Claude Fable 5 refused 52 of 97 benign MultiAgentDojo tasks; one author co-wrote CaMeL", "older": false }, { "source": "Sharma, Jiang, Chen, Lin, \"Ajar: Measuring Open Privilege in Agent Defenses\"", "url": "https://arxiv.org/abs/2609.26900", "date": "2026-09-22", "locator": "§3.3; §6.2.3; Tables 5, 6, 7 and 8", "task": "Offer each defense the calls a task needs, calls it may make and calls it does not need, at every decision point of AgentDojo's tasks, and score what it allows and refuses; also run the 97 benign tasks with and without each defense", "effect": "CaMeL (released implementation and its four suite policies), deciding model Sonnet-5: over-privilege leakage 0.0895 against 0.3551 for a tool-name allowlist and 0.0020 for an argument-exact oracle; 16 of 609 attack sinks left open (allowlist 99, oracle 1); sufficiency 0.700, every required call allowed on 37 of 92 tasks; over-restriction 0.415 (594 entitled calls refused), the highest of the four defenses run under both models (0.412 with Haiku-4.5). Benign completion with and without CaMeL: 64.9% against 86.6% (Sonnet-5, -21.6 points); 73.2% against 77.3% (Haiku-4.5, -4.1).", "direction": "mixed", "measured": true, "conditions": "AgentDojo, 97 benign tasks, 609 attack sinks; Claude Sonnet-5 and Haiku-4.5; the wrapper supplies each candidate's provenance from canaries rather than CaMeL's own run; five defenses (Progent, CaMeL, AC4A, Permission Assistant, Claude Code Auto mode); independent of CaMeL's authors", "older": false }, { "source": "Ma, Li, Xiao, Yu, Zhang, Vorobeychik, \"AutoDojo: A Generative Benchmark for Evaluating Prompt Injection Defenses in LLM Agents\"", "url": "https://arxiv.org/abs/2606.15057", "date": "2026-06-13", "locator": "§4.1; §4.2; App. B; App. D; App. H Table 5", "task": "Evaluate ten injection defenses, including CaMeL with its authors' implementation and policies, against AgentDojo's static attack and an adaptive black-box injection optimizer, on five target models", "effect": "AgentDojo (banking, slack, travel), clean utility / static attack success / adaptive attack success. CaMeL: GPT-4o-mini 26.3 / 0.0 / 0.0; GPT-5.4-mini 52.6 / 1.8 / 1.8; Gemini-2.5-Flash 28.1 / 0.3 / 0.3; DeepSeek-v4-Flash 38.6 / 0.8 / 0.5; Claude-Haiku-4.5 40.4 / 0.3 / 0.5. No defense: 66.7 / 58.6 / 52.4; 77.2 / 6.9 / 8.7; 63.2 / 50.4 / 34.7; 89.5 / 22.6 / 16.5; 70.2 / 0.3 / 1.8. CaMeL, \"being the most effective defense overall, incurs the largest clean utility cost on four of the five models.\" On GPT-4o-mini, Progent 63.2 / 8.2 / 7.7 and ROPE (argument provenance) 59.6 / 6.4 / 4.4. CaMeL was not run on AgentDyn \"because it requires extensive suite-specific policies.\"", "direction": "mixed", "measured": true, "conditions": "57 user tasks, 389 attack cases (workspace suite dropped: agents completed nothing on 52.9% of its cases); adaptive attack optimized by Gemini-3.1-Pro, six iterations, success-only feedback; numbers from v3 (Sep 2026); independent of CaMeL's authors; ROPE is the authors' own defense", "older": false }, { "source": "Garby, Gordon, Sands, \"The LLMbda Calculus: AI Agents, Conversations, and Information Flow\"", "url": "https://arxiv.org/abs/2602.20064", "date": "2026-02-23", "locator": "§1; §7.2; §7.3 Table 1", "task": "Formalize code-generating agents with labeled values in a lambda calculus with machine-checked noninterference, audit CaMeL's label tracking, and compare a verified dual-LLM agent with CaMeL rerun on AgentDojo banking", "effect": "Two leaks in CaMeL: its interpreter raises a variable's label only on the branch taken, so a secret reaches a public guard through an assignment that is skipped; and its retry loop runs outside the checked interpreter, so an adaptive driver can extract a secret one blocked-or-not bit per run. Footnote: \"interpolating untrusted values into string literals implicitly makes them trusted.\" AgentDojo banking, utility without / with attack: CaMeL with policies 37.5% / 30% (GPT-5.2) and 37.5% / 41.5% (MiMo-v2.5); without policies 66.7% / 71.1% and 70.8% / 72.2%. The authors' verified agent (enforcement always on): 75% / 65% (GPT-5.2), with 2 successful attacks in 1,296 attacked runs across three models.", "direction": "mixed", "measured": true, "conditions": "AgentDojo banking only: 16 user tasks x 9 injections, k=3; safe-utility intervals of plus or minus 19 to 27 points; not all CaMeL reruns completed (for GPT-5.2 without policies, 135 of 432 attacked runs); the verified agent is the authors' own; independent of CaMeL's authors", "older": false }, { "source": "Ji, Wang, Li, Ma, Gao, Wu, Yan, Tian, Wang, \"Taxonomy, Evaluation and Exploitation of IPI-Centric LLM Agent Defense Frameworks\"", "url": "https://arxiv.org/abs/2511.15203", "date": "2025-11-19", "locator": "§IV setup; Table II; Table III; §IV-B RC 2, RC 3; App. C; §V-A Table V", "task": "Independent like-for-like evaluation of injection defenses including CaMeL (with the parts that need manual configuration disabled), then defense-aware attacks built from their failure causes", "effect": "GPT-4o, average attack success for CaMeL: 1.21% on AgentDojo (undefended 15.62%), 7.15% on ASB (31.04%), 0.00% on InjecAgent (2.66%); 4.81% overall (23.69%). AgentDojo without attack: utility 59.79% against 80.41% undefended (Travel 25.00% against 70.00%); tokens 5,083 against 8,020 and time 17.30 s against 4.98 s per task. Failures: a recipient the quarantined model parsed from a landlord's notice was used for a scheduled payment (parameter hijack); an error message returned to the privileged model for debugging carried an injection that put reserve_hotel into the rewritten code. A semantic-masquerading attack that reuses the task's tools raised CaMeL from 0.95% to 1.90% (2X).", "direction": "mixed", "measured": true, "conditions": "GPT-4o (2024-11-20), temperature 0; AgentDojo 629, ASB 2,040, InjecAgent 1,054 cases; adaptive-attack success judged by an LLM on 949 rewritten payloads; the paper does not list which CaMeL parts were disabled", "older": false } ], "caveats": "The guarantee holds only where labels are tracked and policies are written: an audit found leaks through untaken branches, the retry loop and string interpolation (Garby, Feb 2026), an error message carried an injection into the privileged model (Ji, Nov 2025), tool output written into another agent's instructions got through (Peters-Gill, Oct 2026), and untrusted data can still choose among the program's branches. Several tests ran CaMeL without its policies (Ji, Nov 2025; Peters-Gill, Oct 2026) or on one 16-task suite with wide intervals (Garby, Feb 2026), the strongest utility figure (77.3% for o3) is the authors' own, and the independent test that kept the policies found the largest utility cost of ten defenses on four of five models (Ma, Jun 2026). Tasks whose actions come from the data cannot be expressed at all, and frontier models already resist static template attacks undefended (0.3% for Claude-Haiku-4.5, 0.0% for GPT-5.5), so the case rests on adaptive attackers and on a hard block on exfiltration.", "also_covers": [], "related": [ { "id": "6.c.ii", "slug": "plan-then-execute", "name": "Plan-then-execute" }, { "id": "10.c.i", "slug": "contain-untrusted-input", "name": "Contain sensitive data and untrusted input" }, { "id": "10.b.i", "slug": "risky-action-policy", "name": "Gate irreversible actions: allow, ask or deny by side effect" }, { "id": "10.a.i", "slug": "constrain-agent-reach", "name": "Constrain what the agent can touch" }, { "id": "2.a.iii", "slug": "choose-action-interface", "name": "Choose how the agent acts: typed tools, code execution, or shell, matched to the model and task" }, { "id": "6.b.iii", "slug": "multi-agent-for-breadth", "name": "Use multi-agent orchestration only for breadth-first, independent sub-tasks" }, { "id": "1.b.i", "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class" }, { "id": "7.d.iii", "slug": "react-loop", "name": "ReAct loop" }, { "id": "6.b.ii", "slug": "llm-map-reduce", "name": "LLM map-reduce" } ], "url": "https://mahalsystems.ai/aot/optimizations/code-then-execute/", "markdown": "https://mahalsystems.ai/aot/optimizations/code-then-execute.md" }, { "id": "6.c.ii", "legacy_id": "P48", "former_ids": [], "slug": "plan-then-execute", "name": "Plan-then-execute", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it fixes the plan before untrusted data is seen", "proposed": true }, "levers": [ "guardrails", "control-flow" ], "domains": [ "general" ], "evidence_domains": [ "tool-use", "computer-use" ], "categories": [ "Separating privileges" ], "category_ids": [ "topologies.trust" ], "summary": "Have the agent fix its tool calls (an ordered list or a dependency graph) from the trusted request before it reads any tool output, and let a controller execute only those calls, so injected text cannot add an action. On AgentDojo this cut attack success below 1% (0.69% against 13.16% undefended for a tool-graph design; 0.52% against 15.62% for another plan-first design in an independent test), but a strict plan cost 15 to 43 points of benign utility; letting read-only lookups be added during execution recovered it in one study. It pays only where the request alone fixes the actions: on a benchmark where every task needs data-dependent planning, plan-first defenses kept 0 to 30% utility against 53% undefended on GPT-4o, while undefended Claude Sonnet 4.5 already held attack success at 0.87%. Injected data can still pick the arguments and branches of planned calls (89.5% attack success against a plan-first computer-use agent), so also bound each planned call's parameters and destinations.", "how": [ "Start from the baseline: your current tool loop (7.d.iii) and, as the cheapest plan-first variant, a tool filter that only fixes the tool set from the request (in an independent GPT-4o test it kept 70.10% of AgentDojo tasks against 80.41% undefended, at 4.59% average attack success against 23.69%). Run both on an injection suite that includes tasks whose actions depend on what the agent reads: AgentDojo has only 6 of 97, and on a benchmark built from such tasks undefended Claude Sonnet 4.5 already held attack success at 0.87%.", "Sort your tasks by whether the request alone fixes the actions. Where the next write depends on what is read (act on the to-do list in this file), do not force a fixed plan: plan-first defenses kept 0 to 30% utility on such tasks against 53.33% undefended (GPT-4o); contain the input (10.c.i) and gate the action (10.b.i) instead.", "Give a planner only the user request, the tool descriptions and trusted context, and have it emit a dependency graph: each node a tool call, each argument a literal or marked as filled from a named earlier node's output. Pin the tool descriptions it reads, since they are planner input too. Spend on the planner rather than the executor: an o4-mini planner over a GPT-4o-mini executor raised utility under attack from 57.07% to 64.39% for $7.99 against $6.73, planning being about 20% of tokens (1.b.i).", "Execute the graph in dependency order with a controller that rejects any call not in the plan, except read-only (query) tools added after reading results; the executor fills only the pending arguments. A strict graph cost 15.5 points of benign utility (52.58% against 68.04%); read-only expansion brought it to 64.95%. Never return raw tool output or error messages to the planner to re-plan: in one study an error message and an email that reached the planner each put an attacker's call into the plan.", "Bound what each planned write may do before execution, in code: recipients, accounts and amounts from the request or a trusted record (ask the user when there is none), allowed hosts and endpoints, and per-branch conditions. Without this, injected data set the recipient of a planned send_money in one test and steered branches at 89.5% success against a plan-first computer-use agent; per-branch constraints took that to 0% at 97% of benign utility. Where an injected instruction targets a write tool the plan already uses, answer it with a simulated call so the executor fills the real one from the request (attack success 4.77% to 0.64% in one study).", "Measure paired against the baseline, same suite and model: benign utility, utility under attack, attack success, tokens and latency per task (one design used 2.4x the input tokens and 1.9x the time, another 0.54x the tokens and 2.7x the time); add adaptive attacks that reuse planned tools with attacker arguments, craft values that meet branch conditions and reach the planner through errors; in production, monitor rejected calls and write arguments taken from tool output." ], "detect": { "static": "A plan (tool calls with their dependencies) is generated from the request before any untrusted data is read; a controller executes only planned calls plus read-only lookups; write-call arguments and destinations are bounded before execution; raw tool output and error messages never go back to the planner.", "traces": "Executed calls equal the planned calls plus read-only lookups; no new write call after data ingestion; each write argument traces to the request or a trusted record, not to tool output.", "evals": "Injection suite attack success, benign utility and utility under attack, on tasks that include data-dependent ones; tokens and latency per task.", "probes": "Injected data that tries to add a tool call, change the arguments of a planned call, or steer a planned branch." }, "impact": { "measured_basis": true, "primary": [ "security", "governance" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "large", "when": "benign utility about even for a tool graph with read-only expansion (67.01% against 68.04%) and higher under attack (58.77% against 54.30%) (An, Aug 2025); 37.11% against 80.41% for a strict plan-first design (Ji, Nov 2025); 0 to 30% against 53.33% on data-dependent tasks (Li, Feb 2026); Sonnet-5 in both roles 38.8% against 51.0% with ReAct, but a gpt-5 planner lifted a small executor from 23.5% to 36.8% (Zingrillo, Oct 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "mixed", "size": "medium", "when": "2.4x the input tokens for a tool-graph design (An, Aug 2025); 0.54x the tokens for a strict plan-first design and 0.65x for a tool filter, from fewer turns (Ji, Nov 2025); $0.627 against $0.291 per task with Sonnet-5 in both roles, $0.332 with a gpt-5 planner over a locally served executor (Zingrillo, Oct 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "medium", "when": "13.88 s against 7.13 s per task (An, Aug 2025); 2.7x the wall-clock time for a strict plan-first design (13.36 against 4.98), while a tool filter was slightly faster (4.28) (Ji, Nov 2025)", "basis": "evidence" }, { "dimension": "security", "effect": "improves", "size": "large", "when": "attack success below 1% on static benchmarks: 0.69% against 13.16% on AgentDojo (An, Aug 2025); 0.52% against 15.62% on AgentDojo and 0.00% against 31.04% on ASB (Ji, Nov 2025). Not against argument or branch steering: 89.5% against a plan-first computer-use agent until each branch's parameters were bounded (0%) (Zingrillo, Oct 2026); an adaptive attack reusing planned tools raised one design 3.9x, to 1.91% (Ji, Nov 2025)", "basis": "evidence" }, { "dimension": "governance", "effect": "improves", "size": "large", "when": "The plan is a record of what the agent may do, reviewable before it runs; no source here measures it.", "basis": "assessment" }, { "dimension": "brand", "effect": "improves", "size": "small", "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired runs, plan-first against the current loop and a tool filter, same model, on an injection suite that includes data-dependent tasks and helpful third-party instructions (AgentDojo or ASB plus AgentDyn-style tasks; WASP or STEER-Bench for browser agents): benign utility, utility under attack, attack success, tokens and latency per task; add adaptive attacks that reuse planned tools with attacker arguments, craft values that meet branch conditions and reach the planner through error messages" }, { "method": "monitor", "tier": null, "how": "calls the controller rejected as unplanned, read-only expansions per task, tasks abandoned because the plan did not fit, and write arguments whose values came from tool output rather than the request or a trusted record" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Control Flow group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Beurer-Kellner et al., \"Design Patterns for Securing LLM Agents against Prompt Injections\"", "url": "https://arxiv.org/abs/2506.08837", "date": "2025-06-10", "locator": "Sec 3.1 (2) The Plan-Then-Execute Pattern; Figure 2 caption", "task": "Names the plan-then-execute pattern: fix the tool-call plan before reading untrusted data", "effect": "The pattern means \"a prompt injection cannot force the LLM into executing a tool that is not part of the defined plan,\" though injected data can still change the inputs to planned calls. Not benchmarked.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Zingrillo, Foerster, Shumailov, Zhao, Mullins, \"Securing Computer-Use Agents Against Branch Steering Attacks\"", "url": "https://arxiv.org/abs/2610.03089", "date": "2026-10-02", "locator": "Abstract; §5 Table 1; §5.1; §5.2 Table 2; App. G Table 5", "task": "Test a plan-first (dual-LLM) computer-use agent, whose trusted planner fixes a branching plan before untrusted pages are read, against branch steering; then add ahead-of-time per-branch constraints on parameters and destinations, enforced at HTTP and MCP proxies (COBRA)", "effect": "STEER-Bench, cell-weighted attack success on the 74 cells generic agents can run: 94.4% for ReAct and 89.5% for the vanilla dual-LLM plan-first baseline; 0% for COBRA on all 199 cells, which keeps 97% of the baseline's benign utility (4 false positives in 138 benign cells). WASP: host-level allowlist alone left 14/19 (73.7%) same-host attacks, endpoint sitemaps 0/19 at 100% benign completion. OSWorld-MCP, 361 tasks, pass@5: gpt-5 planner + UI-TARS-1.5-7B 133 (36.8%) against 85 (23.5%) for UI-TARS ReAct; Sonnet-5 in both roles 140 (38.8%) against 184 (51.0%) with ReAct, kimi-k2.5 92 (25.5%) against 152 (42.1%). API cost per task: $0.332 (gpt-5 + UI-TARS), $0.627 for Sonnet-5 against $0.291 for its ReAct (ReAct billed for one attempt, COBRA for attempts up to early stopping).", "direction": "mixed", "measured": true, "conditions": "computer-use agents; STEER-Bench is the authors' own (101 tasks, 9 domains, 199 attack cells), plus OS-Harm, WASP, MCPTox, MCPSecBench; gpt-5 planner, UI-TARS-1.5-7B perception model; the baseline is the same authors' earlier dual-LLM agent", "older": false }, { "source": "Li, Wen, Shi, Zhang, Vorobeychik, Xiao, \"AgentDyn: Are Your Agent Security Defenses Deployable in Real-World Dynamic Environments?\"", "url": "https://arxiv.org/abs/2602.03117", "date": "2026-02-03", "locator": "§1; Table 3; §4.2 System-level Defense; App. F Table 8; App. G tables", "task": "Evaluate injection defenses, including the plan-first Tool Filter, CaMeL and DRIFT, on tasks that all require dynamic planning and contain helpful third-party instructions", "effect": "Only 6 of 97 AgentDojo tasks require dynamic planning. AgentDyn, GPT-4o, benign utility / utility under attack / attack success: undefended 53.33 / 55.52 / 37.80; Tool Filter 8.33 / 4.91 / 4.22; CaMeL 0.00 / 0.00 / 0.00; DRIFT 30.00 / 27.09 / 0.83. Undefended Claude-Sonnet-4.5 71.67 / 70.74 / 0.87, against benign utility 0.00 with Tool Filter and 28.33 with DRIFT. On undefended GPT-5-mini, GPT-5.1 and Claude-Sonnet-4.5, template attacks reached at most 5.0% and the authors' adaptive attacks 2.1 to 5.6%. Planning-dependent defenses \"rely heavily on initial plans, leading to severe utility drops.\"", "direction": "contradicts", "measured": true, "conditions": "AgentDyn: 60 user tasks, 560 injection cases (Shopping, GitHub, Daily Life), important_instructions attack by default; GPT-4o, Gemini-2.5 Pro, Claude-Sonnet-3.5, Llama-3.3-70B in the main table, 8 more models in the appendix; official defense code; three authors are also DRIFT's", "older": false }, { "source": "Ji, Wang, Li, Ma, Gao, Wu, Yan, Tian, Wang, \"Taxonomy, Evaluation and Exploitation of IPI-Centric LLM Agent Defense Frameworks\"", "url": "https://arxiv.org/abs/2511.15203", "date": "2025-11-19", "locator": "Table II; Table III; §IV-B RC 2, RC 3; §V-A Table V; §V-C", "task": "Independent like-for-like evaluation of injection defenses, including a plan-then-execute design (IsolateGPT), a tool filter and CaMeL, then defense-aware attacks built from their failure causes", "effect": "GPT-4o, average attack success: IsolateGPT 0.52% on AgentDojo (undefended 15.62%), 0.00% on ASB (31.04%) and InjecAgent (2.66%); Tool Filter 4.59% overall (undefended 23.69%). AgentDojo without attack: utility 37.11% for IsolateGPT, 70.10% for Tool Filter, 80.41% undefended; tokens 4,358 / 5,251 / 8,020 and time 13.36 / 4.28 / 4.98. Failures: injected data set the recipient of a planned send_money (parameter hijack); an email that reached the planner added an attacker's tool to the plan (case study). A semantic-masquerading attack that reuses planned tools raised IsolateGPT from 0.48% to 1.91% (3.9X).", "direction": "mixed", "measured": true, "conditions": "GPT-4o (2024-11-20), temperature 0; AgentDojo 629, ASB 2,040, InjecAgent 1,054 cases; adaptive-attack success judged by an LLM on 949 rewritten payloads; the isolation breach is a single case", "older": false }, { "source": "An, Zhang, Du, Zhou, Li, Lin, Ji, \"IPIGuard: A Novel Tool Dependency Graph-Based Defense Against Indirect Prompt Injection in LLM Agents\"", "url": "https://arxiv.org/abs/2508.15310", "date": "2025-08-21", "locator": "§3; Table 1; §4.2.1 Figure 5; Table 2; Table 3; App. G Table 5", "task": "Plan the task as a tool dependency graph before reading tool output, traverse it filling pending arguments from dependencies, allow only read-only tools to be added during execution, and answer injected instructions aimed at planned write tools with a simulated call", "effect": "GPT-4o-mini, AgentDojo, average over four attacks: attack success 0.69% against 13.16% undefended and 2.06% for Tool Filter; utility under attack 58.77% against 54.30% and 53.36%. Benign utility 67.01% against 68.04% undefended. Important Instructions attack: 0.64% against 27.19%. Ablation (benign utility / utility under attack / attack success): graph only 52.58 / 42.13 / 3.18; with read-only expansion 64.95 / 52.46 / 4.77; with both mechanisms 69.07 / 57.07 / 0.64. Per task: input tokens 14,605 against 6,165, output 560 against 179, time 13.88 s against 7.13 s. An o4-mini planner over a GPT-4o-mini executor: utility under attack 64.39% against 57.07%, cost $7.99 against $6.73; planning is about 20% of tokens.", "direction": "supports", "measured": true, "conditions": "AgentDojo, 97 tasks, 629 security cases; GPT-4o-mini for the tables, six models including o4-mini and Qwen3-32B in Figure 4; Ignore Previous, InjecAgent, Tool Knowledge and Important Instructions attacks; no adaptive attack; the authors' own method", "older": true } ], "caveats": "Most attack-success numbers here come from static template attacks: the one independent adaptive test raised a plan-first design 3.9x (to 1.91%) by reusing planned tools with attacker arguments, and an email that reached the planner's context put an attacker's tool into the plan in a case study. Utility depends on how plannable the tasks are: AgentDojo has 6 data-dependent tasks in 97, and on a benchmark where all tasks are, plan-first defenses lost most of their utility; that benchmark shares authors with one of the defenses it tests, and the 89.5% branch-steering figure is on its authors' own benchmark. A fixed plan does not stop injections in the user prompt (Beurer-Kellner, Jun 2025) or in tool descriptions the planner reads, and frontier models already resist template attacks without a defense (0.87% for undefended Claude Sonnet 4.5), so the guarantee matters most against attackers stronger than today's benchmarks.", "also_covers": [], "related": [ { "id": "6.c.i", "slug": "code-then-execute", "name": "Code-then-execute (CaMeL)" }, { "id": "10.c.i", "slug": "contain-untrusted-input", "name": "Contain sensitive data and untrusted input" }, { "id": "10.b.i", "slug": "risky-action-policy", "name": "Gate irreversible actions: allow, ask or deny by side effect" }, { "id": "10.a.i", "slug": "constrain-agent-reach", "name": "Constrain what the agent can touch" }, { "id": "7.d.iii", "slug": "react-loop", "name": "ReAct loop" }, { "id": "1.b.i", "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class" }, { "id": "6.b.ii", "slug": "llm-map-reduce", "name": "LLM map-reduce" }, { "id": "5.c.i", "slug": "discrete-phase-separation", "name": "Discrete phase separation" }, { "id": "7.c.iv", "slug": "graph-of-thought", "name": "Graph-of-Thought" }, { "id": "7.d.ii", "slug": "inversion-of-control", "name": "Inversion of control" }, { "id": "6.a.i", "slug": "oracle-worker-escalation", "name": "Oracle-worker multi-model" }, { "id": "6.b.i", "slug": "parallel-read-only-tools", "name": "Parallel tool execution for read-only calls" }, { "id": "3.c.i", "slug": "self-discover-reasoning", "name": "Self-Discover reasoning structures" } ], "url": "https://mahalsystems.ai/aot/optimizations/plan-then-execute/", "markdown": "https://mahalsystems.ai/aot/optimizations/plan-then-execute.md" }, { "id": "7.a.i", "legacy_id": "T54", "former_ids": [], "slug": "semantic-layer-routing", "name": "Route metric questions through a deterministic semantic layer; fall back to text-to-SQL", "kind": "technique", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "routing within a single agent", "proposed": false }, "levers": [ "control-flow", "knowledge-access" ], "domains": [ "data-analysis" ], "evidence_domains": [ "data-analysis" ], "categories": [ "Routing", "Querying precisely" ], "category_ids": [ "orchestration.route", "retrieval.query" ], "summary": "Where a semantic layer exists, have the agent answer metric questions by choosing metrics and dimensions that the layer compiles to SQL, and write SQL only when the layer cannot answer. The layer fails with an explicit error rather than a plausible wrong answer. Higher reasoning effort or a bigger model did not close the gap; the layer did.", "how": [ "Confirm a semantic layer covers the metrics the agent is asked about.", "Add a first path that maps the question to metrics and dimensions; fall back to SQL on a layer error.", "Paired eval on metric questions, tracking explicit errors against wrong answers." ], "detect": { "static": "a data agent that writes SQL directly although a semantic or metrics layer exists", "traces": "wrong numbers on questions whose metric the layer defines; failure rows labeled reasoning-failure on metric questions", "evals": "accuracy on metric questions, split by whether the layer defines them" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "large", "when": "dbt: 98.2% against 90.0% (Sonnet 4.6), 100% against 84.1% (GPT-5.3 Codex), 11 questions x 20 runs (Apr 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "performance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval on metric questions" } ], "evidence": [ { "source": "Jason Ganz, Benoit Perigaud (dbt Labs), \"Semantic Layer vs. Text-to-SQL: 2026 Benchmark Update\"", "url": "https://docs.getdbt.com/blog/semantic-layer-vs-text-to-sql-2026", "date": "2026-04-07", "locator": "How the two approaches actually work; Which model should you use? table; The main event: 2023 vs. 2026 table", "task": "Route metric questions through a deterministic semantic layer", "effect": "On questions the Semantic Layer can reach (not too many entity hops), 11-question ACME set x 20 runs: SL 100.0% for both Sonnet 4.6 and GPT-5.3 Codex vs text-to-SQL 62.5% (Sonnet 4.6) and 51.2% (GPT-5.3 Codex) on raw tables; 2023 GPT-4 was SL 83.1% vs text-to-SQL 26.9%. Across the model x reasoning-effort sweep on SL-answerable questions, SL scored 87.2-100.0% vs text-to-SQL 50.0-65.0% (SL advantage 24.7-50.0 pp). Before extra modeling, SL scored 0.0% on the too-many-hops questions (it refused), giving 72.7% overall vs text-to-SQL 64.5%. Qualitative: \"the Semantic Layer tells you it can't answer. It never returns invalid data. Text-to-SQL will cheerfully give you a wrong number.\"", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Jason Ganz, Benoit Perigaud (dbt Labs), \"Semantic Layer vs. Text-to-SQL: 2026 Benchmark Update\"", "url": "https://docs.getdbt.com/blog/semantic-layer-vs-text-to-sql-2026", "date": "2026-04-07", "locator": "Which should you use?", "task": "Try semantic layer first, then modeling, then text-to-SQL", "effect": "Recommendation derived from the benchmark above; the routing policy itself was not evaluated. Packaged as a dbt agent skill for answering natural-language questions.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Jason Ganz, Benoit Perigaud (dbt Labs), \"Semantic Layer vs. Text-to-SQL: 2026 Benchmark Update\"", "url": "https://docs.getdbt.com/blog/semantic-layer-vs-text-to-sql-2026", "date": "2026-04-07", "locator": "Which model should you use? (It matters less than you'd think)", "task": "Do not buy semantic-layer accuracy with bigger models or effort", "effect": "Sweep on SL-answerable questions (Opus 4.6, Sonnet 4.6, GPT-5.3 Codex, GPT-5.2; Anthropic low..max, OpenAI none..xhigh): most configurations hit 95-100% SL accuracy regardless of effort; Sonnet 4.6 100.0% at low/medium/max vs Opus 4.6 87.2-87.5% at every effort. GPT models at xhigh averaged over 20 s per query vs 8 s at high with no meaningful accuracy gain; for Anthropic models effort changed neither results nor latency. (The post calls it 16 permutations; the table lists 18 rows.)", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Michael Rumiantsau, Ivan Fokeev (Cube), \"Semantic Layers for Reliable LLM-Powered Data Analytics: A Paired Benchmark of Accuracy and Hallucination Across Three Frontier Models\"", "url": "https://arxiv.org/abs/2604.25149", "date": "2026-04-28", "locator": "Section 5.2 Table 2; Section 5.3", "task": "Write the semantic layer instead of upgrading the model", "effect": "Pairwise McNemar, n=99: within the with-document cluster all p >= 0.79 (67.7-68.7%); within the DDL-only cluster all p >= 0.42 (45.5-50.5%); all nine cross-cluster comparisons p < 0.01. Sonnet 4.6 plus the document used ~28% fewer input tokens than Opus 4.7 plus the document at equal accuracy. Authors caution n=99 cannot resolve differences of about 5 pp.", "direction": "supports", "measured": true, "conditions": null, "older": false } ], "caveats": "Applies only where a semantic layer already covers the metrics being asked about; building one is data-side work and a separate change. Both studies come from semantic-layer vendors (dbt Labs, Cube), and the question sets are small (11 questions x 20 runs; 99 questions). The routing policy itself, layer first and SQL as fallback, was recommended by dbt but not evaluated.", "also_covers": [], "related": [ { "id": "5.b.i", "slug": "ground-queries-in-schema", "name": "Give the model the schema it queries, in context: descriptions, keys, codes, sample values" }, { "id": "3.a.i", "slug": "few-shot-examples", "name": "Add worked examples to the prompt (few-shot), or curated example queries for data agents" }, { "id": "7.d.ii", "slug": "inversion-of-control", "name": "Inversion of control" } ], "url": "https://mahalsystems.ai/aot/optimizations/semantic-layer-routing/", "markdown": "https://mahalsystems.ai/aot/optimizations/semantic-layer-routing.md" }, { "id": "7.a.ii", "legacy_id": "P13", "former_ids": [], "slug": "prompt-chaining-with-gates", "name": "Prompt chaining with gate checks", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it decomposes one call into a gated chain", "proposed": true }, "levers": [ "control-flow" ], "domains": [ "general" ], "evidence_domains": [ "structured-output", "retrieval", "data-analysis", "code-generation", "reasoning", "incident-response" ], "categories": [ "Routing" ], "category_ids": [ "orchestration.route" ], "summary": "Split a task that decomposes cleanly into fixed steps into a chain of model calls, one goal per call, with a gate between steps: a schema check, an assertion, or a run against the real system, and an LLM check only for subjective criteria. On a gate failure, rerun only the failed step with its output and the error. The trade is latency and tokens for accuracy and inspectable intermediate outputs; it pays on tasks beyond what one call does reliably, and less as models improve.", "how": [ "Start from a single call tuned with the same effort; chain only if the task splits cleanly into fixed steps.", "Give each step one goal and a handoff the next step can validate against a schema.", "Gate in code first: schema validation or assertions; where you can, run the output against the real system (execute the query) instead of asking the model whether it is valid; use an LLM check with hard thresholds only for subjective criteria.", "On a failure, rerun only the failed step with its previous output and the error message; cap retries, then stop or continue with a warning; keep outputs that already passed.", "Run new gates in warn-only mode first and review each halt for false alarms.", "Log each step's output and gate result; compare the chain with the single call on success, tokens and latency, per model." ], "detect": { "static": "A fixed sequence of model calls with checks between them rather than one prompt.", "traces": "Stage outputs; gate failures; retries per stage; the whole chain rerun when one step fails.", "evals": "Per-stage and end-to-end quality.", "judge": "Rate each stage's output.", "probes": "Inputs that should fail at each gate." }, "impact": { "measured_basis": true, "primary": [ "quality", "governance" ], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "tasks beyond what one call does reliably: well-formatted JSON 37.6% to 98.8% with assertions and retry (DSPy Assertions, 2023); over a tuned single call, orchestration gained at most 4.6 points (Leins, Aug 2026); 37% of gate halts were false alarms in one study (Elina, Sep 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "medium", "when": "more calls: orchestration took about 2 to 4 times the tokens of a single call (Leins, Aug 2026); a fixed chain that reran downstream steps cost 1,632 tokens per retry against 904 for one prompt, and 436 when only the failed step reran (Asthana, May 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "medium", "when": "serial calls; with gates, one model got faster and the other markedly slower (Dhage, Aug 2026)", "basis": "evidence" }, { "dimension": "security", "effect": "improves", "size": "small", "when": "Gates can check policy.", "basis": "assessment" }, { "dimension": "governance", "effect": "improves", "size": "medium", "when": "Checkpoints are auditable.", "basis": "assessment" }, { "dimension": "brand", "effect": "improves", "size": "small", "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired eval of the chain against a single call tuned with the same effort, per model: success, tokens, latency" }, { "method": "monitor", "tier": null, "how": "gate failure and retry rate per step; false-alarm rate on reviewed halts; each error traced to the step that caused it" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file master-agent-optimization-cookbook.md", "task": "Lists this practice in its agent-optimization pattern library (Workflow Patterns group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Anthropic (Erik Schluntz, Barry Zhang), \"Building effective agents\"", "url": "https://www.anthropic.com/engineering/building-effective-agents", "date": "2024-12-19", "locator": "Workflow: Prompt chaining", "task": "Describes prompt chaining with programmatic gates between steps", "effect": "\"You can add programmatic checks (see \"gate\" in the diagram below) on any intermediate steps to ensure that the process is still on track.\" Use it when \"the task can be easily and cleanly decomposed into fixed subtasks\"; \"The main goal is to trade off latency for higher accuracy, by making each LLM call an easier task.\" No measurement.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Tongshuang Wu, Michael Terry, Carrie J. Cai, \"AI Chains: Transparent and Controllable Human-AI Interaction by Chaining Large Language Model Prompts\"", "url": "https://arxiv.org/abs/2110.01691", "date": "2021-10-04", "locator": "Abstract", "task": "Introduces chaining LLM steps so each output feeds the next", "effect": "\"In a 20-person user study, we found that Chaining not only improved the quality of task outcomes, but also significantly enhanced system transparency, controllability, and sense of collaboration.\"", "direction": "supports", "measured": null, "conditions": "20-person user study", "older": true }, { "source": "Singhvi et al., \"DSPy Assertions: Computational Constraints for Self-Refining Language Model Pipelines\"", "url": "https://arxiv.org/abs/2312.13382", "date": "2023-12-20", "locator": "Abstract; §1; §5", "task": "Assertions between pipeline modules; on failure, backtrack and retry the failing module with the erring output and error message in its prompt", "effect": "Well-formatted JSON quizzes 37.6% to 98.8% with assertions and backtracking; valid quiz questions 30.5% to 87.2% with all assertion-driven optimizations. Across four case studies, constraints passed \"up to 164% more often\" and responses were \"up to 37% more higher-quality\".", "direction": "supports", "measured": true, "conditions": "gpt-3.5-turbo; HotPotQA-based tasks, 500 test examples", "older": true }, { "source": "Sun et al., \"Prompt Chaining or Stepwise Prompt? Refinement in Text Summarization\"", "url": "https://arxiv.org/abs/2406.00507", "date": "2024-06-01", "locator": "Abstract; Table 1", "task": "Draft, critique and refine as three chained prompts against the same steps in one stepwise prompt", "effect": "Chained refinement with GPT-4 won 77 of 100 comparisons against a one-step GPT-4 summary; the stepwise prompt won 53. The authors suggest the stepwise prompt \"might produce a simulated refinement process\".", "direction": "supports", "measured": true, "conditions": "InstruSum, 100 items; GPT-3.5 and GPT-4; LLM judge", "older": true }, { "source": "Prithvi Rajasekaran (Anthropic), \"Harness design for long-running application development\"", "url": "https://www.anthropic.com/engineering/harness-design-long-running-apps", "date": "2026-03-24", "locator": "Running the harness; Removing the sprint construct", "task": "Fixed planner, generator and evaluator steps with hard-threshold gates per sprint, against one agent", "effect": "Each criterion \"had a hard threshold, and if any one fell below it, the sprint failed\". One app: solo 20 min and $9, full harness 6 hr and $200, with better output judged by the author. The sprint breakdown was later removed for a stronger model; the evaluator \"is worth the cost when the task sits beyond what the current model does reliably solo.\"", "direction": "mixed", "measured": true, "conditions": "Claude Opus 4.5; one prompt, one run each", "older": false }, { "source": "Ijaz, \"SQL Query Engine: A Self-Healing LLM Pipeline for Natural Language to PostgreSQL Translation\"", "url": "https://arxiv.org/abs/2604.16511", "date": "2026-04-15", "locator": "Abstract", "task": "Two-stage text-to-SQL pipeline gated by executing the query; on failure or an empty result, retry with the database error", "effect": "The retry loop \"yields up to +9.3pp accuracy gains with zero regressions on the best model (Llama 4 Scout 17B, 57.3%)\". Passing queries are accepted without re-evaluation, and the best partial result is kept across retries.", "direction": "supports", "measured": true, "conditions": "five LLM backends; synthetic benchmark, 75 questions, three databases", "older": false }, { "source": "Asthana, Zhang, DeLuca, Patel, et al., \"Runtime-Structured Task Decomposition for Agentic Coding Systems\"", "url": "https://arxiv.org/abs/2605.15425", "date": "2026-05-14", "locator": "Abstract", "task": "One prompt against a fixed-step chain and against steps with schema-validated outputs where only failed steps rerun", "effect": "Retry cost on Kubernetes root cause analysis: 1,632 ± 145 tokens for the fixed chain against 904 ± 17 for one prompt, \"because failures forced reruns of downstream subtasks\"; 436 ± 132 when only failed subtasks reran. Multi-file debugging: 933, 703 and 460 tokens.", "direction": "mixed", "measured": true, "conditions": "two software engineering workloads; 10 runs per configuration", "older": false }, { "source": "Leins, Pelleriti, Gonnermann-Müller, Pokutta, \"When Does LLM Orchestration Pay Off? A Controlled Evaluation of Accuracy, Cost, and Task Difficulty\"", "url": "https://arxiv.org/abs/2608.00685", "date": "2026-08-01", "locator": "Abstract; §4", "task": "Self-Refine (draft, critique, refine), Best-of-N and Debate against single calls, each tuned with GEPA under the same budget", "effect": "Averaged across backbones, the largest gain was 4.6 points over optimized chain-of-thought and 4.5 over task-only inference, \"while requiring approximately 2 to 4 times the mean total tokens\"; gains were benchmark-dependent and did not grow with task difficulty.", "direction": "mixed", "measured": true, "conditions": "five LLM backbones; competitive programming, chess puzzles, mathematics", "older": false }, { "source": "Dhage, \"Harness Engineering for Predictable Agentic Systems: An Empirical Study of Deterministic Execution Constraints\"", "url": "https://arxiv.org/abs/2608.26197", "date": "2026-08-25", "locator": "Abstract; §6", "task": "Wrap an agent in fixed states, output validation, bounded retry, then a schema check on the plan before any tool runs", "effect": "Validation and bounded retry alone improved reproducibility in one of four model-task cells and degraded it in two. With the plan checked against a schema, three of four cells reached a reproducibility rate of 1.000 and task success of 100%. Token cost fell in every cell; latency fell for one model and rose markedly for the other.", "direction": "mixed", "measured": true, "conditions": "Qwen-2.5-7B-Instruct, Gemma-3-27B; synthetic finance and legal tasks; N=100 per cell", "older": false }, { "source": "Elina, \"Maat: Independent Deterministic Contract-Based Governance for Multi-Agent LLM Workflows\"", "url": "https://arxiv.org/abs/2609.34017", "date": "2026-09-27", "locator": "Abstract; §6", "task": "Check every agent-to-agent handoff against a workflow contract in code, with no model in the check", "effect": "On paired trials, rubric score +7.7% to +29.1% in five workflows and flat in one; model-call cost fell 17-53% where halts came early. Of 94 halts, 35 (37%) were false alarms from validator defects; counting them as failed work, the gated runs scored below ungated ones in four of six workflows.", "direction": "mixed", "measured": true, "conditions": "Claude Haiku 4.5; six workflows of 6-15 agents, 522 trials, injected data defects", "older": false } ], "caveats": "Chains cost latency and tokens, and a fixed chain that reruns downstream steps on a failure can cost more per retry than one prompt; rerun only the failed step. Gates misfire: in one study 37% of halts were false alarms, enough to turn a gain into a loss. The benefit depends on the model and shrinks as models improve: Anthropic dropped a step-by-step breakdown when a stronger model arrived, and tuned single calls came within a few points of orchestration. Several studies are small (one app, one run).", "also_covers": [], "related": [ { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "7.c.iii", "slug": "evaluator-optimizer-loop", "name": "Evaluator-optimizer loop" }, { "id": "5.c.i", "slug": "discrete-phase-separation", "name": "Discrete phase separation" }, { "id": "4.b.ii", "slug": "initializer-maintainer-agents", "name": "Initializer-maintainer dual agent" }, { "id": "7.d.ii", "slug": "inversion-of-control", "name": "Inversion of control" } ], "url": "https://mahalsystems.ai/aot/optimizations/prompt-chaining-with-gates/", "markdown": "https://mahalsystems.ai/aot/optimizations/prompt-chaining-with-gates.md" }, { "id": "7.b.i", "legacy_id": "T05", "former_ids": [], "slug": "detect-loops-and-stalls", "name": "Detect stalls and loops: warn, then stop early", "kind": "technique", "status": "card", "tier": { "tier": 2, "name": "Reliability", "why": "an execution bound: runaway loops are hard failures", "proposed": false }, "levers": [ "control-flow" ], "domains": [ "general" ], "evidence_domains": [ "code-generation", "web-research" ], "categories": [ "Bounding execution" ], "category_ids": [ "orchestration.bound" ], "summary": "Track repeated identical actions (same tool, same arguments; same search query) and turns with no new information. Inject a warning at a small streak (about 5 identical calls), refuse exact-repeat queries, and end the run early at a longer streak (about 8 identical failures) or when progress has stalled, instead of running to the turn cap. A prompt line telling the agent not to repeat itself is a weak version of the same thing and is not a substitute.", "how": [ "In the loop, hash each tool call (name plus arguments) and each query; keep the last N.", "On a streak of 5, inject a 'you are repeating; change approach' message; on 8 identical failures, stop with the best answer so far.", "Define 'stalled' for the task (no new file, result or fact in K turns) and stop on it.", "Measure loop and turn-cap failures before and after." ], "detect": { "static": "no comparison of a tool call or query against previous ones; the loop ends only at a fixed turn cap", "traces": "tool spans with identical name and arguments repeated 3 or more times in a run; the same query string issued twice; runs ending at the turn cap; failure rows labeled redundancy or state-tracking-failure", "evals": "turn-cap and timeout outcomes as a share of failures" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "adaptive termination +5.6% on FACTS; anti-loop prompt line GAIA +0.04, flat on SimpleQA (VeRO)", "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "medium", "when": null, "basis": "evidence" }, { "dimension": "performance", "effect": "improves", "size": "medium", "when": "the template with query dedup ran about 2x faster, bundled", "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "before-after", "tier": 2, "how": "loop and turn-cap failure rate on the same cases; time per case" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper §A.4", "task": "TerminalBench-2", "effect": "Part of the Filesystem-B=178 commit; contributed to AgentTimeoutError rising 16 to 23 as crashing tasks migrated to timeouts rather than passes.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, VeRO code release", "url": "https://github.com/scaleapi/vero", "date": null, "locator": "cookbook (master-agent-optimization-cookbook, Infinite Loop Prevention)", "task": "library pattern", "effect": "Hash recent tool calls, inject a warning on repeat.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Fan et al., \"An Empirical Study of Harness Design for Coding Agents\"", "url": "https://arxiv.org/abs/2609.20804", "date": "2026-09-17", "locator": "§2.4 Stuck detection; §3.1; App 7.4 Fig 20", "task": "Detect identical-call streaks; warn, then terminate", "effect": "Held fixed across all conditions. The reminder texts and thresholds are given.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 6.2; Table 5; Table 11 \"Stuck Detection\"; Recommendation 18", "task": "Hash repeated tool calls to detect stuck loops", "effect": "Hermes hashes the call signature (SHA-256), \"warning after two identical failures and halting after eight—but the hard stop ships disabled\"; Gemini CLI aborts on \"five identical tool calls or ten identical content chunks\"; OpenCode's \"three consecutive byte-identical tool calls raise a doom_loop permission ask\"; OpenHands StuckDetector covers 5 scenarios.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 6.2 (Gemini CLI); Recommendation 18", "task": "Add a periodic LLM self-check for loops", "effect": "Gemini CLI LoopDetectionService: \"after thirty turns in a single prompt an LLM-based self-check runs at an adaptive interval—a two-tier design\"; paper calls this a \"platform-tier investment\".", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 6.2 (Mini-SWE-Agent); Sec 10.10; Recommendation 18", "task": "Cap consecutive malformed responses and wall-clock time", "effect": "Mini-SWE-Agent v2.4 \"added a wall-clock time limit on runs and a cap on consecutive malformed-action responses\"; Rec 18: \"do ship the cheap caps, which cost a dozen lines.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 14", "task": "GAIA", "effect": "\"Strengthened anti-loop prompt instructions\" took validation 0.26 to 0.30 (best of run).", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 15", "task": "SimpleQA", "effect": "\"Loop/entity prompt guidance\" flat (0.86 to 0.85).", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"Can AI Agents Build Better AI Agents?\"", "url": "https://labs.scale.com/blog/vero", "date": null, "locator": "tricks table (Minimal template)", "task": "FACTS", "effect": "+5.6%; \"invented adaptive termination to prevent infinite loops\".", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"Can AI Agents Build Better AI Agents?\"", "url": "https://labs.scale.com/blog/vero", "date": null, "locator": "tricks table (Evidence-Based template)", "task": "GAIA/FACTS/SimpleQA", "effect": "Runtime efficiency; Evidence-Based agents ran about 2x faster than Tool-Centric ones (26.2s vs 56.6s per sample for Knight).", "direction": "supports", "measured": null, "conditions": null, "older": false } ], "caveats": null, "also_covers": [ { "name": "Infinite-loop prevention by hashing recent calls" } ], "related": [ { "id": "7.b.iii", "slug": "size-execution-budgets", "name": "Size run budgets so productive runs aren't cut off: turn caps, deadlines, command timeouts" }, { "id": "5.a.i", "slug": "staged-compaction", "name": "Compact conversation history in stages: elide stale tool outputs, then keep an anchored summary" }, { "id": "7.d.i", "slug": "continuous-task-loop", "name": "Continuous autonomous task loop" }, { "id": "2.b.i", "slug": "fail-soft-tools", "name": "Make tool calls and tool results fail soft, and retry transient failures" }, { "id": "4.b.i", "slug": "filesystem-checkpoints", "name": "Filesystem-based agent state and checkpoints" }, { "id": "7.b.ii", "slug": "keep-errors-in-context", "name": "Keep errors in context" }, { "id": "7.d.iii", "slug": "react-loop", "name": "ReAct loop" }, { "id": "3.a.ii", "slug": "vary-few-shot-examples", "name": "Avoid few-shot ruts" } ], "url": "https://mahalsystems.ai/aot/optimizations/detect-loops-and-stalls/", "markdown": "https://mahalsystems.ai/aot/optimizations/detect-loops-and-stalls.md" }, { "id": "7.b.ii", "legacy_id": "P26", "former_ids": [], "slug": "keep-errors-in-context", "name": "Keep errors in context", "kind": "pattern", "status": "card", "tier": { "tier": 2, "name": "Reliability", "why": "error recovery: failures stay visible to the model", "proposed": true }, "levers": [ "control-flow", "context" ], "domains": [ "general" ], "evidence_domains": [ "tool-use", "code-generation", "planning" ], "categories": [ "Bounding execution" ], "category_ids": [ "orchestration.bound" ], "summary": "Return every failed action to the model as an error it can read (the call, the cause, a capped trace) instead of swallowing the exception, retrying silently, returning an empty result or deleting the failed attempt, and collapse each error once it is resolved. Failures that surface as explicit errors were recovered from far more often than failures that look like valid output (37 points more on average across nine models on one benchmark), and a linter that shows its errors and rejects the bad edit raised SWE-bench Lite from 15.0% to 18.0% on one model. The limits: in 135M-1.7B models the verbatim failed call made exact repeats more likely (though deleting it was worse), a model's own unflagged mistakes in its context raise its later error rate, and on HumanEval restarting fresh after 2-3 failed fixes beat continuing; so keep the diagnosis, cap it, and don't let failures pile up.", "how": [ "Start from the harness as it is and find every place a failure is hidden from the model: exceptions swallowed, silent retries, a failed attempt deleted before the retry, a generic \"operation failed\" in place of the cause, or an empty or default result returned on failure. Inject a few failures (a tool that fails on first use, a silent wrong result, a third-party error message) and record recovery and repeated failed calls.", "Return every failed action to the model in the next turn as a tool result marked as an error: the call (tool and arguments), the error type, the cause message and the exit code. Never turn a failure into an empty or plausible-looking result; on one benchmark agents recovered from silent failures 37 points less often than from explicit ones. Keeping the loop alive when a tool raises is 2.b.i.", "Cap what goes in: keep the exception type and message, the frames in the agent's own code and the last lines of output; write the full trace or log to a file and give the path (2.c.i); make test and lint runs print a few lines, with ERROR and the reason on one line. Write the errors you own so they name the fix, and strip sentences from third-party error text that ask for steps the agent cannot take (a terminal command, a config change, a web page); in one study a one-sentence filter prompt raised recovery on expired credentials from 45% to 82%.", "Do not retry from a context with the failure deleted: under greedy decoding that was the harness that repeated most. If the agent re-emits a failed call verbatim (measured in small models), replace the earlier verbatim call with a short harness-written note of what failed and why, or refuse exact repeats (7.b.i); keep the diagnosis either way.", "Drop errors once they are resolved: when a later call succeeds or the tests pass, collapse the old error observation to a one-line stub, keep only the first of a run of identical format errors, and carry only unresolved errors through compaction (5.a.i). Don't let failures pile up: after 2-3 failed fixes of the same error, restart from the original task (optionally with a one-line note of what failed) or hand off, since non-thinking models make more mistakes as their own uncorrected errors accumulate in context.", "Measure with paired runs on the same tasks and injected failures, the harness as is against errors kept, capped and dropped once resolved, per model: task success, recovery rate per injected failure by type (the agent's own call, credentials, permissions, rate limits, silent wrong results), exact and canonical repeat rate of failed calls, runs ending in a loop, steps and tokens. Re-run after a model change: the repetition effect shrank with model size, and thinking models did not self-condition." ], "detect": { "static": "Exceptions swallowed or retried silently; failures returned as empty or default results; a generic \"operation failed\" in place of the cause; failed attempts deleted before a retry; full stack traces or test logs passed untruncated; third-party error text passed through with steps the agent cannot take (a terminal command, a config change, a web page).", "traces": "Errors absent from the next model input; exact repeats of a call that already failed; runs of failed fixes of the same error; agents stopping after error text that asks for a step they cannot take; error observations thousands of lines long.", "evals": "Recovery rate per injected failure and repeat rate of failed calls, alongside task success.", "probes": "A tool that fails deterministically on first use." }, "impact": { "measured_basis": true, "primary": [ "quality" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "recovery from silent failures 37.15 points below explicit errors, averaged over nine models (Zhu, Jun 2026); lint errors shown with the bad edit rejected 18.0% vs 15.0% on SWE-bench Lite (Yang, May 2024); in 135M-1.7B models a kept verbatim failed call raised repeat probability from 0.06 to 0.54, while deleting it raised exact repeats from 31% to 80% (Gumaan, Aug 2026); wrong answers in a model's own history lowered later accuracy in non-thinking models of every size tested (Sinha, Sep 2025); fresh starts after 2-3 failed debug attempts beat continuing, e.g. 72.56% to 82.82% (Adnan, Jun 2025); a developer-facing step in the error text cut recovery from 82% to 45% (Xu, Sep 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "mixed", "size": "small", "when": "error text and full traces add tokens unless capped (Carlini, Feb 2026); masking old observations, errors included, cut cost per instance by more than half in most configurations against the full history (Lindenbauer, Aug 2025); fresh starts used similar or fewer tokens, e.g. 108,289 to about 89,000 (Adnan, Jun 2025); filtering 949 error messages with a prompt cost $0.09 (Xu, Sep 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "unknown", "size": null, "when": "no study times error retention directly; LLM summaries that may hide failure signals lengthened Gemini 2.5 Flash runs to 52 turns against 44 with observation masking (Lindenbauer, Aug 2025)", "basis": null }, { "dimension": "security", "effect": "mixed", "size": "small", "when": "Stack traces may leak internals.", "basis": "assessment" }, { "dimension": "governance", "effect": "improves", "size": "small", "when": null, "basis": "assessment" }, { "dimension": "brand", "effect": "mixed", "size": "small", "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 2, "how": "paired runs on a fixed task set with failures injected at fixed steps (a tool that fails deterministically on first use, a silent failure that returns a plausible but wrong result, a third-party error that names a step the agent cannot take): the harness as is against failures returned as error results with capped traces and collapsed once resolved, and optionally against a harness-written note in place of the verbatim failed call; per model: task success, recovery rate per injected failure, exact and canonical repeat rate of failed calls, runs ending in a loop, steps and tokens" }, { "method": "monitor", "tier": null, "how": "in production traces: the share of failed tool calls whose error text reaches the next model input; exact repeats of a call that already failed; consecutive failed attempts on the same error; the size of error observations; and recovery after errors by failure type (the agent's own call against credentials, permissions and rate limits)" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file master-agent-optimization-cookbook.md", "task": "Lists this practice in its agent-optimization pattern library (Production group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Yichao 'Peak' Ji (Manus), \"Context Engineering for AI Agents: Lessons from Building Manus\"", "url": "https://manus.im/blog/Context-Engineering-for-AI-Agents-Lessons-from-Building-Manus", "date": "2025-07-18", "locator": "Keep the Wrong Stuff In", "task": "Leave failed actions and their errors in the context instead of cleaning up the trace", "effect": "\"one of the most effective ways to improve agent behavior is deceptively simple: leave the wrong turns in the context.\" No numbers.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Akshit Sinha, Arvindh Arun, Shashwat Goel, Steffen Staab, Jonas Geiping, \"The Illusion of Diminishing Returns: Measuring Long Horizon Execution in LLMs\"", "url": "https://arxiv.org/abs/2509.09677", "date": "2025-09-11", "locator": "§3.2 Results 3-5, Figures 5-6; Appendix A; Appendix C.1-C.2, Figures 9-10", "task": "Counterfactually inject a chosen rate of wrong answers into the model's own earlier turns of a long, fully specified execution task (a running sum over key lookups) and measure accuracy at turn 100; test thinking, a self-verification prompt and a sliding context window as fixes", "effect": "\"as we increase the rate of injected errors into the context, accuracy at turn 100 consistently degrades further.\" Frontier non-thinking models (Kimi-K2, DeepSeek-V3, Qwen3-235B-Instruct-2507) \"remain susceptible to self-conditioning\"; Qwen3 thinking models \"do not self-condition—the accuracy of the models at turn 100 remains stable, regardless of the error rate in its context.\" \"Prompting to self-verify does not suffice to fix the self-conditioning effect completely\"; keeping only the most recent turns (a sliding window) improved performance \"significantly\" on Gemma3 12B. From a manual reading of annotated agent failures the authors \"estimate that roughly 20% of GAIA, 48% of ALFWorld, 33% of WebShop failures are similar to self-conditioning.\" The text gives no per-model numbers for the injection experiment (plotted in Figure 5 only).", "direction": "contradicts", "measured": true, "conditions": "Qwen3 4B-32B and Gemma3 4B-27B, plus frontier models via OpenRouter; synthetic key-value task, 100 samples; the injected errors are unflagged wrong answers in the model's own turns, not tool error messages; the sliding window suits only tasks without long-range dependencies", "older": true }, { "source": "Esmail Gumaan, \"Feedback That Backfires: Why Small Language Model Agents Repeat the Call They Just Watched Fail\"", "url": "https://arxiv.org/abs/2608.23651", "date": "2026-08-24", "locator": "Abstract; §6.1 Table 3; §6.4-6.5; §8.3 Table 8; §10-11", "task": "Measure whether a failed tool call and its error message in the transcript make the model less likely to emit the same call again; compare harnesses that keep the transcript verbatim, add a do-not-repeat instruction, delete the failed step (clean restart), replace the failed call with a runtime-generated description of the failure, or block previously failed strings at the decoder", "effect": "Keeping the failed call and its error made a repeat more likely for all 6 models: \"Over a fixed candidate set the probability of repeating the failed call rises from 0.06 to 0.54\"; \"the failed call's surface form accounts for 83% of the damage, while the semantic contribution of marking it failed is small and inconsistent in sign\". The runtime description \"removes 76% of the inversion at no token cost\". In rollouts (Qwen2.5-0.5B, 24 tasks) success and exact-repeat rate were 0.42 and 31% verbatim, 0.33 and 80% with the failed step deleted, 0.33 and 16% with the description, 0.42 and 8% with the decoder ban, 0.58 and 24% with a do-not-repeat instruction; deleting the failed attempt \"is the worst harness we measured for repetition\".", "direction": "mixed", "measured": true, "conditions": "6 instruction-tuned checkpoints, 135M-1.7B, 4 families; the author's own synthetic tool-calling environment and MBPP program repair (real Python tracebacks, which quote the failing line); greedy decoding; single author; within each family the effect shrinks with model size and is untested above 1.7B; the description and the ban cut repeats but did not raise task success", "older": false }, { "source": "Tobias Lindenbauer, Igor Slinko, Ludwig Felder, Egor Bogomolov, Yaroslav Zharov, \"The Complexity Trap: Simple Observation Masking Is as Efficient as LLM Summarization for Agent Context Management\"", "url": "https://arxiv.org/abs/2508.21433", "date": "2025-08-29", "locator": "Abstract; §4.1 Table 1; §4.4 Figure 4; §5.1", "task": "Replace environment observations older than a window with a placeholder while keeping all reasoning and actions (observation masking), or summarize old turns with an LLM, against the full raw history", "effect": "Observation masking \"halves cost relative to the Raw Agent while matching, and sometimes slightly exceeding, the solve rate of LLM-Summary\" (Qwen3-Coder 480B: 53.4% raw, 54.8% masking, 53.8% summary; Gemini 2.5 Flash with thinking: 40.4%, 36.4%, 31.4%). Summaries lengthened runs (Gemini 2.5 Flash: 52 turns against 44 with masking and 50 raw), \"suggesting they mask failure signals that would otherwise prompt earlier termination.\" On error content: \"SWE-agent directly elides retries due to syntax errors from the dialog history. However, OpenHands retains such retry turns\", so OpenHands needed a larger masking window.", "direction": "mixed", "measured": true, "conditions": "SWE-agent on SWE-bench Verified, five model configurations (Qwen3-32B with and without thinking, Qwen3-Coder 480B, Gemini 2.5 Flash with and without thinking); OpenHands probe on 50 instances; masks all old observations, not errors specifically; the failure-signal reading is the authors' interpretation", "older": true }, { "source": "John Yang, Carlos E. Jimenez, Alexander Wettig, Kilian Lieret, Shunyu Yao, Karthik Narasimhan, Ofir Press, \"SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering\"", "url": "https://arxiv.org/abs/2405.15793", "date": "2024-05-06", "locator": "§3 (Context management); §5.1 Table 3; §5.2 (Editing remains challenging)", "task": "Lint every edit, discard edits that introduce errors and show the agent the lint errors with the file before and after; after a run of malformed outputs keep only the first error message; collapse observations older than the last 5 to one line", "effect": "Edit with linting 18.0% against 15.0% without on SWE-bench Lite; keeping the last 5 observations 18.0% against 15.0% for the full history. For malformed generations, \"Once received, all past error messages except the first are omitted.\" 51.7% of trajectories had one or more failed edits; \"Any attempt at editing has a 90.5% chance of eventually being successful. This probability drops off to 57.2% after a single failed edit.\"", "direction": "supports", "measured": true, "conditions": "GPT-4 Turbo; ablations on SWE-bench Lite (300); the linting condition bundles showing the error with rejecting the bad edit; failed-edit statistics over the full SWE-bench test set (2,294)", "older": true }, { "source": "Xiaonan Xu, Wenjing Wu, \"MCP Error Messages Written for Developers Hurt the Most Capable Agents Most\"", "url": "https://arxiv.org/abs/2609.35381", "date": "2026-09-28", "locator": "Abstract; §4.2-4.3; §5.3; §5.4 Table 6; §6", "task": "Return different error texts for the same failed call (a generic notice, the cause only, the cause plus a correct next step, the cause plus a step the agent cannot take) and, on the harness side, strip next-step sentences from the error text with a one-sentence prompt before the model reads it", "effect": "On expired credentials, recovery was 61% with \"Operation failed.\", 82% with the cause, and 45% with the cause plus a terminal command; \"deleting the step with a one-sentence prompt before the model reads it raised recovery on expired credentials to 82%\" (+37.5 points), and where the removed step had been correct recovery changed by -1.7 to -0.8 points, intervals including zero. For failures in the agent's own call (wrong unit or format, missing field, wrong tool, missing resource) recovery \"stayed within four points\" of the generic notice under every text. On a rate limit the cause alone left 5% recovered; naming the call to repeat gave 88%.", "direction": "mixed", "measured": true, "conditions": "Five OpenAI models (GPT-5.5, GPT-5.6 Sol, GPT-6 Sol, GPT-6 Astra, GPT-6 Luna), reasoning effort high; BFCL multi-turn tasks, seven failure types, 15,120 trials plus 1,440 for the filter; agents limited to the task's tools; survey of 3,001 error messages in 150 MCP servers", "older": false }, { "source": "Dongsheng Zhu, Xuchen Ma, Yucheng Shen, Xiang Li, Yukun Zhao, Shuaiqiang Wang, Lingyong Yan, Dawei Yin, \"When Tools Fail: Benchmarking Dynamic Replanning and Anomaly Recovery in LLM Agents\"", "url": "https://arxiv.org/abs/2606.05806", "date": "2026-06-04", "locator": "Abstract; §3.2; §4.1-4.2; §4.3 (The implicit–explicit trust gap), Figure 6", "task": "Inject tool failures at fixed nodes of multi-step tool tasks, either explicit (machine-readable exceptions such as HTTP 404) or implicit (structurally valid but semantically wrong output), transient or permanent, and measure how often agents recover", "effect": "Recovery from implicit failures was lower than from explicit ones for every model; the gap was \"averaging 37.15% overall (53.75% for transient and 20.54% for permanent conditions)\", which the authors attribute to over-trust in \"semantically perturbed tool outputs\". A failure-aware prompt yielded \"improvements ranging from +1.5% to +20.8%\" but success under perturbation stayed \"substantially lower\" than without it.", "direction": "supports", "measured": true, "conditions": "Nine models (GLM-5.1, DeepSeek-V4-Pro, MiniMax-M2.7, Qwen3.5-35B-A3B, Qwen3.5-397B-A17B, Qwen3.6-27B, GPT-5.5, Gemini-3.1-Pro-Preview, Claude-Sonnet-4-6); 2,000 instances from 400 synthesized tasks over 270 tools; at most 25 steps; an implicit failure stands in for a harness that hides an error, it is not a harness ablation", "older": false }, { "source": "Muntasir Adnan, Carlos C. N. Kuhn, \"The Debugging Decay Index: Rethinking Debugging Strategies for Code LLMs\"", "url": "https://arxiv.org/abs/2506.18403", "date": "2025-06-23", "locator": "Abstract; RQ3; Table 1; Table 2 and its discussion", "task": "Debug generated code on test feedback in one conversation for up to six attempts, against clearing the conversation and restarting from the problem statement alone at the attempt where debugging effectiveness has decayed by 50% or 80%", "effect": "\"most models lose 60-80% of their debugging capability within just 2-3 attempts\". Iterative debugging itself raised llama3.1:8b from 56.707% at the first attempt to 72.561% within six; with fresh starts at the computed points, \"Of the six models evaluated, all showed performance improvements\": llama3.1:8b 72.56% to 82.82%, deepseek-coder-v2:16b 84.1% to 92.1%, Mistral:Instruct 54.3% to 62.8% and 57.3%, at the same six-attempt budget with similar or fewer tokens.", "direction": "contradicts", "measured": true, "conditions": "HumanEval (164 problems); decay fitted for 18 models, fresh starts tested on 6, mostly small open models; the paper frames a fresh start as a shift \"from exploitation to exploration\", so the gain is not attributed to removing errors alone", "older": true }, { "source": "Nicholas Carlini (Anthropic), \"Building a C compiler with a team of parallel Claudes\"", "url": "https://www.anthropic.com/engineering/building-c-compiler", "date": "2026-02-05", "locator": "Lessons from programming with Claude agent teams: Put yourself in Claude's shoes (Context window pollution)", "task": "Keep test-harness output short for long-running autonomous agents: print a few lines, log the rest to files, and make errors greppable", "effect": "\"The test harness should not print thousands of useless bytes. At most, it should print a few lines of output and log all important information to a file so Claude can find it when needed. Logfiles should be easy to process automatically: if there are errors, Claude should write ERROR and put the reason on the same line so grep will find it.\" No measured comparison.", "direction": "supports", "measured": false, "conditions": "Claude Opus 4.6 agent teams, nearly 2,000 Claude Code sessions over two weeks; the author's own project", "older": false }, { "source": "Ryan Lopopolo (OpenAI), \"Harness engineering: leveraging Codex in an agent-first world\"", "url": "https://openai.com/index/harness-engineering/", "date": "2026-02-11", "locator": "Enforcing architecture and taste", "task": "Enforce architecture and taste rules with custom linters and structural tests whose error messages are written for the agent", "effect": "\"Because the lints are custom, we write the error messages to inject remediation instructions into agent context.\" No measured comparison.", "direction": "supports", "measured": false, "conditions": "Codex agents building an internal product of about a million lines over five months; the team's own report", "older": false } ], "caveats": "No study isolates keeping versus hiding an error on a frontier agent: the closest measurements are a bundled linter change on one model (+3 points on SWE-bench Lite), a benchmark that compares explicit with silent tool failures, and a single-author study of 135M-1.7B models in which the verbatim failed call raised repeats while deleting it was worse. Failures that accumulate hurt: wrong answers in a model's own history lowered its later accuracy in non-thinking models (not fixed by scale, absent in one thinking family), and on HumanEval restarting fresh after 2-3 failed fixes beat continuing for all six models tested. What the error says matters as much as whether it is shown: a step the agent cannot take cut recovery from 82% to 45%, and for failures in the agent's own call the wording barely mattered.", "also_covers": [], "related": [ { "id": "2.b.i", "slug": "fail-soft-tools", "name": "Make tool calls and tool results fail soft, and retry transient failures" }, { "id": "7.b.i", "slug": "detect-loops-and-stalls", "name": "Detect stalls and loops: warn, then stop early" }, { "id": "2.c.i", "slug": "bound-tool-output", "name": "Bound tool output: cap it, or write it to a file the agent can read" }, { "id": "5.a.i", "slug": "staged-compaction", "name": "Compact conversation history in stages: elide stale tool outputs, then keep an anchored summary" }, { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "2.a.ii", "slug": "rewrite-tool-surface", "name": "Rewrite the tool surface: names, schemas, descriptions" }, { "id": "7.c.i", "slug": "reflection-loop", "name": "Reflection loop" }, { "id": "9.b.iii", "slug": "verify-completion", "name": "Verify completion against the environment before the agent stops" }, { "id": "9.b.i", "slug": "extractable-final-answer", "name": "Make the final answer reliably extractable: state the format, extract robustly, force an answer at the limit" }, { "id": "3.a.ii", "slug": "vary-few-shot-examples", "name": "Avoid few-shot ruts" } ], "url": "https://mahalsystems.ai/aot/optimizations/keep-errors-in-context/", "markdown": "https://mahalsystems.ai/aot/optimizations/keep-errors-in-context.md" }, { "id": "7.b.iii", "legacy_id": "T29", "former_ids": [], "slug": "size-execution-budgets", "name": "Size run budgets so productive runs aren't cut off: turn caps, deadlines, command timeouts", "kind": "technique", "status": "card", "tier": { "tier": 2, "name": "Reliability", "why": "budgets are execution bounds", "proposed": false }, "levers": [ "control-flow" ], "domains": [ "general" ], "evidence_domains": [ "web-research" ], "categories": [ "Bounding execution" ], "category_ids": [ "orchestration.bound" ], "summary": "Set the turn or step cap, the wall-clock deadline and per-command timeouts from what runs that succeeded actually needed, not from framework defaults, so a run that is still making progress is not stopped short. This is one of three changes that bound a run, each for a different failure: this one lets productive runs finish, 7.b.i stops runs that have stalled, and 9.b.i makes sure a run that does hit the limit still returns its best answer. Raise a cap only together with 7.b.i, or a stall becomes a longer stall. Timeouts are tuned both ways: a timeout that kills commands that would have finished is too low, one that lets a hung command eat the run is too high.", "how": [ "From traces, take the distribution of turns, wall-clock and command durations on runs that succeeded.", "Find the runs that ended at a limit while still progressing: new tool results, new files or a changing answer in their last turns.", "Set each cap above the successful range (for example the 95th percentile) and each command timeout above the longest command that finished, with 7.b.i in place before raising a cap.", "Re-run the truncated cases and compare limit-ended outcomes, cost and turns on the same cases." ], "detect": { "static": "caps, deadlines and command timeouts at framework defaults, unchanged since the agent was built", "traces": "runs ending at the turn cap or deadline while still producing new information in their last turns (a run that ends at the cap after repeating itself is 7.b.i's signal, not this one); commands killed by a timeout that finish when re-run with a longer one", "evals": "limit-ended outcomes (turn cap, deadline, command timeout) on cases where the run was still progressing" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "unknown", "size": null, "when": "the only evidence is how often optimizers changed these limits, not what the change did", "basis": null }, { "dimension": "cost", "effect": "mixed", "size": "medium", "when": "more turns per case when a cap is raised", "basis": "evidence" }, { "dimension": "performance", "effect": "mixed", "size": "medium", "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "before-after", "tier": 2, "how": "limit-ended outcomes, cost and turns on the same cases, before and after" } ], "evidence": [ { "source": "Scale AI, VeRO code release", "url": "https://github.com/scaleapi/vero", "date": null, "locator": "figure_04", "task": "five follow-up benchmarks", "effect": "MAX_TURNS raised 63 times, lowered 14; MAX_STEPS raised 26, lowered 5. The most-edited constants in the corpus.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, VeRO code release", "url": "https://github.com/scaleapi/vero", "date": null, "locator": "figure_01", "task": "five follow-up benchmarks", "effect": "Turn-budget edits in 11 to 18 of 20 runs per benchmark.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 10", "task": "GAIA", "effect": "Knight 40 max turns vs Pawn 20.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, VeRO code release", "url": "https://github.com/scaleapi/vero", "date": null, "locator": "figure_04", "task": "five follow-up benchmarks", "effect": "SOFT_DEADLINE_SEC 2 up / 3 down, COMMAND_TIMEOUT_SEC 1 up / 4 down, RESEARCH_DEADLINE_SEC 0 up / 2 down. Direction is mixed, unlike turn budgets.", "direction": "supports", "measured": null, "conditions": null, "older": false } ], "caveats": "The evidence is frequency only: VeRO's optimizers raised turn caps far more often than they lowered them (MAX_TURNS 63 up, 14 down) and moved timeouts both ways, but no run measured the effect. Client retries for rate limits and transient errors are not a budget; they are in 2.b.i.", "also_covers": [], "related": [ { "id": "7.b.i", "slug": "detect-loops-and-stalls", "name": "Detect stalls and loops: warn, then stop early" }, { "id": "9.b.i", "slug": "extractable-final-answer", "name": "Make the final answer reliably extractable: state the format, extract robustly, force an answer at the limit" }, { "id": "2.b.i", "slug": "fail-soft-tools", "name": "Make tool calls and tool results fail soft, and retry transient failures" }, { "id": "5.a.i", "slug": "staged-compaction", "name": "Compact conversation history in stages: elide stale tool outputs, then keep an anchored summary" }, { "id": "7.d.i", "slug": "continuous-task-loop", "name": "Continuous autonomous task loop" }, { "id": "4.c.i", "slug": "episodic-memory", "name": "Episodic memory retrieval" }, { "id": "7.c.iii", "slug": "evaluator-optimizer-loop", "name": "Evaluator-optimizer loop" }, { "id": "8.b.ii", "slug": "link-large-schemas", "name": "Link large schemas by exploration: start broad, grow the linked set with probing queries, favor recall" }, { "id": "6.b.i", "slug": "parallel-read-only-tools", "name": "Parallel tool execution for read-only calls" }, { "id": "7.d.iii", "slug": "react-loop", "name": "ReAct loop" } ], "url": "https://mahalsystems.ai/aot/optimizations/size-execution-budgets/", "markdown": "https://mahalsystems.ai/aot/optimizations/size-execution-budgets.md" }, { "id": "7.b.iv", "legacy_id": "T62", "former_ids": [], "slug": "track-task-progress", "name": "Track explicit progress: a task list with a status per item, re-injected each turn", "kind": "technique", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes what the model sees each turn", "proposed": false }, "levers": [ "control-flow", "context" ], "domains": [ "planning" ], "evidence_domains": [ "code-generation" ], "categories": [ "Bounding execution" ], "category_ids": [ "orchestration.bound" ], "summary": "Give the agent a task-list tool (each item with a status, one in progress at a time), ask for the list on the first turn of non-trivial tasks, and re-inject the current list after the system prompt each turn instead of keeping old copies in history. The effect depends on the model: the one controlled study found a large success gain for a weaker model, none for a mid-size one, and a small success loss with a large cost saving for strong ones, whose removed turns were mostly post-edit verification.", "how": [ "Add the task-list tool and the first-turn requirement.", "Inject the current list after the system prompt each turn; strip old copies from history.", "Paired eval per model you run: success, cost and turns. Re-run it when the model changes." ], "detect": { "static": "no task list or todo mechanism in the harness", "traces": "runs ending with no edit or output, and short aimless trajectories (weaker models); long post-edit loops (stronger models); failure rows labeled goal-drift or state-tracking-failure", "evals": "success by trajectory length, per model" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "weaker model +11.6 points on SWE tasks and +4.5 on TerminalBench (a 30B model); no consistent gain at 120B; strong models about -2 points (Fan, Sep 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "mixed", "size": "medium", "when": "strong models about 30% cheaper (fewer post-edit turns); the weaker model up to 75% dearer, because it now works the task instead of stopping early", "basis": "evidence" }, { "dimension": "performance", "effect": "mixed", "size": null, "when": "follows turns: fewer for strong models, more for the weaker one", "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval with and without the task list, separately for each model in use" } ], "evidence": [ { "source": "Fan et al., \"An Empirical Study of Harness Design for Coding Agents\"", "url": "https://arxiv.org/abs/2609.20804", "date": "2026-09-17", "locator": "§2.1; §3.2 'Planning improves success rate for the weaker model...', Fig 6, Table 5; §4 Table 6, Figs 8-9; App 7.2 Figs 15-17, App 8.4 Fig 30", "task": "Add a persistent plan tool re-injected each turn", "effect": "T4/128k, full tools. Nemotron-3 30B: planning +11.6 pp SWE and +4.5 pp TB at higher cost (TB cost +75%). Without planning its median SWE trajectory is 5 turns (vs 40), and 68.6% of runs end without an edit (vs 27.8%). Nemotron-3 120B: no consistent SR gain; cost up on SWE and down about 26% on TB. Nemotron-3 550B / Mistral: SWE cost down about 30% / 32% with SR -2.0 / -0.4 pp. Median turns 108->74 and 68->53, the removed turns being mostly post-edit verification. Mistral TB cost down about 40%.", "direction": "supports", "measured": true, "conditions": null, "older": false } ], "caveats": "Model strength decides the sign. On weaker models the list keeps the agent working the task; on strong models it mainly trades a little success for a lot of cost, by cutting verification turns. Measure it per model, and again after a model upgrade. Keeping the plan in a file only matters when work outlives one context window; that is 4.b.iii.", "also_covers": [ { "name": "Attention recitation" } ], "related": [ { "id": "4.b.iii", "slug": "multi-window-task-state", "name": "Keep plan, progress and results in files when work outlives one context window" }, { "id": "5.a.i", "slug": "staged-compaction", "name": "Compact conversation history in stages: elide stale tool outputs, then keep an anchored summary" }, { "id": "7.d.i", "slug": "continuous-task-loop", "name": "Continuous autonomous task loop" }, { "id": "5.c.i", "slug": "discrete-phase-separation", "name": "Discrete phase separation" }, { "id": "4.b.ii", "slug": "initializer-maintainer-agents", "name": "Initializer-maintainer dual agent" }, { "id": "3.a.ii", "slug": "vary-few-shot-examples", "name": "Avoid few-shot ruts" } ], "url": "https://mahalsystems.ai/aot/optimizations/track-task-progress/", "markdown": "https://mahalsystems.ai/aot/optimizations/track-task-progress.md" }, { "id": "7.c.i", "legacy_id": "P37", "former_ids": [], "slug": "reflection-loop", "name": "Reflection loop", "kind": "pattern", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it adds a refine step graded against a metric", "proposed": true }, "levers": [ "control-flow" ], "domains": [ "general" ], "evidence_domains": [ "reasoning", "code-generation", "planning", "conversation" ], "categories": [ "Scaling compute" ], "category_ids": [ "orchestration.extra-compute" ], "summary": "The same model critiques and revises its own output for a few rounds (Self-Refine), or after a failed attempt writes a short lesson it carries into the next try (Reflexion). It pays when the critique has something to stand on, such as a failing test, an error, an environment reward or a checkable constraint (Reflexion on hard Rust problems went from 0.60 to 0.68 with its tests and fell to 0.52 without them), or when it judges form rather than correctness. Asked only to re-check whether an answer is right, models often break right ones as well as fix wrong ones: one revision pass lowered accuracy in 20 of 28 BoolQ settings across open-weight models, every RefineBench model (GPT-5 and o1 included) moved by -2.5 to +2.6 points over five turns, and plain retries against the tests beat Reflexion on HumanEval at lower cost. Revision can still help long reasoning (one study found significant gains on BBEH Non-Choice for all five models and on HLE for two), but there it turned up to 13.25% of right answers wrong against at most 2.1% on checkable SAT constraints, so gate the loop by how checkable the answer is. On reasoning models an outer loop mostly adds tokens: over five self-refinement turns on RefineBench, o1 went from 18.5 to 18.3 and DeepSeek-R1 from 8.1 to 7.9.", "how": [ "Start from the baselines on your own items: the single pass with the model's own thinking on or a higher effort (1.b.i), and retries or best-of-N at the same tokens (9.c.ii). Plain retries against HumanEval's example tests scored 92.0% for $2.51 where Reflexion scored 87.8% for $3.90.", "Add the loop only where the critique has a signal to work from (a failing test, a compiler or runtime error, an environment reward, a constraint the model can check such as a formula, required fields or a length limit) or where it judges form (tone, format, coverage of required items) rather than correctness. If the check is sound, feed its verdict back instead (9.a.i); if the critic needs written criteria and its own context, use 7.c.iii. Do not add an outer \"check your answer\" step on a reasoning model with thinking on.", "Put the signal or the checklist into the critique prompt: the error, the failing test, the list of requirements (given the full checklist, Gemini 2.5 Pro scored 44.5 points above open self-refinement, which gained 1.8 over five turns). On a non-reasoning model, try appending \"Wait\" after its own output before the critique: 14 such models fixed injected errors in their own output 64.5% less often than the same errors in the user's message, and appending \"Wait\" cut that gap by 89.3%. Use a neutral instruction that keeps the answer unless something contradicts it; on BoolQ every refinement prompt lowered mean accuracy, and more forceful ones added more harmful flips than helpful ones.", "Cap rounds at one to three and stop when the check passes or the critique finds nothing to fix; Self-Refine capped at four, with most of the gain in the first iterations.", "Keep the first answer unless the revision is better by the metric: accept it only when the check passes or the score rises, otherwise return the original. A trained gate took Qwen2.5-7B on BoolQ to 86.02% where always revising gave 72.32%, though the author flags the gate's gains as unverified.", "When attempts repeat (agent episodes, test-driven retries), store each failure's short verbal reflection and give the last few to the next attempt (Reflexion kept 1 to 3); that is where reflection beat plain retry: 130 of 134 ALFWorld tasks while the retry-only agent stopped improving after six trials.", "Measure on held-out items with ground truth the loop never sees: accuracy per round against both baselines at matched tokens, the share of right answers turned wrong and of wrong ones fixed per round (one untrained model turned 15.8% of MATH problems from right to wrong), and tokens and latency per item; re-run on every model upgrade." ], "detect": { "static": "A step where the same model critiques its own output and a step that revises it (a reflect, critique or \"double-check your answer\" call), or verbal reflections stored between attempts, with a round cap. Flag a loop that re-checks correctness with no test, error, environment signal or explicit checklist in the critique, no cap, one that always returns the last revision, and an outer reflect step on a reasoning model with thinking on.", "traces": "Rounds per item; share of rounds whose critique changes the answer; on items with ground truth, right-to-wrong and wrong-to-right flips per round; whether the critique cites a test, error or requirement or only restates the answer; tokens per round.", "evals": "Accuracy per round against the single pass and against retries or best-of-N at equal tokens; share of initially right answers the loop breaks.", "judge": "Rate critiques for specificity.", "probes": "Correct outputs mixed with outputs carrying one detectable defect." }, "impact": { "measured_basis": true, "primary": [ "quality", "cost" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "gains where a signal or checkable constraint exists: hard Rust problems 0.60 to 0.68 with self-written tests, 0.52 without (Shinn, Mar 2023); SAT 34.62% to 64.96% on Mistral Large with at most 2.1% of right answers broken (Stav, Jun 2026); constrained generation 15.0 to 45.0 on GPT-4 (Madaan, Mar 2023). Mixed on re-checks of correctness: significant gains on BBEH Non-Choice for all five models and on HLE for two, with up to 13.25% of right answers turned wrong (Stav, Jun 2026); MATH 52.8% to 51.8% with Self-Refine (Kumar, Sep 2024); lower accuracy in 20 of 28 BoolQ settings (Zhang, Sep 2026); -2.5 to +2.6 points over five turns for every RefineBench model (Lee, Nov 2025); Reflexion 87.8% against plain retry 92.0% on HumanEval (Kapoor, Jul 2024); over 90% of in-thinking reflections confirmatory in reasoning models (Kang, Oct 2025)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "medium", "when": "at least one critique and one revision call per round: Reflexion cost $3.90 on HumanEval against $1.93 for one GPT-4 call and $2.51 for plain retry (Kapoor, Jul 2024); inside reasoning models, reflections after the first answer took 16.8% to 47.8% of tokens for 1.4 to 3.5 points (Kang, Oct 2025)", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "medium", "when": "each round is a serial critique and revision call; no source here measures the latency", "basis": "assessment" }, { "dimension": "security", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "governance", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "brand", "effect": "improves", "size": "small", "when": "Polished output where the critique is about form (tone, format, required items).", "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval on held-out items with ground truth the loop never sees: the loop against the single pass with thinking on and against retries or best-of-N at matched tokens, per model; accuracy per round, share of right answers turned wrong and of wrong answers fixed per round, tokens and latency" }, { "method": "monitor", "tier": null, "how": "rounds per item, share of rounds that change the answer, gate acceptance rate, check-pass rate after each round, cap-hit rate; on a regularly audited sample, right-to-wrong flips" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Feedback & Iteration group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Madaan et al., \"Self-Refine: Iterative Refinement with Self-Feedback\"", "url": "https://arxiv.org/abs/2303.17651", "date": "2023-03-30", "locator": "Abstract; §3.3 Table 1; §4 Figure 4", "task": "The same model generates, critiques its output and refines it iteratively", "effect": "\"improving by ~20% absolute on average in task performance\" over one-step generation. Per task, GPT-4 base to Self-Refine: dialogue response preference 25.4 to 74.6, constrained generation 15.0 to 45.0, code optimization 27.3 to 36.0, math reasoning 92.9 to 93.1 (GPT-3.5 64.1 to 64.1), the math gain being limited by \"the inability to accurately identify whether there is any error\". Up to 4 iterations; most of the gain came early (code optimization 22.0 at the first output to 28.8 after three iterations).", "direction": "supports", "measured": true, "conditions": "7 tasks from dialog response generation to math reasoning; GPT-3.5, ChatGPT, GPT-4 as generator, critic and refiner; baseline is one-step generation with the same model, not compute-matched", "older": true }, { "source": "Noah Shinn, Federico Cassano, Edward Berman, Ashwin Gopinath, Karthik Narasimhan, Shunyu Yao, \"Reflexion: Language Agents with Verbal Reinforcement Learning\"", "url": "https://arxiv.org/abs/2303.11366", "date": "2023-03-20", "locator": "§4.1 Figure 3; §4.2 Figure 4; §4.3 Tables 1-3", "task": "After a failed trial (signaled by the environment, a stall heuristic, exact-match grading or the agent's own generated unit tests) the same model writes a verbal self-reflection, kept in memory and given to the next trial; ablations remove the reflection or the tests", "effect": "ALFWorld: ReAct + Reflexion completed \"130 out of 134 tasks\"; the ReAct-only agent, which also reset and retried, stopped improving: \"performance increase halts between trials 6 and 7\". HotpotQA (CoT with the gold context, exact-match success signal): self-reflection gave \"an 8% absolute boost over the episodic memory learning advantage\". Pass@1, GPT-4 single sample vs Reflexion: HumanEval Python 80.1 vs 91.0; MBPP Python 80.1 vs 77.1, where its self-generated tests passed wrong solutions (false-positive rate 16.3% against 1.4% on HumanEval). Ablation on the 50 hardest HumanEval Rust problems: base 0.60, self-reflection without test generation 0.52, tests without self-reflection 0.60, both 0.68.", "direction": "mixed", "measured": true, "conditions": "GPT-3 on ALFWorld (134 tasks, up to 12 trials, memory of the last 3 reflections); 100 HotpotQA questions; GPT-4 on HumanEval, MBPP and their Rust translations, up to 6 self-generated tests, memory of 1 reflection", "older": true }, { "source": "Sayash Kapoor, Benedikt Stroebl, Zachary S. Siegel, Nitya Nadgir, Arvind Narayanan, \"AI Agents That Matter\"", "url": "https://arxiv.org/abs/2407.01502", "date": "2024-07-01", "locator": "§2.2-2.3; Figure 1; Appendix A Table A1; Table A7; Appendix E.1", "task": "Reran Reflexion as released (at most 2 iterations, temperature 0.2 for its internal tests and self-reflections) on HumanEval against plain baselines: retry, which re-invokes GPT-4 at temperature 0 up to five times when it fails the problem's example tests, and warming, which also raises the temperature from 0 to 0.5 across retries", "effect": "Mean of 5 runs, accuracy and total cost: Reflexion (GPT-4) 87.8% and $3.90 (91.0 in its own paper); retry 92.0% and $2.51; warming 93.2% and $2.45; one GPT-4 call 89.6% and $1.93. \"Reflexion and LDB cost over 50% more than the warming strategy\"; \"the question of whether debugging, reflection, and other such “System 2” approaches ... are useful for code generation remains open\".", "direction": "contradicts", "measured": true, "conditions": "gpt-4-turbo-2024-04-09 at April 2024 prices; HumanEval, 164 problems (LDB version with example tests for all); 5 runs", "older": true }, { "source": "Aviral Kumar, Vincent Zhuang, Rishabh Agarwal, Yi Su, John D Co-Reyes, Avi Singh, et al. (Google DeepMind), \"Training Language Models to Self-Correct via Reinforcement Learning\"", "url": "https://arxiv.org/abs/2409.12917", "date": "2024-09-19", "locator": "§6.1 Tables 3-4; §6.2 Figure 1 (right)", "task": "Two attempts per problem; the second-attempt instruction asks the model to find any mistake in its first answer without saying whether it is correct. The untrained model and prompted Self-Refine compared with fine-tuned variants and SCoRe, multi-turn RL that trains the same model to self-correct", "effect": "MATH, Gemini 1.5 Flash, first to second attempt: base model 52.6% to 41.4% (-11.2; 15.8% of problems went from correct to incorrect, 4.6% the other way); Self-Refine 52.8% to 51.8% (-1.0; 4.2% vs 3.2%); SCoRe 60.0% to 64.4% (+4.4; 1.4% vs 5.8%), \"the first significantly positive delta\". HumanEval, Gemini 1.0 Pro: base 53.7% to 56.7%; Self-Refine 53.7% to 52.5% (11.0% correct to incorrect); SCoRe 52.4% to 64.6%. With the trained model and a 32-solution budget, \"parallel sampling shows a 7.4% accuracy gain, while combining it with sequential sampling using self-correction yields a 10.5% improvement\".", "direction": "mixed", "measured": true, "conditions": "Gemini 1.5 Flash (MATH) and Gemini 1.0 Pro (trained on MBPP, tested on HumanEval); greedy decoding; one round of self-correction", "older": true }, { "source": "Soumya Suvra Ghosal, Souradip Chakraborty, Avinash Reddy, Yifu Lu, Mengdi Wang, Dinesh Manocha, Furong Huang, Mohammad Ghavamzadeh, Amrit Singh Bedi, \"Does Thinking More always Help? Mirage of Test-Time Scaling in Reasoning Models\"", "url": "https://arxiv.org/abs/2506.04210", "date": "2025-06-04", "locator": "§2.3 Figure 2; §4 Figures 6-7", "task": "Reasoning models made to keep re-checking by suppressing the end-of-thinking token and appending \"Wait\", against parallel thinking: the same token budget spread over independent traces with a majority vote", "effect": "DeepSeek-R1-Distill-Qwen-1.5B on GSM8K: \"accuracy increases from 82.2% to 87.3% as the average number of thinking tokens increases from 385 to 1100\", then \"pushing the average thinking token count from 1100 to 15980 reduces accuracy from 87.3% to 70.3%\". At a 16,000-token budget, \"Wait & Think More\" gave \"a 11.8% performance drop\" against standard thinking, while parallel thinking with the same budget \"boosts accuracy by 10.1%\". The authors report \"a consistent non-monotonic trend in performance across multiple tasks and datasets\".", "direction": "contradicts", "measured": true, "conditions": "DeepSeek-R1-Distill-Qwen-1.5B, -Qwen-7B and -Llama-8B; GSM8K (1,320), MATH-500, AIME 2024 (30)", "older": true }, { "source": "Ken Tsui, \"Self-Correction Bench: Uncovering and Addressing the Self-Correction Blind Spot in Large Language Models\"", "url": "https://arxiv.org/abs/2507.02778", "date": "2025-07-03", "locator": "Abstract; §4.1; §4.4 Table 3; §6", "task": "The same error is injected either into the user's message or into the model's own previous output, all else identical, and the model's correction rate is compared; then \"Wait\" is appended after the model's erroneous output", "effect": "Across 14 open-source non-reasoning models, a \"64.5% average Self-Correction Blind Spot\": \"models reliably correct external errors but fail on identical internal ones\". Appending \"Wait\" \"reduces the blind spot by 89.3% on average\" and \"increases mean accuracy by 156%\"; reasoning models \"exhibit a small, even negative\" blind spot. Macro accuracy, base / with \"Wait\" / reasoning sibling: DeepSeek-V3-0324 0.567 / 0.902 / DeepSeek-R1-0528 0.908; Qwen3-32B non-thinking 0.045 / 0.793 / thinking 0.894. On models' own natural errors, \"at least 4.3–10.8%\" were ones they had the knowledge to catch.", "direction": "mixed", "measured": true, "conditions": "14 open-source non-reasoning models plus reasoning counterparts; injected errors in SCLI5, GSM8K-SC and PRM800K-SC; figures from v3 (Aug 2026), headline numbers unchanged since v1", "older": true }, { "source": "Liwei Kang, Yue Deng, Yao Xiao, Zhanfeng Mo, Wee Sun Lee, Lidong Bing, \"First Try Matters: Revisiting the Role of Reflection in Reasoning Models\"", "url": "https://arxiv.org/abs/2510.08308", "date": "2025-10-09", "locator": "§2.2 Figures 3-4", "task": "Splits reasoning-model rollouts at the first candidate answer and labels each later reflection by whether it changes the answer's correctness", "effect": "\"over 90% of the reflections are confirmatory\"; corrective ones (wrong to right) are \"mostly less than 2%\". Reflections after the first candidate took 16.8% to 47.8% of the tokens for a gain of 1.4% to 3.5%.", "direction": "contradicts", "measured": true, "conditions": "eight reasoning models from 7B to 685B (DeepSeek-R1-0528, gpt-oss-20b, Magistral-Small-2506, MiMo-7B-RL, Qwen3-8B, three R1 distills); AIME 2024 and 2025, AMC, Olympiad Bench, MATH-500; 3,427 rollouts per model; answers checked with Math-Verify", "older": true }, { "source": "Young-Jun Lee, Seungone Kim, Byung-Kwan Lee, Minkyeong Moon, Yechan Hwang, Jong Myoung Kim, Graham Neubig, Sean Welleck, Ho-Jin Choi, \"RefineBench: Evaluating Refinement Capability of Language Models via Checklists\"", "url": "https://arxiv.org/abs/2511.22173", "date": "2025-11-27", "locator": "§4.2 Table 3; §5.1 Table 4; §5.3 Figure 9", "task": "Up to five turns of self-refinement (the model is asked to improve its own answer with no feedback and decides when to stop) against guided refinement (told which checklist items failed) and against self-refinement given the full checklist; each answer graded item by item by GPT-4.1", "effect": "Pass (all checklist items met), turn 1 to turn 5, self-refinement: Gemini 2.5 Pro 29.5 to 31.3; GPT-5 27.5 to 29.1; o1 18.5 to 18.3; DeepSeek-R1 8.1 to 7.9; across all models in Table 3 the change ranged from -2.5 to +2.6. Guided refinement: Gemini 2.5 Pro 29.5 to 94.7, Claude-Opus-4.1 18.7 to 98.4. Given the full checklist but no feedback, \"LLaMA-3.1-70B-Instruct improves by +43.6% and Gemini 2.5 Pro by +44.5%\" at turn 5 over plain self-refinement. DeepSeek-R1 from turn 2 to 3: 19.1% of answers went from correct to incorrect, 8.1% the other way.", "direction": "contradicts", "measured": true, "conditions": "1,000 problems in 11 domains; 34 models including o1, o3-mini, o4-mini, GPT-5, Claude 4 and 4.1, Gemini 2.5, DeepSeek-R1; OpenAI reasoning effort medium, 10K-token limit for reasoning models", "older": false }, { "source": "Elroy Stav, Dvir Berlowitz, Maayan Orner, Sarit Kraus, \"When Does Intrinsic Self-Correction Help? A Task-Sensitive Analysis\"", "url": "https://arxiv.org/abs/2606.23196", "date": "2026-06-22", "locator": "§4 Tables 1-2; Appendix A Tables 3-4; Appendix B.5 Table 5", "task": "The same model reviews and revises its own answer with no labels, tools or new information (prompts checked for fairness), on tasks chosen so the revision can check explicit constraints (SAT), revisit long reasoning (BBEH, HLE) or weigh alternatives (Wordle, Hangman, Codenames)", "effect": "SAT accuracy, first answer to self-correction: Mistral Large 34.62 to 64.96, Claude Haiku 4.5 75.64 to 93.59, DeepSeek V3.1 78.63 to 92.74, Gemini 3.1 Flash Lite 60.68 to 82.91, Gemini 3 Pro 97.00 to 100; it \"eliminates an average of 65.66% of the baseline errors\", and turned right answers wrong \"at a rate of at most 2.1%\" on SAT against \"up to 13.25%\" on BBEH and HLE. BBEH Choice: Mistral Large 27.75 to 26.50, Claude Haiku 4.5 35.00 to 34.50. BBEH Non-Choice improved significantly for all five models, HLE for two. Gains on BBEH sub-tasks correlated with judged verifiability (Spearman 0.65) and reasoning intensity (0.54); sub-tasks low on both gave \"negligible or negative\" gains. Gemini Pro gained least; Claude Haiku 4.5 changed its Codenames clue in 98.7% of turns and its score fell from 0.990 to 0.889.", "direction": "supports", "measured": true, "conditions": "five models at temperature 0; SAT 234, BBEH Choice 400, BBEH Non-Choice about 500, HLE text 200 items, plus three word games; baseline is one zero-shot answer, no sampling baseline at equal compute", "older": false }, { "source": "Tianzhu Zhang, \"When Should LLMs Trust Their Own Revisions? A Risk-Aware Study of Intrinsic Self-Correction\"", "url": "https://arxiv.org/abs/2609.35832", "date": "2026-09-23", "locator": "§3.1-3.3; Table 2; Figure 2; §4.3", "task": "One revision pass by the same model with no new evidence; tracks right-to-wrong and wrong-to-right transitions and compares keeping the first answer, always taking the revision, and a gate trained to decide per item from first-pass features", "effect": "\"Unconditional revision reduces accuracy in 20 of 28 BoolQ settings, 10 of 28 GSM8K settings, and 9 of 26 Corr2Cause settings.\" Llama-3.1-8B on GSM8K rose 25.5 points (17.44% to 42.99%) while turning 19.1% of its initially correct answers wrong; DeepSeek-R1-Distill-Qwen-14B +6.5 points with 4.2% harm. Qwen2.5-7B on BoolQ: always revising 80.83% to 72.32%, gate 86.02%; Qwen3-Coder-30B-A3B on Corr2Cause: always revising 56.06% to 83.70%, gate stayed at 56.06%. All five BoolQ refinement prompts lowered mean accuracy, the neutral one least; \"More forceful instructions to switch answers increase harmful transitions more than helpful ones.\" The author flags the gate gains as unverified.", "direction": "mixed", "measured": true, "conditions": "29 open-weight models up to 123B (no closed models), 82 model-task settings; BoolQ (3,270), GSM8K (1,319), Corr2Cause (2,246); decoding and protocol differ across settings; the gate needs labeled development data; one author", "older": false } ], "caveats": "Most of the evidence is math, code and puzzles with ground truth; the form-critique gains come from Self-Refine's 2023 GPT-3.5 and GPT-4 runs against one-step generation, not against best-of-N at equal compute, and no source here measures the loop's latency. Reflexion's signals are sometimes close to an oracle (HotpotQA graded by exact match against the gold answer), and its HumanEval gain did not hold against plain retries in an independent rerun. The supporting 2026 study runs at temperature 0 against a single zero-shot answer with no sampling baseline, and the 29-model study mixes protocols across settings, its largest gain starting from an unusually low 17.44% on GSM8K. Trained self-correction (SCoRe) is the one result where same-model revision beat parallel sampling at the same number of samples, but it needs multi-turn RL on the model itself.", "also_covers": [], "related": [ { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "7.c.iii", "slug": "evaluator-optimizer-loop", "name": "Evaluator-optimizer loop" }, { "id": "7.c.ii", "slug": "self-critique-evaluator", "name": "Self-critique evaluator loop" }, { "id": "9.c.ii", "slug": "samples-before-verification", "name": "Spend extra samples before extra verification: parallel attempts with a vote or judge; verify only where it pays" }, { "id": "1.b.i", "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class" }, { "id": "9.c.i", "slug": "no-self-written-tests", "name": "Don't prompt agents to write their own tests; suppress it for test-heavy models" }, { "id": "7.b.ii", "slug": "keep-errors-in-context", "name": "Keep errors in context" }, { "id": "4.c.i", "slug": "episodic-memory", "name": "Episodic memory retrieval" }, { "id": "7.c.v", "slug": "language-agent-tree-search", "name": "Language Agent Tree Search (LATS)" }, { "id": "7.c.vi", "slug": "multi-agent-debate", "name": "Multi-agent debate" } ], "url": "https://mahalsystems.ai/aot/optimizations/reflection-loop/", "markdown": "https://mahalsystems.ai/aot/optimizations/reflection-loop.md" }, { "id": "7.c.ii", "legacy_id": "P38", "former_ids": [], "slug": "self-critique-evaluator", "name": "Self-critique evaluator loop", "kind": "pattern", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it adds a self-bootstrapped judge", "proposed": true }, "levers": [ "control-flow" ], "domains": [ "general" ], "evidence_domains": [ "judging", "reasoning", "code-generation", "data-analysis" ], "categories": [ "Scaling compute" ], "category_ids": [ "orchestration.extra-compute" ], "summary": "Train or tune the system's own judge from its own outputs and synthetic judgments (contrastive pairs with one side deliberately degraded, the judge's own verdicts kept only where they match the construction label) instead of human labels, then use it to score, select or train in the loop. It pays where no sound check exists and a prompted judge is too weak or too costly: synthetic-only training lifted a 70B judge from 75.4 to 88.3 on RewardBench and another 70B judge from 48.6 to 60.0 on JudgeBench, and an agent-loop verifier evolved from ten anchored examples agreed with held-out labels 0.625 of the time against 0.55 for the bare LLM judge it contains, a gap that was not significant. The limits are the point: gains concentrate on the benchmark used for selection (human agreement on MT-Bench moved only from 77.8 to 78.9), and a policy optimized against its own reference-free judge drove the judge's pass rate from 0.72 to 0.94 while true accuracy stayed at 0.20. Keep a human-labeled set the judge never trains or selects on; the agent's downstream score cannot validate the judge.", "how": [ "Start from the baseline: a prompted judge from another model family with written criteria (7.c.iii), or a sound external check where one exists (9.a.i). Bootstrap your own judge only where that baseline disagrees with human labels too often or costs too much per call. First label two small human sets: one for calibration and selection, and a locked one the judge never trains or selects on (one agent-loop verifier was selected on ten anchored items and validated on locked sets of 40-48).", "Build contrastive pairs from your own traces so the label comes from construction, not from the judge's opinion: a good output against a degraded one, such as an answer to a subtly modified instruction, a sampled answer that fails a known-answer check, or an output cut to a bare opener ('Thought process:'). Include both orderings of every pair.", "Train the judge on the pairs (fine-tune on its own verdicts that match the label and iterate, or RL on verdict correctness), or tune a judge prompt against them. Prefer a base from another family than the generator, since self-preference rises with self-recognition, but check it: judges also favor any model whose mistakes resemble theirs. Where answers are checkable, make the judge commit its own answer before it compares (false-positive rate on wrong answers 0.719 to 0.012 in one study).", "Gate it before use: chance-corrected agreement and true- and false-positive rates on the locked set; judge benchmarks other than the one used for selection; accuracy on the generator's outputs against equal-quality outputs from other sources; false-positive rate on adversarial probes (bare openers, punctuation, padded and plausible-but-wrong answers). Never validate it by the agent's downstream score: an always-pass verifier trained skills as well as a valid one.", "In the loop, control length when picking preferred outputs, track the judge's pass rate against audited accuracy, and keep a human-audited sample and an outer judge outside the loop (9.c.iii).", "Refresh on a schedule from new traces: promote a retrained judge only if it agrees better with the locked set, add outputs the old judge wrongly accepted as negatives, and discard scores that depended on the replaced judge.", "Measure with a paired eval on the locked set against the prompted judge and against best-of-N or majority vote at equal compute (9.c.ii); report the gap between the judge's pass rate and audited accuracy at every iteration." ], "detect": { "static": "A pipeline that trains or prompt-tunes a judge on the system's own outputs or its own verdicts, with periodic refresh. Flag a judge from the same model or family as the generator, no human-labeled set held out from both training and selection, and a judge validated only by the agent's downstream score.", "traces": "Judge pass rate and score distribution per training or refresh iteration next to accuracy on an audited sample; a widening gap, scores saturating near the top of the scale, and output length rising across iterations.", "evals": "Chance-corrected agreement with a locked human-labeled set; the judge's accuracy on the generator's own outputs against equal-quality outputs from other sources; results on judge benchmarks other than the one used for model selection.", "judge": "An outer judge kept outside every loop, from another family, given the task contract and checked against humans.", "probes": "Outputs built to fool the judge: content-free answers, plausible-but-wrong answers and padded answers, plus the generator's outputs paired with equal-quality outputs from other sources." }, "impact": { "measured_basis": true, "primary": [ "quality", "governance" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "gains on judge benchmarks and on what is trained against them: RewardBench 75.4 to 88.3 (Wang, Aug 2024); JudgeBench 48.6 to 60.0 for a 70B judge trained on 22K synthetic pairs, but +0.6 (70.8 to 71.4) on a Qwen3-32B thinking base, still below o3's 75.7 (Whitehouse, May 2025); AlpacaEval 2 LC win rate 25.5% to 39.4% (Wu, Jul 2024); search with a co-evolved code reviewer found a coder passing 23 of 28 held-out DeepSWE tasks against 21 of 28 with a fixed evaluator, one run (Iacob, Jun 2026). Gains that do not transfer: MT-Bench human agreement 77.8 to 78.9 (Wang, Aug 2024); a self-trained 8B judge scored 83.0 against 69.5 on RewardBench but 30.2 against 32.3 on JudgeBench (Whitehouse, May 2025); self-play against its own judge took the pass rate from 0.72 to 0.94 at 0.20 true accuracy (Zhou, Jul 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "medium", "when": "up-front generation, training and validation, repeated at every refresh: Meta-Rewarding sampled 7 responses for each of 5,000 prompts and 11 judgments of each response per iteration, for four iterations (Wu, Jul 2024); Self-Taught Evaluators retrained a 70B judge for five iterations (Wang, Aug 2024); one co-evolution search with a learned reviewer used 587M tokens to its best coder against 429M with a fixed evaluator on DeepSWE, but 1.35 to 1.72 times fewer on Polyglot (Iacob, Jun 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "security", "effect": "mixed", "size": "medium", "when": "a learned judge is a gate that can be gamed: content-free answers such as 'Thought process:' or ':' drew false-positive rates as high as 80%, one trained verifier 66.8% on a single blank space, and adding 20k truncated outputs as negatives brought trained judges near 0% (Zhao, Jul 2025); errors a policy learned against its own judge fooled judges of other families, and a three-family ensemble still accepted 55% of wrong answers on one seed, 65% over three seeds (Zhou, Jul 2026)", "basis": "evidence" }, { "dimension": "governance", "effect": "degrades", "size": "medium", "when": "self-referential evaluation: the system's own score cannot show whether it improved. A collapsed always-pass verifier trained skills as well as a valid one, and rubric gaming that no in-loop signal saw was caught by an outer judge (Zhang, Oct 2026); without a held-out check a self-play judge reported 94% correct when 20% were (Zhou, Jul 2026)", "basis": "evidence" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval on a locked human-labeled set the judge never trains or selects on: the bootstrapped judge against the prompted judge it would replace and against majority vote or best-of-N at equal compute; chance-corrected agreement, true- and false-positive rates, accuracy on the generator's own outputs against equal-quality outputs from other sources, false-positive rate on adversarial probes; then the loop's end quality graded on the same locked set, never by the judge" }, { "method": "monitor", "tier": null, "how": "per training iteration and per refresh: the judge's pass rate against accuracy on a regularly audited human sample, score saturation near the top of the scale, output length, and the judge's acceptance rate on the generator's outputs against outputs from other sources; promote a retrained judge only if it agrees better with the locked set" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Feedback & Iteration group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Wang et al., \"Self-Taught Evaluators\"", "url": "https://arxiv.org/abs/2408.02666", "date": "2024-08-05", "locator": "Abstract; §3; §5.1 Table 1; §5.2 Table 2", "task": "Train an LLM judge on its own synthetic judgments, iteratively, without human labels", "effect": "\"can improve a strong LLM (Llama3-70B-Instruct) from 75.4 to 88.3 (88.7 with majority vote) on RewardBench\"; the same judge setup trained on human-labeled HelpSteer2 data reached 85.6. Agreement with human judgments on MT-Bench: 77.8 for the seed, 78.9 after five iterations (79.5 with 32-sample majority vote; GPT4-0125 79.1). Iteration-0 judgments were sampled from Mixtral 22Bx8 Instruct.", "direction": "mixed", "measured": true, "conditions": "Llama3-70B-Instruct; RewardBench, MT-Bench human judgments, HelpSteer2 baseline; five iterations", "older": true }, { "source": "Weizhe Yuan, Richard Yuanzhe Pang, Kyunghyun Cho, Xian Li, Sainbayar Sukhbaatar, Jing Xu, Jason Weston, \"Self-Rewarding Language Models\"", "url": "https://arxiv.org/abs/2401.10020", "date": "2024-01-18", "locator": "§3.2.1; §3.2.2; Table 1; Table 4; §6 Limitations", "task": "One model writes responses to new prompts, scores them as its own LLM-as-a-judge with a five-criterion additive prompt, and is trained by iterative DPO on the resulting pairs; seeded with human instruction data and human-ranked evaluation examples", "effect": "AlpacaEval 2.0 win rate over GPT-4 Turbo \"from 9.94% in Iteration 1, to 15.38% in Iteration 2, to 20.44% in Iteration 3\". Judge pairwise accuracy against held-out human rankings: 65.1% with instruction data only, 78.7% after adding the human evaluation seed data, then 80.4% and 81.7% after self-reward iterations 2 and 3. Average AlpacaEval generation length 1092, 1552 and 2552 across the three iterations, \"which we note may be a factor in relative performance\"; no gain on \"mathematics and logical reasoning\". The authors leave open \"if so-called “reward-hacking” can happen within our framework\".", "direction": "mixed", "measured": true, "conditions": "Llama 2 70B; Open Assistant seed data; AlpacaEval 2.0 (805 prompts), held-out Open Assistant human rankings; three iterations", "older": true }, { "source": "Tianhao Wu, Weizhe Yuan, Olga Golovneva, Jing Xu, Yuandong Tian, Jiantao Jiao, Jason Weston, Sainbayar Sukhbaatar, \"Meta-Rewarding Language Models: Self-Improving Alignment with LLM-as-a-Meta-Judge\"", "url": "https://arxiv.org/abs/2407.19594", "date": "2024-07-28", "locator": "Abstract; §3.3; §3.4 Table 3; §3.5 Table 4; §5 Limitations", "task": "The model judges its own responses and then judges its own judgments (meta-judge), and is trained by DPO on both; a length-control rule picks the shortest top-scored response; compared with Self-Rewarding given the same length control", "effect": "AlpacaEval 2 length-controlled win rate 25.47% (after fine-tuning on the human-derived evaluation data; 22.9% before it) to 39.4% and Arena-Hard 20.6% to 29.1% over four iterations; Self-Rewarding with the same length control reached 35.5%. Judge agreement with GPT-4 (ties removed) rose from 61.03% to 79.33% on pairs the judge picked from seven seed-model responses, but from 56.49% to 59.54% on 170 pairs GPT-4 labeled consistently. Correlation with human rankings rose, but \"this improvement is not sustained over later training iterations\"; the judge \"demonstrated a tendency to assign higher scores, which accelerated score saturation\" and \"showed limited improvement in evaluating non-self-generated responses\". Without length control, training \"makes the model excessively verbose\".", "direction": "mixed", "measured": true, "conditions": "Llama-3-8B-Instruct, after SFT on the human-derived evaluation (EFT) data; 20,000 synthetic prompts, 5,000 per iteration; AlpacaEval 2 (805), Arena-Hard, Open Assistant test prompts", "older": true }, { "source": "Chenxi Whitehouse, Tianlu Wang, Ping Yu, Xian Li, Jason Weston, Ilia Kulikov, Swarnadeep Saha, \"J1: Incentivizing Thinking in LLM-as-a-Judge via Reinforcement Learning\"", "url": "https://arxiv.org/abs/2505.10320", "date": "2025-05-15", "locator": "§2.1; §3; §4.1 Table 2", "task": "Train thinking judges by online RL on 22K synthetic preference pairs only (the rejected side a response to a noisy version of the instruction, or a wrong MATH solution), both orderings in every batch, rewarded for verdict correctness and consistency; compared with base models, EvalPlanner (same data, DPO) and reasoning models", "effect": "Position-consistent JudgeBench accuracy: Llama-3.3-70B-Instruct 48.6, J1-Llama-70B 60.0; Llama-3.1-8B-Instruct 32.3, J1-Llama-8B 42.0; Qwen3-32B 70.8, J1-Qwen-32B-MultiTask 71.4 (+0.6); o3 75.7. RewardBench for the same pairs 85.4 to 93.3, 69.5 to 85.7 and 90.9 to 93.6; RM-Bench 69.5 to 82.7 for the 70B. In the same table EvalPlanner-Llama-8B, trained on the same data with DPO, scored 83.0 on RewardBench against 69.5 for Llama-3.1-8B-Instruct, but 30.2 against 32.3 on JudgeBench and 54.3 against 55.6 on PPE. \"Model selection is based on overall accuracy on RewardBench.\"", "direction": "mixed", "measured": true, "conditions": "bases Llama-3.1-8B-Instruct, Llama-3.3-70B-Instruct, Qwen3-32B; 17K WildChat and 5K MATH prompts; PPE, RewardBench (3K), RM-Bench (4K), JudgeBench (350 GPT-4o pairs), FollowBenchEval", "older": true }, { "source": "Arjun Panickssery, Samuel R. Bowman, Shi Feng, \"LLM Evaluators Recognize and Favor Their Own Generations\"", "url": "https://arxiv.org/abs/2404.13076", "date": "2024-04-15", "locator": "§1; §2.3; §3.2; Figure 7; Table 1", "task": "Measure whether LLM evaluators recognize their own summaries and prefer them over other models' and human summaries; fine-tune evaluators to raise or lower self-recognition and track self-preference", "effect": "Out of the box all three evaluators exceed 50% accuracy at telling their own outputs apart; \"GPT-4 is 73.5% accurate distinguishing itself from two other LLMs and humans\". GPT-3.5 and Llama 2 exceed 90% self-recognition \"after fine-tuning on 500 examples\". \"fine-tuning for self-recognition results in a strong linear correlation between a model’s self-recognition ability and it’s self-preference\"; the example-level Kendall's tau between recognizing and preferring its own summary was 0.41 for GPT-3.5 on XSUM before fine-tuning and positive in every fine-tuning configuration.", "direction": "contradicts", "measured": true, "conditions": "GPT-4, GPT-3.5, Llama-2-7b-chat; news summarization on XSUM and CNN/DailyMail, 1,000 articles each (500 to fine-tune, 500 to evaluate)", "older": true }, { "source": "Shashwat Goel, Joschka Struber, Ilze Amanda Auzina, Karuna K Chandra, Ponnurangam Kumaraguru, Douwe Kiela, Ameya Prabhu, Matthias Bethge, Jonas Geiping, \"Great Models Think Alike and this Undermines AI Oversight\"", "url": "https://arxiv.org/abs/2502.04313", "date": "2025-02-06", "locator": "§3.2 (Q1, Q2), Figure 3, Table 2; §5.2, Figure 6", "task": "Measure how a judge's scores depend on how similar the judged model's mistakes are to the judge's own (CAPA), controlling for accuracy, and whether frontier models' mistakes are converging", "effect": "\"LLM-as-a-judge scores favor models similar to the judge, generalizing recent self-preference results\"; \"model favoritism extends beyond self- or family- preference to all models that are functionally similar\". Example: Qwen2.5-72B-Instruct as judge scored Qwen2.5-7B-Instruct 71% correct and the more capable Llama-3.1-70B-Instruct (51% against 41% MCQ accuracy) 67%. In multiple regression both accuracy and similarity had a significant positive effect on the judge score; \"model mistakes are becoming more similar with increasing capabilities\".", "direction": "contradicts", "measured": true, "conditions": "MMLU-Pro, 8,707 questions answered free-form and judged without a reference; 9 open-weight judges from four families, 39 evaluated models; AlpacaEval check; similarity trend over 130 models", "older": true }, { "source": "Chenyu Zhou, \"More Convincing, Not More Correct: Self-Play Reward Hacking of Reference-Free LLM Judges\"", "url": "https://arxiv.org/abs/2607.05904", "date": "2026-07-07", "locator": "Abstract; §5.1; §5.2; §5.3", "task": "A model answers, judges its own answers without a reference and is trained by DPO on the verdicts; a held-out exact-match or unit-test anchor the judge never sees audits the result; defenses compared (recompute prompt, larger and other-family judges, ensembles, committing an answer before judging, solving blind)", "effect": "Full GSM8K test set, Qwen3-4B: \"self-play drives the judge’s pass rate from 0.716 to 0.938±0.016 while anchor accuracy stays at 0.209→0.202±0.005\"; \"the judge reports 94% correct when 20% are\". Under natural chain-of-thought (base accuracy 0.76-0.88) the gap fell to 0.086-0.125. Training against a different-family judge left a gap of 0.40±0.09; a three-family ensemble that accepts only when all agree \"still accepts 55% of full-test wrong answers\" on seed 0, 65% averaged over three seeds. Making the judge write its own answer first cut the false-positive rate on wrong answers \"from the verify prompt’s 0.719 to 0.012\"; used as the training reward, the blind-solve verdict accepted 0 of about 380 wrong answers. Best-of-16 selection by the self-judge on LiveCodeBench widened the gap from 0.20 to 0.588 while unit-test pass of the selected candidates went from 0.27 to 0.29.", "direction": "contradicts", "measured": true, "conditions": "Qwen3 1.7B-14B policies and self-judges, Llama, Gemma and Mistral judges; GSM8K (n=1319 full test, n=128 audit subset), reasoning-suppressed JSON answers by default, three seeds; LiveCodeBench best-of-16 with Qwen3-1.7B; single author", "older": false }, { "source": "Yulai Zhao, Haolin Liu, Dian Yu, Sunyuan Kung, Meijia Chen, Haitao Mi, Dong Yu, \"One Token to Fool LLM-as-a-Judge\"", "url": "https://arxiv.org/abs/2507.08794", "date": "2025-07-11", "locator": "§1; §3.2; §3.3.1 Table 1; §3.3.2 Table 2", "task": "Probe general-purpose and trained generative reward models with content-free 'master key' answers (punctuation, reasoning openers) in reference-based grading; retrain judges with 20k truncated outputs added as negatives", "effect": "\"such simple hacks lead to false positive rates (FPRs) as high as 80%\"; \"Thought process:\" led to \"FPRs as high as 60-90%\" in LLaMA3-70B-Instruct and Qwen2.5-72B-Instruct, and \":\" to \"up to 35% FPRs\" in GPT-4o; the trained General-Verifier \"shows an alarming FPR of 66.8% on the MATH dataset using a naive single blank space\". It surfaced when an RLVR policy trained against a Qwen2.5-72B-Instruct judge collapsed into short reasoning openers. Adding 20k first-sentence truncations labeled NO to 160k examples gave Master-RMs \"near 0% FPR\" on unseen master keys, with Cohen's kappa 0.91 against GPT-4o and 0.90 against humans for the 7B. The authors \"confirm the ineffectiveness of common inference-time techniques such as chain-of-thought and majority voting\".", "direction": "contradicts", "measured": true, "conditions": "judges Qwen2.5-72B-Instruct, LLaMA3-70B-Instruct, GPT-4o, GPT-o1, Claude-4-Sonnet, Multi-sub RM, General-Verifier, Omni-Judge; GSM8K, MATH, AIME, Multi-subject RLVR, NaturalReasoning; Master-RM-7B/32B from Qwen2.5-Instruct on labels from a Qwen2.5-72B-Instruct grader", "older": true }, { "source": "Xing Zhang, Guanghui Wang, Yanwei Cui, Ziyuan Li, Wei Qiu, Bing Zhu, Peiyang He, \"Who Verifies the Verifier? Co-Evolving Inspectable Graders with Self-Improving Agents\"", "url": "https://arxiv.org/abs/2610.11464", "date": "2026-10-08", "locator": "Abstract; §5.1; §5.2 Table 3; §5.3 Table 1; §5.4; §5.5", "task": "Evolve an agent's verifier from clustered failures as a composition of small, mostly deterministic detectors plus narrow judge queries, selected for agreement with a ten-item anchored set and detector consensus on unlabeled outputs, never for the agent's score; co-evolve it with a skill-learning loop and compare with the same loop given ground truth or a rubric", "effect": "MBPP+ held-out agreement 0.625±0.050 against 0.417±0.058 for the hand-authored seed composition and 0.55±0.04 for the bare LLM judge it contains (p=0.12 against the seed, with verdicts clustered by task). Removing the anchor guards \"collapses the verifier into a vacuous always-pass grader, yet that collapsed verifier trains skills just as well\"; \"Downstream task score cannot certify a self-evolved verifier.\" The co-evolved loop retained 106%, 110% and 88% of the reference loop's lift on MBPP+, Spider 2.0 and report generation; Spider agreement fell to 0.500±0.026 once residual failures were wrong values under clean execution. On reports evolved skills gamed the rubric and \"no in-loop signal saw it\"; an outer judge preferred the pre-evolution baseline in 88% of decided pairs, and was itself wrong until given the task contract.", "direction": "mixed", "measured": true, "conditions": "Claude Opus 4.7 in every in-loop role, Claude Opus 4.8 as outer judge; MBPP+, Spider 2.0-Snow, report generation; locked sets of 40-48 items, three seeds", "older": false }, { "source": "Alex Iacob, Andrej Jovanović, William F. Shen, et al., \"The Red Queen Gödel Machine: Co-Evolving Agents and Their Evaluators\"", "url": "https://arxiv.org/abs/2606.26294", "date": "2026-06-24", "locator": "§1; §5.1; §5.4 Figure 7; Appendix C.5 Table 6; C.6-C.7", "task": "Self-improvement search that co-evolves learned evaluators (code reviewer, paper reviewer, proof grader) with the agents they score; the evaluator is frozen within an epoch, a challenger replaces it only if it scores better against a held-out ground-truth anchor, scores that depended on it are erased, and papers the displaced reviewer accepted become an adversarial pool", "effect": "DeepSWE held-out pass rate at low reasoning effort 82.1% against 75.0% for the fixed-evaluator baseline (HGM-H); search tokens to the winner 587M against 429M. Polyglot 71.7% against 69.9% with 1.35×-1.72× fewer search tokens. The fixed reviewer \"accepts AI-generated papers at 1.42× – 1.91× the rate of human ones\"; rewarding challengers for rejecting papers the displaced reviewer had accepted gave a reviewer that accepts AI and human papers at similar rates \"while retaining 80% ground-truth accuracy\", which \"costs raw accuracy by construction\".", "direction": "mixed", "measured": true, "conditions": "GPT-6 Astra on DeepSWE (28 held-out tasks), GPT-5.5 (low) elsewhere; Polyglot 166 held-out tasks; one run per arm; code reviewer anchored on CRAVE accepted and rejected pull requests", "older": false } ], "caveats": "Most evidence comes from instruction-following benchmarks graded by GPT-4 (AlpacaEval, Arena-Hard) or from judge benchmarks also used for model selection, and the agent-loop studies are small: 28 held-out DeepSWE tasks with one run per arm, locked sets of 40-48 items over three seeds, one model family. Gains shrink on strong reasoning bases (+0.6 on JudgeBench for a Qwen3-32B thinking model, which o3 still beat), and the largest self-play hacking appeared in a regime built to hold accuracy low; under natural chain-of-thought the gap between judge pass rate and true accuracy fell to 0.086-0.125. A judge from another family cuts self-preference but not shared blind spots: judges favor any model whose mistakes resemble theirs, and errors a policy learned against one judge fooled judges of other families. Several methods sold as label-free still used human data (Self-Rewarding's seed evaluation examples, a ten-item anchor, accepted and rejected pull requests), so treat a small human set as part of the method.", "also_covers": [], "related": [ { "id": "7.c.iii", "slug": "evaluator-optimizer-loop", "name": "Evaluator-optimizer loop" }, { "id": "7.c.i", "slug": "reflection-loop", "name": "Reflection loop" }, { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "9.c.iii", "slug": "audit-verifier-cascade", "name": "Audit a verifier-gated cascade against a gold sample the verifier never sees" }, { "id": "9.c.ii", "slug": "samples-before-verification", "name": "Spend extra samples before extra verification: parallel attempts with a vote or judge; verify only where it pays" }, { "id": "9.b.ii", "slug": "false-success-detector", "name": "Triage false success with a cheap trained detector, not an LLM judge" }, { "id": "7.c.v", "slug": "language-agent-tree-search", "name": "Language Agent Tree Search (LATS)" }, { "id": "7.c.vii", "slug": "tree-of-thought", "name": "Tree-of-Thought" } ], "url": "https://mahalsystems.ai/aot/optimizations/self-critique-evaluator/", "markdown": "https://mahalsystems.ai/aot/optimizations/self-critique-evaluator.md" }, { "id": "7.c.iii", "legacy_id": "P17", "former_ids": [], "slug": "evaluator-optimizer-loop", "name": "Evaluator-optimizer loop", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it adds a generate-evaluate loop to the workflow", "proposed": true }, "levers": [ "control-flow" ], "domains": [ "general" ], "evidence_domains": [ "code-generation", "reasoning", "document-extraction", "judging" ], "categories": [ "Scaling compute" ], "category_ids": [ "orchestration.extra-compute" ], "summary": "A separate evaluator (an LLM judge with written criteria, a different model, or an agent that tests the running app) scores the generator's output and returns specific findings; the generator revises until the output passes or a round cap is hit. It pays when the task is beyond what the generator does reliably in one pass and the evaluator is more reliable than the generator: a production diff judge vetoed about a quarter of coding-agent sessions, and half of those recovered. It fails when the judge is weak or shares the generator's blind spots: judge loops left table extraction worse than the first output and raised essay scores while human scores fell. Most of the gain comes in the first two to four rounds, at roughly twice the tokens of one call.", "how": [ "Start from the single pass, tuned with the same effort, and from best-of-N at equal tokens; add the loop only where outputs often fall short and no compiler, test or execution check can judge them (if one exists, feed its verdict back instead, 9.a.i).", "Write the evaluator's criteria as a few gradable items, each with a hard pass threshold, and calibrate it with a few scored examples; where possible let it check by acting (drive the app with Playwright, compare the diff with the original request). The same criteria in the generator's prompt also steer the first draft.", "Make the evaluator separate and skeptical: its own call and context, not the generator's dialogue history; a model at least as capable as the generator; tune its prompt by reading its logs against your own verdicts until it stops approving problems it found. Check it on labeled good and bad outputs and planted-defect probes before wiring it in. A judge from another model family does not remove favoritism (judges favor any model whose mistakes resemble theirs), and content-free answers such as ':' or 'Thought process:' can pass it, so include them among the probes.", "Return findings the generator can act on (failing criterion or checklist item, where, expected against actual; told which checklist items failed, Gemini 2.5 Pro went from 29.5 to 94.7, against -2.5 to +2.6 with no feedback) and tell it to keep what already passed; unconstrained regeneration breaks parts that were right.", "Stop on pass or after a low cap (most gains came in the first 2 to 4 rounds); raise the cap only if the per-round curve keeps rising. Return the best version rather than the last, and stop early when the score stalls or drops.", "Measure against a check the loop never sees (a human-labeled sample or a deterministic metric), per round: quality, share of items that end worse than their first output, evaluator agreement with the audit, tokens and latency." ], "detect": { "static": "A generate, evaluate, revise loop: a separate evaluator call, model or tool-using agent scores the output against written criteria and returns feedback, with a pass threshold and a round cap. Flag an evaluator that shares the generator's dialogue history, one weaker than the generator, no cap, or a loop that returns the last version instead of the best.", "traces": "Rounds per item; evaluator score per round next to an independent check; veto rate and recovery after a veto; items whose evaluator score rose while the independent check fell; revisions that rewrote parts that had passed.", "evals": "Quality per round on a grader the loop never sees; share of items that end worse than their first output; evaluator agreement with human or gold labels.", "judge": "The evaluator is a judge; a meta-judge checks its agreement with humans.", "probes": "Items with a clear rubric and a known defect." }, "impact": { "measured_basis": true, "primary": [ "quality", "performance" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "gains where the generator falls short and the critic is reliable: CodeContests 7.88 to 16.24 with a trained critic over five rounds, 12.48 with GPT-4o as critic (Xie, Feb 2025); Self-Refine +4.6 and +1.9 points over a tuned single call, -0.3 on chess (Leins, Aug 2026); losses where the judge is weak: table extraction 0.792 to 0.772 TEDS after 8 rounds (Kim, Jul 2026), essay judge scores rose while human scores fell (Pan, Jul 2024)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "medium", "when": "an evaluator call plus a revision per round: Self-Refine took 1.82 to 2.13 times the tokens of a tuned single call (Leins, Aug 2026); in one app build the QA rounds cost $3.24, $3.09 and $4.06 of $124.70, and the rebuilds they triggered $36.89 and $5.88 (Rajasekaran, Mar 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "large", "when": "serial rounds; an evaluator that clicks through the running app took 6.8 to 9.6 minutes per round, and a 5 to 15 round design loop ran up to four hours (Rajasekaran, Mar 2026)", "basis": "evidence" }, { "dimension": "security", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "governance", "effect": "improves", "size": "small", "when": "Criteria are explicit.", "basis": "assessment" }, { "dimension": "brand", "effect": "improves", "size": "medium", "when": "Polished output.", "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired eval of the loop against the single pass and against best-of-N at equal tokens, per model, graded by a check the loop never sees (human-labeled sample or deterministic metric): quality per round, share of items ending worse than their first output, tokens, latency" }, { "method": "monitor", "tier": null, "how": "rounds per item, first-pass rate, veto rate and recovery rate after a veto, cap-hit rate; on a regularly audited sample, evaluator verdicts against human labels and drift between evaluator scores and the audit" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file master-agent-optimization-cookbook.md", "task": "Lists this practice in its agent-optimization pattern library (Workflow Patterns group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Anthropic (Erik Schluntz, Barry Zhang), \"Building effective agents\"", "url": "https://www.anthropic.com/engineering/building-effective-agents", "date": "2024-12-19", "locator": "Workflow: Evaluator-optimizer", "task": "Names the evaluator-optimizer workflow: one call generates, another evaluates and gives feedback in a loop", "effect": "\"one LLM call generates a response while another provides evaluation and feedback in a loop\"; \"particularly effective when we have clear evaluation criteria, and when iterative refinement provides measurable value.\"", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Prithvi Rajasekaran (Anthropic), \"Harness design for long-running application development\"", "url": "https://www.anthropic.com/engineering/harness-design-long-running-apps", "date": "2026-03-24", "locator": "Why naive implementations fall short; Frontend design: making subjective quality gradable; Running the harness; Removing the sprint construct; Results from the updated harness", "task": "A separate evaluator agent with written criteria, few-shot calibration and the Playwright MCP grades a generator's frontend or full-stack app and sends its critique back for the next round", "effect": "\"tuning a standalone evaluator to be skeptical turns out to be far more tractable than making a generator critical of its own work\". Out of the box the evaluator would \"identify legitimate issues, then talk itself into deciding they weren't a big deal and approve the work anyway\"; it took several rounds of prompt tuning. Design loop: 5 to 15 iterations; scores \"improved over iterations before plateauing\", and the author \"regularly saw cases where I preferred a middle iteration over the last one\". App build on Opus 4.6: QA rounds cost $3.24, $3.09 and $4.06 of a $124.70 run and caught display-only and stubbed core features; the evaluator \"is worth the cost when the task sits beyond what the current model does reliably solo.\"", "direction": "supports", "measured": true, "conditions": "Claude Opus 4.5 and Opus 4.6, Claude Agent SDK; frontend designs and two full-stack apps, one run each; quality judged by the author", "older": false }, { "source": "Max Charas, Marc Bruggmann (Spotify), \"Background Coding Agents: Predictable Results Through Strong Feedback Loops (Honk, Part 3)\"", "url": "https://engineering.atspotify.com/2025/12/feedback-loops-background-coding-agents-part-3", "date": "2025-12-09", "locator": "Using LLMs in the verification loops", "task": "After the build, format and test verifiers pass, an LLM judge reviews the diff against the original prompt; a veto sends the coding agent back to fix its change before a PR opens", "effect": "\"out of thousands of agent sessions, the judge vetoes about a quarter of them. When that happens, the agent is able to course correct half the time.\" The most common trigger is \"the agent going outside the instructions outlined in the prompt\". \"We have yet to invest in evals for our judge.\"", "direction": "supports", "measured": true, "conditions": "production background coding agent (Claude Code) for fleet-wide code changes; internal metrics; judge accuracy not evaluated", "older": false }, { "source": "Zhihui Xie, Jie Chen, Liyu Chen, Weichao Mao, Jingjing Xu, Lingpeng Kong, \"Teaching Language Models to Critique via Reinforcement Learning\"", "url": "https://arxiv.org/abs/2502.03492", "date": "2025-02-05", "locator": "Table 1; Table 3; Table 4 and \"The Effect of Generator Ability\" (§4.4)", "task": "A separate critic gives feedback and the generator revises, for one or five rounds; the generator's own critique, GPT-4o as critic, and a critic trained with RL (CTRL) compared", "effect": "Qwen2.5-Coder generator, Pass@1 averaged over CodeContests, LiveCodeBench and MBPP+: zero-shot 38.75; after five critique rounds 38.30 with its own critique, 39.98 with GPT-4o, 42.77 with the trained critic. CodeContests 7.88 to 12.48 (GPT-4o critic, five rounds) and 16.24 (trained critic); trained critic after one, two and three rounds 11.76, 14.18, 15.15. MBPP+ fell from 77.83 to 74.60 after five GPT-4o critique rounds. GPT-4o generator: 43.53 zero-shot, 44.13 after five rounds of its own critique, 45.83 with the trained critic. In an inference-only size sweep, \"critics less capable than their generators typically degrade performance\".", "direction": "mixed", "measured": true, "conditions": "generators Qwen2.5-Coder-32B-Instruct and GPT-4o; trained critic from Qwen2.5-Coder-32B-Instruct; CodeContests, LiveCodeBench, MBPP+; 5 seeds", "older": true }, { "source": "Jane Pan, He He, Samuel R. Bowman, Shi Feng, \"Spontaneous Reward Hacking in Iterative Self-Refinement\"", "url": "https://arxiv.org/abs/2407.04549", "date": "2024-07-05", "locator": "§3.3-3.5; §4; Figures 3-5", "task": "An LLM judge scores essays against a four-item rubric and gives feedback, an LLM author revises, for five rounds; same model in both roles with separate prompts; expert humans score every version offline", "effect": "With GPT-3.5, \"the Online LLM Judge scores the model-edited essays much higher compared to the ground-truth Human scores\", and \"the Human scores demonstrate a decrease in quality in the last iteration\"; an offline judge without the loop history tracked the human scores. The divergence appeared when judge and author saw the same dialogue history; with different histories \"the gap between the Online LLM Judge and Human scores becomes statistically insignificant\". Only Conventions kept improving for humans; on the other items human-judged improvement \"either plateaus after a single iteration or even degrades\". GPT-4 showed less divergence.", "direction": "contradicts", "measured": true, "conditions": "gpt-3.5-turbo-1106 and gpt-4; 23 seed college essays, 92 trajectories per model, 5 rounds; 23 annotators, 3 per essay", "older": true }, { "source": "Dongwei Jiang, Alvin Zhang, Andrew Wang, Nicholas Andrews, Daniel Khashabi, \"Feedback Friction: LLMs Struggle to Fully Incorporate External Feedback\"", "url": "https://arxiv.org/abs/2506.11930", "date": "2025-06-13", "locator": "§2.1; §3.2; Figure 3; Figure 4; Table 1", "task": "A separate feedback model that sees the ground-truth solution critiques each wrong answer; the solver retries for up to 10 rounds", "effect": "\"The performance typically improves rapidly through the first 2-4 iterations before significantly slowing down\", and every model plateaus below the accuracy it would reach if it used all the feedback. Feedback resistance accounted for \"62.8-100% of errors\" left after 10 rounds. Richer feedback mattered: with strong-model reflective feedback Llama-4-Maverick reached 73.3% on AIME and 96.5% on GPQA, \"improvements of +26.7% and +10.6% over binary feedback\". Models \"claim to understand feedback and express willingness to update their beliefs (>95%) yet fail to actually incorporate corrections\".", "direction": "mixed", "measured": true, "conditions": "solvers Llama-3.3-70B, Llama-4-Scout, Llama-4-Maverick, Claude 3.7 Sonnet with and without extended thinking; feedback by GPT-4.1 mini; AIME 2024, MATH-500, TriviaQA, PopQA, MMLU, MMLU Pro, GPQA, two multiplication tasks; ground truth decides which answers get another round", "older": true }, { "source": "Donghwan Kim, \"LLM-as-a-Judge Scores Are Unreliable Optimization Signals in Closed-Loop Table Recognition\"", "url": "https://arxiv.org/abs/2607.13347", "date": "2026-07-15", "locator": "Abstract; §4.1; §4.2; Appendix E Tables 19-20", "task": "A reference-free LLM judge scores each table extraction and feeds its error claims back for 8 rounds of regeneration; judge scores also used to pick the best candidate or to accept revisions", "effect": "FinTabNet TEDS fell from 0.792 to 0.772 after 8 rounds (p<10^-7), 44% of the drop at the first regeneration; 41.6% of tables worsened and 23.9% improved; OmniDocBench +0.002, not significant. The loop's judge tied on 49.1% and 37.5% of distinct candidate pairs; cross judges from other families (GPT-5.4, GPT-5.4-nano, Claude Opus 4.6) tied less but \"still showed near-zero or negative TEDS rank correlations\", and on FinTabNet \"the trivial always-iter0 baseline beats every judge policy\". A loop with the judge's feedback removed degraded by the same amount (99 paired tables). Across six generators the loop helped only Claude Haiku 4.5, whose first outputs were weakest (+0.0917 from 0.5675); Spearman 0.771 between headroom and net effect (n=6).", "direction": "contradicts", "measured": true, "conditions": "Gemini 3.1 Flash Lite as generator and judge; FinTabNet 476 tables, OmniDocBench 272 tables; 100-table check across six generators", "older": false }, { "source": "Nicolas Leins, Nico Pelleriti, Jana Gonnermann-Müller, Sebastian Pokutta, \"When Does LLM Orchestration Pay Off? A Controlled Evaluation of Accuracy, Cost, and Task Difficulty\"", "url": "https://arxiv.org/abs/2608.00685", "date": "2026-08-01", "locator": "§3.3; §4.2; §4.5; §5; Appendix B.4", "task": "Self-Refine with a separate critic call (\"Act only as a critic of the proposed solution ... Do not rewrite the solution\"), at most two feedback-refinement rounds, stopping when the critic says no refinement is needed, against a single chain-of-thought call; all prompts tuned with GEPA under the same budget", "effect": "Accuracy, Self-Refine against optimized CoT: Codeforces 66.1% vs 61.5% (+4.6, 95% CI [2.1, 7.1]); AMC 89.8% vs 87.9% (+1.9, [0.5, 3.3]); Lichess 30.7% vs 31.0%. Weighted tokens 2.12, 2.13 and 1.82 times CoT. Benefits \"did not increase systematically with difficulty\", and \"additional computation has little value when the backbone cannot generate a viable solution\".", "direction": "mixed", "measured": true, "conditions": "five open-weight backbones (DeepSeek V4 Flash, Gemma 4, GLM 4.7 Flash, Qwen 3.6, GPT-OSS Puzzle 88B), same backbone as generator and critic; E2H-Codeforces, E2H-Lichess, AMC", "older": false }, { "source": "Ngoc Phuoc An Vo, Brent Paulovicks, Vadim Sheinin, \"LLM-as-a-Judge for Reference-less Automatic Code Validation and Refinement for Natural Language to Bash in IT Automation\"", "url": "https://arxiv.org/abs/2506.11237", "date": "2025-06-12", "locator": "§4.2; §5; Table 1", "task": "A dedicated evaluator scores generated Bash with two LLM-as-judge checks (bidirectional functionality matching, logic representation) and sends its judgment and feedback to the generator; against no refinement and against self-reflection", "effect": "Execution accuracy on three suites, baseline / self-reflection / dedicated evaluator: GPT-4o 86/84/88%, 72/76/82%, 68/72/74%; Llama-3.3-70B 68/74/72%, 52/64/76%, 46/52/50%; Mistral-large 76/76/80%, 66/70/76%, 54/58/58%. The evaluator loop beat no refinement in all nine cells and self-reflection in six; it helps \"but not always; it still depends on the specific model and dataset\".", "direction": "supports", "measured": true, "conditions": "GPT-4o, Llama-3.3-70B, Mistral-large in CrewAI; three Bash test suites, 150 questions; execution as ground truth", "older": true }, { "source": "Jiajun Chen, Haoyu Wu, Mingda Jia, Xihui Liu, \"Recursive Game Creator: An Agentic Product-Level Experience-Oriented Game Harness\"", "url": "https://arxiv.org/abs/2610.08621", "date": "2026-10-06", "locator": "§3.3; §4.1.1; §4.1.2; Tables 1-2", "task": "A Reviewer grades each game version from trajectories a coding-native Player collects, plus screenshots, against a game-specific rubric, keeps the better version and sends revision feedback to the Designer and Builder; three rounds", "effect": "GameCraft-Bench overall: same-model baseline 71.26; rounds one, two and three 72.70, 75.35, 77.89. Action stayed below the baseline's 73.33 for two rounds (67.96, 70.33) before 77.62 in round three. GameASG-Bench task success 25/47 (53.2%) against 9/47 (19.1%) for the same model in Codex CLI, a baseline taken from another paper.", "direction": "supports", "measured": true, "conditions": "GPT-6 Astra (high); 45 GameCraft-Bench tasks, 47 GameASG-Bench tasks; whole harness, the Reviewer not ablated", "older": false }, { "source": "Shashwat Goel, Joschka Struber, Ilze Amanda Auzina, Karuna K Chandra, Ponnurangam Kumaraguru, Douwe Kiela, Ameya Prabhu, Matthias Bethge, Jonas Geiping, \"Great Models Think Alike and this Undermines AI Oversight\"", "url": "https://arxiv.org/abs/2502.04313", "date": "2025-02-06", "locator": "§3.2 (Q1, Q2), Figure 3, Table 2; §5.2, Figure 6", "task": "Measure how a judge's scores depend on how similar the judged model's mistakes are to the judge's own (CAPA), controlling for accuracy, and whether frontier models' mistakes are converging", "effect": "\"LLM-as-a-judge scores favor models similar to the judge, generalizing recent self-preference results\"; \"model favoritism extends beyond self- or family- preference to all models that are functionally similar\". Example: Qwen2.5-72B-Instruct as judge scored Qwen2.5-7B-Instruct 71% correct and the more capable Llama-3.1-70B-Instruct (51% against 41% MCQ accuracy) 67%. In multiple regression both accuracy and similarity had a significant positive effect on the judge score; \"model mistakes are becoming more similar with increasing capabilities\".", "direction": "contradicts", "measured": true, "conditions": "MMLU-Pro, 8,707 questions answered free-form and judged without a reference; 9 open-weight judges from four families, 39 evaluated models; AlpacaEval check; similarity trend over 130 models", "older": true }, { "source": "Chenyu Zhou, \"More Convincing, Not More Correct: Self-Play Reward Hacking of Reference-Free LLM Judges\"", "url": "https://arxiv.org/abs/2607.05904", "date": "2026-07-07", "locator": "Abstract; §5.1; §5.2; §5.3", "task": "A model answers, judges its own answers without a reference and is trained by DPO on the verdicts; a held-out exact-match or unit-test anchor the judge never sees audits the result; defenses compared (recompute prompt, larger and other-family judges, ensembles, committing an answer before judging, solving blind)", "effect": "Full GSM8K test set, Qwen3-4B: \"self-play drives the judge’s pass rate from 0.716 to 0.938±0.016 while anchor accuracy stays at 0.209→0.202±0.005\"; \"the judge reports 94% correct when 20% are\". Under natural chain-of-thought (base accuracy 0.76-0.88) the gap fell to 0.086-0.125. Training against a different-family judge left a gap of 0.40±0.09; a three-family ensemble that accepts only when all agree \"still accepts 55% of full-test wrong answers\" on seed 0, 65% averaged over three seeds. Making the judge write its own answer first cut the false-positive rate on wrong answers \"from the verify prompt’s 0.719 to 0.012\"; used as the training reward, the blind-solve verdict accepted 0 of about 380 wrong answers. Best-of-16 selection by the self-judge on LiveCodeBench widened the gap from 0.20 to 0.588 while unit-test pass of the selected candidates went from 0.27 to 0.29.", "direction": "contradicts", "measured": true, "conditions": "Qwen3 1.7B-14B policies and self-judges, Llama, Gemma and Mistral judges; GSM8K (n=1319 full test, n=128 audit subset), reasoning-suppressed JSON answers by default, three seeds; LiveCodeBench best-of-16 with Qwen3-1.7B; single author", "older": false }, { "source": "Yulai Zhao, Haolin Liu, Dian Yu, Sunyuan Kung, Meijia Chen, Haitao Mi, Dong Yu, \"One Token to Fool LLM-as-a-Judge\"", "url": "https://arxiv.org/abs/2507.08794", "date": "2025-07-11", "locator": "§1; §3.2; §3.3.1 Table 1; §3.3.2 Table 2", "task": "Probe general-purpose and trained generative reward models with content-free 'master key' answers (punctuation, reasoning openers) in reference-based grading; retrain judges with 20k truncated outputs added as negatives", "effect": "\"such simple hacks lead to false positive rates (FPRs) as high as 80%\"; \"Thought process:\" led to \"FPRs as high as 60-90%\" in LLaMA3-70B-Instruct and Qwen2.5-72B-Instruct, and \":\" to \"up to 35% FPRs\" in GPT-4o; the trained General-Verifier \"shows an alarming FPR of 66.8% on the MATH dataset using a naive single blank space\". It surfaced when an RLVR policy trained against a Qwen2.5-72B-Instruct judge collapsed into short reasoning openers. Adding 20k first-sentence truncations labeled NO to 160k examples gave Master-RMs \"near 0% FPR\" on unseen master keys, with Cohen's kappa 0.91 against GPT-4o and 0.90 against humans for the 7B. The authors \"confirm the ineffectiveness of common inference-time techniques such as chain-of-thought and majority voting\".", "direction": "contradicts", "measured": true, "conditions": "judges Qwen2.5-72B-Instruct, LLaMA3-70B-Instruct, GPT-4o, GPT-o1, Claude-4-Sonnet, Multi-sub RM, General-Verifier, Omni-Judge; GSM8K, MATH, AIME, Multi-subject RLVR, NaturalReasoning; Master-RM-7B/32B from Qwen2.5-Instruct on labels from a Qwen2.5-72B-Instruct grader", "older": true }, { "source": "Young-Jun Lee, Seungone Kim, Byung-Kwan Lee, Minkyeong Moon, Yechan Hwang, Jong Myoung Kim, Graham Neubig, Sean Welleck, Ho-Jin Choi, \"RefineBench: Evaluating Refinement Capability of Language Models via Checklists\"", "url": "https://arxiv.org/abs/2511.22173", "date": "2025-11-27", "locator": "§4.2 Table 3; §5.1 Table 4; §5.3 Figure 9", "task": "Up to five turns of guided refinement (told which checklist items failed) against self-refinement (asked to improve its own answer with no feedback, deciding when to stop) and against self-refinement given the full checklist; each answer graded item by item by GPT-4.1", "effect": "Pass (all checklist items met), turn 1 to turn 5, guided refinement: Gemini 2.5 Pro 29.5 to 94.7, Claude-Opus-4.1 18.7 to 98.4. Self-refinement: Gemini 2.5 Pro 29.5 to 31.3; GPT-5 27.5 to 29.1; o1 18.5 to 18.3; DeepSeek-R1 8.1 to 7.9; across all models in Table 3 the change ranged from -2.5 to +2.6. Given the full checklist but no feedback, \"LLaMA-3.1-70B-Instruct improves by +43.6% and Gemini 2.5 Pro by +44.5%\" at turn 5 over plain self-refinement.", "direction": "supports", "measured": true, "conditions": "1,000 problems in 11 domains; 34 models including o1, o3-mini, o4-mini, GPT-5, Claude 4 and 4.1, Gemini 2.5, DeepSeek-R1; OpenAI reasoning effort medium, 10K-token limit for reasoning models", "older": false } ], "caveats": "Most studies are small or single-run (one app per harness, 150 Bash questions, 23 essays), and the largest critic gain came from a critic trained for its generator, which is closer to 7.c.ii; prompted critics, GPT-4o included, helped less and over five rounds cut MBPP+ from 77.83 to 74.60. On outputs with no deterministic check the evaluator's own error decides the sign: judges tied, inflated scores and were gamed when they shared the generator's blind spots or context. Gains shrink as generators improve: Anthropic found the evaluator unnecessary overhead for tasks within the newer model's reach, and in one study the loop helped only the weakest of six generators. The 2 to 4 round finding comes from a setup where ground truth decided which answers got feedback, so it describes a best case.", "also_covers": [], "related": [ { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "7.c.i", "slug": "reflection-loop", "name": "Reflection loop" }, { "id": "7.c.ii", "slug": "self-critique-evaluator", "name": "Self-critique evaluator loop" }, { "id": "7.a.ii", "slug": "prompt-chaining-with-gates", "name": "Prompt chaining with gate checks" }, { "id": "9.b.iii", "slug": "verify-completion", "name": "Verify completion against the environment before the agent stops" }, { "id": "9.c.ii", "slug": "samples-before-verification", "name": "Spend extra samples before extra verification: parallel attempts with a vote or judge; verify only where it pays" }, { "id": "9.c.iii", "slug": "audit-verifier-cascade", "name": "Audit a verifier-gated cascade against a gold sample the verifier never sees" }, { "id": "7.b.iii", "slug": "size-execution-budgets", "name": "Size run budgets so productive runs aren't cut off: turn caps, deadlines, command timeouts" }, { "id": "7.c.iv", "slug": "graph-of-thought", "name": "Graph-of-Thought" }, { "id": "4.b.ii", "slug": "initializer-maintainer-agents", "name": "Initializer-maintainer dual agent" }, { "id": "7.c.vi", "slug": "multi-agent-debate", "name": "Multi-agent debate" }, { "id": "7.c.vii", "slug": "tree-of-thought", "name": "Tree-of-Thought" } ], "url": "https://mahalsystems.ai/aot/optimizations/evaluator-optimizer-loop/", "markdown": "https://mahalsystems.ai/aot/optimizations/evaluator-optimizer-loop.md" }, { "id": "7.c.iv", "legacy_id": "P33", "former_ids": [], "slug": "graph-of-thought", "name": "Graph-of-Thought", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it replaces a single chain with a search", "proposed": true }, "levers": [ "control-flow" ], "domains": [ "reasoning" ], "evidence_domains": [ "reasoning", "document-extraction" ], "categories": [ "Scaling compute" ], "category_ids": [ "orchestration.extra-compute" ], "summary": "Graph-of-Thought runs one task as a graph of model calls: split the input into parts, generate and score several candidates per part, keep the best, then merge the partial results and refine the merged answer. It paid on merge-shaped tasks that one call gets wrong as inputs grow: on sorting it made fewer errors than Tree-of-Thought at lower cost (median error about 62% lower in the original paper; 12.7 against 18.4 mistakes at 5.0 against 15.9 cents in a re-implementation), but it cost far more than one chain-of-thought call. Independent 2025-2026 comparisons are mixed: graph methods mostly beat chain-of-thought on Game of 24 and multi-hop QA but trailed it or direct answering on GPQA and step-by-step math at several times the tokens; on a six-task suite GoT ranked below tree search and had the highest share of runs both costlier and worse. No study compares GoT with self-consistency at matched tokens.", "how": [ "Start from one chain-of-thought call and from self-consistency or best-of-N with a vote at the same total tokens (9.c.ii); add a graph only where one call fails on inputs that split into parts whose results can be merged (sorting, set intersection, keyword counting, merging documents, questions that combine several sources) and the failures grow with input size. On step-by-step math (GSM8K, MATH-500) graphs lost to a single call with GPT-4o-mini and Qwen2.5-7B, at far more tokens where counted.", "Write the graph of operations for the task class up front, as the original framework does (a controller executes a fixed plan through a per-task prompter, parser and scorer): split the input into parts small enough that one call usually gets each right (16 elements in the paper), Generate several candidates per part (k=5), Score them and KeepBestN(1), Aggregate pairs of partial results with several attempts each, Score and keep the best again, and add a refine step on the merged result; merge level by level up to the final answer.", "Score with code wherever the task allows (the paper's sorting and set scorers count errors); an LLM scorer is noisier, and in one benchmark replacing search strategies' heuristic evaluator with an exact one raised quality and cut run-to-run variance. Without a trustworthy score, keeping the best of k is close to a random pick.", "Hold cost down: run independent nodes in parallel and cache repeated calls (parallel execution took GoT sorting from 259 s to 30 s per instance); trim few-shot examples in node prompts, which dominate once the task is split finely; pass compact summaries instead of whole parent contexts; route leaf subtasks to a smaller model while planning and the final merge stay on the strong one, under a token budget that caps depth, branching and node count (one router cut output tokens 79.1% against Adaptive GoT).", "Tune k, part size and depth on a held-out set with cost capped at the untuned level (one framework cut sorting mistakes from 12.7 to 12.1 at no extra cost).", "Measure on a paired set against chain-of-thought and self-consistency at matched input-plus-output tokens, per model and per input size, over repeated runs (the top strategy won only 77% of head-to-head runs in one benchmark): accuracy on a deterministic grader, tokens, wall-clock latency and cost per correct answer." ], "detect": { "static": "A controller that runs a fixed or generated graph of model calls on one task: split the input, generate several candidates per part, score and keep the best, aggregate partial results and refine the merge (GoT's Generate, Score, KeepBestN, Aggregate). Flag independent nodes run one after another, every node on the strongest model, an LLM scorer where code could check, and no cap on depth, branching, nodes or tokens.", "traces": "Per item: nodes and model calls, input and output tokens per node, aggregation and refine steps, scorer verdicts beside an independent check, budget-cap hits, wall-clock time. In a single-call agent, errors that grow with input size (dropped, duplicated or miscounted elements) mark where a split-and-merge graph could help.", "evals": "Accuracy against chain-of-thought and self-consistency at matched tokens, sliced by input size; cost and latency per correct answer.", "judge": "Rate whether each merge kept all information from its inputs without duplication (retention and redundancy, as in GoT's document-merging score).", "probes": "Tasks whose sub-results must be merged, at growing input sizes, plus a sequential control." }, "impact": { "measured_basis": true, "primary": [ "quality", "cost" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "gains on merge-shaped tasks: sorting 128 numbers, median error about 62% below ToT (Besta, Aug 2023), 12.7 against 18.4 mistakes in a re-implementation (Fricke, Feb 2026); a generalized GoT on Qwen3-30B beat CoT on multi-hop QA (MoreHopQA 73.0 vs 68.0) and Game of 24 (72.0 vs 58.0) but trailed it on GPQA (59.6 vs 63.1) (Liu, Mar 2026); GPT-4.1 Nano six-task suite 0.336 vs CoT 0.279 and CoT-SC 0.227, below ToT-BFS 0.415 (Potamitis, Dec 2025); below direct answering on MATH-500 (73.24 vs 74.96) and GSM8K (88.76 vs 90.95) with Qwen2.5-7B (Hao, May 2025); graph reasoning 82.0 vs CoT 89.5 on GSM8K under the same grader (Huang, Mar 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "large", "when": "many calls per item: costs \"much higher\" than IO and CoT (Besta, Aug 2023); 9,468 output and 16,826 input tokens per GPQA question against 4,965 and 247 for CoT (Liu, Mar 2026); mean cost 0.0099 against 0.0003 for CoT and 0.0014 for CoT-SC, and the highest joint cost-quality failure rate of nine strategies, 43.3% (Potamitis, Dec 2025); cheaper than ToT: over 31% at 128 elements (Besta, Aug 2023), 5.0 against 15.9 cents (Fricke, Feb 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "large", "when": "rounds of dependent calls: 259 s per sorting instance and 145 s per document merge run sequentially, 30 s and 32 s with parallel nodes (Fricke, Feb 2026); 22.89 to 116.03 s per question for a generalized GoT on Qwen3-30B (Liu, Mar 2026)", "basis": "evidence" }, { "dimension": "security", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "governance", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired eval of the graph against one chain-of-thought call and against self-consistency or best-of-N at matched input-plus-output tokens, per model and sliced by input size, over repeated runs with a deterministic grader: accuracy, tokens, wall-clock latency, cost per correct answer, and the share of runs both costlier and worse" }, { "method": "monitor", "tier": null, "how": "per item: nodes and model calls, input and output tokens per node, aggregation and refine steps, budget-cap hits, latency p50 and p95; on a sampled subset, scorer verdicts against an independent check" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Reasoning & Search group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Besta et al., \"Graph of Thoughts: Solving Elaborate Problems with Large Language Models\"", "url": "https://arxiv.org/abs/2308.09687", "date": "2023-08-18", "locator": "Abstract; §7.1-7.3; Appendix F", "task": "Models LLM thoughts as a graph so partial results can be merged and refined", "effect": "\"increasing the quality of sorting by 62% over ToT, while simultaneously reducing costs by >31%\" (median error at 128 elements). Against the cheaper ToT variant GoT \"only negligibly improves\" at 32 elements and has ≈61% and ≈69% fewer errors at 64 and 128. Against CoT and IO at 64 elements median error is ≈65% and ≈83% lower, but \"the costs of GoT – and ToT – are much higher than in IO and CoT.\"", "direction": "supports", "measured": true, "conditions": "Sorting digits 0-9 with duplicates (32, 64, 128 elements), also set intersection, keyword counting and NDA merging; against IO, CoT and ToT; GPT-3.5, 100 samples per task and baseline (50 for document merging), temperature 1.0; hand-built graph per task (sorting and set intersection split into 16-element parts, 5 candidates each, error-counting score, keep best, merge, improve)", "older": true }, { "source": "Maciej Besta et al., \"Demystifying Chains, Trees, and Graphs of Thoughts\"", "url": "https://arxiv.org/abs/2401.14295", "date": "2024-01-25", "locator": "§VII-D3 Performance; §VIII Chains vs. Trees vs. Graphs of Thoughts", "task": "Survey and taxonomy of chain, tree and graph reasoning topologies, comparing published schemes on design, performance and cost", "effect": "\"In terms of cost-effectiveness, chain-based prompting schemes generally outperform their tree-based and graph-based counterparts\", since tree and graph schemes \"often involve multiple rounds of prompting\"; \"When it comes to the quality of outcomes, tree-based and graph-based prompting schemes typically surpass chain-based ones.\"", "direction": "supports", "measured": false, "conditions": "Literature survey by the GoT authors' group; no new experiments", "older": true }, { "source": "Tushar Pandey, Ara Ghukasyan, Oktay Goktas, Santosh Kumar Radha, \"Adaptive Graph of Thoughts: Test-Time Adaptive Reasoning Unifying Chain, Tree, and Graph Structures\"", "url": "https://arxiv.org/abs/2502.05078", "date": "2025-02-07", "locator": "Table 1; Table 2; §5.1.1; §5.2; §5.3", "task": "Builds the graph per query instead of by hand: recursively decomposes the question into a DAG of subproblems, expands only nodes the LLM judges complex, and stops when it judges an answer good enough; compared with direct answering (IO), CoT and AIoT", "effect": "GPQA Diamond with shuffled options: gpt-4o-mini 37.4 (IO) to 49.5 (+32.4%, relative), CoT 38.6; gpt-4o 39.4 to 57.6 (+46.2%). Standard unshuffled GPQA Diamond: 54.6 to 55.1 (+0.9%), where CoT (49.0) and AIoT (48.0) fell below IO. HotpotQA LLM-judged accuracy: IO 72.0, CoT 75.0, AGoT 80.0; Game of 24: 10.0, 20.0, 50.0. Mini-crosswords: AGoT 34.7% letters and 11.1% words against AIoT 33.5% and 11.0%. No token, cost or latency figures are reported.", "direction": "mixed", "measured": true, "conditions": "gpt-4o-mini for all methods (gpt-4o for one GPQA row); GPQA Diamond 198 questions, 100 hard HotpotQA questions, the 20 hardest Game of 24 problems; d_max, l_max, n_max = 1, 3, 3", "older": true }, { "source": "Qianyue Hao, Sibo Li, Jian Yuan, Yong Li, \"RL of Thoughts: Navigating LLM Reasoning with Inference-time Reinforcement Learning\"", "url": "https://arxiv.org/abs/2505.14140", "date": "2025-05-20", "locator": "Appendix E.1, Table 10", "task": "Runs GoT as a baseline (\"We carefully design graphs for each specific dataset\") against direct answering and other inference-time methods", "effect": "Qwen2.5-7B-Instruct, mean of 5 runs, GoT against direct answering (DirectQA): MATH-500 73.24 vs 74.96, GSM8K 88.76 vs 90.95, GPQA 33.26 vs 31.16, StrategyQA 75.69 vs 68.53; average 67.74 vs 66.40. On average GoT ranked below rStar (70.80), AFlow (70.28), Q* (69.30) and Buffer of Thoughts (69.11). GoT's token use is not reported.", "direction": "mixed", "measured": true, "conditions": "Qwen2.5-7B-Instruct; MATH-500, GSM8K, GPQA, StrategyQA; GoT graphs built by the authors of a competing method", "older": true }, { "source": "Nearchos Potamitis, Vansh Ramani, Har Ashish Arora, Dhairya Kuchhal, Lars Klein, Akhil Arora, \"ReasonBENCH: Benchmarking the (In)Stability of LLM Reasoning\"", "url": "https://arxiv.org/abs/2512.07795", "date": "2025-12-08", "locator": "Table 1; §4.1; Appendix, joint cost-quality failure (Table 14)", "task": "Re-implements ten reasoning strategies, GoT among them, behind one interface and runs each 30 times on six tasks with the same backbone, recording quality and cost per run", "effect": "GPT-4.1 Nano, suite mean quality: GoT 0.3364 [0.33, 0.35] against IO 0.1341, CoT 0.2792, CoT-SC 0.2265, ToT-BFS 0.4145, RAP 0.3713 and FoA 0.4549; mean cost GoT 0.0099 against CoT 0.0003 and CoT-SC 0.0014. Joint cost-quality failure (quality below and cost above the pooled median): 43.3% of GoT runs, the highest among strategies, against 0% for IO and CoT. \"the highest-performing strategy wins only 77% of head-to-head runs against its nearest competitor\".", "direction": "mixed", "measured": true, "conditions": "GPT-4.1 Nano for the strategy comparison; Game of 24, HumanEval, HotpotQA, Humanity's Last Exam, SciBench, sonnet writing; 30 runs per cell; figures from v2 (May 2026)", "older": false }, { "source": "Felix Fricke, Simon Malberg, Georg Groh, \"Framework of Thoughts: A Foundation Framework for Dynamic and Optimized Reasoning based on Chains, Trees, and Graphs\"", "url": "https://arxiv.org/abs/2602.16512", "date": "2026-02-18", "locator": "Table 2; Table 3; §4; §4.1", "task": "Re-implements GoT (sorting 128 digits, NDA document merging) and ToT in a framework that runs independent operations in parallel, caches repeated calls, and tunes hyperparameters and prompts with cost capped at the untuned level", "effect": "GPT-3.5-Turbo. Per-instance runtime: GoT sorting 259 s sequential against 30 s parallel (8.5x), document merging 145 s against 32 s (4.6x); caching cut document-merging cost 14%, from 6.9 to 5.9 US cents with a persistent cache (6.1 with a process cache), and left sorting at 5.0 cents. ToT on the same sorting task: 18.4 mistakes at 15.9 cents against GoT's 12.7 at 5.0. Tuning: GoT sorting mistakes 12.7 to 12.1, document-merging F1 8.4 to 8.8, with costs constrained \"to not exceed those of the unoptimized variant\".", "direction": "supports", "measured": true, "conditions": "GPT-3.5-Turbo, as in the original GoT paper; 50 test instances per GoT task, 50 for tuning", "older": false }, { "source": "Yuhang Liu, Ruijie Wang, Yunlong Chu, Bing Hao et al., \"RouteGoT: Node-Adaptive Routing for Cost-Efficient Graph of Thoughts Reasoning\"", "url": "https://arxiv.org/abs/2603.05818", "date": "2026-03-06", "locator": "Abstract; §1; §4.1.2; Table 2; Table 3; Table 5", "task": "Compares a generalized GoT (dynamic graph expansion, every node on the 30B model), Adaptive GoT, ToT, CoT and direct answering in one executor, then routes leaf subtasks to smaller models under a token budget while planning and synthesis stay on the large model", "effect": "Qwen3-30B, accuracy (output tokens), GoT* against CoT: GPQA 59.6 (9,468) vs 63.1 (4,965); HotpotQA 87.0 (812) vs 86.0 (431); MoreHopQA 73.0 vs 68.0; HybridQA 88.0 vs 84.0; Game of 24 72.0 (17,396) vs 58.0 (1,948); crossword words 6.5 vs 13.5. Input tokens per GPQA question: 16,826 (GoT*) vs 247 (CoT). GoT* took 22.89 to 116.03 s per question. \"increasing the structural complexity of inference does not consistently improve system-level returns\"; on HotpotQA Adaptive GoT \"consumes substantially more tokens than CoT or Direct IO yet performs worse\" (72.0 vs 86.0 and 81.0). RouteGoT: \"an average 8.1 percentage points accuracy improvement and 79.1% output token reduction compared to AGoT\".", "direction": "mixed", "measured": true, "conditions": "Qwen3-30B for the baselines, Qwen3-4B/8B/30B for routing; GPQA Diamond (198, shuffled), 100 each of HotpotQA, MoreHopQA, HybridQA and Game of 24, 20 mini-crosswords; graph depth capped at 3, branching at 5, 50 nodes; open-ended answers graded by a Qwen3-30B judge", "older": false }, { "source": "Fan Huang, \"Reasoning Topology Matters: Network-of-Thought for Complex Reasoning Tasks\"", "url": "https://arxiv.org/abs/2603.20730", "date": "2026-03-21", "locator": "Table 3; Table 5; Table 6; §5.1; §5.3; §5.4", "task": "Graph-structured reasoning (typed nodes and edges that merge evidence, expanded by a controller) against CoT and ToT, with tokens per instance", "effect": "GPT-4o-mini: GSM8K CoT 89.5 vs graph 82.0 under the same grader (85.0 LLM-judged) at 490 vs 7,748 tokens; Game of 24 CoT 58.0, ToT 11.0, graph 75.0 under the same grader (86.0 LLM-judged, 16,004 tokens); HotpotQA with an LLM judge for all methods, graph 91.0 vs ToT 88.0 and CoT 95.7; ProofWriter CoT 43.3, ToT 51.7, graph 50.3 at 27,096 tokens. With Llama-3.3-70B and Qwen2.5-72B the graph reached 91.5 on GSM8K (CoT 91.0 and 82.0) but CoT was best on ProofWriter (79.7 and 76.3 vs 57.7 and 65.0). \"CoT is the most token-efficient method across all datasets\".", "direction": "mixed", "measured": true, "conditions": "GPT-4o-mini, Llama-3.3-70B-Instruct, Qwen2.5-72B-Instruct; GSM8K 200, Game of 24 100, HotpotQA 300, ProofWriter 300; 5 to 20 calls per instance; graph scored by an LLM judge and CoT and ToT by string match except where noted; one author", "older": false } ], "caveats": "Most gains come from hand-built graphs on synthetic merge tasks run with GPT-3.5 on 100 samples per task, or from papers proposing a successor method; in some independent comparisons graph methods trailed chain-of-thought or direct answering on GPQA and step-by-step math, and GoT ranked below tree search on a six-task suite and had the highest share of runs both costlier and worse. No source compares GoT with self-consistency or best-of-N at matched tokens, and none tests the original GoT on reasoning models. The later graph methods (Adaptive GoT, Network-of-Thought) are different designs; Adaptive GoT reports no cost figures, and an independent rerun with token counts found it worse than chain-of-thought on HotpotQA (72.0 vs 86.0) at substantially more tokens. Results vary run to run (the top strategy won only 77% of head-to-head runs in one benchmark), and only two of these sources report repeated runs.", "also_covers": [], "related": [ { "id": "7.c.vii", "slug": "tree-of-thought", "name": "Tree-of-Thought" }, { "id": "9.c.ii", "slug": "samples-before-verification", "name": "Spend extra samples before extra verification: parallel attempts with a vote or judge; verify only where it pays" }, { "id": "6.b.ii", "slug": "llm-map-reduce", "name": "LLM map-reduce" }, { "id": "7.c.v", "slug": "language-agent-tree-search", "name": "Language Agent Tree Search (LATS)" }, { "id": "1.b.i", "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class" }, { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "6.c.ii", "slug": "plan-then-execute", "name": "Plan-then-execute" }, { "id": "7.c.iii", "slug": "evaluator-optimizer-loop", "name": "Evaluator-optimizer loop" } ], "url": "https://mahalsystems.ai/aot/optimizations/graph-of-thought/", "markdown": "https://mahalsystems.ai/aot/optimizations/graph-of-thought.md" }, { "id": "7.c.v", "legacy_id": "P34", "former_ids": [], "slug": "language-agent-tree-search", "name": "Language Agent Tree Search (LATS)", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it replaces a single chain with a search", "proposed": true }, "levers": [ "control-flow" ], "domains": [ "reasoning", "planning" ], "evidence_domains": [ "code-generation", "computer-use", "web-research", "reasoning", "incident-response" ], "categories": [ "Scaling compute" ], "category_ids": [ "orchestration.extra-compute" ], "summary": "Language Agent Tree Search runs the agent as a Monte Carlo tree search over its own actions: sample several next actions, execute each in the environment, score the resulting states with the model, reflect on failed endings, and back the scores up the tree until a branch succeeds or the budget runs out. Against a single ReAct run it raised success on web, coding and incident-diagnosis tasks (VisualWebArena 18.9% to 26.4%, SWE-bench Lite +23% relative across five models, root-cause accuracy 39.8% to 91.3%, though those baselines got about a quarter of the tokens). At equal spend the results split: on HumanEval plain retries scored 92.0% at $2.51 against LATS's 88.0% at $134.50, while on VisualWebArena tree search (32.0%) beat re-ranking whole runs (30%) at about equal compute. It needs an environment whose states can be restored and kept free of irreversible side effects. It cost 3.8 to 14.1 times the tokens or dollars of a single run in most studies (over 50 times simple retries on HumanEval) and 7.6 to about 10 times the wall time.", "how": [ "Start from the cheap baselines on the same model and tasks: the ReAct loop (7.d.iii) with retries on failed checks, and best-of-N whole runs picked by a vote, a judge or a test (9.c.ii), each given the tokens and wall time the search would get; add search only where a gap remains. Read-only diagnosis (searching over hypotheses with tools that only list, search and compare logs and metrics) suits it best, since every branch is safe to explore.", "Make every node restorable before searching: reset the environment and replay the action prefix, commit the workspace per node (git), or snapshot the sandbox, and check that a restored state matches the original. Docker containers with background processes do not checkpoint reliably and replays can diverge; where states cannot be restored, use one-step lookahead (sample K actions, a critic picks one) plus selection among whole runs, or simulate candidate actions with an LLM world model instead of executing them.", "Validate each proposed action before running it, before adding any search: that alone raised WebArena-lite success from 47.74% to 53.55%, most of the gain, while adding naive tree search lowered it to 51.61% until destructive actions were handled. Keep exploration off irreversible actions: flag likely destructive actions before execution (form submits, Enter, buttons that are not navigation) and confirm after (POST, PUT, DELETE or PATCH requests), defer them until the search commits, drop the states they make unreachable, and on live systems restrict expansion to non-destructive actions or gate them (10.b.i).", "Build the search: select by UCT, expand 3 to 5 sampled actions and execute each, score the new state with the model after it sees the observation plus self-consistency over several samples (with state-specific value prompts; where a sound signal such as tests or the environment's reward exists, use it, 9.a.i), write a reflection on each failed ending and feed it to later expansions, and backpropagate.", "Bound and parallelize: cap iterations or nodes per task and search time per state, expand siblings concurrently with prefix caching on, and on reasoning models choose the effort setting that leaves room for enough nodes in the time budget (maximum effort completed 18.5 search nodes per run against 62.8 at medium, 1.b.i).", "Choose the final answer with a checked selector: the value function picked the resolving patch 73% of the time and a separate discriminator 84%, so compare the selector's choice with outcomes.", "Measure against step 1 at matched tokens and wall time, per model and task difficulty: success, tokens, LLM calls, wall time, restore time and destructive actions executed during exploration; re-check after a model upgrade, since the gain flipped with backbone capability." ], "detect": { "static": "A search loop over agent actions: UCT or best-first selection, expansion of several sampled actions executed in the environment, a model-scored value (often with self-consistency), reflections on failed endings, backpropagation, and a way to restore earlier states (reset and replay, a commit per node, snapshots). Flag search over actions with side effects and no restore path or destructive-action gate, and search with no simpler baseline at a matched budget.", "traces": "Nodes expanded and LLM calls per item; tokens and wall time per item; restore or replay time; value of the chosen trajectory against its actual outcome; destructive actions executed during exploration.", "evals": "Success against ReAct with retries and best-of-N with a vote or judge at equal tokens and wall time, per model and task difficulty.", "judge": "Rate reflections used as value estimates.", "probes": "Multi-step tasks with recoverable mistakes." }, "impact": { "measured_basis": true, "primary": [ "quality", "cost" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "gains over a single ReAct run: VisualWebArena 18.9% to 26.4% (Koh, Jul 2024), SWE-bench Lite +23% relative across five models (Antoniades, Oct 2024), root-cause accuracy 39.8% to 91.3% (Naakka, May 2026); at equal spend simpler methods matched or won: HumanEval 88.0% against 92.0% for retries (Kapoor, Jul 2024), GAIA best-of-N 63.03 against beam search 56.97 and tree search 55.76 at equal sampling width (Zhu, Jun 2025); search won: VisualWebArena 32.0% against 30% for re-ranking at about equal compute (Koh, Jul 2024), MCTS beat best-of-N at equal trajectory count on 3 of 4 backbones (Chen, Feb 2025); no search policy significantly better than best-of-N within one harness (Brilliantov, Sep 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "large", "when": "LATS cost $134.50 against $2.51 for retries on HumanEval (Kapoor, Jul 2024); SWE-Search $576.00 against $40.86 with GPT-4o and about 5x on four other models (Antoniades, Oct 2024); MCTS agents used 7.2x to 10.1x ReAct's tokens (Yu, Oct 2024); LATS averaged 71.0 LLM calls per request (Kim, Jun 2025); 156K against 41K tokens per investigation (Naakka, May 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "large", "when": "tree search took 749.2 to 972.1 s per VisualWebArena task against 68.3 to 87.7 s for a reactive agent (Gu, Nov 2024); 9.1 against 1.2 minutes per root-cause investigation (Naakka, May 2026); some LATS tasks ran over 2 hours (Kapoor, Jul 2024); expanding 16 children in parallel instead of 1 cut LATS latency by 196.3 s on average (Kim, Jun 2025)", "basis": "evidence" }, { "dimension": "security", "effect": "degrades", "size": "medium", "when": "exploratory branches execute real actions; irreversible ones (placing an order, submitting a form) cannot be backtracked, so search outside a sandbox needs a non-destructive action space or gating (Koh, Jul 2024; Gu, Nov 2024); a conservative pre-execution check flagged actions of which about 37% were truly destructive (Dihan, Dec 2025)", "basis": "evidence" }, { "dimension": "governance", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired eval of the tree search against ReAct with retries and against best-of-N with a vote or judge, at matched tokens and wall time, same model and value signal, split by task difficulty and per model and reasoning-effort setting: success, tokens, LLM calls, wall time, destructive actions executed during exploration" }, { "method": "monitor", "tier": null, "how": "nodes expanded and budget-hit rate per task; tokens, LLM calls and p50/p95 wall time; state restore or replay time and replay divergence; agreement between the chosen trajectory's value and its actual outcome; destructive actions flagged, blocked and executed" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Reasoning & Search group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Zhou et al., \"Language Agent Tree Search Unifies Reasoning Acting and Planning in Language Models\"", "url": "https://arxiv.org/abs/2310.04406", "date": "2023-10-06", "locator": "Abstract", "task": "Monte Carlo tree search over agent actions with LM value functions and self-reflection", "effect": "\"state-of-the-art pass@1 accuracy (92.7%) for programming on HumanEval with GPT-4\"; \"average score of 75.9\" on WebShop with GPT-3.5, \"comparable to gradient-based fine-tuning\"", "direction": "supports", "measured": true, "conditions": "HumanEval, 164 problems, GPT-4: model-generated tests as feedback, 8 iterations, 5 solutions per expansion; WebShop (GPT-3.5)", "older": true }, { "source": "Sayash Kapoor, Benedikt Stroebl, Zachary S. Siegel, Nitya Nadgir, Arvind Narayanan, \"AI Agents That Matter\"", "url": "https://arxiv.org/abs/2407.01502", "date": "2024-07-01", "locator": "§2.3; Appendix A Table A1 and A.2; Appendix E.1", "task": "Reran LATS, Reflexion and LDB on HumanEval next to simple baselines (retry up to five times when the example tests fail, warming, model escalation), five runs each, reporting accuracy and dollar cost", "effect": "Mean of five runs: LATS (GPT-4) 88.0% at $134.50 total; Retry (GPT-4) 92.0% at $2.51; Warming (GPT-4) 93.2% at $2.45; GPT-4 alone 89.6% at $1.93; LATS (GPT-3.5) 80.4% at $9.49. \"Reflexion and LDB cost over 50% more than the warming strategy, and LATS over 50 times more.\" LATS had been scored on \"only a subset of the test cases\" per problem, \"responsible for a 3% difference in accuracy\". With June 2023 models some LATS tasks took \"more than 2 hours to complete for GPT-3.5\".", "direction": "contradicts", "measured": true, "conditions": "gpt-4-turbo-2024-04-09 and gpt-3.5-turbo-0125; LDB's 164-problem HumanEval; LATS at 8 iterations and expansion factor 3; April 2024 prices", "older": true }, { "source": "Jing Yu Koh, Stephen McAleer, Daniel Fried, Ruslan Salakhutdinov, \"Tree Search for Language Model Agents\"", "url": "https://arxiv.org/abs/2407.01476", "date": "2024-07-01", "locator": "Abstract; §4.2 Table 2; §5.1 Figures 2-3, Table 4; §5.2 Table 5; §5.4", "task": "Best-first tree search over web actions, scored by a GPT-4o value function with self-consistency; backtracking resets the environment and replays the action sequence; compared with no search and with re-ranking whole trajectories at about equal compute", "effect": "GPT-4o agent: VisualWebArena 18.9% to 26.4% (\"39.7% relative\"), WebArena 15.0% to 19.2%; Llama-3-70B: VWA 7.6% to 16.7%, WA 7.6% to 10.1%. On 200 VWA tasks: no search 24.5%, budget c=5 32.0%, c=20 37.0%; re-ranking \"starts to plateau around 7 runs\" at 30%, below search at c=5, where the two \"spend approximately equal inference compute\". Value function: GPT-4o without self-consistency 28.5%, with 5 samples 32.5%, with 20 samples 37.0%, ground-truth reward 43.5%. By difficulty: easy 34.2% to 42.3%, medium 12.7% to 22.2%, hard 10.2% to 14.9%. Limits: c=20 can mean \"20× more LM calls\"; \"we will need to restrict the search space to actions that are not destructive\".", "direction": "supports", "measured": true, "conditions": "GPT-4o with set-of-marks and Llama-3-70B-Instruct agents; depth 5, branching 5; VisualWebArena and WebArena; read at v4", "older": true }, { "source": "Antonis Antoniades, Albert Örwall, Kexun Zhang, Yuxi Xie, Anirudh Goyal, William Wang, \"SWE-Search: Enhancing Software Agents with Monte Carlo Tree Search and Iterative Refinement\"", "url": "https://arxiv.org/abs/2410.20285", "date": "2024-10-26", "locator": "Table 1; §4.2; Appendix B; Appendix I Table 3; Appendix J Table 4", "task": "MCTS over a coding agent's states with an LLM value agent that returns a score and written feedback, and a discriminator that debates up to five final candidates; each state is a git commit so the search can revert the repository; against the same agent without search", "effect": "SWE-bench Lite resolve rate, without and with search: GPT-4o 25.7% to 31.0%, GPT-4o-mini 13.0% to 17.0%, Qwen-2.5-72B 18.0% to 24.7%, DeepSeek-V2.5 16.3% to 21.0%, Llama-3.1-70B 13.6% to 17.7%; \"a 23% relative improvement\". API cost: GPT-4o $576.00 against $40.86, GPT-4o-mini $52.34 against $9.90, the three open models about 5x (estimated). Compute-matched: the baseline's Pass@5 over five runs against SWE-Search Pass@1 was 17.0 vs 17.0, 22.3 vs 24.7, 22.0 vs 21.0 and 21.7 vs 17.7, but that pits the baseline's any-of-5 against search's single pick: SWE-Search's own Pass@5 (22.3, 25.7, 23.3, 22.3) beat the baseline's on all four models. The value function picked the resolving solution \"73% of the time on average\", the discriminator 84%.", "direction": "mixed", "measured": true, "conditions": "moatless-tools adapted for tree states; SWE-bench Lite, 300 issues; up to 100 iterations, 3 expansions per node; open-model costs estimated; the paper gives Llama's Pass@1 as 17.7 in Table 1 and 21.0 in Figure 3", "older": true }, { "source": "Xiao Yu, Baolin Peng, Vineeth Vajipey, Hao Cheng, Michel Galley, Jianfeng Gao, Zhou Yu, \"ExACT: Teaching AI Agents to Explore with Reflective-MCTS and Exploratory Learning\"", "url": "https://arxiv.org/abs/2410.02052", "date": "2024-10-02", "locator": "§4 Search Parameters; §4.1 Tables 2-3; §5 Table 5; §8 Limitations", "task": "Reflective MCTS (MCTS plus contrastive reflections stored and retrieved across tasks, and a multi-agent-debate value function) against ReAct and other search methods on web and computer tasks, with tokens per task", "effect": "Success (tokens relative to ReAct): Classifieds ReAct 28.6% (1x), ToT-BFS 30.7% (3.2x), best-first Search Agent 33.8% (4.2x), MCTS 37.6% (7.2x), R-MCTS 40.2% (7.3x), R-MCTS with debate 41.0% (7.4x); with debate, Reddit 13.8% to 28.7% (9.7x), Shopping 23.2% to 32.3% (10.1x), GitLab 11.3% to 23.5% (8.8x). OSWorld: ReAct 11.21%, R-MCTS 19.39%. \"MCTS-based agents ... consume nearly 10x more tokens compared to ReAct.\" On 234 text-only Classifieds tasks, ReAct with o1-mini scored 23.9% against 32.1% for GPT-4o R-MCTS at 9.6x tokens.", "direction": "supports", "measured": true, "conditions": "GPT-4o; VisualWebArena Classifieds, Reddit, Shopping and WebArena GitLab; breadth and depth 5, at most 5 actions, 5 minutes of search per state; OSWorld at 15 steps", "older": true }, { "source": "Yu Gu, Kai Zhang, Yuting Ning, Boyuan Zheng, Boyu Gou, Tianci Xue, Cheng Chang, Sanjari Srivastava, Yanan Xie, Peng Qi, Huan Sun, Yu Su, \"Is Your LLM Secretly a World Model of the Internet? Model-Based Planning for Web Agents\"", "url": "https://arxiv.org/abs/2411.06559", "date": "2024-11-10", "locator": "§1; §4.1; §4.2 Tables 1-2", "task": "A reactive agent, tree search with real interactions (backtracking by reset and replay) and WebDreamer, which simulates each candidate action's outcome with an LLM world model before executing one, all on GPT-4o", "effect": "Success: VisualWebArena reactive 17.6%, tree search 26.2%, WebDreamer 23.6%; on live sites WebDreamer raised Online-Mind2Web 26.0% to 37.0% and Mind2Web-Live 20.2% to 25.0%, where tree search could not run: \"resetting the environment to undo effects is not feasible on real-world websites\". Wall time per task, reactive / tree search / WebDreamer: Classifieds 68.3 / 749.2 / 183.6 s, Reddit 83.5 / 972.1 / 233.7 s, Shopping 87.7 / 785.7 / 179.4 s; action steps on Classifieds 3.4 / 9.9 / 4.1. Tree search \"introduces about 10 times greater latency\".", "direction": "mixed", "measured": true, "conditions": "GPT-4o for all methods; VisualWebArena (sandbox), Online-Mind2Web and Mind2Web-Live (live websites)", "older": true }, { "source": "Zhenfang Chen, Delin Chen, Rui Sun, Wenjun Liu, Chuang Gan, \"Scaling Autonomous Agents via Automatic Reward Modeling And Planning\"", "url": "https://arxiv.org/abs/2502.12130", "date": "2025-02-17", "locator": "§4.2 Table 1; Appendix A (Planning Algorithm Details)", "task": "One automatically trained reward model guides Reflexion, best-of-N or MCTS, each capped at the same number of explored trajectories (10 on WebShop and ScienceWorld, 100 on Game of 24); against sampling and greedy decoding", "effect": "Average over WebShop, ScienceWorld (seen, unseen) and Game of 24, best-of-N vs MCTS: Llama-3.1-70B 46.1 vs 49.3; Llama-3.1-8B 34.1 vs 32.6; Mistral-7B 26.4 vs 29.6; Phi-3.8B 26.5 vs 30.0 (greedy 37.5, 28.9, 19.5, 17.5). \"MCTS performs the best on average\"; Reflexion did worst on the weaker models.", "direction": "mixed", "measured": true, "conditions": "Llama-3.1-70B and 8B, Mistral-7B, Phi-3.8B policies; a 3B reward model trained on synthetic trajectories; at most 10 actions per trajectory", "older": true }, { "source": "Karina Zainullina, Alexander Golubev, Maria Trofimova, Sergei Polezhaev, Ibragim Badertdinov, Daria Litvintseva, Simon Karasik, Filipp Fisin, Sergei Skvortsov, Maksim Nekrashevich, Anton Shevtsov, Boris Yangel, \"Guided Search Strategies in Non-Serializable Environments with Applications to Software Engineering Agents\"", "url": "https://arxiv.org/abs/2505.13652", "date": "2025-05-19", "locator": "§1; §2.3; §3.1-3.2; §4.2 Tables 1-2", "task": "Search for a coding agent in Docker containers whose state cannot be saved and restored: one-step lookahead (sample K candidate actions, a learned critic picks one) and trajectory selection (N full runs, the critic picks one), in place of MCTS", "effect": "MCTS \"becomes impossible in these settings\"; Docker checkpointing with CRIU \"cannot guarantee consistent preservation\" of process state, and replaying actions in a fresh container is expensive and nondeterministic, \"making it impossible to reliably reproduce a specific state\". Verified-50 success: Qwen-based policy 16.2%, with lookahead 26.8%, with selection (N=5) 27.2%, with both (N=10) 41.7%; GPT-4o 22.0%, 27.2%, 34.0% (N=5) and 40.0% (both, N=5). Full SWE-bench Verified 40.8% (K=8, N=15). Lookahead \"only increases the number of generated output tokens\"; selection runs in parallel with \"minimal impact on inference latency\".", "direction": "mixed", "measured": true, "conditions": "SWE-agent; fine-tuned Qwen-72B-based policy and GPT-4o; critic trained on 80,000 trajectories; Verified-50 (50 random SWE-bench Verified tasks) and the full set", "older": true }, { "source": "King Zhu, Hanhao Li, Siwei Wu, Tianshun Xing, Dehua Ma, Xiangru Tang, Minghao Liu, Jian Yang, et al., \"Scaling Test-time Compute for LLM Agents\"", "url": "https://arxiv.org/abs/2506.12928", "date": "2025-06-15", "locator": "§1; §3.1; §4.1 Table 1; §4.3 Table 4", "task": "Parallel test-time scaling methods for one agent under the same settings (sampling width 4, beam size 2): best-of-N over whole runs, step-wise best-of-N, beam search, DVTS and tree search, each picked by an LLM verifier", "effect": "GAIA validation average: baseline 55.76, best-of-N 63.03, step-wise best-of-N 58.79, beam search 56.97, DVTS 55.76; tree search 55.76 with list-wise verification and 50.91 with scoring. \"despite simplicity, BoN achieves the optimal performance\"; beam search and DVTS \"show no significant improvement over baseline\" because their exploration \"depends on the accuracy of signals provided by the verify model\".", "direction": "contradicts", "measured": true, "conditions": "GPT-4.1 in a smolagents CodeAgent; GAIA validation, 165 tasks; equal sampling width, not equal tokens", "older": true }, { "source": "Jiin Kim, Byeongjun Shin, Jinha Chung, Minsoo Rhu, \"The Cost of Dynamic Reasoning: Demystifying AI Agents and Test-Time Scaling from an AI Infrastructure Perspective\"", "url": "https://arxiv.org/abs/2506.04301", "date": "2025-06-04", "locator": "§III; §IV-A Figure 4; §IV-B (prefix caching); §V Figure 16; Table III", "task": "System-level measurement of CoT, ReAct, Reflexion, LATS and LLMCompiler served on vLLM: LLM calls, latency, KV-cache memory and GPU energy, and LATS with more child nodes expanded in parallel", "effect": "LATS \"exhibits the highest LLM invocation count, with an average of 71.0 LLM calls per request\". HotpotQA with the 8B model: LATS 80% accuracy, 380.90 s and 22.76 Wh per query against Reflexion 38%, 649.34 s and 41.53 Wh; with 70B, LATS 82%, 305.67 s and 158.48 Wh; a single-turn chat query 0.32 Wh (8B). Raising LATS child nodes from 1 to 16 \"improves accuracy by 14.4 percentage points while simultaneously reducing latency by 196.3s on average\". Prefix caching cut LATS KV-cache memory \"by an average of 64.8%\".", "direction": "mixed", "measured": true, "conditions": "Llama-3.1-8B and 70B-Instruct on A100s; HotpotQA, WebShop, MATH, HumanEval; LATS reimplemented with concurrent LLM calls and parallel tool calls", "older": true }, { "source": "Mahir Labib Dihan, Tanzima Hashem, Mohammed Eunus Ali, Md Rizwan Parvez, \"WebOperator: Action-Aware Tree Search for Autonomous Agents in Web Environment\"", "url": "https://arxiv.org/abs/2512.12692", "date": "2025-12-14", "locator": "§3.2; §4.2 Table 3; §4.4; §4.5 Table 4", "task": "Best-first tree search for a web agent that classifies actions as safe or destructive (a pre-execution heuristic on element and key, a post-execution check of POST, PUT, DELETE and PATCH requests), defers destructive actions, invalidates states they make unreachable and jumps to refresh-stable checkpoints instead of replaying from the root; ablated step by step", "effect": "WebArena-lite ablation: ReAct 47.74%; with action validation 53.55%; with multiple, merged and varied action proposals 54.84%; adding naive tree search lowered it to 51.61%; with destructive-action handling and context-aware selection 58.71%; full system 60.00%, at 31.34 generated actions per task against 9.30 for ReAct. Full WebArena: 24.4%, 42.7%, 48.4% and 54.6% at search budgets 5, 10, 15 and 20. \"only about 37% of pre-flagged actions are confirmed as truly destructive\".", "direction": "mixed", "measured": true, "conditions": "gpt-4o in BrowserGym; WebArena-lite 155 tasks for the ablation; comparisons with other tree-search agents use their reported numbers", "older": false }, { "source": "Alexander Naakka, Yuqing Wang, Mika V Mäntylä, \"LATS-RCA: Language Agent Tree Search for Root Cause Analysis in Microservices\"", "url": "https://arxiv.org/abs/2605.03505", "date": "2026-05-05", "locator": "§4.2 RQ1-RQ2, Table 1; §4.3 Table 2; Table 3; §5", "task": "LATS over root-cause hypotheses: log and metric agents expand five candidate investigative actions per node, score them by LLM reflection (evidence quality, completeness, consistency) plus self-consistency, select by UCT and backpropagate; against single- and multi-agent ReAct baselines built by the authors", "effect": "LO2 accuracy: LATS-RCA 91.3% (SD 1.2% over five runs), multi-agent ReAct 57.4%, single-agent ReAct 39.8%. Per investigation: 53.1 calls, 156K tokens, 9.1 minutes against 13.5 calls, 41K tokens, 1.2 minutes for single ReAct; \"approximately 3.8× and 2.1× the tokens, and 7.6× and 5.4× the runtime, of single- and multi-agent ReAct\". Ablation: without candidate batching 84.3%, without backpropagation 84.8%, without reflection 87.6%. Claude Sonnet 4.5, GPT-5 and Gemini 3 Pro scored 89.7% to 91.3%. On 37 production incidents: 65.1% on average (62% to 70%), about 75 API calls, 220K tokens and 13 minutes each.", "direction": "supports", "measured": true, "conditions": "Claude Sonnet 4.5 unless noted; LO2 sample, 100 anomaly cases over 53 failure types; tools only list, retrieve, search and compare recorded logs and metrics; ReAct baselines not given an equal budget; production system of a web host", "older": false }, { "source": "Kirill Brilliantov, Alejandro Hernández-Cano, Emmanuel Abbé, \"How Much of a Harness Does a Strong Agent Need for Autonomous ML Engineering?\"", "url": "https://arxiv.org/abs/2609.40303", "date": "2026-09-30", "locator": "§1; §4.2; Appendix D.2 Tables 15-16; Appendix D.3 Table 17", "task": "Holds backbone, hardware (one A100) and a 24-hour budget fixed and compares one long coding-agent session (Malena) with harnesses including MLEvolve, a UCT tree search over complete training scripts; also compares search policies (chain, greedy, UCB1, best-of-N) over coding-agent iterations", "effect": "MLE-bench mean percentile, single session vs MLEvolve: Kimi K3 72.75 vs 63.89, GLM 5.2 69.74 vs 54.32, DeepSeek V4 Flash 58.53 vs 42.53, Gemma 4 31B 32.35 vs 42.88 (paired difference -10.53 [-14.72, -6.36], the only loss; medal rate 17.8% vs 22.2%, not significant). Across search policies at GLM 5.2 \"no strategy provides a statistically significant advantage\"; UCB1 led best-of-N by 2.71 percentile points, 95% CI (-1.44, +6.60). On DeepSeek V4 Flash, MLEvolve at maximum reasoning effort completed 18.5 search nodes per run against 62.8 at medium, and its medal rate was 0% against 25.0% (10-task subset). \"weaker backbones still benefit from hand-crafted workflow priors\".", "direction": "mixed", "measured": true, "conditions": "open-weight backbones on vLLM; MLE-bench fixed30 split, 24 hours per run, 95% bootstrap CIs over seeds; search over whole solutions, not single actions; MLEvolve at medium effort on DeepSeek V4 Flash", "older": false } ], "caveats": "Most gains are measured against a single ReAct run, not against retries or best-of-N at the same spend; where that comparison was made, results split: retries beat LATS on HumanEval, best-of-N led on GAIA (matched on samples, not tokens, one run) and no search policy beat best-of-N significantly on MLE-bench, while tree search beat re-ranking at about equal compute on VisualWebArena and MCTS beat best-of-N on three of four backbones in one study that used a trained reward model. The value signal is the bottleneck: a ground-truth reward reached 43.5% against 37.0% with GPT-4o's estimate, naive tree search lowered WebArena-lite success from 54.84% to 51.61% until destructive-action handling was added, and SWE-Search's value function picked the resolving patch 73% of the time. Search needs states that can be restored, which Docker containers and live websites do not reliably provide; the positive results come from sandboxes, and the one production study fell from 91.3% on its benchmark to 65.1% on 37 incidents. Most action-level results use GPT-4o-era models; on stronger open-weight reasoning backbones one study found no search policy significantly better than another and that maximum reasoning effort starved the search of nodes.", "also_covers": [], "related": [ { "id": "9.c.ii", "slug": "samples-before-verification", "name": "Spend extra samples before extra verification: parallel attempts with a vote or judge; verify only where it pays" }, { "id": "7.c.vii", "slug": "tree-of-thought", "name": "Tree-of-Thought" }, { "id": "7.d.iii", "slug": "react-loop", "name": "ReAct loop" }, { "id": "7.c.i", "slug": "reflection-loop", "name": "Reflection loop" }, { "id": "7.c.ii", "slug": "self-critique-evaluator", "name": "Self-critique evaluator loop" }, { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "10.b.i", "slug": "risky-action-policy", "name": "Gate irreversible actions: allow, ask or deny by side effect" }, { "id": "4.b.i", "slug": "filesystem-checkpoints", "name": "Filesystem-based agent state and checkpoints" }, { "id": "1.b.i", "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class" }, { "id": "3.b.i", "slug": "cache-stable-prefix", "name": "Keep the prompt prefix cache-stable" }, { "id": "7.c.iv", "slug": "graph-of-thought", "name": "Graph-of-Thought" } ], "url": "https://mahalsystems.ai/aot/optimizations/language-agent-tree-search/", "markdown": "https://mahalsystems.ai/aot/optimizations/language-agent-tree-search.md" }, { "id": "7.c.vi", "legacy_id": "P43", "former_ids": [], "slug": "multi-agent-debate", "name": "Multi-agent debate", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it adds a multi-agent topology", "proposed": true }, "levers": [ "control-flow" ], "domains": [ "reasoning" ], "evidence_domains": [ "reasoning", "code-generation", "judging", "document-extraction" ], "categories": [ "Scaling compute" ], "category_ids": [ "orchestration.extra-compute" ], "summary": "Several model instances answer the same question, read each other's answers and revise over one or more rounds, and a vote or judge settles the final answer. It beats a single answer on closed-form reasoning (+3.3 to +6.6 points across 23 open models where tasks had headroom), but a plain majority vote over the same number of independent samples gets the same gain: in budget-matched tests debate tied or lost to the vote on every task, GPQA Diamond at full thinking budget included, at 1.6x the wall-clock and 3.4x the tokens, and a vote over the first-round answers alone beat every debate variant on average, though mostly on one arithmetic task (0.99 vs 0.76; Choi, NeurIPS 2025). Mixing model families lifted debate above the average of its single-model runs in one study but lost to a vote over the same mixed roster in another (-1.04 points over 80 teams), and with frontier models on real paper feedback, authors ranked a single pass above two debate tools that spent 9x and 30x the tokens. Treat debate as an experiment against a vote at matched compute, worth keeping only where the vote is often wrong while some agent has the right answer.", "how": [ "Start from the cheap baselines on your own items: the single pass at full thinking budget (1.b.i) and a majority vote over as many independent generations as the debate would spend (agents x rounds) from the same model or roster (9.c.ii). Across 23 open models debate with 3 agents and 3 rounds tied or lost to a 9-sample vote on every task, never significantly ahead (GPQA Diamond 54.9 vs 54.3, CommonsenseQA 80.6 vs 78.3), and a vote over 5 first-round answers beat every debate variant on average (Qwen2.5-7B 0.7691 vs at best 0.7377), mostly on one arithmetic task (0.99 vs 0.76); on the other six benchmarks the two averaged about 0.73 each. Build debate only where the vote leaves headroom.", "Give every arm the same final-answer format and extractor, and parse the last answer an agent states, since debate replies quote their peers' answers (9.b.i). With Du et al.'s parser debate beat the vote on GSM8K (0.7533 vs 0.6700); with a fixed answer format the vote won (0.9400 vs 0.8867).", "To use several models, pool them and vote before adding debate, and compare any mixed debate against a vote over the same mixed roster and against the strongest model alone. Mixing GPT-4o-mini and Llama-3.1-70B lifted every debate framework over the mean of its single-model runs (Society of Minds 76.7 vs 74.3 and 69.9), but no vote over that mix was run; across 80 three-model teams where it was, debate lost 1.04 points to it, and in the 19 team-task cells where mixed debate beat the team's best member by 10 points or more, the roster vote gained more (+16.4 vs +13.5). Do not add persona prompts on small models: they cost 2.1 to 4.4 points.", "If debate stays, run one exchange after the first answers and stop. In mixed-team debate accuracy went from 84.6% (vote over first answers) to 87.2% after one exchange and stayed at 87.0-87.5% through five rounds; Decentralized debate fell from 0.7377 at two rounds to 0.7050 at five. Letting an agent keep an answer that matched the last majority raised debate to 0.7625, still below the vote (0.7691).", "Size the context window to agents x rounds x per-turn budget, thinking tokens included, and give the debate and the vote the same thinking budget. With long-thinking models at 16-32K windows, 93-100% of some cells' final-round answers came back empty (one GPQA cell 0.24 at 16K vs 0.82 at 131K); three 40K-budget thinkers reached 100-112K-token prompts by round 3; and a 4,096-token thinking cap left on some reasoning models hurt the single-shot arm more than debate, creating a 3.2-point debate lead that became -0.6 at the full budget.", "For open-ended output with no answer to vote on (reviews, critiques, reports), compare against a single pass of equal length, ranked blind by the people who use it, not by an LLM judge: authors ranked a single Claude Opus 4.8 pass (mean rank 1.59) above a cross-family debate (2.25) and a multi-agent workshop (2.16) that cost 9x and 30x the tokens, and an external AI judge agreed with them at only 0.14 and would have reversed the verdict.", "Measure on held-out items with ground truth: paired accuracy of debate against the vote at matched generations, with confidence intervals; total tokens including re-read transcripts (3.4x the vote's, though generated tokens matched at 1.06x) and wall-clock (1.6x); empty or truncated final answers; and how often debate changes the vote's answer and in which direction. Re-run on model upgrades." ], "detect": { "static": "Several agents answer the same question, see each other's answers and revise over rounds, with a vote, judge or synthesizer at the end. Flag it when the eval has no majority-vote arm at the same number of generations, when more than one exchange round runs, when transcripts can outgrow the context window (agents x rounds x per-turn budget, thinking included), when agents get persona prompts, or when arms parse answers differently.", "traces": "Rounds per item; share of items where the final answer differs from the first-round vote and, on items with ground truth, whether the change fixed or broke it; right-to-wrong and wrong-to-right flips per round; empty or truncated final-round answers; prompt and total tokens per round.", "evals": "Accuracy against a majority vote over the same number of independent generations (same model, or the same mixed roster) and against the single pass at full thinking budget, with the same answer extractor; for open-ended output, blind ranking by intended users against a single pass of equal length.", "judge": "Secondary only: an external AI judge agreed with paper authors at 0.14 and would have reversed their ranking of debate against a single pass; for open-ended output rank with intended users, and balance any judge panel across vendors.", "probes": "Items where the first-round majority is wrong but at least one agent is right, mixed with items where the majority is right." }, "impact": { "measured_basis": true, "primary": [ "quality", "cost" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "over one answer +3.3 to +6.6 points where tasks had headroom (Ferreira, Sep 2026; GSM 85.0 vs 77.0, Du, May 2023); against a vote at matched or lower compute none or negative: ties or losses on every task, -1.04 for mixed-model teams (Ferreira, Sep 2026), vote 0.7691 vs best debate 0.7377 (Choi, Aug 2025), debate beat CoT in at most 5 of 36 scenarios (13.9%) (Zhang, Feb 2025); paper feedback from frontier models: single pass mean rank 1.59 vs 2.25 and 2.16 for debate tools (Havranek, Jul 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "large", "when": "3.4x the total tokens of a matched 9-sample vote (2.3x to 4.1x by task) because each round re-reads peers' transcripts (Ferreira, Sep 2026); 9x and 30x the tokens of a single pass, $1.87 and $7.20 vs $0.20 per paper (Havranek, Jul 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "large", "when": "1.6x the wall-clock of a matched vote; 20 vs 12 vs 5 minutes per 100-problem job for debate, vote and single pass on local serving (Ferreira, Sep 2026); rounds run in series", "basis": "evidence" }, { "dimension": "security", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "governance", "effect": "improves", "size": "small", "when": "Recorded arguments.", "basis": "assessment" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired eval on held-out items with ground truth: debate (agents x rounds) against a majority vote over the same number of independent generations from the same model or the same mixed roster, and against the single pass at full thinking budget; identical answer format and extractor in every arm; context window at least agents x rounds x per-turn budget; accuracy with paired confidence intervals, total tokens including re-read transcripts, wall-clock; for open-ended output, blind ranking by the intended users against a single pass of equal length" }, { "method": "monitor", "tier": null, "how": "per item: rounds run, empty or truncated final-round answers, prompt and total tokens, latency, share of items where debate changes the first-round vote; on a regularly audited sample, whether those changes fix or break the answer" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Multi-Agent group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Du et al., \"Improving Factuality and Reasoning in Language Models through Multiagent Debate\"", "url": "https://arxiv.org/abs/2305.14325", "date": "2023-05-23", "locator": "Table 1", "task": "Several model instances propose answers and debate over rounds before agreeing on one", "effect": "Arithmetic: debate 81.8 vs 67.0 single agent and 69.0 majority vote; Grade School Math: 85.0 vs 77.0 and 81.0; chess pawn-score gain 122.9 vs 91.4 and 102.2 (Table 1)", "direction": "supports", "measured": null, "conditions": "ChatGPT-based model; 3 agents and two rounds of debate, against one agent and a majority vote over the 3 agents' answers; arithmetic, GSM8K, chess move prediction", "older": true }, { "source": "Tomas Havranek, Zuzana Irsova, \"Does Multi-Agent Debate Improve AI Feedback on Research Papers?\"", "url": "https://arxiv.org/abs/2607.14713", "date": "2026-07-16", "locator": "§2.2-2.4 Table 1; §4.1 Table 3; §4.2 Table 4; §4.5", "task": "Pre-registered, identity-masked comparison of three feedback reports per paper, all rewritten to a common length and template: a single pass by one frontier model; mad-research, where two model families critique the paper, argue each other's points in an anonymized round and a fresh instance synthesizes; and paper-workshop, a single-model panel that argues contested points. The papers' authors ranked the three by usefulness for improving the paper", "effect": "Mean rank (1 = most useful), 44 papers: single pass 1.59 [1.39, 1.82], mad-research 2.25, paper-workshop 2.16; the single pass was first on 55% of papers and ranked above mad-research on 32 of 44 and above paper-workshop on 30 of 44 (contrasts -0.66 [-1.00, -0.32], Holm p=0.005; -0.57 [-0.95, -0.16], Holm p=0.026; pre-registered hypothesis that a debate tool would win not supported). Cost per paper: 1 call, 27,000 tokens, $0.20; about 6 calls, 238,000 tokens, $1.87; about 10 calls, 800,000 tokens, $7.20 (tokens 1:9:30). An external AI judge (Gemini) agreed with the authors at rank correlation 0.14 (95% CI -0.05 to 0.32) and would have ranked paper-workshop first.", "direction": "contradicts", "measured": true, "conditions": "Claude Opus 4.8 for the single pass and the workshop, Claude Opus 4.8 with GPT-5.5 for mad-research, late June 2026; 55 published or accepted economics meta-analyses, rankings for 44 from 47 authors; the workshop ran in a light configuration; reports normalized to about 1,000 words; measures perceived usefulness, not correctness; the authors built both debate tools and expected them to win", "older": false }, { "source": "Leonardo Ferreira, Gardenia Liu, Kaden Zheng, \"Beyond Symmetric Agents: Cognitive Diversity and Multi-Agent Debate in Small Language Models\"", "url": "https://arxiv.org/abs/2609.35875", "date": "2026-09-26", "locator": "§5.1 Table 1; §5.2 Figure 1; §5.3 Table 2; §5.5; Appendix D Tables 10, 13; Appendix G; Appendix H Table 18", "task": "Du et al.-style debate (3 agents x 3 rounds, majority vote at the end) against a generation-budget-matched self-consistency control (9 independent samples, majority vote), varying persona, temperature and model diversity; mixed-model teams compared with votes over the same rosters in matched proportions", "effect": "Mean accuracy over 23 models, solo / debate / SC-9: GSM8K 87.9 / 91.2 / 91.0; MMLU 69.9 / 76.5 / 76.8; CommonsenseQA 72.6 / 78.3 / 80.6 (sampling +2.3, p=0.011); arithmetic 91.2 / 91.5 / 90.6; biography claim precision 54.6 / 55.6 / 64.6. GPQA Diamond at full thinking budget 49.1 / 54.3 / 54.9 (debate - SC-9 = -0.6 [-2.2, +1.0]; a 4,096-token thinking cap left on some reasoning models had shown debate +3.2). Debate took 1.6x the wall-clock and 3.4x the total tokens of SC-9 (generated tokens 1.06x) because each round re-reads peers' transcripts. 80 three-model teams: debate 88.4 vs roster vote 89.4 (-1.04 [-1.61, -0.52]; second seed -0.93); five-agent quads tied (+0.04, +0.56); one cell favored debate (weak-duplicated quads on MMLU, +2.80). Rounds, not budget-matched: 84.6 for a vote over first answers, 87.2 after one exchange, 87.0-87.5 through five rounds. Mixed teams overturned 63.7% of wrong round-one majorities against 42.9% for homogeneous ones. Persona prompts cost 2.1 to 4.4 points. Context overflow left Qwen3.5-27B seven-agent cells 93-100% empty in the final round at 16-32K windows; fixing it moved debate vs sampling from -1.84 to -0.06 points.", "direction": "contradicts", "measured": true, "conditions": "23 open-weight models from 0.6B to 109B (17B active), eleven vendor families, including reasoning models (R1-Distill-Qwen-14B and -32B, Qwen3 and Qwen3.5 thinkers); 100 problems per task (biography 40 subjects), seed 42 plus a second-seed slice; paired bootstrap CIs; local serving on 2x RTX 3090; preprint submitted to ICLR 2027", "older": false }, { "source": "Hyeong Kyu Choi, Xiaojin Zhu, Sharon Li, \"Debate or Vote: Which Yields Better Decisions in Multi-Agent Large Language Models?\"", "url": "https://arxiv.org/abs/2508.17536", "date": "2025-08-24", "locator": "§3.2 Table 1; §4 Figure 4, Appendix E; §5.2 Table 2; §6 Tables 3-4; Appendix F Table 9; Appendix G Table 10", "task": "Separates voting from debate: a majority vote over 5 agents' first-round answers against Decentralized, Sparse and Centralized debate after 2, 3 and 5 rounds; models debate as a martingale over agents' beliefs; tests updates biased toward the last majority", "effect": "Average over seven benchmarks, Qwen2.5-7B-Instruct: vote 0.7691; best debate 0.7377 (Decentralized, 2 rounds), 0.7050 at 5 rounds; single agent 0.7205. Llama3.1-8B-Instruct: vote 0.7242; best debate 0.6990 (Sparse, 2 rounds); single agent 0.6203. Qwen2.5-32B GSM8K: vote 0.9433 vs best debate 0.9400. Three GPT-4 agents: vote equal or ahead on three of four benchmarks (CommonsenseQA debate 0.8788 vs vote 0.8721). Mean per-agent accuracy stayed flat across rounds. Most of the vote's average lead comes from arithmetic (0.99 vs 0.76); on the other six benchmarks vote and best debate average about 0.73 each, debate slightly higher on four. Answer extraction reversed the GSM8K result: with Du et al.'s parser debate 0.7533 vs vote 0.6700; with a fixed final-answer format debate 0.8867 vs vote 0.9400. Persona-diverse agents: vote ahead on GSM8K (0.9367 vs 0.8900), debate ahead on MMLU professional medicine (0.8419 vs 0.8235). Keeping an answer that matched the last majority (MAD-Conformist) raised two-round debate to 0.7625, below the vote.", "direction": "contradicts", "measured": true, "conditions": "N=5 agents (3 for GPT-4), simultaneous talk, temperature 1.0, 512-token limit; the vote uses only the 5 first-round answers while debate adds 5 generations per round; 100 to 300 items per benchmark (arithmetic, GSM8K, MMLU professional medicine and formal logic, HellaSwag, CommonsenseQA, HH-RLHF labeling); one run per debate configuration; NeurIPS 2025 spotlight", "older": true }, { "source": "Hangfan Zhang, Zhiyao Cui, Jianhao Chen, Xinrun Wang, Qiaosheng Zhang, Zhen Wang, Dinghao Wu, Shuyue Hu, \"Stop Overvaluing Multi-Agent Debate -- We Must Rethink Evaluation and Embrace Model Heterogeneity\"", "url": "https://arxiv.org/abs/2502.08788", "date": "2025-02-12", "locator": "§3.2 Figure 2, Table 3, Figure 4, Figure 6 (Appendix D); §4.2 Table 4", "task": "Five debate frameworks (Society of Minds, Multi-Persona, Exchange-of-Thoughts, ChatEval, AgentVerse) at about 6 calls against single-agent, chain-of-thought and self-consistency baselines; then Heter-MAD, where each agent call goes at random to GPT-4o-mini or Llama-3.1-70B", "effect": "Debate frameworks significantly beat chain-of-thought in at most 5 of 36 model-benchmark scenarios (13.9%; ANOVA, p<0.05); Multi-Persona in none. Self-consistency scored highest in most cases where it applies and used added tokens more efficiently than debate. GPT-4o-mini, CoT vs Society of Minds: MMLU 80.73 vs 74.73, HumanEval 78.05 vs 68.09, GSM8K 93.60 vs 94.93, MATH 72.87 vs 75.40. Mixing models raised every framework over the mean of its two single-model runs (+2.7% to +8.2% relative; Heter-SoM 76.7 average vs 74.3 on GPT-4o-mini and 69.9 on Llama-3.1-70B), but Heter-AgentVerse (74.0) stayed below AgentVerse on GPT-4o-mini alone (75.7). No vote over the mixed models was run.", "direction": "mixed", "measured": true, "conditions": "gpt-4o-mini-2024-07-18, claude-3-5-haiku, Llama-3.1-8B and -70B; MMLU, MMLU-Pro, CommonsenseQA, ARC-Challenge, AGIEval, GSM8K, MATH, HumanEval, MBPP; temperature 1; 3 runs; position paper, v1 titled \"If Multi-Agent Debate is the Answer, What is the Question?\"", "older": true } ], "caveats": "Most matched-compute evidence is closed-form QA and math on open models from 0.6B to 109B (100 problems per task in the 23-model study, an unreviewed preprint), where there is an answer to vote on; the one frontier-model study measures how useful 44 authors found length-normalized reports, not correctness, and its authors built the debate tools. Heterogeneity is contested: mixing GPT-4o-mini with Llama-3.1-70B lifted every debate framework over the two models' average, but no vote over that mix was run and one mixed framework stayed below its GPT-4o-mini-only run (74.0 vs 75.7), while the study that did run a roster vote found the vote ahead (-1.04 points for debate). Debate results are fragile to evaluation details: a different answer parser reversed debate against the vote on GSM8K, and in one study a capped thinking budget and context overflow each moved the comparison by about two to four points before correction. Biasing agents toward the last majority narrowed the gap to the vote but did not close it (0.7625 vs 0.7691), and Du et al.'s original gains (GSM 85.0 vs an 81.0 vote) came from a 2023 ChatGPT model with an answer parser that, rerun on Qwen2.5-7B, cut the vote's GSM8K score far more than debate's (0.6700 vs 0.9400 for the vote, 0.7533 vs 0.8867 for debate).", "also_covers": [], "related": [ { "id": "9.c.ii", "slug": "samples-before-verification", "name": "Spend extra samples before extra verification: parallel attempts with a vote or judge; verify only where it pays" }, { "id": "1.b.i", "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class" }, { "id": "9.b.i", "slug": "extractable-final-answer", "name": "Make the final answer reliably extractable: state the format, extract robustly, force an answer at the limit" }, { "id": "7.c.i", "slug": "reflection-loop", "name": "Reflection loop" }, { "id": "7.c.iii", "slug": "evaluator-optimizer-loop", "name": "Evaluator-optimizer loop" }, { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "6.b.iii", "slug": "multi-agent-for-breadth", "name": "Use multi-agent orchestration only for breadth-first, independent sub-tasks" }, { "id": "6.a.i", "slug": "oracle-worker-escalation", "name": "Oracle-worker multi-model" } ], "url": "https://mahalsystems.ai/aot/optimizations/multi-agent-debate/", "markdown": "https://mahalsystems.ai/aot/optimizations/multi-agent-debate.md" }, { "id": "7.c.vii", "legacy_id": "P32", "former_ids": [], "slug": "tree-of-thought", "name": "Tree-of-Thought", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it replaces a single chain with a search", "proposed": true }, "levers": [ "control-flow" ], "domains": [ "reasoning" ], "evidence_domains": [ "reasoning", "planning", "data-analysis" ], "categories": [ "Scaling compute" ], "category_ids": [ "orchestration.extra-compute" ], "summary": "Tree-of-Thought replaces one reasoning chain with a search: the model proposes several next steps, an evaluator (the model itself, a trained verifier or a rule check) scores the partial solutions, and breadth-first or depth-first search keeps the best or backtracks. It pays on narrow tasks where early steps decide the outcome and partial states can be judged reliably: Game of 24 rose from 4% with CoT to 74% on GPT-4, and logic-proof accuracy on GPT-4o from 72.0 to 83.1. Against cheaper baselines the gain often disappears: tree search needed an evaluator at least 90% accurate to beat re-ranking, PRM-guided tree search showed no significant gain over best-of-N despite higher cost, and DeepSeek-V3.2 with thinking on solved 92% of Game of 24 in one call against 30% for ToT on the same model with thinking off. Its authors put the cost at 5 to 100 times the generated tokens of CoT.", "how": [ "Start from the cheap baselines on your own items: the same model in one call with thinking on or higher effort (1.b.i), and self-consistency or best-of-N at the same token budget (9.c.ii); consider a search only for the items those miss, where an early wrong step dooms the answer and partial states can be checked.", "If the moves and the goal can be written down, have the model write the successor function and goal test in code and run plain BFS or DFS: that solved all 1,362 Game of 24 instances with about 2 LLM calls in total, where ToT makes about 100 calls per problem.", "Otherwise define a thought as one checkable step (an equation, a SQL clause, a line of code, one derived fact), propose several candidates per node and keep the best b (ToT kept b = 5), scoring partial states with the most reliable evaluator you have: a rule or execution check first (execution feedback raised discriminator accuracy by up to 30.2 points), then a trained verifier, then the model's own value or vote. Measure the evaluator's ranking accuracy on labeled partial solutions before wiring it in; tree search needed at least 90% to beat re-ranking.", "Use a search that can back up: depth-first or best-first with pruning of states rated impossible, not a fixed-depth breadth-first pass (ToT's BFS won 9.47% of 5-number Countdown games against 60.00% for MCTS). Cap nodes, tokens and wall-clock, stop on the first verified solution, and return a fallback answer when the search ends empty.", "Branch only where a step needs it: a cheap check before each expansion cut ToT beam-search output tokens by 78.3% and 78.9% on Qwen3-32B, at accuracy 0.97 and 0.86 against 0.98 and 0.87, but on LLaMA3-8B the cuts were 37.8% to 73.4% with accuracy drops, so re-check accuracy on your model; run the b expansions and their evaluations in parallel.", "Route by difficulty: send only the items the cheap path is unsure about to the search (a low average verifier score across samples served as the difficulty estimate), and re-check the need on every model upgrade, since stronger models gained less (best-of-N beat search at 72B).", "Measure on held-out items with ground truth the search never sees: accuracy against both baselines at matched tokens, split by difficulty, plus tokens, LLM calls, p50 and p95 latency, and the share of searches that end without a solution." ], "detect": { "static": "A search controller: several candidate next steps per node, an evaluator that scores partial solutions (the model's value or vote, a trained verifier, or a rule or execution check), breadth-first or beam search keeping the best b, or depth-first with backtracking, and a node or token budget. Flag a one-prompt imitation (\"imagine three experts\") presented as ToT, a model self-evaluator where a rule or execution check exists, a search with no fallback answer when it ends empty, and no comparison against self-consistency or best-of-N at equal tokens.", "traces": "Nodes expanded, depth and branching per item; pruned and backtracked branches; LLM calls, tokens and wall-clock per item; evaluator score of the chosen path next to the ground truth; share of searches that hit the budget or end without a solution.", "evals": "Accuracy against the same model in one call with thinking on and against self-consistency or best-of-N at matched tokens, split by difficulty; evaluator ranking accuracy on labeled partial solutions; tokens and latency.", "judge": "The partial-solution evaluator is the judge: check its ranking accuracy against ground truth before relying on it (tree search needed at least 90% to beat re-ranking).", "probes": "Puzzles with dead ends." }, "impact": { "measured_basis": true, "primary": [ "quality", "cost" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "gains where one pass fails early and partial states can be judged: Game of 24 4% to 74% on GPT-4 (Yao, May 2023); logic proofs 72.0 to 83.1 on GPT-4o (He, Apr 2025); Math500 0.79 to 0.87 on Qwen3-32B (Li, Sep 2025); no gain over cheaper selection when the evaluator is weak or the model strong: tree search 67.5 vs re-ranking 70.3 on Spider (Chen, Feb 2024), PRM beam search 71.8 vs best-of-8 72.7 (Cinquin, Oct 2025), best-of-N best at 72B (Liu, Feb 2025); thinking on alone 92% vs ToT with thinking off 30% on Game of 24 (Hao, Mar 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "large", "when": "$0.74 and 5.5k completion tokens per Game of 24 case vs $0.47 for 100 CoT samples; 5 to 100 times CoT's tokens by the authors' estimate (Yao, May 2023); about 100 LLM calls per Game of 24 problem vs about 2 in total for a coded search (Katz, Apr 2024); PRM beam search 20.2 vs 8.2 (x10^7) generated tokens for best-of-8 at the same accuracy (Cinquin, Oct 2025); branching only where needed cut ToT output tokens 78.3% to 78.9% on Qwen3-32B; on LLaMA3-8B cuts were 37.8% to 73.4%, with accuracy drops (Li, Sep 2025)", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "large", "when": "serial generate-then-evaluate rounds: tree search 262.8 to 405.2 s per Bird example vs 16.4 to 24.4 s for re-ranking (Chen, Feb 2024); ToT on GPT-4o 0.30 to 2.13 min per logic problem vs 0.08 to 0.12 for CoT and 0.21 to 1.02 for o3-mini (He, Apr 2025)", "basis": "evidence" }, { "dimension": "security", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "governance", "effect": "none", "size": null, "when": null, "basis": "assessment" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired eval on held-out items with ground truth the search never sees: ToT against the same model in one call with thinking on or higher effort, and against self-consistency or best-of-N at matched tokens, split by difficulty; accuracy, tokens, LLM calls, p50 and p95 latency, share of searches that end without a solution" }, { "method": "monitor", "tier": null, "how": "per item: nodes expanded, depth, branching and pruning, LLM calls, tokens and wall-clock, budget-hit and fallback rates; on a regularly audited sample, the evaluator's scores of partial states against a rule check or ground truth" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Reasoning & Search group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Yao et al., \"Tree of Thoughts: Deliberate Problem Solving with Large Language Models\"", "url": "https://arxiv.org/abs/2305.10601", "date": "2023-05-17", "locator": "Abstract; §4.1, Table 2; Appendix B.1-B.3, Tables 4, 5, 7", "task": "Search over a tree of intermediate thoughts with self-evaluation, lookahead and backtracking", "effect": "\"in Game of 24, while GPT-4 with chain-of-thought prompting only solved 4% of tasks, our method achieved a success rate of 74%\". Other Game of 24 results: CoT self-consistency over 100 samples 9.0%, an oracle best of 100 CoT samples 49%, ToT with breadth b = 1 45% and b = 5 74%. Per case, ToT used 5.5k completion tokens and $0.74 against 6.7k and $0.47 for 100 CoT samples; the authors say ToT \"could require 5-100 times more generated tokens than CoT\". With GPT-3.5, ToT solved 19%; GPT-4 generation with GPT-3.5 evaluation 64%, GPT-3.5 generation with GPT-4 evaluation 31%. Zero-shot ToT on 100 GSM8K and 100 StrategyQA questions scored 90 and 83 against 86 and 82 for CoT (\"only slightly, given GPT-4 + CoT is already very good\").", "direction": "supports", "measured": null, "conditions": "Game of 24, 100 hard games (indices 901-1,000 of 1,362); GPT-4, and GPT-3.5-turbo in the appendix; BFS keeping the best b candidates per step, the model's own value or vote as evaluator; GSM8K and StrategyQA subsets of 100", "older": true }, { "source": "Ziru Chen, Michael White, Raymond Mooney, Ali Payani, Yu Su, Huan Sun, \"When is Tree Search Useful for LLM Planning? It Depends on the Discriminator\"", "url": "https://arxiv.org/abs/2402.10890", "date": "2024-02-16", "locator": "Abstract; §1; §5.2; §7; Tables 3-5", "task": "A generator proposes SQL queries or Python programs step by step, a discriminator scores them, and three planners use the scores: re-ranking complete samples, iterative correction, and tree search over partial programs (best partial plan expanded first, scored by Monte Carlo completions); first with oracle discriminators of controlled accuracy, then with LLM discriminators", "effect": "\"advanced planning methods demand discriminators with at least 90% accuracy to achieve significant improvements over re-ranking\"; with oracle discriminators, tree search agents \"consistently underperform the other two when the discrimination accuracy threshold\" is 0.8 or below. With the best LLM discriminators, tree search trailed re-ranking on text-to-SQL (Spider 67.5 vs 70.3, Bird 21.7 vs 23.7) and on GSM8K scored 51.0 against 47.6, a difference McNemar's test did not find significant. Tree search \"is at least 10–20 times slower\"; on Bird it took 262.8 to 405.2 seconds per example against 16.4 to 24.4 for re-ranking. Execution feedback raised discriminator accuracy \"by up to 30.2 and 8.4 absolute points\" on text-to-SQL and math and cut tree search latency.", "direction": "contradicts", "measured": true, "conditions": "generator CodeLlama-13B-Instruct; discriminators CodeLlama-7B/13B (prompted and fine-tuned), GPT-3.5-Turbo, GPT-4-Turbo; Spider (400), Bird (300) and GSM8K (500, tree search on 100) dev examples", "older": true }, { "source": "Michael Katz, Harsha Kokel, Kavitha Srinivas, Shirin Sohrabi, \"Thought of Search: Planning with Language Models Through The Lens of Efficiency\"", "url": "https://arxiv.org/abs/2404.11833", "date": "2024-04-18", "locator": "§4 (24 game, Mini crosswords); §5, Table 1; §6", "task": "Counts the LLM calls ToT and other LLM-in-the-loop search methods make on four search problems; instead has GPT-4 write the successor function and goal test in Python once per domain and runs plain BFS or DFS with them", "effect": "ToT \"performs ≈100 calls to the LLM\" per Game of 24 problem and about 200 per crossword in the worst case. Projected over the datasets, ToT would make 102K calls on Game of 24 (1,362 problems), 5K on crosswords, 201K on BlocksWorld and 600K on PrOntoQA, against 2.2, 3.8, 3.8 and 2.6 calls in total for the coded search, which solved every instance; BFS solved all 1,362 Game of 24 instances in 1.92 to 6.83 seconds. The authors call the LLM-search methods \"unsound, incomplete, and quite expensive\".", "direction": "contradicts", "measured": true, "conditions": "GPT-4 with a human feeding back errors (1 to 2.8 interactions per function); Game of 24 (1,362), 5x5 crosswords (20, with 10 candidate answers per clue supplied), BlocksWorld (502), PrOntoQA (4,000); ToT call counts are analytical estimates, not runs", "older": true }, { "source": "Charlie Snell, Jaehoon Lee, Kelvin Xu, Aviral Kumar, \"Scaling LLM Test-Time Compute Optimally can be More Effective than Scaling Model Parameters\"", "url": "https://arxiv.org/abs/2408.03314", "date": "2024-08-06", "locator": "§5.2-5.3; Figures 3-4", "task": "Search against a process reward model: step-level beam search (BFS over steps with a value model) and lookahead search compared with best-of-N weighted at equal generation budgets, binned by question difficulty", "effect": "\"with smaller generation budgets, beam search significantly outperforms best-of-N. However, as the budget is scaled up, these improvements greatly diminish, with beam search often underperforming the best-of-N baseline.\" On easy questions beam search \"degrades performance as the generation budget increases, suggesting signs of exploitation of the PRM signal\"; on levels 3 and 4 it \"consistently outperforms best-of-N\"; on the hardest \"no method makes much meaningful progress\". Lookahead search \"generally underperforms other methods at the same generation budget\". Choosing the strategy per difficulty \"can nearly outperform best-of-N using up to 4x less test-time compute\".", "direction": "mixed", "measured": true, "conditions": "PaLM 2-S* (Codey); PRM trained from Monte Carlo rollouts; MATH 500-question test split; the evaluator is a trained PRM, not the model's self-evaluation", "older": true }, { "source": "Runze Liu, Junqi Gao, Jian Zhao, Kaiyan Zhang, Xiu Li, Biqing Qi, Wanli Ouyang, Bowen Zhou, \"Can 1B LLM Surpass 405B LLM? Rethinking Compute-Optimal Test-Time Scaling\"", "url": "https://arxiv.org/abs/2502.06703", "date": "2025-02-10", "locator": "§4.2-4.3; Figures 4, 7-9", "task": "Best-of-N, step-level beam search and diverse verifier tree search (DVTS) compared at equal budgets of 4 to 256 across policy models from 0.5B to 72B, several PRMs and three difficulty levels", "effect": "\"For small policy models, search-based methods outperform BoN, while for large policy models, BoN is more effective than search-based methods.\" \"For policy models with 72B parameters, BoN is the best method for all difficulty levels\"; below 7B, beam search won on harder problems and BoN on easy ones. \"BoN outperforms other strategies most of the time when using Math-Shepherd and RLHFlow PRMs\"; for Llama-3.1-8B-Instruct, search with those PRMs stayed \"relatively poor, even worse than majority voting\".", "direction": "mixed", "measured": true, "conditions": "Qwen2.5 0.5B to 72B and Llama 3 policy models; Math-Shepherd, RLHFlow, Skywork and Qwen2.5-Math PRMs; MATH-500 and AIME24; beam width 4; evaluator is a trained PRM", "older": true }, { "source": "Kang He, Kaushik Roy, \"LogicTree: Structured Proof Exploration for Coherent and Rigorous Logical Reasoning with Large Language Models\"", "url": "https://arxiv.org/abs/2504.14089", "date": "2025-04-18", "locator": "Table 1; §4.2; Appendix E, Table 7", "task": "ToT (tree search over derivation steps, step limit 80) run as a baseline against CoT and other modular methods on multi-step logical proofs, with o1-mini and o3-mini run alone for comparison", "effect": "Average proof accuracy, CoT vs ToT: GPT-4o 72.0 vs 83.1, GPT-4o-mini 51.2 vs 66.5, Llama-3.3 70B 73.6 vs 83.4. The authors' own search (LogicTree) reached 95.6 on GPT-4o, 12.5 above ToT, and beat o3-mini alone by 7.6 on average. Per example on GPT-4o, ToT generated 258.4 to 1,896.7 tokens and took 0.30 to 2.13 minutes, against 103.2 to 171.9 tokens and 0.08 to 0.12 minutes for CoT; o3-mini alone generated 481.9 to 2,412.3 tokens in 0.21 to 1.02 minutes.", "direction": "supports", "measured": true, "conditions": "GPT-4o, GPT-4o-mini, Llama-3.3 70B; o1-mini and o3-mini (medium) alone; RobustLR, PrOntoQA-OOD, ProofWriter, ParaRules, LogicNLI (1,790 sampled examples); proofs checked by hand", "older": true }, { "source": "Nathan Herr, Tim Rocktäschel, Roberta Raileanu, \"LLM-First Search: Self-Guided Exploration of the Solution Space\"", "url": "https://arxiv.org/abs/2506.05213", "date": "2025-06-05", "locator": "§5.1; §6.1; Tables 2-3", "task": "ToT's breadth-first search (top k = 5 states kept per level, fixed depth, no backtracking) against best-first search, MCTS and the authors' LLM-guided search; the environment supplies the valid moves and the LLM scores them", "effect": "Win rate with GPT-4o, ToT-BFS vs MCTS (c=0.5): Countdown with 3, 5 and 7 numbers 82.11, 9.47, 0.00 vs 100, 60.00, 32.63; Sudoku 4x4 53.68 vs 100; 6x6 0.00 for both. \"ToT-BFS-GPT4o lags behind due to the lack of backtracking\", and \"due to compute constraints, ToT-BFS-o3mini is not tested\". On win rate per token (area under the performance profile) ToT-BFS tied MCTS at 3.68; the LLM-guided search scored 4.70.", "direction": "mixed", "measured": true, "conditions": "GPT-4o and o3-mini at temperature 0, 5 runs per game; Countdown (3, 5, 7 numbers) and Sudoku (4x4, 6x6)", "older": true }, { "source": "Xinzhe Li, \"Chain-in-Tree: Back to Sequential Reasoning in LLM Tree Search\"", "url": "https://arxiv.org/abs/2509.25835", "date": "2025-09-30", "locator": "Abstract; §4; Tables 1-2; §5", "task": "ToT beam search (ToT-BS: an LLM proposes steps, an LLM evaluator scores them) compared with CoT, then with a cheap branching-necessity check that expands the tree only where a step needs it", "effect": "Accuracy, ToT-BS vs CoT: Qwen3-32B GSM8K 0.98 vs 0.96, Math500 0.87 vs 0.79; LLaMA3-8B 0.79 vs 0.68 and 0.39 vs 0.34. With the direct-prompt check (BN-DP) on Qwen3-32B, ToT-BS output tokens fell 78.3% and 78.9%, policy calls 78.3% and 80.0%, total runtime 77.3% and 78.2%, at accuracy 0.97 and 0.86. Across frameworks BN-DP \"reduces token generation, model calls, and runtime by 75-85%\"; a self-consistency check instead was unstable \"in 1-4 out of 14 settings\", all within ToT-BS, and \"poor BN evaluation can negate the benefits of chaining or even harm performance\". On LLaMA3-8B (Table 2) the cuts were 37.8% to 73.4%, with accuracy drops; the 75-85% is the abstract's claim.", "direction": "supports", "measured": true, "conditions": "Qwen3-32B-AWQ and LLaMA3-8B-Instruct; first 100 test items of GSM8K and of Math500 (single-number answers); CoT not matched for compute", "older": true }, { "source": "Tristan Cinquin, Geoff Pleiss, Agustinus Kristiadi, \"Limits of PRM-Guided Tree Search for Mathematical Reasoning with LLMs\"", "url": "https://arxiv.org/abs/2510.20272", "date": "2025-10-23", "locator": "Abstract; §4.2; Tables 1-3", "task": "PRM-guided tree search (greedy, beam, greedy best-first, MCTS, a Gittins-index method) against best-of-8 and majority vote", "effect": "\"PRM-guided tree search shows no statistically significant improvements over BoN despite higher costs.\" Mean accuracy and generated tokens (x10^7): best-of-8 by final PRM score 72.7 with 8.2; beam search 71.8 and 71.9 with 20.2 and 21.0; MCTS 71.2 with 89.4; greedy best-first 46.0 with 65.2; majority vote 71.4. \"PRMs poorly approximate state values and their reliability degrades with reasoning depth\".", "direction": "contradicts", "measured": true, "conditions": "Qwen2.5-Math-7B-Instruct with Qwen2.5-Math-PRM-7B; 23 math benchmarks including AIME24, AIME25 and AMC23; evaluator is a trained PRM", "older": true }, { "source": "Guangfu Hao, Yuming Dai, Xianzhe Qin, Shan Yu, \"Brain-Inspired Graph Multi-Agent Systems for LLM Reasoning\"", "url": "https://arxiv.org/abs/2603.15371", "date": "2026-03-16", "locator": "§III-E; §IV-A; §IV-B, Table I", "task": "ToT (4 rounds, 3 thoughts, temperature 0.7) and ReAct run as baselines on DeepSeek-V3.2 against the same model in one call; the single call also run with thinking on and on other frontier models", "effect": "Accuracy on Game 24, Six Fives and Tower of London, DeepSeek-V3.2: single call 25.0, 12.0, 6.0; ReAct 26.0, 18.0, 10.0; ToT 30.0, 25.0, 18.0. The same model with thinking on, single call: 92.0, 96.0, 79.0; GPT-5 single call 96.0, 95.0, 91.0. ToT was not run with thinking on.", "direction": "mixed", "measured": true, "conditions": "DeepSeek-V3.2 with and without thinking, Claude 4.5 Sonnet, Gemini 2.5 Pro, GPT-5; 100 instances per task (Game 24 sampled from the ToT paper's 1,362)", "older": false } ], "caveats": "Most gains come from puzzles built to break left-to-right decoding (Game of 24, crosswords, Sudoku, Countdown) or synthetic logic, often with a programmatic move generator or checker doing part of the work, and the headline 74% is against single-sample CoT: self-consistency over 100 samples scored 9.0%, and the 49% best-of-100 baseline is an oracle. The studies that compare against re-ranking or best-of-N (Chen, Snell, Liu, Cinquin) find tree search ties or trails re-ranking and best-of-N except on harder items with weaker models and a reliable evaluator; three of them use a trained PRM rather than the model's own evaluation. None of these sources runs ToT on a reasoning model with thinking on; the closest shows thinking alone far above ToT on the same model with thinking off. Several studies are small (100 items per task in Hao and Li, one model in Cinquin, 1,790 items in He), and Katz's ToT call counts are estimates, not runs. Branching only where a check calls for it held accuracy on Qwen3-32B but cost accuracy on LLaMA3-8B (Li).", "also_covers": [], "related": [ { "id": "9.c.ii", "slug": "samples-before-verification", "name": "Spend extra samples before extra verification: parallel attempts with a vote or judge; verify only where it pays" }, { "id": "7.c.iv", "slug": "graph-of-thought", "name": "Graph-of-Thought" }, { "id": "7.c.v", "slug": "language-agent-tree-search", "name": "Language Agent Tree Search (LATS)" }, { "id": "7.c.iii", "slug": "evaluator-optimizer-loop", "name": "Evaluator-optimizer loop" }, { "id": "7.c.ii", "slug": "self-critique-evaluator", "name": "Self-critique evaluator loop" }, { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "1.b.i", "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class" }, { "id": "3.c.i", "slug": "self-discover-reasoning", "name": "Self-Discover reasoning structures" } ], "url": "https://mahalsystems.ai/aot/optimizations/tree-of-thought/", "markdown": "https://mahalsystems.ai/aot/optimizations/tree-of-thought.md" }, { "id": "7.d.i", "legacy_id": "P51", "former_ids": [], "slug": "continuous-task-loop", "name": "Continuous autonomous task loop", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it changes the control loop", "proposed": true }, "levers": [ "control-flow" ], "domains": [ "planning" ], "evidence_domains": [ "code-generation", "planning" ], "categories": [ "Shaping the loop" ], "category_ids": [ "orchestration.loop" ], "summary": "Run the coding agent from a shell or harness loop that starts a fresh headless session every iteration with the same short prompt; each session reads the plan, progress notes and git history, does one item, runs its tests, commits and exits, until an iteration cap, a budget or a stop rule ends the run. It pays on long backlogs with a near-perfect automatic check: 16 such loops on Opus 4.6 built a 100,000-line C compiler that builds Linux 6.9 over nearly 2,000 sessions for just under $20,000, though new features kept breaking existing ones and the author had to add a CI gate. In one controlled study, a harness that rebuilt a fresh context for every invocation and assigned each next experiment reached the strongest baseline's kernel result with 93.1% fewer tokens, while a framework that resumed long-running sessions kept spending until 1B tokens though its last improvement came at 231M. No study compares a plain loop with one long session on the same backlog, and fresh contexts do not stop drift: across fresh-container iterations code erosion rose in 77% of agent trajectories.", "how": [ "Start from the baseline: the same backlog run as one long agent session with the harness's own compaction, or an operator re-prompting it, and log where it stops: it waits for input, declares the job done early, or fills its context. Use the loop only for a long list of checkable items with an automatic check you trust, since the agent \"will solve the wrong problem\" unless the verifier is \"nearly perfect\" (Carlini, Feb 2026); tests, type checkers and builds are the loop's back pressure (Huntley, Jul 2025).", "Start a new headless process every iteration with the same short prompt file, inside a container that holds only the repository, and log each iteration to its own file: while true; do claude --dangerously-skip-permissions -p \"$(cat AGENT_PROMPT.md)\" &> agent_logs/agent_$(git rev-parse --short=6 HEAD).log; done (Carlini, Feb 2026). Do not resume the previous session: the official Claude Code plugin re-feeds the prompt inside one session through a stop hook (Anthropic, Nov 2025), and in a framework that resumed each agent's own session after restarts, agents that had declared their work optimal kept repeating that decision, with 98% of the last 1,500 sessions making no tool calls (Zhang, Oct 2026).", "Keep state in the repository, not the conversation, and load the same files every iteration: the spec, a plan or fix list, a progress file and README the agent updates, and the git log (see 4.b.iii). Ask for one item per iteration and have the agent search the code before implementing, since a wrong \"not implemented\" conclusion produces duplicate implementations (Huntley, Jul 2025). Letting the agent pick \"the next most obvious\" problem works while failing tests are independent (Carlini, Feb 2026); in one controlled study a stateless planner that assigned each next experiment from a harness-built summary beat independent worker choices in all four checkpoint comparisons (Zhang, Oct 2026). For parallel loops, claim an item by pushing a lock file through git so a second claim fails (Carlini, Feb 2026); posted notes did not stop two agents from building the same feature twice (Zhang, Oct 2026).", "Gate every commit: a CI run so new commits can't break existing tests (added once the agents began breaking existing functionality with each new feature), a --fast mode that runs a 1% or 10% sample of the tests, deterministic per agent, for checks inside an iteration, short test output with details in log files (see 7.b.ii), and a known-good oracle to split a monolithic target (GCC compiled most of the kernel so agents could fix different files) (Carlini, Feb 2026). Schedule cleanup as well: across fresh-container iterations structural erosion rose in 77% of trajectories and verbosity in 75.5%, and quality prompts lowered the level but not the rate of decline (Orlanski, Mar 2026); Carlini gave one agent the job of coalescing duplicate code.", "Bound and stop the run: always set an iteration cap (the plugin's default is unlimited), a token or dollar budget and a time limit per iteration; stop when the list is done and a check the agent does not control passes, or when several iterations add no passing test or score gain, not on the agent's own completion phrase (the plugin's exact-match phrase cannot express \"blocked\") (see 7.b.i, 7.b.iii, 9.b.iii). Budget for scale: 16 loops used 2 billion input and 140 million output tokens, just under $20,000, over two weeks (Carlini, Feb 2026).", "Measure: run the baseline and the loop on the same backlog, model and dollar budget; score with an end-to-end suite the agents never see plus complexity and duplication of the final code, read at several budget points (one harness trailed on all four software tasks at 25% of the budget and led on all four at 100%, Zhang, Oct 2026); per iteration, log tokens, dollars, commits, tests gained and broken, iterations with no tool calls and tokens since the last improvement." ], "detect": { "static": "A shell or harness loop that starts a headless agent each iteration with the same prompt file over a plan, task list or progress file, committing per iteration; whether it resumes the previous session (a stop hook or resume flag) or starts a fresh one; iteration, budget and time caps; permission prompts turned off outside a container.", "traces": "Iterations; commits; rate-limit events; iterations with no tool calls; tokens since the last test or score gain; the same item claimed or built twice; previously passing tests broken; completion claims while hidden tests fail.", "evals": "Throughput; defect rate of produced work.", "judge": "Rate produced commits.", "probes": "A queue with a poison task." }, "impact": { "measured_basis": true, "primary": [ "cost", "quality" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "long backlogs with a near-perfect check: 16 loops built a 100,000-line C compiler that builds Linux 6.9 on x86, ARM and RISC-V and passes 99% of most compiler test suites, but its output is less efficient than GCC with all optimizations disabled and new features frequently broke existing ones (Carlini, Feb 2026); a fresh-context harness had the best final result on every task, 1112.0 against 1275.7 kernel cycles for the strongest baseline (Zhang, Oct 2026); across fresh-container iterations erosion rose in 77% of trajectories and core pass rate fell from 64.6% to 35.5% (Orlanski, Mar 2026); no study against one long session on the same backlog", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "large", "when": "nearly 2,000 sessions, 2 billion input and 140 million output tokens, just under $20,000 (Carlini, Feb 2026); a framework resuming long sessions kept spending until 1B tokens after its last improvement at 231M, while a fresh-context harness reached the strongest baseline's kernel result with 67.9M against 986.3M tokens and its planner used under 0.6% of tokens (Zhang, Oct 2026); mean cost per iteration grew 2.2x from first to last checkpoint (Orlanski, Mar 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "mixed", "size": "small", "when": "unattended runs of hours to weeks (two weeks for the compiler) replace operator turns, but each fresh session spends significant time re-orienting on a large project and agents will spend hours running tests without a sampled fast mode (Carlini, Feb 2026); going from 1 to 15 parallel workers cut elapsed time 6.6 to 7.3x (Zhang, Oct 2026); no source compares wall time with one long session", "basis": "assessment" }, { "dimension": "security", "effect": "degrades", "size": "medium", "when": "loops run with permission prompts off (--dangerously-skip-permissions) and push their own commits; Carlini says to run it \"in a container, not your actual machine\" and saw an agent run pkill -9 bash and end its own loop (Carlini, Feb 2026); one harness ran workers as unprivileged processes under Landlock filesystem limits with restricted network (Zhang, Oct 2026)", "basis": "evidence" }, { "dimension": "governance", "effect": "mixed", "size": "medium", "when": "each iteration leaves a commit, a log named by commit and lock files in git history (Carlini, Feb 2026), but no one reviews each commit and the agent's own \"done\" is weak: \"it is easy to see tests pass and assume the job is done, when this is rarely the case\" (Carlini, Feb 2026); the vendor plugin stops on an exact completion phrase and has no iteration cap by default (Anthropic, Nov 2025)", "basis": "evidence" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired runs on the same backlog, model and dollar budget: one long session with compaction (or resumed sessions), the fresh-process loop with state files, and the loop with a CI gate, assigned next items and a stop-on-no-progress rule; score with an end-to-end suite the agents never see plus cyclomatic complexity and duplicated code of the final tree, read at 25%, 50% and 100% of the budget; report dollars per accepted commit, regressions introduced and human hours needed to finish" }, { "method": "monitor", "tier": null, "how": "per iteration: tokens and dollars, wall time, tool calls (zero means idle), commits, tests gained and broken, CI rejections, items claimed or built twice, rate-limit waits, tokens since the last improvement, complexity and duplication trend, and completion claims checked against the hidden suite" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Control Flow group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Geoffrey Huntley, \"Ralph Wiggum as a “software engineer”\"", "url": "https://ghuntley.com/ralph/", "date": "2025-07-14", "locator": "Opening section; first some fundamentals; one item per loop; don't assume it's not implemented; phase two: backpressure", "task": "Introduces \"Ralph\": run a coding agent in an endless loop over the same prompt file", "effect": "\"In its purest form, Ralph is a Bash loop. while :; do cat PROMPT.md | claude-code ; done\". Says it has defects that are \"identifiable and resolvable through various styles of prompts.\" Each loop loads the same plan and specifications, and \"you need to ask Ralph to do one thing per loop. Only one thing\", trusting it to choose \"the most important thing\"; \"A common failure scenario for Ralph is when the LLM runs ripgrep and comes to the incorrect conclusion that the code has not been implemented\", which shows up as \"multiple implementations\"; tests, type checkers and builds are the \"back pressure\" that rejects bad output. No numbers. A note added at the top of the page links a video \"to learn why the claude code plugin isn't it\".", "direction": "supports", "measured": false, "conditions": "One practitioner's account of his own projects (a new programming language built with the loop); no numbers or comparison", "older": true }, { "source": "Qizheng Zhang, Changxiu Ji, Isaac Sun, Yuetai Li, Shubhangi Upasani, Sherry Ruan, Boyuan Ma, Fenglu Hong, Vamsidhar Kamanuru, Yoonho Lee, Yuzhen Mao, Genghan Zhang, Rulin Shao, Qiuyang Mang, Andy Dimnaku, Changran Hu, Radha Poovendran, Kunle Olukotun (Stanford, CMU, University of Washington, SambaNova, UC Berkeley), \"Stateless Language Agents: Scaling Long-Horizon Automated Research\"", "url": "https://arxiv.org/abs/2610.07625", "date": "2026-10-06", "locator": "§1 Fig. 1; §2.2 Fig. 2(b); §3; §4.1; §4.2 Tables 1-2; §4.3 Table 4; §4.4 Table 5; Appendix D.2 Table 9", "task": "Long unattended automated-research runs of up to 1B tokens in which no agent carries its conversation across invocations: the harness keeps candidates and scores, rebuilds a fresh role-specific context for every invocation, and a stateless Advisor assigns each next experiment to parallel Workers. Compared with EvoX (a fresh coding agent per generation), CORAL (each agent keeps one long-running session, resumed after every heartbeat, stall or restart) and SwarmResearch (a long-running coordinator; workers resume a parent's conversation) under the same coding agents, models, evaluators and external token counter; ablations continue from shared checkpoints", "effect": "Best full-budget result on every task. Anthropic kernel task with Codex (three runs): 1112.0 ± 8.9 cycles against 1275.7 ± 133.9 for the strongest baseline, whose final result it matched at 67.9M against 986.3M tokens (93.1% fewer); with Claude Code (one run) 84.4% fewer. FrontierSWE (four tasks, one run each): behind the strongest baseline on all four at 25% of the budget, ahead on all four at 100%, +4.95 points on average. In one CORAL run the last improvement came at 231M tokens while spending continued to 1B; agents \"repeatedly declare their candidates optimal and wait, despite prompts to continue and restarts\", and 98% of the final 1,500 sessions made no tool calls. In a CORAL trace two agents built the same feature twice within 13 minutes, the second time despite a posted claim. Removing the Advisor's assignments reduced progress in every checkpoint comparison; a full run without Worker context isolation ended at 1336 cycles, 224 worse. The Advisor used under 0.6% of tokens; 1 to 15 Workers cut elapsed time 6.6 to 7.3x.", "direction": "supports", "measured": true, "conditions": "Codex CLI with GPT-5.5 and Claude Code with Claude Opus 4.8 at high reasoning effort, the same model for Advisor and Workers; FrontierSWE (4 tasks), Anthropic's VLIW SIMD kernel task, SOL-ExecBench (3), FrontierCS (1); three runs only for the kernel task with Codex, one run elsewhere; the authors' own framework; the baselines differ in more than session reuse, so the gain is not attributable to fresh contexts alone; automated research against a scored objective rather than a task backlog", "older": false }, { "source": "Gabriel Orlanski, Devjeet Roy, Alexander Yun, Changho Shin, Alex Gu, Albert Ge, Dyah Adila, Nicholas Roberts, Frederic Sala, Aws Albarghouthi, \"SlopCodeBench: Benchmarking How Coding Agents Degrade Over Long-Horizon Iterative Tasks\"", "url": "https://arxiv.org/abs/2603.24755", "date": "2026-03-25", "locator": "Abstract; §1 RQ1-RQ4; §3 Setup; §3.1 Table 1, Fig. 2; §3.2 Fig. 3; §3.3 Fig. 4; §3.4 Table 3, Fig. 5", "task": "Agents repeatedly extend their own code over 3 to 8 checkpoints per problem (36 problems, 196 checkpoints); each checkpoint runs in a fresh Docker container with only the working directory carried over and no prior conversation; structural erosion and verbosity tracked per checkpoint and compared with the commit histories of 473 open-source Python repositories; two quality prompts tested against a plain one", "effect": "No agent solved any problem end to end; best strict checkpoint solve rate 14.8% (GPT 5.5). Structural erosion rose in 77% of trajectories (53% of human repositories) and verbosity in 75.5%; agent code was 2.3x more verbose and 2.0x more eroded than the human repositories. From first to last checkpoint, core pass rate fell from 64.6% to 35.5% and mean cost per checkpoint grew 2.2x. An anti-slop prompt cut verbosity by 27.5% to 35.6% and erosion by 34.3% to 57.6% against the plain prompt, but quality improved from start to end only on GPT 5.4; the two quality prompts raised cost per checkpoint by 12.1% on average and cut the strict solve rate by 2.4 (anti-slop) and 3.6 points (plan-first).", "direction": "mixed", "measured": true, "conditions": "15 models from six providers in their native CLI harnesses, a two-hour wall-clock limit per run, no turn or cost cap; the prompt study on three OpenAI models (GPT 5.4, GPT 5.5, GPT 5.3 Codex); the benchmark supplies each checkpoint's spec, so the agent does not pick its next task, and there is no per-checkpoint quality gate; numbers from v2 (May 2026)", "older": false }, { "source": "Nicholas Carlini (Anthropic), \"Building a C compiler with a team of parallel Claudes\"", "url": "https://www.anthropic.com/engineering/building-c-compiler", "date": "2026-02-05", "locator": "Intro; Enabling long-running Claudes; Running Claude in parallel; Lessons from programming with Claude agent teams; Stress testing the limits of agent teams (Evaluation); Looking forward", "task": "Sixteen Claude Code loops build a Rust C compiler from scratch: a bash while-true loop starts a fresh headless session per iteration, each in its own container, from the same prompt file; each session claims a task with a lock file pushed through git, works, merges, pushes and exits, with progress files and READMEs carrying state; the author added a CI gate, a sampled fast test mode, a GCC oracle and specialist roles as failure modes appeared", "effect": "\"Over nearly 2,000 Claude Code sessions across two weeks, Opus 4.6 consumed 2 billion input tokens and generated 140 million output tokens, a total cost just under $20,000.\" The 100,000-line compiler builds a bootable Linux 6.9 on x86, ARM and RISC-V and has \"a 99% pass rate on most compiler test suites including the GCC torture test suite\", but outputs code less efficient than GCC with all optimizations disabled, calls GCC for 16-bit x86, lacks its own assembler and linker, and \"New features and bugfixes frequently broke existing functionality.\" Near the end \"Claude started to frequently break existing functionality each time it implemented a new feature\", so the author built a CI pipeline \"so that new commits can't break existing code\"; 16 agents on the single kernel build kept overwriting each other's fixes until GCC served as a known-good oracle; one agent ran pkill -9 bash and ended its loop. \"it is easy to see tests pass and assume the job is done, when this is rarely the case.\"", "direction": "supports", "measured": true, "conditions": "Claude Opus 4.6, 16 parallel agents, no orchestration agent, no internet access; one project run as a capability stress test and judged by its author, a vendor demonstrating its own model; no comparison with another harness; earlier Opus 4 models \"were barely capable of producing a functional compiler\" and Opus 4.5 could not compile real large projects", "older": false }, { "source": "Anthropic (Claude Code repository), \"Ralph Wiggum Plugin\"", "url": "https://github.com/anthropics/claude-code/blob/main/plugins/ralph-wiggum/README.md", "date": "2025-11-16", "locator": "Core Concept; Commands; Prompt Writing Best Practices (4. Escape Hatches); When to Use Ralph; Real-World Results; hooks/stop-hook.sh", "task": "Anthropic's official Claude Code plugin for the technique: /ralph-loop runs the loop inside the current session, where a Stop hook blocks exit and feeds the same prompt back each iteration until an iteration cap or an exact completion phrase", "effect": "\"The loop happens inside your current session - you don't need external bash loops.\" \"--max-iterations - Stop after N iterations (default: unlimited)\"; \"Always use --max-iterations as a safety net to prevent infinite loops on impossible tasks\"; the completion promise \"uses exact string matching, so you cannot use it for multiple completion conditions (like \"SUCCESS\" vs \"BLOCKED\")\". Good fits: \"Well-defined tasks with clear success criteria\" and \"Tasks with automatic verification (tests, linters)\"; poor fits: \"Tasks requiring human judgment or design decisions\" and \"Tasks with unclear success criteria\". Its \"Real-World Results\" are relayed claims without a source or method. No measurement.", "direction": "mixed", "measured": false, "conditions": "Claude Code plugin documentation and hook script; no evaluation", "older": false } ], "caveats": "No study compares a plain fresh-process loop with one long session, or an operator re-prompting, on the same backlog, model and budget: the largest account is one build judged by its author on the vendor's own model, and the one controlled comparison is an automated-research harness whose gain mixes fresh contexts with planner-assigned work and harness summaries, with three runs on one task and one run elsewhere. Fresh contexts do not stop drift: across fresh-container iterations code erosion rose in 77% of agent trajectories (against 53% of human repositories) and quality prompts lowered the level but not the rate, on a benchmark with a two-hour limit per run. The vendor's own plugin loops inside one session rather than restarting, and no source measures that design against a fresh process. What the loop can finish depends on the model: earlier Opus 4 models \"were barely capable of producing a functional compiler\" and Opus 4.5 could not compile real large projects.", "also_covers": [], "related": [ { "id": "4.b.iii", "slug": "multi-window-task-state", "name": "Keep plan, progress and results in files when work outlives one context window" }, { "id": "4.b.ii", "slug": "initializer-maintainer-agents", "name": "Initializer-maintainer dual agent" }, { "id": "7.b.i", "slug": "detect-loops-and-stalls", "name": "Detect stalls and loops: warn, then stop early" }, { "id": "7.b.iii", "slug": "size-execution-budgets", "name": "Size run budgets so productive runs aren't cut off: turn caps, deadlines, command timeouts" }, { "id": "9.b.iii", "slug": "verify-completion", "name": "Verify completion against the environment before the agent stops" }, { "id": "7.b.iv", "slug": "track-task-progress", "name": "Track explicit progress: a task list with a status per item, re-injected each turn" }, { "id": "6.b.iii", "slug": "multi-agent-for-breadth", "name": "Use multi-agent orchestration only for breadth-first, independent sub-tasks" }, { "id": "5.c.ii", "slug": "isolated-subagent-context", "name": "Sub-agent architectures with clean contexts" }, { "id": "10.a.i", "slug": "constrain-agent-reach", "name": "Constrain what the agent can touch" }, { "id": "4.b.i", "slug": "filesystem-checkpoints", "name": "Filesystem-based agent state and checkpoints" } ], "url": "https://mahalsystems.ai/aot/optimizations/continuous-task-loop/", "markdown": "https://mahalsystems.ai/aot/optimizations/continuous-task-loop.md" }, { "id": "7.d.ii", "legacy_id": "P50", "former_ids": [], "slug": "inversion-of-control", "name": "Inversion of control", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it changes who owns the control flow", "proposed": true }, "levers": [ "control-flow" ], "domains": [ "general" ], "evidence_domains": [ "planning", "computer-use", "web-research", "retrieval", "data-analysis", "reasoning" ], "categories": [ "Shaping the loop" ], "category_ids": [ "orchestration.loop" ], "summary": "Give the agent the goal, the tools and the acceptance checks and let it choose the sequence of steps, instead of coding a fixed workflow; the goal, permissions, budget and evaluation stay outside the agent. It pays when the path depends on what the agent finds and the model is strong: a GPT-5.5 optimizer that composed its own process beat two prescribed pipelines in 12 of 14 comparisons at a median 34.3% of the target-interaction budget, but a medium model did better with the pipeline. Where the procedure is known and the constraints are hard, or the model is small, a coded workflow wins: 35.56% against 10.40% for same-model ReAct on TravelPlanner, and 45.3% against 6.7% for a 4B tool-calling agent on WebArena-Verified at 98.6% lower cost. Agent-decided steps also cost more per task: 2.7x to 3.6x the input tokens in one RAG comparison.", "how": [ "Start from two baselines on the same model, tools and task set: the fixed workflow you have or would write (code picks the steps, the model fills bounded nodes) and an agent loop (7.d.iii) that gets the goal, the same tools and the same acceptance checks and picks its own steps.", "Keep the ends outside the agent in both designs: the goal, permissions, budget caps and the acceptance check or evaluator, so only the route is delegated.", "Sort tasks by whether the path is known. Where the steps and constraints are stable (a runbook, a recurring task family, hard business rules), code the path and enforce the constraints in code: on TravelPlanner a coded workflow passed 35.56% against 10.40% for same-model ReAct, and writing the same rules into the baselines' prompts left them 12.7 to 20.2 points behind (37.20 against 17.00 to 24.50). Where the next step depends on what is found, let the agent choose: it rewrote retrieval queries better than a fixed rewrite step (NDCG@10 55.6 against 52.8).", "Delegate the route only to a model that can carry it: a GPT-5.5 optimizer that composed its own process beat two prescribed pipelines in 12 of 14 comparisons, but with a medium model the pipeline won by 3.50 to 9.68 points and a 4B model produced no valid action; on WebArena-Verified, code-owned control held 44.7 to 45.3% while a tool-calling agent scored 30.0% on a 120B model and 12.7% and 6.7% on 20B and 4B models.", "Mix within one task: keep fixed steps the agent does worse (a fixed reranker scored 49.5 NDCG@10 against 43.9 when the agent re-retrieved on its own), and enforce preconditions on irreversible actions in code (10.b.i); for a task family you run often, move recurring decisions out of the loop into code and keep the model for the semantic steps (76.0 to 91.8% fewer LLM calls in one study).", "Measure paired runs on the deployed model and one cheaper tier: success, constraint violations, LLM calls, tokens and latency per task, and spread over at least three runs; agent-orchestrated RAG used 2.7x to 3.6x the input tokens and 1.4x to 1.5x the time of a fixed pipeline. Re-run the comparison when the model changes." ], "detect": { "static": "Minimal orchestration code; the agent decides the sequence.", "traces": "Variety of action sequences across tasks of the same type: one sequence recurring across most tasks suggests code could own the path; widely varying sequences on a known procedure suggest constraints are being missed.", "evals": "Success, constraint violations, LLM calls, tokens and latency per task for the agent loop against a fixed workflow, per model tier.", "judge": "Rate whether autonomy was used well.", "probes": "Tasks with a known procedure and hard constraints, and tasks whose next step depends on what is found." }, "impact": { "measured_basis": true, "primary": [ "quality", "governance" ], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "large", "when": "an agent-decided route wins with a strong model where the path depends on findings: 12 of 14 comparisons against prescribed pipelines with a GPT-5.5 optimizer (Xue, Aug 2026); query rewriting NDCG@10 55.6 against 52.8 for a fixed step (Ferrazzi, Jan 2026). A coded workflow wins on known procedures and with smaller models: 35.56% against 10.40% for ReAct on TravelPlanner (Qiu, Aug 2025); a medium optimizer did 3.50 to 9.68 points better with the pipeline (Xue, Aug 2026); 45.3% against 6.7% for a 4B tool-calling agent on WebArena-Verified (Li, Sep 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "medium", "when": "more model calls per task: code-owned control cut LLM calls by 76.0 to 91.8% and deployed cost by 74.4 to 98.6% against a tool-calling agent, offline training excluded (Li, Sep 2026); agent-orchestrated RAG used 2.7x to 3.6x the input tokens of a fixed pipeline (Ferrazzi, Jan 2026). Not always: an agent-composed optimizer used a median 34.3% of a fixed pipeline's target-interaction budget (Xue, Aug 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "medium", "when": "agent-orchestrated RAG took 1.4x to 1.5x the end-to-end time (Ferrazzi, Jan 2026); a tool-calling agent took longer per task than a code-owned harness in all six settings (Li, Sep 2026); 14.0 steps against 10.2 for a coded workflow (Qiu, Aug 2025)", "basis": "evidence" }, { "dimension": "security", "effect": "degrades", "size": "small", "when": "The agent picks each next action, so text in tool output can steer the sequence; contain input (10.c.i) or fix the plan before reading it (6.c.ii).", "basis": "assessment" }, { "dimension": "governance", "effect": "degrades", "size": "medium", "when": "rules stated only in the prompt get broken: 835 commonsense-constraint violations for ReAct against 11 for a coded workflow on 1,000 tasks (Qiu, Aug 2025); the agent's route varied more than a staged pipeline's (larger edits and more churn in all 8 paired runs), though final pass/fail agreement was at least 0.78 in 7 of 8 (Xue, Aug 2026)", "basis": "evidence" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired runs of the agent loop against a fixed workflow on the same model, tools, task set, budget and acceptance checks, on the deployed model and one cheaper tier: success, constraint violations, LLM calls, tokens and latency per task, and spread over at least three runs; report tasks with a known procedure and tasks whose path depends on findings separately" }, { "method": "monitor", "tier": null, "how": "distinct action sequences per task type, steps and cost per task, constraint and policy violations caught by checks, cap-hit rate, and sequences that recur often enough to move into code" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file agentic-patterns-catalog.md", "task": "Lists this practice in its agent-optimization pattern library (Control Flow group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Simon Willison, \"I think “agent” may finally have a widely enough agreed upon definition to be useful jargon now\"", "url": "https://simonwillison.net/2025/Sep/18/agents/", "date": "2025-09-18", "locator": "Opening", "task": "Defines an agent as the model choosing which tools to run, in a loop, toward a goal", "effect": "\"An LLM agent runs tools in a loop to achieve a goal.\"", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Li et al., \"Grow the Harness, Not the Context: From Strategy-Free Scaffolds to Reusable Specialist Agents\"", "url": "https://arxiv.org/abs/2609.26760", "date": "2026-09-22", "locator": "Abstract; §4.2; Table 1; App. A.3, A.5", "task": "Learn recurring control as harness code from failure traces (starting from a scaffold with no controller), keeping LLM calls for semantic steps, and compare it with a tool-calling (ReAct-style) agent and other inference-time agents on the same models, tools and call budget", "effect": "Against the tool-calling agent, LLM calls fell 76.0-91.8% and deployed-agent cost 74.4-98.6%, with the highest mean success in five of six settings (0.7 points behind in the sixth). WebArena-Verified success 45.3 / 44.7 / 45.3% against 30.0 / 12.7 / 6.7% for tool-calling on gpt-oss-120b / gpt-oss-20b / Qwen3.5-4B; BrowseComp-Plus 49.3 / 39.3 / 29.3% against 40.0 / 40.0 / 12.0%. The tool-calling agent took longer per task in all six settings. 95% confidence-interval half-widths on success are 7 to 13 points.", "direction": "contradicts", "measured": true, "conditions": "BrowseComp-Plus and WebArena-Verified; 200 training, 50 gate and 50 final tasks each; deployed on gpt-oss-120b, gpt-oss-20b and Qwen3.5-4B; harness trained offline with GPT-5.6-terra or GPT-5.4 as optimizer, offline cost excluded; 50-call budget per task; three evaluation runs; the authors' own method", "older": false }, { "source": "Xue, Yang, \"Rethinking Self-Evolving Agents: Do We Still Need Prescribed Optimization Pipelines?\"", "url": "https://arxiv.org/abs/2608.09629", "date": "2026-08-10", "locator": "Abstract; §3.1 Table 2; §3.3 Table 4; §4; App. A.1", "task": "Let an optimizer model compose its own skill-improvement process (OEO) under the same objective, data, operations, budget and sealed evaluator as two prescribed pipelines (SkillOpt, GEPA), then vary optimizer capability", "effect": "With GPT-5.5 as optimizer, OEO beat SkillOpt in 7 of 8 settings and tied 1, and beat GEPA in 5 of 6 (GEPA's one lead 0.21 points), using a median 34.3% of SkillOpt's target-interaction token budget; optimizer tokens slightly exceeded SkillOpt's budget in 3 settings. With a medium optimizer (Qwen3.5-27B) SkillOpt led by 9.68 points on LiveMath and 3.50 on SearchQA; a weak optimizer (Qwen3.5-4B) produced no executable action under the open interface, while the pipeline still ran. OEO made larger edits and had more revision churn in all 8 paired runs, but pass/fail agreement with SkillOpt's result was at least 0.78 in 7 of 8.", "direction": "supports", "measured": true, "conditions": "SearchQA (n=1,400), SpreadsheetBench (280), OfficeQA (172), LiveMathematicianBench (124); target models Qwen3.5-4B and GPT-5.5; one optimization run per cell; the authors' own method", "older": false }, { "source": "Ferrazzi, Cvjeticanin, Piraccini, Giannuzzi, \"Is Agentic RAG worth it? An experimental comparison of RAG approaches\"", "url": "https://arxiv.org/abs/2601.07711", "date": "2026-01-12", "locator": "Tables 3, 4, 5 and 9; §3.5; §4", "task": "Compare a fixed enhanced RAG pipeline (semantic router, HyDE rewriting, reranker) with an agent that decides whether and how to retrieve, rewrite and re-retrieve", "effect": "Query rewriting NDCG@10 55.6 for the agent against 52.8 for the fixed pipeline on average (51.7 against 43.9 on NQ). Intent routing F1 98.8 and 99.8 against 95.7 and 96.6 on two narrow domains, but 64.6 against 87.9 on FEVER. Document refinement 43.9 against 49.5 for a fixed reranker; the agent re-retrieved in 10% of cases. The agent used 2.7x and 3.6x the input tokens, 1.7x and 1.8x the output tokens and 1.5x and 1.4x the time (FiQA, CQADupStack-EN, averaged over five models). Answer quality rose with model size at similar rates for both designs.", "direction": "mixed", "measured": true, "conditions": "FiQA, NQ, FEVER, CQADupStack-EN; 500 valid and 500 invalid queries per dataset for routing; single-tool agent capped at three turns; GPT-4.1-nano and Qwen3 0.6B to 32B in the model and cost tests; answer quality judged by Selene-70B", "older": false }, { "source": "Qiu et al., \"Blueprint First, Model Second: A Framework for Deterministic LLM Workflow\"", "url": "https://arxiv.org/abs/2508.02721", "date": "2025-08-01", "locator": "Table I; Table II; Table IV; Table V; §IV-F", "task": "Code an expert procedure as a workflow run by a deterministic engine, calling the model only for bounded sub-tasks, against agents that choose their own steps on the same model", "effect": "TravelPlanner test, 1,000 tasks, Claude-Sonnet-4: final pass 35.56% against 10.40% for ReAct and 18.00% for the ATLAS multi-agent framework; commonsense-constraint violations 11 against 835 (ReAct) and 275 (ATLAS); 10.2 steps against 14.0 for ReAct. With the same constraints written into their prompts (100-task subset, mean constraint-satisfaction score): ReAct 17.00, CodeAct 19.00, ATLAS 24.50, workflow 37.20. Final pass with GPT-5.2: 34.10 against 13.82 for ReAct; Gemini-2.5-Pro: 40.00 against 19.50 (ReAct) and 35.00 (ATLAS). ScienceWorld (30 tasks): 56.7% against 43.3% for ReAct; ALFWorld (134): 97.0% against 91.0%, with more steps (16.8 against 14.0). The authors report the advantage narrows on tasks that need discovery.", "direction": "contradicts", "measured": true, "conditions": "Claude-Sonnet-4 at temperature 0.7, three seeds on TravelPlanner, single seed on ScienceWorld and ALFWorld; GPT-5.2 and Gemini-2.5-Pro; the blueprint encodes the benchmark scorer's constraints; the authors' own method; cross-model, ablation and ScienceWorld/ALFWorld results added in v2 (Jun 2026)", "older": true } ], "caveats": "Each study is small or tests the authors' own method: the optimizer comparison is one run per cell, the RAG comparison caps a single-tool agent at three turns, the code-harness results have confidence-interval half-widths of 7 to 13 points on 50 tasks and exclude the offline optimization cost, and the coded-workflow paper encodes the benchmark scorer's constraints and has a cross-model table whose caption and figures disagree. The winner depends on the model and the task, so a result on one model tier does not transfer: prescribed structure helped a medium model and was required for a weak one. The coded workflow's advantage narrowed on open-ended ScienceWorld tasks that need discovery, where both designs failed. The older evidence here (a pattern-library listing and a definition) is unmeasured.", "also_covers": [], "related": [ { "id": "7.d.iii", "slug": "react-loop", "name": "ReAct loop" }, { "id": "7.a.ii", "slug": "prompt-chaining-with-gates", "name": "Prompt chaining with gate checks" }, { "id": "7.a.i", "slug": "semantic-layer-routing", "name": "Route metric questions through a deterministic semantic layer; fall back to text-to-SQL" }, { "id": "6.c.ii", "slug": "plan-then-execute", "name": "Plan-then-execute" }, { "id": "1.b.i", "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class" }, { "id": "10.b.i", "slug": "risky-action-policy", "name": "Gate irreversible actions: allow, ask or deny by side effect" }, { "id": "4.c.ii", "slug": "skill-library", "name": "Skill library evolution" } ], "url": "https://mahalsystems.ai/aot/optimizations/inversion-of-control/", "markdown": "https://mahalsystems.ai/aot/optimizations/inversion-of-control.md" }, { "id": "7.d.iii", "legacy_id": "P04", "former_ids": [], "slug": "react-loop", "name": "ReAct loop", "kind": "pattern", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it changes the control loop", "proposed": true }, "levers": [ "control-flow" ], "domains": [ "general" ], "evidence_domains": [ "computer-use", "web-research", "tool-use", "conversation", "data-analysis", "code-generation", "retrieval", "planning" ], "categories": [ "Shaping the loop" ], "category_ids": [ "orchestration.loop" ], "summary": "Run the agent as a loop: the model reasons, calls a tool, reads the result and picks the next step, until it answers or hits a cap. The loop is the common default for tasks that need several dependent tool calls; the 2022 text template (Thought/Action/Observation lines in a few-shot prompt) is not. On current models, use native tool calling, carry the model's reasoning across tool calls, bound the whole feedback path, and keep the per-step preamble small: with the model fixed, the harness around the loop moved cost per task by up to 3x and pass rate by less than the noise.", "how": [ "Use the provider's native tool calling, not a text Thought/Action/Observation template.", "Loop: call the model, run every tool call it returns, append each result to the history (errors included, as observations), and call again; stop on a final answer or an explicit finish tool.", "Let the model reason before each action (interleaved or adaptive thinking; a think tool on models without it), and pass its reasoning back unchanged across tool calls.", "Cap steps, cost and wall time on the whole feedback path, retries and handoffs included; count a cap hit as a failure (7.b.iii sizes the caps, 7.b.i catches stalls).", "Keep the per-step preamble (system prompt and tool schemas) small; it is re-sent every step.", "Paired eval against the current design (one call or a fixed workflow), per model: success, steps, cost per task, cap-hit rate." ], "detect": { "static": "A loop that alternates model call and tool dispatch, feeds results back, and has a max-iteration guard; check that the guard bounds the whole feedback path (retries, handoffs), not just the outer loop.", "traces": "Thought before each action; observations used in the next thought; loops that ignore results.", "evals": "Success on tool-use tasks; steps per task; tool error rate.", "judge": "Do thoughts inform the next action or are they boilerplate.", "probes": "Tasks needing two to four dependent tool calls." }, "impact": { "measured_basis": true, "primary": [ "quality", "performance" ], "rows": [ { "dimension": "quality", "effect": "improves", "size": "large", "when": "tasks that need several dependent tool calls: text-to-SQL 45.9 to 72.2 on EHR-SQL over a single shot, but 78.5 to 79.1 on BIRD (Lu, Aug 2026); ALFWorld 71 against 45 for acting without reasoning (Yao, 2022)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "medium", "when": "one model call per step, each re-sending the preamble and history; with the model fixed, cost per task differed by up to 3x across harnesses (Liu and Han, Oct 2026)", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "medium", "when": "serial round trips, one per step", "basis": "evidence" }, { "dimension": "security", "effect": "mixed", "size": "medium", "when": "Autonomous tool actions need scoping.", "basis": "assessment" }, { "dimension": "governance", "effect": "mixed", "size": "medium", "when": "Every action must be logged to be auditable.", "basis": "assessment" }, { "dimension": "brand", "effect": "none", "size": null, "when": null, "basis": "assessment" } ] }, "measure": [ { "method": "experiment", "tier": 4, "how": "paired eval of the loop against the current design, per model: success, steps, cost per task, cap-hit rate; use the full task pool, since 45 tasks catch a 12.9-point gap only half the time" }, { "method": "monitor", "tier": null, "how": "steps per task, cap-hit rate, tool error rate, cost per task" } ], "evidence": [ { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 6; pattern library file master-agent-optimization-cookbook.md", "task": "Lists this practice in its agent-optimization pattern library (Prompting Strategies group)", "effect": "Names the practice as a known agent optimization; reports no measured effect for it.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Yao et al., \"ReAct: Synergizing Reasoning and Acting in Language Models\"", "url": "https://arxiv.org/abs/2210.03629", "date": "2022-10-06", "locator": "Abstract; Table 1, Table 3, Table 4", "task": "Interleave reasoning traces with actions, against acting only, chain-of-thought, and imitation or RL agents", "effect": "ALFWorld success 71 (best of 6) against 45 for acting only and 37 for BUTLER; WebShop success 40.0 against 30.1 (acting only), 29.1 (IL) and 28.7 (IL+RL). HotpotQA EM 27.4 against 29.4 for chain-of-thought, 35.1 when ReAct falls back to CoT-SC; FEVER 60.9 against 56.3 for chain-of-thought.", "direction": "supports", "measured": true, "conditions": "PaLM-540B, few-shot prompts (one or two examples on ALFWorld and WebShop), greedy decoding", "older": true }, { "source": "Verma, Bhambri, Kambhampati, \"On the Brittle Foundations of ReAct Prompting for Agentic Large Language Models\"", "url": "https://arxiv.org/abs/2405.13966", "date": "2024-05-22", "locator": "Abstract; Table 1", "task": "Vary ReAct's few-shot prompt on ALFWorld: move the reasoning out of the loop, change its content, change the examples", "effect": "Reasoning given once before acting, instead of interleaved, beat base ReAct for GPT-3.5-Instruct (44.7 to 61.9) and GPT-4 (23.3 to 43.3), not for Claude-Opus (56.6 to 50). Performance is \"minimally influenced by the 'interleaving reasoning trace with action execution'\" and driven by how similar the examples are to the query.", "direction": "contradicts", "measured": true, "conditions": "ALFWorld, six task types; few-shot text prompting", "older": true }, { "source": "Yao, Shinn, Razavi, Narasimhan, \"τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains\"", "url": "https://arxiv.org/abs/2406.12045", "date": "2024-06-17", "locator": "Method comparison; Figure 3", "task": "Native function calling against text-formatted ReAct and Act agents", "effect": "\"natively supported function calling consistently outperforms text-formatted agent methods with the state-of-the-art models. For text-formatted agent methods, adding reasoning traces still consistently helps.\" A \"think\" function added to function-calling agents \"did not boost performance\".", "direction": "mixed", "measured": true, "conditions": "τ-bench retail and airline; gpt-4o and other API models", "older": true }, { "source": "Trivedi et al., \"AppWorld: A Controllable World of Apps and People for Benchmarking Interactive Coding Agents\"", "url": "https://arxiv.org/abs/2407.18901", "date": "2024-07-26", "locator": "Table 3; App. E.8", "task": "ReAct against plan-then-execute and other agent methods on everyday app tasks", "effect": "Task goal completion, normal / challenge test sets: GPT-4o ReAct 48.8 / 30.2 against plan-then-execute 44.6 / 19.7; GPT-4-Turbo ReAct 26.8 / 17.5 against 32.7 / 11.0. GPT-4o cost about $0.7 per example with ReAct and $1.33 with plan-then-execute.", "direction": "mixed", "measured": true, "conditions": "GPT-4o, GPT-4-Turbo; greedy decoding", "older": true }, { "source": "Anthropic, The \"think\" tool: Enabling Claude to stop and think in complex tool use situations", "url": "https://www.anthropic.com/engineering/claude-think-tool", "date": "2025-03-20", "locator": "Evaluation sections; Dec 15, 2025 update note", "task": "Give the model a think tool to reason between tool calls", "effect": "τ-bench airline pass^1 0.570 with the think tool and an optimized prompt against 0.370 baseline; retail 0.812 against 0.783; SWE-bench +1.6% on average. A Dec 2025 update recommends extended thinking \"instead of a dedicated think tool in most cases\".", "direction": "supports", "measured": true, "conditions": "Claude 3.7 Sonnet; SWE-bench n=30 with the tool, 144 without", "older": true }, { "source": "OpenAI, \"GPT-5 prompting guide\"", "url": "https://cookbook.openai.com/examples/gpt-5/gpt-5_prompting_guide", "date": "2025-08-07", "locator": "Responses API paragraph", "task": "Pass the model's previous reasoning items back across tool calls (Responses API) instead of dropping them", "effect": "\"Tau-Bench Retail score increases from 73.9% to 78.2% just by switching to the Responses API and including previous_response_id to pass back previous reasoning items into subsequent requests.\"", "direction": "supports", "measured": true, "conditions": "GPT-5; vendor-reported, no sample size given", "older": true }, { "source": "Lu, Yu, Xiong, Zhang, \"Iteration Without Elaboration: A Simple ReAct Architecture Suffices for Text-to-SQL Generation\"", "url": "https://arxiv.org/abs/2608.22651", "date": "2026-08-23", "locator": "Abstract; Table 4", "task": "Add a ReAct loop (SQL actions with execution feedback) to a zero-shot single-shot baseline, then a typed DSL of 15 operations", "effect": "Single shot to loop: EHR-SQL 45.9 to 72.2, BIRD 78.5 to 79.1; with the typed DSL, 73.9 and 84.5. Matches more elaborate pipelines while running up to 8x faster.", "direction": "supports", "measured": true, "conditions": "Gemini-2.5-Flash; corrected BIRD mini-dev and EHR-SQL", "older": false }, { "source": "Liu, Han, \"What Does a Harness Buy? Tokens, Mostly\"", "url": "https://arxiv.org/abs/2610.04433", "date": "2026-10-03", "locator": "Abstract; Table 1; §4.4; App. J", "task": "Hold the model fixed and swap the harness around the loop: Claude Code, OpenCode, and the bash-only mini-SWE-agent", "effect": "Claude Code and mini-SWE-agent \"are equivalent within five points\" (-1.8 and -1.4 points on two models). Cost per task differed by up to 3x across harnesses, set by the per-step preamble (16k tokens in Claude Code, 7k in OpenCode, under a thousand in mini-SWE-agent) times the number of steps. Switching thinking off cost mini-SWE-agent 11 points on one model and moved the other two by less than the noise.", "direction": "supports", "measured": true, "conditions": "SWE-bench Verified, 447 tasks, two models; a 45-task hard subset, five models", "older": false }, { "source": "Hou, Wang, Zhao, Wang, \"When Agents Do Not Stop: Uncovering Infinite Agentic Loops in LLM Agents\"", "url": "https://arxiv.org/abs/2607.01641", "date": "2026-07-02", "locator": "Abstract; §1", "task": "Scan agent code for feedback paths that can repeat costly operations without an effective bound", "effect": "68 confirmed infinite agentic loops across 47 projects, from 6,549 agent repositories scanned (91.9% precision). Developers \"omit them, misuse them, configure them with ineffective bounds, or place them outside the actual feedback path.\"", "direction": "supports", "measured": true, "conditions": "open-source LLM agent repositories; static analysis with manual review", "older": false }, { "source": "Piet et al., \"Web Agents Should Adopt the Plan-Then-Execute Paradigm\"", "url": "https://arxiv.org/abs/2605.14290", "date": "2026-05-14", "locator": "Abstract", "task": "Position paper: whether web agents should default to ReAct or to plan-then-execute", "effect": "Argues ReAct is \"the wrong default for web agents\" because untrusted page content flows into every next-action decision. All WebArena tasks are compatible with plan-then-execute, and 80% can be done with a purely programmatic plan.", "direction": "contradicts", "measured": false, "conditions": "WebArena task analysis; no agent success rates", "older": false } ], "caveats": "The 2022 gains came from few-shot text prompting on PaLM-540B, and ReAct alone trailed chain-of-thought on HotpotQA; a 2024 study found the interleaved reasoning in that template mattered little next to the choice of examples. Whether reasoning between calls helps depends on the model and harness: switching thinking off cost one harness 11 points and moved two others by less than the noise. Every tool result feeds the next action, so untrusted content can steer the loop: contain it (10.c.i) or commit to a plan before reading it (6.c.ii). Harness comparisons need large task pools.", "also_covers": [], "related": [ { "id": "7.b.iii", "slug": "size-execution-budgets", "name": "Size run budgets so productive runs aren't cut off: turn caps, deadlines, command timeouts" }, { "id": "7.b.i", "slug": "detect-loops-and-stalls", "name": "Detect stalls and loops: warn, then stop early" }, { "id": "2.a.iii", "slug": "choose-action-interface", "name": "Choose how the agent acts: typed tools, code execution, or shell, matched to the model and task" }, { "id": "6.c.ii", "slug": "plan-then-execute", "name": "Plan-then-execute" }, { "id": "10.c.i", "slug": "contain-untrusted-input", "name": "Contain sensitive data and untrusted input" }, { "id": "6.c.i", "slug": "code-then-execute", "name": "Code-then-execute (CaMeL)" }, { "id": "7.d.ii", "slug": "inversion-of-control", "name": "Inversion of control" }, { "id": "7.c.v", "slug": "language-agent-tree-search", "name": "Language Agent Tree Search (LATS)" }, { "id": "6.b.i", "slug": "parallel-read-only-tools", "name": "Parallel tool execution for read-only calls" } ], "url": "https://mahalsystems.ai/aot/optimizations/react-loop/", "markdown": "https://mahalsystems.ai/aot/optimizations/react-loop.md" }, { "id": "8.a.i", "legacy_id": "T57", "former_ids": [], "slug": "choose-retriever", "name": "Choose the retriever by corpus size and question type", "kind": "technique", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "routing between retrieval paths that exist", "proposed": false }, "levers": [ "knowledge-access", "control-flow" ], "domains": [ "retrieval" ], "evidence_domains": [ "retrieval", "web-research", "code-generation" ], "categories": [ "Expanding sources", "Querying precisely", "Routing" ], "category_ids": [ "retrieval.sources", "retrieval.query", "orchestration.route" ], "summary": "On a bounded corpus (up to about 100K documents), grep and file reads over the raw text beat an embedding retriever: the agent narrows into documents it has found. As the corpus grows, grep degrades (37.5% at 400K); add semantic search beside it, and on large codebases semantic search plus grep beats grep alone. A dense retriever beats BM25. Use a knowledge-graph retriever only for multi-hop and synthesis questions, routed per query; keep chunk retrieval for single-fact lookup, return source text with any graph triplets, and avoid global community-summary search for detail questions.", "how": [ "Size the corpus and classify the questions (single-fact, multi-hop).", "Pick the retriever by the table in the summary.", "Paired eval, retriever A against B, on the agent's own corpus, measuring evidence localization as well as document recall." ], "detect": { "static": "the retriever in use and the corpus size; a single retrieval path for all question types", "traces": "repeated searches that fail to narrow; documents found but never read in full; failure rows labeled missed-read on multi-hop questions", "evals": "retrieval recall and answer accuracy split by corpus size and question type" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "large", "when": "grep 80.0% against 69.0% for an 8B embedding retriever at ~100K documents (May 2026); Cursor +12.5% from semantic search plus grep on large codebases; GraphRAG routing +1.1%, concatenating both +6.4% (Mar 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "medium", "when": "grep 29% cheaper at ~100K documents; a graph index costs to build", "basis": "evidence" }, { "dimension": "performance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval, retriever A against B, split by question type" }, { "method": "production-test", "tier": 4, "how": "when a new index is introduced" } ], "evidence": [ { "source": "Li et al., \"Beyond Semantic Similarity: Rethinking Retrieval for Agentic Search via Direct Corpus Interaction\"", "url": "https://arxiv.org/abs/2605.05242", "date": "2026-05-03", "locator": "Abstract; Sec 4.2 (Fig 1, Tables 2-3); Sec 4.3 RQ2 (Fig 4)", "task": "Replace the retriever tool with grep and file reads", "effect": "BrowseComp-Plus (830 q, 100,195 docs, avg 5,179 words), same Sonnet 4.6 backbone: Qwen3-Embedding-8B retrieval agent 69.0% -> DCI-Agent-CC 80.0% (+11.0), cost $1,440 -> $1,016 (-29.4%); beats best retrieval baseline GPT-5 + Qwen3-Embed-8B (71.7%). Per-question: 176 DCI-only wins vs 76 retrieval-only wins; of the 176, 142 already had at least one gold doc surfaced by the retriever. DCI-Agent-Lite (GPT-5.4 nano) 62.9% at $93 vs o3 + Qwen3-Embed-8B 66.0% at ~$740. Multi-hop QA over 21M-passage Wikipedia (50 q/dataset): DCI-CC avg 83.0 vs ASearcher-Local-14B 52.3 (MuSiQue 74 vs 24). IR ranking NDCG@10 avg 68.5 (CC) / 56.7 (Lite) vs ReasonRank-32B 47.0; Lite loses to ReasonRank on Economics (32.3 vs 36.6) and SciFact (72.7 vs 75.5). Wins attributed to fine-grained local inspection of documents already reached, not to higher recall.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Li et al., \"Beyond Semantic Similarity: Rethinking Retrieval for Agentic Search via Direct Corpus Interaction\"", "url": "https://arxiv.org/abs/2605.05242", "date": "2026-05-03", "locator": "Sec 4.3 RQ6; Table 5", "task": "Give a read plus grep tool pair as minimum retrieval surface", "effect": "BrowseComp-Plus 100-q subset, GPT-5.4 nano, L4 context policy: read+grep 61% acc, 19 tool calls, $0.0355/q vs Qwen3-Embedding-8B retrieval 45%, 18 calls, $0.0498/q vs BM25 32%, 19 calls, $0.0527/q.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Li et al., \"Beyond Semantic Similarity: Rethinking Retrieval for Agentic Search via Direct Corpus Interaction\"", "url": "https://arxiv.org/abs/2605.05242", "date": "2026-05-03", "locator": "Sec 4.3 RQ6, Table 5; Appendix B.1, Fig 6; Fig 4 (right)", "task": "Allow piped shell commands for chained lexical search", "effect": "BrowseComp-Plus 100-q subset, GPT-5.4 nano: open bash 73% vs read+grep 61% (+12), but 35 vs 19 tool calls and $0.1021 vs $0.0355 per q. Lite bash usage (3,168 commands): rg | head 56.2%, rg | rg 20.6%, wc 7.8%, single rg 6.3%, ls 4.3%, python -c 2.4%, cat 0.1%. CC: Bash 62.4% / Grep 33.0% of calls; within Bash, chained search 22.3%, local context peek 18.0%, regex 17.0%, file localization 14.0%, full-document read 9.1%.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Li et al., \"Beyond Semantic Similarity: Rethinking Retrieval for Agentic Search via Direct Corpus Interaction\"", "url": "https://arxiv.org/abs/2605.05242", "date": "2026-05-03", "locator": "Sec 4.3 RQ4; Fig 5", "task": "Do not rely on grep-only search as the corpus grows", "effect": "DCI-Agent-CC, BrowseComp-Plus 100-q subset, corpus padded with FineWeb distractors: 100K -> 200K docs: tool calls 38.5 -> 86.9, latency and cost more than doubled, accuracy -13.6 points. At 400K docs: accuracy 37.5%, 122.4 calls on average, 20 of 100 examples terminated at the max tool budget. (By contrast the paper's multi-hop QA runs over 21M short Wikipedia passages still worked, avg 83.0.)", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Li et al., \"Beyond Semantic Similarity: Rethinking Retrieval for Agentic Search via Direct Corpus Interaction\"", "url": "https://arxiv.org/abs/2605.05242", "date": "2026-05-03", "locator": "Sec 3.2, Table 1; Sec 4.3 RQ5, Table 6", "task": "Cap tool output and compact old tool results moderately", "effect": "DCI-Agent-Lite, GPT-5.4 nano, BrowseComp-Plus 100-q subset. L0 none: 72% acc, $0.0716/q, 2,226 s. L1 truncate 50K: 75%, fastest (1,820 s), highest retained gold coverage 31.3. L2 truncate 20K: 69% (worst), cheapest $0.0590. L3 truncate 20K + compaction (trigger at 240K chars of tool output, keep last 12 turns as-is): 77% (best) but $0.1109/q and 8,712 s. L4 = L3 + LLM summary keeping last 20K tokens: 73%. Non-monotonic; retaining more verbatim evidence did not give the best accuracy.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Li et al., \"Beyond Semantic Similarity: Rethinking Retrieval for Agentic Search via Direct Corpus Interaction\"", "url": "https://arxiv.org/abs/2605.05242", "date": "2026-05-03", "locator": "Sec 3.3; Sec 4.3 RQ3, Table 4", "task": "Measure evidence localization, not only gold-document recall", "effect": "BrowseComp-Plus 100-q subset, GPT-5.4 nano: Qwen3-Embed-8B agent coverage-any 74.0, mean recall 56.7, localization 21.7, acc 45; DCI-Agent-Lite coverage-any 70.0, mean recall 28.0, localization 48.4, acc 73; BM25 63.0 / 42.8 / 23.5 / 32. Questions have 1-4 gold docs, which favors reaching one and mining it.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Stefan Heule, Emily Jia, Naman Jain (Cursor), \"Improving agent with semantic search\"", "url": "https://cursor.com/blog/semsearch", "date": "2025-11-06", "locator": "Intro bullets; Offline evals; Online A/B tests; Conclusion", "task": "Add a semantic code-search tool alongside grep", "effect": "Offline (Cursor Context Bench): on average 12.5% higher accuracy answering questions with semantic search available (6.5%-23.5% depending on model); improved accuracy for every model tested, including all frontier coding models. Online A/B (same model, one arm grep-style tools only): agent code retention +0.3% with semantic search, +2.6% on codebases with 1,000+ files; dissatisfied follow-up requests +2.2% when semantic search was absent. Effect sizes online are diluted because not all requests need search. \"The combination of these two leads to the best outcomes.\"", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Stefan Heule, Emily Jia, Naman Jain (Cursor), \"Improving agent with semantic search\"", "url": "https://cursor.com/blog/semsearch", "date": "2025-11-06", "locator": "Custom retrieval models", "task": "Train the embedder on agent traces ranked by an LLM", "effect": "Described as \"one piece that enables these results\"; training data is agent traces where the agent \"performs multiple searches and opens files before finding the right code\"; an LLM ranks what \"would have been most helpful at each step\" and the embedding model is trained \"to align its similarity scores with these LLM-generated rankings\". No ablation against an off-the-shelf embedder is reported.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Chen et al., \"BrowseComp-Plus: A More Fair and Transparent Evaluation Benchmark of Deep-Research Agent\"", "url": "https://arxiv.org/abs/2508.06600", "date": "2025-08-08", "locator": "Sec 4.5-4.7, Tables 1-3; Sec 4.8.1; Appendix H Table 8", "task": "Use a strong dense embedder rather than BM25", "effect": "BM25 -> Qwen3-Embed-8B, accuracy: gpt-5 55.90 -> 70.12%; o3 49.28 -> 63.49%; gpt-4.1 14.58 -> 35.42%; Sonnet 4 14.34 -> 36.75%; Opus 4 15.54 -> 36.14%; Gemini 2.5 Pro 19.04 -> 28.67%; gpt-oss-120B-high 28.67 -> 42.89%. Search calls fall ~1-3 per query for proprietary models (gpt-5 23.23 -> 21.74). API cost falls (gpt-5 $400.36 -> $360.71; o3 $836.35 -> $740.79; Opus 4 $2,043.95 -> $1,842.48). Standalone gold-doc Recall@100: BM25 6.1 vs Qwen3-Embed-8B 55.8. Oracle (all positive docs in context) gives gpt-4.1 93.49% vs 14.58% with BM25, showing retrieval is the bottleneck.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Chen et al., \"BrowseComp-Plus: A More Fair and Transparent Evaluation Benchmark of Deep-Research Agent\"", "url": "https://arxiv.org/abs/2508.06600", "date": "2025-08-08", "locator": "Tables 1-2; Sec 4.7", "task": "Scale the embedder; reasoning-tuned embedder did not help", "effect": "Evidence-doc Recall@100: Qwen3-Embed 0.6B 26.5, 4B 40.2, 8B 47.7, ReasonIR-8B 43.6; nDCG@10 8.0 / 14.0 / 20.3 / 16.8. End-to-end with Qwen3-32B agent: 4.10 / 7.83 / 10.36 / 9.16%; with Search-R1-32B: 5.66 / 9.40 / 10.36 / 9.43%.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Chen et al., \"BrowseComp-Plus: A More Fair and Transparent Evaluation Benchmark of Deep-Research Agent\"", "url": "https://arxiv.org/abs/2508.06600", "date": "2025-08-08", "locator": "Sec 4.8.3, Table 5", "task": "Add a full-document reader tool beside truncated search", "effect": "Qwen3-Embed-8B retriever, 512-token previews: gpt-4.1 35.42 -> 43.61% with get-doc (search calls 8.67 -> 10.03, 1.85 get-doc calls/q); Qwen3-32B 10.36 -> 11.69% (0.27 get-doc calls/q).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Chen et al., \"BrowseComp-Plus: A More Fair and Transparent Evaluation Benchmark of Deep-Research Agent\"", "url": "https://arxiv.org/abs/2508.06600", "date": "2025-08-08", "locator": "Sec 4.8.2, Table 4", "task": "Raise reasoning effort for search agents", "effect": "gpt-oss-20B with Qwen3-Embed-8B low -> medium -> high: accuracy 13.37 -> 29.88 -> 34.58%, recall 17.37 -> 41.31 -> 49.29%, search calls 1.87 -> 13.64 -> 23.87. gpt-oss-120B: 24.94 -> 37.59 -> 42.89%. Calibration error also falls. Qwen3-Embed-8B beats BM25 at every effort level.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Chen et al., \"BrowseComp-Plus: A More Fair and Transparent Evaluation Benchmark of Deep-Research Agent\"", "url": "https://arxiv.org/abs/2508.06600", "date": "2025-08-08", "locator": "Sec 4.8.4, Tables 6-7", "task": "Expect dense retrieval to degrade as distractors grow", "effect": "Corpus 100K -> 9,771,311 docs (FineWeb-edu 10BT added). Evidence Recall@100: BM25 4.7 -> 8.0; Qwen3-Embed-8B 47.7 -> 37.6; ReasonIR-8B 43.6 -> 30.7 (added docs are unjudged, so partly measurement artifact). Accuracy with Qwen3-Embed-8B: Qwen3-32B 10.36 -> 7.11%, Search-R1-32B 10.36 -> 8.33%; with BM25 both rose slightly (3.49 -> 5.42%, 3.86 -> 4.72%).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 13.2; Observation 9; Recommendations 8 and 16; Table 13", "task": "Retrieve code with ripgrep, glob and tree-sitter, not embeddings", "effect": "\"0/11 systems use vector embeddings for code retrieval\"; code \"changes minute-to-minute, making embeddings stale\"; the cited CodeRAG-Bench finds RAG gains \"highly variable across tasks\". No controlled comparison is run.", "direction": "mixed", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 9.7; Table 13; Sec 7.2 (Aider)", "task": "Provide a token-budgeted ranked repository map", "effect": "Aider RepoMap: \"Tree-sitter-based symbol indexing ... still the only ranked repository map in the corpus. Token budget is adaptive (map_tokens, default 1024) with a map_mul_no_files multiplier.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Zhishang Xiang et al. (2506.05690); Haoyu Han et al. (2502.11371), \"When to use Graphs in RAG: A Comprehensive Analysis for Graph Retrieval-Augmented Generation (GraphRAG-Bench), arXiv 2506.05690; and RAG vs. GraphRAG: A Systematic Evaluation and Key Insights, arXiv 2502.11371\"", "url": "https://arxiv.org/abs/2502.11371", "date": "2026-03-04", "locator": "Han Sec 4.2, Tables 1-2; Xiang Sec 4.1-4.2, Tables 2-3, Obs 2 and 5", "task": "Build a text-linked knowledge graph for multi-hop questions", "effect": "Han (Llama-3.1-8B): HotpotQA F1 HippoRAG2 63.01, Community-Local 61.66 vs RAG 60.04; MultiHop-RAG acc HippoRAG2 70.27, Community-Local 69.01 vs RAG 67.02 (Temporal 49.91 vs 30.70). GraphRAG-Bench complex reasoning ACC, novels: HippoRAG2 53.38 vs RAG+rerank 42.93; medical 61.98 vs 58.64; contextual summarize (novels) MS-GraphRAG 64.40 vs 51.30. Level 2-3 evidence recall on novels: HippoRAG 87.9-90.9%.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Zhishang Xiang et al. (2506.05690); Haoyu Han et al. (2502.11371), \"When to use Graphs in RAG: A Comprehensive Analysis for Graph Retrieval-Augmented Generation (GraphRAG-Bench), arXiv 2506.05690; and RAG vs. GraphRAG: A Systematic Evaluation and Key Insights, arXiv 2502.11371\"", "url": "https://arxiv.org/abs/2502.11371", "date": "2026-03-04", "locator": "Han Sec 4.2 obs (1), Table 1; Xiang Obs 1 and 4, Tables 2-3; Xiang Sec 1", "task": "Keep plain chunk RAG for single-hop fact lookup", "effect": "Han NQ F1: RAG 64.78 vs Community-Local 63.01, HippoRAG2 61.03, RAPTOR 60.04, Community-Global 54.48, KG triplets+text 50.27. GraphRAG-Bench fact retrieval ACC, novels: RAG+rerank 60.92 vs HippoRAG2 60.14, MS-GraphRAG 49.29; medical: 64.73 vs 66.28 (HippoRAG2) but MS-GraphRAG 38.63. RAG+rerank evidence recall on novel fact questions 83.21%. Citing prior work: GraphRAG 13.4% lower on NQ, 16.6% lower on time-sensitive queries.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Zhishang Xiang et al. (2506.05690); Haoyu Han et al. (2502.11371), \"When to use Graphs in RAG: A Comprehensive Analysis for Graph Retrieval-Augmented Generation (GraphRAG-Bench), arXiv 2506.05690; and RAG vs. GraphRAG: A Systematic Evaluation and Key Insights, arXiv 2502.11371\"", "url": "https://arxiv.org/abs/2502.11371", "date": "2026-03-04", "locator": "Han Sec 4.4-4.5, Figs 2-3", "task": "Route each query to RAG or GraphRAG by type", "effect": "Consistent improvement across datasets; MultiHop-RAG with Llama-3.1-70B: +1.1% over the best single baseline. Cheaper than Integration since each query runs one method. Motivated by complementarity: on MultiHop-RAG 13.6% of queries are GraphRAG-only correct and 11.6% RAG-only.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Zhishang Xiang et al. (2506.05690); Haoyu Han et al. (2502.11371), \"When to use Graphs in RAG: A Comprehensive Analysis for Graph Retrieval-Augmented Generation (GraphRAG-Bench), arXiv 2506.05690; and RAG vs. GraphRAG: A Systematic Evaluation and Key Insights, arXiv 2502.11371\"", "url": "https://arxiv.org/abs/2502.11371", "date": "2026-03-04", "locator": "Han Sec 4.5, Fig 3; Sec 5.2 obs (4), Tables 6-7", "task": "Concatenate RAG and GraphRAG evidence for QA", "effect": "MultiHop-RAG, Llama-3.1-70B: +6.4% over best baseline (vs +1.1% for Selection); generally above Selection across QA datasets. Query-based summarization: Integration ROUGE-2 F1 10.67 vs RAG 10.08 on SQuALITY but 9.53 vs 9.81 on ODSum-story, i.e. \"comparably to RAG alone\".", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Zhishang Xiang et al. (2506.05690); Haoyu Han et al. (2502.11371), \"When to use Graphs in RAG: A Comprehensive Analysis for Graph Retrieval-Augmented Generation (GraphRAG-Bench), arXiv 2506.05690; and RAG vs. GraphRAG: A Systematic Evaluation and Key Insights, arXiv 2502.11371\"", "url": "https://arxiv.org/abs/2502.11371", "date": "2026-03-04", "locator": "Han Sec 4.2 obs (4), Tables 1-2, Appendix C", "task": "Return source text with graph triplets, not triplets alone", "effect": "Han NQ F1: KG triplets only 34.28 vs triplets+text 50.27 (RAG 64.78); HotpotQA 25.02 vs 42.60 (RAG 60.04); MultiHop-RAG 41.24 vs 48.51. Only ~65.8% (HotpotQA) and 65.5% (NQ) of answer entities appear in the constructed KG.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Zhishang Xiang et al. (2506.05690); Haoyu Han et al. (2502.11371), \"When to use Graphs in RAG: A Comprehensive Analysis for Graph Retrieval-Augmented Generation (GraphRAG-Bench), arXiv 2506.05690; and RAG vs. GraphRAG: A Systematic Evaluation and Key Insights, arXiv 2502.11371\"", "url": "https://arxiv.org/abs/2502.11371", "date": "2026-03-04", "locator": "Han Sec 4.2 obs (3), Table 2, Sec 5.3; Xiang Obs 8-9, Figs 8-9", "task": "Avoid global community-summary search for detail questions", "effect": "Han MultiHop-RAG: Community-Global Null-query acc 19.27 vs RAG 96.01, but Comparison 64.02 vs 57.59 and Temporal 53.34 vs 30.70. NQ F1 54.48 vs 64.78. Summarization: LLM-judge prefers RAG on comprehensiveness, Global on diversity. GraphRAG-Bench: MS-GraphRAG(global) prompts ~331K tokens avg vs 879 for vanilla RAG, growing 7,800 -> 40,000 tokens with task difficulty.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Zhishang Xiang et al. (2506.05690); Haoyu Han et al. (2502.11371), \"When to use Graphs in RAG: A Comprehensive Analysis for Graph Retrieval-Augmented Generation (GraphRAG-Bench), arXiv 2506.05690; and RAG vs. GraphRAG: A Systematic Evaluation and Key Insights, arXiv 2502.11371\"", "url": "https://arxiv.org/abs/2502.11371", "date": "2026-03-04", "locator": "Han Sec 4.7, Table 5", "task": "Use a stronger LLM to construct the graph", "effect": "Han, Community-GraphRAG (Local), MultiHop-RAG, Llama-3.1-70B generator: no graph (RAG) 65.77 overall; GPT-4o-mini graph 71.17; GPT-4o graph 75.08 (Temporal 25.73 / 49.06 / 58.49; Comparison 56.31 / 60.16 / 66.59), while Null falls 91.36 / 88.70 / 81.06.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Zhishang Xiang et al. (2506.05690); Haoyu Han et al. (2502.11371), \"When to use Graphs in RAG: A Comprehensive Analysis for Graph Retrieval-Augmented Generation (GraphRAG-Bench), arXiv 2506.05690; and RAG vs. GraphRAG: A Systematic Evaluation and Key Insights, arXiv 2502.11371\"", "url": "https://arxiv.org/abs/2502.11371", "date": "2026-03-04", "locator": "Xiang Sec 4.3-4.4, Obs 7-9, Figs 7-9; Han Sec 4.6, Table 4", "task": "Prefer dense, compact-prompt graph retrievers", "effect": "GraphRAG-Bench avg prompt tokens (novel): vanilla RAG 879, HippoRAG2 1,008, RAPTOR 3,441, Fast-GraphRAG 4,204, HippoRAG 7,208, MS-GraphRAG local 38,707, LightRAG 100,832, MS-GraphRAG global 331,375. HippoRAG2 avg degree 8.75 (novel) / 13.31 (medical) vs 1.48-5.50 for others; 2,310 edges and 523 nodes per 10K corpus tokens (novel). Han construction time on MultiHop-RAG: RAG 135 s, KG-GraphRAG 7,702 s, Community-GraphRAG 5,560 s; retrieval time 1,724 / 14,434 / 1,249 s.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Zhishang Xiang et al. (2506.05690); Haoyu Han et al. (2502.11371), \"When to use Graphs in RAG: A Comprehensive Analysis for Graph Retrieval-Augmented Generation (GraphRAG-Bench), arXiv 2506.05690; and RAG vs. GraphRAG: A Systematic Evaluation and Key Insights, arXiv 2502.11371\"", "url": "https://arxiv.org/abs/2502.11371", "date": "2026-03-04", "locator": "Han Sec 4.3, Fig 1; Xiang Tables 2-3", "task": "Add reranking and iterative retrieval to either pipeline", "effect": "Han: reranking (bge-reranker-large) and IRCoT improve all methods on NQ and MultiHop-RAG, with larger gains on MultiHop-RAG (numbers in Fig 1 / appendices, not in text); RAG still best on single-hop. GraphRAG-Bench novels, fact retrieval: RAG evidence recall 61.37 -> 83.21 with rerank, ACC 58.76 -> 60.92.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Zhishang Xiang et al. (2506.05690); Haoyu Han et al. (2502.11371), \"When to use Graphs in RAG: A Comprehensive Analysis for Graph Retrieval-Augmented Generation (GraphRAG-Bench), arXiv 2506.05690; and RAG vs. GraphRAG: A Systematic Evaluation and Key Insights, arXiv 2502.11371\"", "url": "https://arxiv.org/abs/2502.11371", "date": "2026-03-04", "locator": "Han Sec 5.3, Fig 4", "task": "Swap candidate order when LLM-judging retrieval outputs", "effect": "Han: reversing which summary is shown first gives \"substantially different, and in some cases opposite\" preferences, most for RAG vs GraphRAG-Local (QMSum, ODSum-story). Explains why prior LLM-judged work found Community-Global best while reference metrics (ROUGE-2, BERTScore) favor RAG.", "direction": "supports", "measured": true, "conditions": null, "older": false } ], "caveats": null, "also_covers": [], "related": [ { "id": "8.b.i", "slug": "tune-search-tool", "name": "Tune the search tool: snippets, enough results, retries, reformulation, filters, source quality" }, { "id": "8.a.ii", "slug": "surface-unreadable-information", "name": "Surface information the source holds but the agent cannot read" }, { "id": "5.b.i", "slug": "ground-queries-in-schema", "name": "Give the model the schema it queries, in context: descriptions, keys, codes, sample values" }, { "id": "4.c.i", "slug": "episodic-memory", "name": "Episodic memory retrieval" }, { "id": "5.c.ii", "slug": "isolated-subagent-context", "name": "Sub-agent architectures with clean contexts" }, { "id": "8.b.ii", "slug": "link-large-schemas", "name": "Link large schemas by exploration: start broad, grow the linked set with probing queries, favor recall" }, { "id": "6.b.ii", "slug": "llm-map-reduce", "name": "LLM map-reduce" } ], "url": "https://mahalsystems.ai/aot/optimizations/choose-retriever/", "markdown": "https://mahalsystems.ai/aot/optimizations/choose-retriever.md" }, { "id": "8.a.ii", "legacy_id": "T39", "former_ids": [], "slug": "surface-unreadable-information", "name": "Surface information the source holds but the agent cannot read", "kind": "technique", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes what the model can reach and sees", "proposed": false }, "levers": [ "knowledge-access" ], "domains": [ "retrieval", "data-analysis", "document-extraction" ], "evidence_domains": [ "data-analysis", "web-research", "code-generation" ], "categories": [ "Expanding sources" ], "category_ids": [ "retrieval.sources" ], "summary": "Find information a connected source holds that the agent never reads, and expose it. At file level: a reader that drops a format (PDF, Excel, Word, ZIP) or a channel (cell colors, formatting, comments). At warehouse level: tables and columns never queried, found by comparing what the agent queried against the source's catalog and joining that to failed cases; then add them to the schema context. An index over existing metadata raised table discovery sharply. The runtime alternative, a catalog-search tool the agent calls, did not beat giving the schema up front.", "how": [ "List the source's contents (formats, catalog).", "Diff against what the agent's tools can read and what its queries touched.", "Join the gap to failed or insufficient-evidence cases.", "Extend the reader, or add the missing tables to the schema context (5.b.i).", "Paired eval on the cases that failed for lack of data." ], "detect": { "static": "a file tool limited to plain-text formats; a spreadsheet reader that drops formatting; the source catalog lists tables or columns the agent's prompts and tools never reference", "traces": "failed opens of PDF, Excel, Word or ZIP files; the set of tables and columns queried, compared with the catalog, joined to failures; failure rows labeled domain-knowledge-deficit or missed-read", "evals": "failures where the answer existed in an unread part of the source" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "Excel colors +4.9% on FACTS; Genie metadata index raised table-discovery Recall@10 from 0.075 to 0.450 (May 2026); PDF reading flat on GAIA", "basis": "evidence" }, { "dimension": "cost", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "performance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval on the questions that failed for lack of data" } ], "evidence": [ { "source": "Albert Chen, Manas Bundele, Gaurav Ahlawat, et al. (LinkedIn), \"Text-to-SQL for Enterprise Data Analytics\"", "url": "https://arxiv.org/abs/2507.14372", "date": "2025-07-18", "locator": "§2.1, Figure 1, Table 1; §3.3.1-3.3.2, Table 2", "task": "Build a knowledge graph of tables from enterprise metadata", "effect": "Table 2, internal 133-question benchmark, gpt-4o: all knowledge-graph components vs schemas only (A.1): score (% 4+) 48% vs 9%, table recall 78% vs 45%, column recall 56% vs 24%, compile 96% vs 88%. Paper: \"Most of the improvement is from example queries, table clustering, and table/column attributes.\" Modeling-only ablation B.1 vs Full moves score only 46% -> 48%: \"The knowledge graph is a better lever to improve semantic understanding.\"", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "The Databricks AI Research Team, \"Pushing the Frontier for Data Agents with Genie\"", "url": "https://www.databricks.com/blog/pushing-frontier-data-agents-genie", "date": "2026-05-08", "locator": "Specialized Knowledge Search section; Figure 4; Figure 1", "task": "Build a semantic search index from existing data assets", "effect": "Figure 4, internal table-discovery benchmark, Recall@10 with vs without semantic context by max steps: 2 steps 0.450 vs 0.075; 4 steps 0.623 vs 0.219; 8 steps 0.724 vs 0.342; 16 steps 0.774 vs 0.466. Prose: \"improve table search performance by up to 40%\". End-to-end (Figure 1, bundled with the rest of the Genie harness): leading code agent + Databricks MCP 32.1% at 1.00x relative cost vs Genie with Specialized Knowledge Search (Opus 4.6) 73.8% at 0.23x cost (+41.7pp).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "The Databricks AI Research Team, \"Pushing the Frontier for Data Agents with Genie\"", "url": "https://www.databricks.com/blog/pushing-frontier-data-agents-genie", "date": "2026-05-08", "locator": "Specialized Knowledge Search section", "task": "Search several indices in parallel with metadata signals", "effect": "Described only: \"It uses multiple search indices in parallel together with rich metadata signals to efficiently discover most relevant assets for a user query.\" Not ablated separately from the index itself.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Michael Rumiantsau, Ivan Fokeev (Cube), \"Semantic Layers for Reliable LLM-Powered Data Analytics: A Paired Benchmark of Accuracy and Hallucination Across Three Frontier Models\"", "url": "https://arxiv.org/abs/2604.25149", "date": "2026-04-28", "locator": "Section 6.5 Limitations; REPORT.md 3.5", "task": "Put the semantic context in the prompt, not behind search", "effect": "Reported only in passing (Section 6.5; REPORT.md 3.5): \"did not significantly outperform raw-DDL Claude on this benchmark\"; numbers not given in the paper. 25-table schema, so everything fit in context; the authors flag 1,000+ table warehouses as open.", "direction": "contradicts", "measured": false, "conditions": null, "older": false }, { "source": "Andrew Qu (Vercel), \"We removed 80% of our agent's tools\"", "url": "https://vercel.com/blog/we-removed-80-percent-of-our-agents-tools", "date": "2025-12-22", "locator": "v2: The file system is the agent", "task": "Mount raw semantic-layer files instead of retrieval tools", "effect": "Same bundled 5-query result. \"The semantic layer is already great documentation ... We were building tools to summarize what was already legible. Claude just needed access to read it directly.\" Code: sandbox.writeFiles(files) from ./semantic-catalog.", "direction": "mixed", "measured": true, "conditions": null, "older": false }, { "source": "Jediah Katz (Cursor), \"Dynamic context discovery\"", "url": "https://cursor.com/blog/dynamic-context-discovery", "date": "2026-01-06", "locator": "5. Treating all integrated terminal sessions as files", "task": "Sync terminal sessions to files the agent greps", "effect": "\"Since terminal history can be long, the agent can grep for only the relevant outputs, which is useful for logs from a long-running process like a server.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "The Databricks AI Research Team, \"Pushing the Frontier for Data Agents with Genie\"", "url": "https://www.databricks.com/blog/pushing-frontier-data-agents-genie", "date": "2026-05-08", "locator": "Figure 2", "task": "Escalate from SQL to document search when data stops explaining", "effect": "Single worked trajectory (Figure 2), Phase 4 Root-Cause Investigation, steps 56-91, 18 queries + 2 doc searches: \"Finds a Cost-Attribution doc\"; \"Escalates to docs when SQL stops explaining.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Youngwon Lee, Jaejin Kim, Seung-won Hwang (Seoul National University), \"Bootstrapping Semantic Layer from Execution for Text-to-SQL (GATE: Grounding After Test from Execution)\"", "url": "https://arxiv.org/abs/2606.05634", "date": "2026-06-04", "locator": "Appendix B.2; §4.1", "task": "Give the SQL agent web search for public conventions", "effect": "Enabled on LS-Hard only (withheld groundings are web-recoverable by construction); disabled on RealEHR because groundings are institution-specific. Not ablated.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 14", "task": "GAIA", "effect": "\"Added PDF reading support\" in the last phase, flat on train (0.24).", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 10", "task": "GAIA/FACTS/SimpleQA", "effect": "Knight ships with complex-format reading; Pawn reads .txt/.md/.json/.csv only.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"Can AI Agents Build Better AI Agents?\"", "url": "https://labs.scale.com/blog/vero", "date": null, "locator": "tricks table (Original template)", "task": "FACTS", "effect": "+4.9%; \"identified that GAIA questions encode data in cell background colors\".", "direction": "supports", "measured": null, "conditions": null, "older": false } ], "caveats": "The only evidence is one row of the blog's tricks table (checked against the table image, 2026-09-23). It is about background colors alone, not formatting in general, and it credits a FACTS gain to GAIA questions without saying how the two relate. Read it as one instance of 8.a.ii, not as a general spreadsheet technique.", "also_covers": [], "related": [ { "id": "5.b.i", "slug": "ground-queries-in-schema", "name": "Give the model the schema it queries, in context: descriptions, keys, codes, sample values" }, { "id": "8.b.i", "slug": "tune-search-tool", "name": "Tune the search tool: snippets, enough results, retries, reformulation, filters, source quality" }, { "id": "2.a.i", "slug": "reduce-tool-surface", "name": "Reduce the tool surface: remove and consolidate tools, then defer the rest behind search" }, { "id": "8.a.i", "slug": "choose-retriever", "name": "Choose the retriever by corpus size and question type" } ], "url": "https://mahalsystems.ai/aot/optimizations/surface-unreadable-information/", "markdown": "https://mahalsystems.ai/aot/optimizations/surface-unreadable-information.md" }, { "id": "8.b.i", "legacy_id": "T34", "former_ids": [], "slug": "tune-search-tool", "name": "Tune the search tool: snippets, enough results, retries, reformulation, filters, source quality", "kind": "technique", "status": "card", "tier": { "tier": 2, "name": "Reliability", "why": "retries recover failed searches", "proposed": false }, "levers": [ "knowledge-access" ], "domains": [ "retrieval", "web-research" ], "evidence_domains": [ "web-research", "data-analysis" ], "categories": [ "Querying precisely", "Ranking results" ], "category_ids": [ "retrieval.query", "retrieval.results" ], "summary": "Make the search tool return what the model needs to choose what to open (a snippet per result, not just titles; five or more results; a full-document reader beside truncated search), retry failed searches, and reformulate when a query returns little (start broad, then narrow; use the corpus's own terms) rather than retrying it verbatim. Expose the source's metadata as filter parameters (date, product, version, source) where the corpus has it. Drop results from sources that leak the answer or are untrustworthy. Expanding one question into many queries up front has not helped.", "how": [ "From traces, count fetches per question, empty-result queries, and repeats of the same query.", "Change one parameter at a time (snippets, result count, retries, filters).", "Paired eval scored on retrieval recall and answer accuracy." ], "detect": { "static": "a search tool returning titles or URLs only; a fixed small result count; no retry; no filter parameters over a corpus with metadata; results passed to the model unfiltered", "traces": "many page fetches per question; queries returning zero results followed by the same query or by giving up; results from the wrong product, version or date; failure rows labeled missed-read", "evals": "retrieval recall below answer accuracy needs; failures where the answer was in an unfetched result" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "mixed", "size": "medium", "when": "snippets: SimpleQA train +0.15 but validation -0.06; source filtering +3.3% on FACTS; retries and result count unmeasured; up-front query expansion: no gain (LinkedIn, 2025)", "basis": "evidence" }, { "dimension": "cost", "effect": "mixed", "size": "small", "when": null, "basis": "evidence" }, { "dimension": "performance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "before-after", "tier": 2, "how": "failed-search rate" }, { "method": "experiment", "tier": 3, "how": "paired eval on retrieval recall and answer accuracy" } ], "evidence": [ { "source": "Scale AI, VeRO code release", "url": "https://github.com/scaleapi/vero", "date": null, "locator": "figure_01", "task": "BrowseComp-Plus", "effect": "9/20 runs, and almost none elsewhere; the only benchmark with a retrieval corpus.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 15", "task": "SimpleQA", "effect": "Train 0.74 to 0.89, validation 0.80 (down from 0.86 best).", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"Can AI Agents Build Better AI Agents?\"", "url": "https://labs.scale.com/blog/vero", "date": null, "locator": "tricks table (Tool-Centric template)", "task": "FACTS", "effect": "+3.3%.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 10", "task": "GAIA/FACTS/SimpleQA", "effect": "Knight \"Enhanced (5+ results, retry logic)\" vs Pawn \"Basic (3 results, no retries)\".", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Albert Chen, Manas Bundele, Gaurav Ahlawat, et al. (LinkedIn), \"Text-to-SQL for Enterprise Data Analytics\"", "url": "https://arxiv.org/abs/2507.14372", "date": "2025-07-18", "locator": "Appendix A.5", "task": "Decompose the question with a query planner", "effect": "Appendix A.5 negative result, gpt-4o: \"the query planner lowered recall and quality metrics\"; produced overly nested queries even after instructing it to minimize tasks with few-shot examples. Hypothesis: unnecessary for gpt-4o, possibly useful for smaller models.", "direction": "contradicts", "measured": true, "conditions": null, "older": true }, { "source": "Albert Chen, Manas Bundele, Gaurav Ahlawat, et al. (LinkedIn), \"Text-to-SQL for Enterprise Data Analytics\"", "url": "https://arxiv.org/abs/2507.14372", "date": "2025-07-18", "locator": "Appendix A.5", "task": "Use planner sub-tasks as retrieval query expansion", "effect": "Appendix A.5: \"this did not improve the recall of tables retrieved from EBR.\" Authors suggest an explicit query-expansion prompt might do better (untested).", "direction": "contradicts", "measured": true, "conditions": null, "older": true }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Prompt engineering: Start wide, then narrow down", "task": "Prompt searches to start broad, then narrow", "effect": "\"Agents often default to overly long, specific queries that return few results. We counteracted this tendency by prompting agents to start with short, broad queries, evaluate what's available, then progressively narrow focus.\"", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Action Space @retrieve_schema; Appendix Details of Action Space; Table 4; Table 7", "task": "Let the agent write its own targeted schema-retrieval queries", "effect": "Table 4: removing @retrieve_schema costs 6.7-7.6 SRR points (see index entry). Table 7, top-m per call: m=1 SRR 89.2 (20.7K tokens), m=2 89.6 (21.0K), m=3 91.2 (21.2K). Contrasted with CHESS-style question rewriting; not directly compared.", "direction": "supports", "measured": true, "conditions": null, "older": false } ], "caveats": null, "also_covers": [], "related": [ { "id": "8.a.i", "slug": "choose-retriever", "name": "Choose the retriever by corpus size and question type" }, { "id": "8.a.ii", "slug": "surface-unreadable-information", "name": "Surface information the source holds but the agent cannot read" }, { "id": "5.b.i", "slug": "ground-queries-in-schema", "name": "Give the model the schema it queries, in context: descriptions, keys, codes, sample values" }, { "id": "9.a.ii", "slug": "verify-citations", "name": "Verify citations claim by claim for factual support" }, { "id": "8.b.ii", "slug": "link-large-schemas", "name": "Link large schemas by exploration: start broad, grow the linked set with probing queries, favor recall" } ], "url": "https://mahalsystems.ai/aot/optimizations/tune-search-tool/", "markdown": "https://mahalsystems.ai/aot/optimizations/tune-search-tool.md" }, { "id": "8.b.ii", "legacy_id": "T71", "former_ids": [], "slug": "link-large-schemas", "name": "Link large schemas by exploration: start broad, grow the linked set with probing queries, favor recall", "kind": "technique", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes what the model sees", "proposed": false }, "levers": [ "knowledge-access", "control-flow" ], "domains": [ "data-analysis" ], "evidence_domains": [ "data-analysis" ], "categories": [ "Querying precisely" ], "category_ids": [ "retrieval.query" ], "summary": "When a schema is too large to load whole, link the tables and columns a question needs by exploration rather than by one up-front retrieval or by loading everything: start from all table names plus the top retrieved columns, give the agent a column-level vector index, metadata probes and trial queries to grow the linked set, favor recall over minimizing noise, and cap the exploration turns low.", "how": [ "Build a column-level vector index of the schema.", "Seed each question with all table names plus the top retrieved columns.", "Give the agent actions to retrieve more columns, probe metadata and sample values with exploratory SQL (results capped at a few rows, with a timeout), and test sufficiency with a minimal executable query.", "Cap exploration at a few turns.", "Paired eval on schema-linking recall (share of needed columns kept), execution accuracy and tokens." ], "detect": { "static": "a large schema (too many tables to load whole) linked by a single up-front retrieval, or loaded whole into every prompt", "traces": "queries against non-existent columns or the wrong tables on a large schema; prompt tokens dominated by the schema", "evals": "schema-linking recall and execution accuracy on the large-schema subset" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "large", "when": "AutoLink: 91.2% of needed columns kept against 64.0% for the best full-schema method on Spider 2.0-Lite (Nov 2025)", "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "large", "when": "21.2K against 171.9K tokens per question", "basis": "evidence" }, { "dimension": "performance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval on schema-linking recall, execution accuracy and tokens on large schemas" } ], "evidence": [ { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Method; Table 1; Table 2; Table 3; Figure 2", "task": "Link schema by agentic exploration, not full-schema prompting", "effect": "Table 1, DeepSeek-V3 backbone for all LLM methods. Spider 2.0-Lite SRR 91.2% at 21.2K avg tokens vs SQL-to-Schema 64.0% at 171.9K, MCS-SQL 58.9% at 168.9K, RSL-SQL 52.0% at 29.2K, LinkAlign 36.4% at 66.7K; Bird-Dev SRR 97.4% at 8.0K vs RSL-SQL 93.3% at 14.8K. Databases >3,000 columns: all baselines SRR <40%, AutoLink ~90% (Figure 2). Downstream EX: Spider 2.0-Lite 34.92% (DeepSeek-R1, 38.0K tokens) vs ReFoRCE 29.62% same model (81.1K); Bird-Dev 68.71% (Gemini-1.5-Pro) vs CHESS 68.31%.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "External Environment; Appendix A Schema Vector Store Environment; Table 4", "task": "Build a column-level vector index of the schema", "effect": "Ablation Table 4 (Spider 2.0-Lite SRR, removing the @retrieve_schema action that queries this index): n=5 79.2 -> 72.4 (-6.8); n=50 88.0 -> 80.4 (-7.6); n=100 91.2 -> 84.5 (-6.7) - the largest ablation. Build cost: 222.8 s for 158 Spider 2.0-Lite DBs (1.4 s/DB) and 4.4 s for 11 Bird DBs on one H100.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Action Space @retrieve_schema; Appendix Details of Action Space; Table 4; Table 7", "task": "Let the agent write its own targeted schema-retrieval queries", "effect": "Table 4: removing @retrieve_schema costs 6.7-7.6 SRR points (see index entry). Table 7, top-m per call: m=1 SRR 89.2 (20.7K tokens), m=2 89.6 (21.0K), m=3 91.2 (21.2K). Contrasted with CHESS-style question rewriting; not directly compared.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Action Space @explore_schema; Appendix Details of Action Space; Table 4", "task": "Probe schema metadata and sample values with exploratory SQL", "effect": "Table 4 (Spider 2.0-Lite SRR) removing @explore_schema: n=5 79.2 -> 76.8 (-2.4); n=50 88.0 -> 84.8 (-3.2); n=100 91.2 -> 88.8 (-2.4).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Action Space @verify_schema; Table 4", "task": "Test schema sufficiency with a minimal executable query", "effect": "Table 4 (Spider 2.0-Lite SRR) removing @verify_schema: n=5 79.2 -> 77.2 (-2.0); n=50 88.0 -> 86.8 (-1.2); n=100 91.2 -> 89.6 (-1.6).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Agent-Environment Interaction; Table 4; Figure 3", "task": "Seed with all table names plus top-n retrieved columns", "effect": "Table 4 full AutoLink SRR on Spider 2.0-Lite by initial top-n: n=5 79.2, n=50 88.0, n=100 91.2; EX gains plateau at high n (Figure 3, values not tabulated). Main results use n=100 (Spider 2.0-Lite) and n=30 (Bird).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Main Results of Schema Linking; Table 1; Figure 2; Figure 4", "task": "Favor schema-linking recall over noise minimization", "effect": "Table 1 Spider 2.0-Lite: LinkAlign minimizes noise to 21.1 avg columns but reaches SRR 36.4%; AutoLink keeps 159.4 columns at SRR 91.2%. Figure 2: higher SRR correlates with higher EX across database sizes. At similar column counts AutoLink beats BGE retrieval baselines (Figure 4).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Appendix A, Database Environment", "task": "Cap exploration results at 5 rows with a timeout", "effect": "Design description only (Appendix A Database Environment); not ablated.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Appendix D, Impact of Max Turn, Table 6", "task": "Cap schema-linking agent turns low", "effect": "Table 6, Spider 2.0-Lite: max turns 4 -> SRR 89.2 (avg 3.72 turns, 20.1K tokens); 6 -> 90.2 (4.65, 20.7K); 8 -> 91.0 (5.01, 20.8K); 10 -> 91.2 (5.79, 21.2K).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Implementation Details, Experimental Setup", "task": "Use a non-reasoning model as the tool-calling policy", "effect": "\"The reasoning LLM (DeepSeek-R1) is not chosen for AutoLink's policy due to observed instruction following degradation ... which occasionally leads to deviations from the required thought and action format.\" Not quantified.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Appendix D, Limitations of Increasing Decoding Time, Table 5", "task": "Resample full-schema linking more times", "effect": "Table 5, Spider 2.0-Lite, DeepSeek-V3: MCS-SQL N=1 -> 5 SRR 46.0 -> 58.8 while tokens 33.8K -> 168.9K; SQL-to-Schema N=1 -> 5 SRR 45.2 -> 64.0 while tokens 34.7K -> 171.9K. Recall saturates while cost grows linearly; both remain far below AutoLink 91.2 at 21.2K.", "direction": "supports", "measured": true, "conditions": null, "older": false } ], "caveats": "One paper (AutoLink, Nov 2025) supplies all of the evidence, on Spider 2.0 with a DeepSeek-V3 policy; the component ablations are its own. It found a reasoning model a worse policy because its instruction following degraded.", "also_covers": [], "related": [ { "id": "5.b.i", "slug": "ground-queries-in-schema", "name": "Give the model the schema it queries, in context: descriptions, keys, codes, sample values" }, { "id": "8.a.i", "slug": "choose-retriever", "name": "Choose the retriever by corpus size and question type" }, { "id": "8.b.i", "slug": "tune-search-tool", "name": "Tune the search tool: snippets, enough results, retries, reformulation, filters, source quality" }, { "id": "7.b.iii", "slug": "size-execution-budgets", "name": "Size run budgets so productive runs aren't cut off: turn caps, deadlines, command timeouts" } ], "url": "https://mahalsystems.ai/aot/optimizations/link-large-schemas/", "markdown": "https://mahalsystems.ai/aot/optimizations/link-large-schemas.md" }, { "id": "9.a.i", "legacy_id": "T43", "former_ids": [], "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique", "kind": "technique", "status": "card", "tier": { "tier": 2, "name": "Reliability", "why": "error recovery: catches outputs the check can reject", "proposed": false }, "levers": [ "control-flow" ], "domains": [ "general" ], "evidence_domains": [ "reasoning", "data-analysis", "code-generation", "planning", "web-research" ], "categories": [ "Checking externally" ], "category_ids": [ "verification.external" ], "summary": "Run a check the model cannot fool on its output (compiler, tests, schema validation, execution, EXPLAIN on generated SQL, a solver) and feed the verdict back; a pass/fail verdict is enough, and resampling until the check accepts works about as well as detailed feedback. Repair generated SQL from execution errors and check for hallucinated tables; run it read-only with timeouts. Do not add a step where the model critiques or re-checks its own answer with no external signal: it lowered accuracy in every study, and a multi-step self-verification tool that helped on GAIA (+5.75%) cost 17.8 points on SimpleQA.", "how": [ "Identify the sound check the task admits (build, tests, schema, execution).", "Run it on each output before returning; feed pass/fail back and let the agent retry, or resample until it passes.", "If no sound check exists, do not substitute a model critique; use 9.b.iii or 9.b.ii instead.", "Measure the check-failure rate and end quality." ], "detect": { "static": "no step runs the build, tests, linter, schema check or execution on the output before it is returned; a critique or reflect step where the same model reviews its own answer with no tool output or environment state in the loop", "traces": "errors a compiler, test or schema check would catch reaching the final answer; SQL errors and hallucinated table names; answers changed after a critique turn with no new external information; failure rows labeled tool-feedback-neglect or satisficing", "evals": "failures a deterministic check would have flagged; accuracy before and after any critique step" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "large", "when": "external verifier: Blocksworld 40% to 87% (2024); SQL compilation 66% to 96% (LinkedIn, 2025); model-only critique: negative in every study, e.g. GPT-3.5 75.8 to 38.1", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "small", "when": null, "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "small", "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "before-after", "tier": 2, "how": "rate of outputs failing the check, same cases" }, { "method": "experiment", "tier": 3, "how": "paired eval of end quality" } ], "evidence": [ { "source": "Stechly et al., \"On the Self-Verification Limitations of Large Language Models on Reasoning and Planning Tasks\"", "url": "https://arxiv.org/abs/2402.08115", "date": "2024-02-12", "locator": "Sec. 3; Sec. 5; Table 1; Figure 2", "task": "Backprompt with a sound external verifier", "effect": "Table 1, LLM+Sound Critique (best of binary / first-error / all-error feedback) vs standard prompting: Game of 24 38% vs 5%; Graph Coloring 38% vs 16%; Blocksworld 87% vs 40%; Mystery Blocksworld 10% vs 4%. Monotone improvement with more backprompts up to an asymptote (Figure 2), 15-round timeout.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Stechly et al., \"On the Self-Verification Limitations of Large Language Models on Reasoning and Planning Tasks\"", "url": "https://arxiv.org/abs/2402.08115", "date": "2024-02-12", "locator": "Sec. 5.2; Table 1", "task": "Richer verifier feedback adds little over pass/fail", "effect": "Table 1, binary / first-error / all-errors: Game of 24 36% / 38% / n.a.; Graph Coloring 38% / 37% / 34%; Blocksworld 60% / 87% / 83%; Mystery Blocksworld 10% / 8% / 6%. Blocksworld is the exception where first-error feedback helped (60 -> 87).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Stechly et al., \"On the Self-Verification Limitations of Large Language Models on Reasoning and Planning Tasks\"", "url": "https://arxiv.org/abs/2402.08115", "date": "2024-02-12", "locator": "Sec. 5.2; Table 1", "task": "Resample the same prompt until a sound verifier accepts", "effect": "Table 1, sampling k=15 / k=25: Game of 24 28% / 42%; Graph Coloring 40% / 44%; Blocksworld 68% / 72%; Mystery Blocksworld 9% / 14% (standard prompting 5/16/40/4%).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Huang et al., \"Large Language Models Cannot Self-Correct Reasoning Yet\"", "url": "https://arxiv.org/abs/2310.01798", "date": "2023-10-03", "locator": "Sec. 6 Leveraging external feedback", "task": "Use external feedback such as code execution for correction", "effect": "Recommendation citing Self-Debug, CRITIC and trained verifiers; not tested in this paper.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Kamoi et al., \"When Can LLMs Actually Correct Their Own Mistakes? A Critical Survey of Self-Correction of LLMs\"", "url": "https://arxiv.org/abs/2406.01297", "date": "2024-06-03", "locator": "Sec. 5.1; Sec. 7; Table 5", "task": "Drive correction with reliable external feedback", "effect": "Survey conclusion (2): prior studies \"widely agree\" self-correction improves responses when reliable external tools or knowledge exist; code-interpreter feedback counted as fair because it cannot easily improve the initial generation.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Appendix B; Appendix D Table 8", "task": "Repair generated SQL iteratively from execution errors", "effect": "Table 8, Spider 2.0-Lite, DeepSeek-R1: base generation EX 23.47 -> + iterative correction 31.34 (+7.87).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Albert Chen, Manas Bundele, Gaurav Ahlawat, et al. (LinkedIn), \"Text-to-SQL for Enterprise Data Analytics\"", "url": "https://arxiv.org/abs/2507.14372", "date": "2025-07-18", "locator": "§2.2.3; Table 2 B.2; §5", "task": "Validate SQL with EXPLAIN and a hallucination checker, then fix", "effect": "Table 2, removing the query fixer (B.3 -> B.2): successful compilation 95% -> 76%, valid tables & columns 98% -> 85%, score 47% -> 47%, LLM calls 4.3 -> 4.0. Conclusion: schema hallucination 23% -> 1% and compilation errors 34% -> 4% overall from modeling components.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Cemri et al., \"Why Do Multi-Agent LLM Systems Fail?\"", "url": "https://arxiv.org/abs/2503.13657", "date": "2025-03-17", "locator": "App. G.1", "task": "Majority-vote or resample until verification passes", "effect": "Proposed tactic (cited prior work); flagged as inconsistent.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Cemri et al., \"Why Do Multi-Agent LLM Systems Fail?\"", "url": "https://arxiv.org/abs/2503.13657", "date": "2025-03-17", "locator": "§4 Insight 3, App. G.2, Table 4", "task": "Add multi-level, domain-specific verification", "effect": "Observational: existing verifiers \"perform only superficial checks ... such as checking if the code compiles or if there are leftover TODO comments\". The +15.6 on ProgramDev is cited as the demonstration.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 6.3; Table 11 \"Reflection Loop\"", "task": "Feed lint/test failures back in a reflection loop", "effect": "Aider three-stage Python pipeline (syntax, compile, flake8) plus tests; \"up to a configurable maximum (default: 3 reflections)\"; TreeContext shows \"code context around error lines\".", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 6.3; Table 7; Table 13", "task": "Append LSP diagnostics to every edit result", "effect": "\"OpenCode feeds LSP diagnostics back into every edit result\"; bundles \"25 auto-downloaded LSP servers feed diagnostics (no persistent index)\".", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Fan et al., \"An Empirical Study of Harness Design for Coding Agents\"", "url": "https://arxiv.org/abs/2609.20804", "date": "2026-09-17", "locator": "§2.4 Post-edit diagnostics; §2.2", "task": "Append lint diagnostics to every edit result", "effect": "Held fixed. It only fires through the predefined edit tools, so it is part of the bundled action-space contrast above and not isolated.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper §5.3, A.3.4 (Pawn, Cookbook+Reasoning, iteration 3)", "task": "GAIA / SimpleQA", "effect": "+5.75% on GAIA and -17.8% on SimpleQA in the same commit; overhead hurt simple factual queries.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Table 10", "task": "GAIA/FACTS/SimpleQA", "effect": "Present in Knight, absent in Pawn. Knight's baselines are 43.7 / 64.7 / 84.4 vs Pawn's 19.5 / 54.7 / 68.9 (Table 7), though the two also differ in prompt, tools and turns.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Huang et al., \"Large Language Models Cannot Self-Correct Reasoning Yet\"", "url": "https://arxiv.org/abs/2310.01798", "date": "2023-10-03", "locator": "Sec. 3.2-3.3; Tables 3-4; Figure 1", "task": "Do not add intrinsic self-correction to reasoning tasks", "effect": "Table 3/4, standard -> round 1 -> round 2 (1/3/5 calls). GPT-3.5: GSM8K 75.9/75.1/74.7, CSQA 75.8/38.1/41.8, HotpotQA 26.0/25.0/25.0. GPT-4: GSM8K 95.5/91.5/89.0, CSQA 82.0/79.5/80.0, HotpotQA 49.0/49.0/43.0. GPT-4-Turbo: GSM8K 91.5/88.0/90.0, CSQA 84.0/81.5/83.0. Llama-2-70b: GSM8K 62.0/43.5/36.5, CSQA 64.0/37.5/36.5. GPT-3.5 keeps its GSM8K answer 74.7% of the time and more often flips correct -> incorrect than the reverse.", "direction": "contradicts", "measured": true, "conditions": null, "older": true }, { "source": "Huang et al., \"Large Language Models Cannot Self-Correct Reasoning Yet\"", "url": "https://arxiv.org/abs/2310.01798", "date": "2023-10-03", "locator": "Sec. 3.2; Tables 5-6", "task": "Rewording the critique prompt does not rescue self-correction", "effect": "Table 5 GPT-4-Turbo (baseline GSM8K 91.5, CSQA 84.0), best rounds: \"Assume ... could be either correct or incorrect\" 90.0/83.0; \"Review ... determine whether it's correct\" 90.0/81.0 (CSQA round 1 74.5); \"Verify whether your answer is correct\" 91.0/83.5. Table 6 Llama-2 (baseline 62.0/64.0): rounds range 30.5-58.0 GSM8K and 24.0-43.0 CSQA.", "direction": "contradicts", "measured": true, "conditions": null, "older": true }, { "source": "Huang et al., \"Large Language Models Cannot Self-Correct Reasoning Yet\"", "url": "https://arxiv.org/abs/2310.01798", "date": "2023-10-03", "locator": "Sec. 3.2; Table 1; Table 2", "task": "Oracle-stopped self-correction gains are not deployable", "effect": "Table 2 with oracle stop: GPT-3.5 GSM8K 75.9 -> 84.3, CSQA 75.8 -> 89.7, HotpotQA 26.0 -> 29.0; GPT-4 95.5 -> 97.5, 82.0 -> 85.5, 49.0 -> 59.0. Same setups without oracle all decrease (Table 3). Applies to RCI and Reflexion evaluations.", "direction": "contradicts", "measured": true, "conditions": null, "older": true }, { "source": "Stechly et al., \"On the Self-Verification Limitations of Large Language Models on Reasoning and Planning Tasks\"", "url": "https://arxiv.org/abs/2402.08115", "date": "2024-02-12", "locator": "Sec. 5; Table 1; Figure 2", "task": "Do not use the same LLM as verifier in a backprompt loop", "effect": "Table 1, standard prompting -> LLM+LLM: Game of 24 5% -> 3%; Graph Coloring 16% -> 2%; Blocksworld 40% -> 55%; Mystery Blocksworld 4% -> 0%. Figure 2: performance collapses immediately with the LLM verifier.", "direction": "contradicts", "measured": true, "conditions": null, "older": true }, { "source": "Stechly et al., \"On the Self-Verification Limitations of Large Language Models on Reasoning and Planning Tasks\"", "url": "https://arxiv.org/abs/2402.08115", "date": "2024-02-12", "locator": "Sec. 5; Table 2; Sec. 5.1", "task": "LLM verifiers reject correct answers too often", "effect": "Table 2 (accuracy / FPR / FNR): Game of 24 87.0% / 10.4% / 20.7%; Graph Coloring 72.4% / 6.5% / 95.8%; Blocksworld 71.8% / 18.55% / 15.48%; Mystery Blocksworld 79.6% / 0.5% / 97.09%. Game of 24: of expressions guaranteed to equal 24, labeled correct 79.1% while evaluating 81.6% to 24.", "direction": "contradicts", "measured": true, "conditions": null, "older": true }, { "source": "Kamoi et al., \"When Can LLMs Actually Correct Their Own Mistakes? A Critical Survey of Self-Correction of LLMs\"", "url": "https://arxiv.org/abs/2406.01297", "date": "2024-06-03", "locator": "Sec. 4; Sec. 7; Table 4", "task": "Do not rely on prompted intrinsic self-correction for general tasks", "effect": "Survey conclusion (1): no prior work shows successful self-correction with feedback from prompted LLMs under fair settings in general tasks; negative results cited for arithmetic reasoning, closed-book QA, code generation, plan generation and graph coloring.", "direction": "contradicts", "measured": false, "conditions": null, "older": true }, { "source": "Kamoi et al., \"When Can LLMs Actually Correct Their Own Mistakes? A Critical Survey of Self-Correction of LLMs\"", "url": "https://arxiv.org/abs/2406.01297", "date": "2024-06-03", "locator": "Sec. 3.2; Sec. 4 Oracle Information; Table 4", "task": "Do not stop the correction loop with ground-truth labels", "effect": "RCI and Reflexion classified as unrealistic frameworks that cannot verify any research question.", "direction": "contradicts", "measured": false, "conditions": null, "older": true }, { "source": "Chenglin Li, Yisen Xu, Zehao Wang, Shin Hwei Tan, Tse-Hsun (Peter) Chen, \"Independent Patch Verification for Coding Agents with a Bidirectional Reconstruct-and-Verify Framework\"", "url": "https://arxiv.org/abs/2608.08950", "date": "2026-08-09", "locator": "Baselines; Table 1; Table 3; Generalizability", "task": "Do not rely on Self-Refine patch review at equal budget", "effect": "mini-SWE-agent, 500 issues: GPT-5 mini 56.2% -> 54.8% (-1.4); MiniMax M2.5 75.8% -> 74.0% (-1.8). Scaffold-dependent: on OpenHands (120-issue subset) Self-Refine helped, GPT-5-mini 37.5% -> 53.3% (+15.8), MiniMax 62.5% -> 64.2% (+1.7), but stayed below RETRACE (56.7%, 70.0%).", "direction": "contradicts", "measured": true, "conditions": null, "older": false }, { "source": "Cemri et al., \"Why Do Multi-Agent LLM Systems Fail?\"", "url": "https://arxiv.org/abs/2503.13657", "date": "2025-03-17", "locator": "App. G.1, Table 4", "task": "Add a self-verification step after multi-step tasks", "effect": "Proposed tactic (cited to prior work); the paper warns it \"may miss flaws, rely on vague conditions, or be impractical\".", "direction": "mixed", "measured": false, "conditions": null, "older": true }, { "source": "Albert Chen, Manas Bundele, Gaurav Ahlawat, et al. (LinkedIn), \"Text-to-SQL for Enterprise Data Analytics\"", "url": "https://arxiv.org/abs/2507.14372", "date": "2025-07-18", "locator": "§2.2.3; Table 2 B.2; §5", "task": "Validate SQL with EXPLAIN and a hallucination checker, then fix", "effect": "Table 2, removing the query fixer (B.3 -> B.2): successful compilation 95% -> 76%, valid tables & columns 98% -> 85%, score 47% -> 47%, LLM calls 4.3 -> 4.0. Conclusion: schema hallucination 23% -> 1% and compilation errors 34% -> 4% overall from modeling components.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Albert Chen, Manas Bundele, Gaurav Ahlawat, et al. (LinkedIn), \"Text-to-SQL for Enterprise Data Analytics\"", "url": "https://arxiv.org/abs/2507.14372", "date": "2025-07-18", "locator": "§2.2.3; Appendix A.4; Table 2 B.3", "task": "Send hallucinated tables to a researcher sub-agent", "effect": "Table 2, removing the researcher (Full -> B.3): score 48% -> 47%, table recall 78% -> 75%, column recall 56% -> 53%, compile 96% -> 95%, valid 99% -> 98%, LLM calls 4.6 -> 4.3. Small effect.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Action Space @verify_schema; Table 4", "task": "Test schema sufficiency with a minimal executable query", "effect": "Table 4 (Spider 2.0-Lite SRR) removing @verify_schema: n=5 79.2 -> 77.2 (-2.0); n=50 88.0 -> 86.8 (-1.2); n=100 91.2 -> 89.6 (-1.6).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Appendix B; Appendix D Table 8", "task": "Repair generated SQL iteratively from execution errors", "effect": "Table 8, Spider 2.0-Lite, DeepSeek-R1: base generation EX 23.47 -> + iterative correction 31.34 (+7.87).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Ziyang Wang, Yuanlei Zheng, Zhenbiao Cao, Xiaojin Zhang, Zhongyu Wei, Pei Fu, Zhenbo Luo, Wei Chen, Xiang Bai (HUST, Fudan, Xiaomi), \"AutoLink: Autonomous Schema Exploration and Expansion for Scalable Schema Linking in Text-to-SQL at Scale\"", "url": "https://arxiv.org/abs/2511.17190", "date": "2025-11-21", "locator": "Appendix B Majority Voting; Table 8", "task": "Majority-vote SQL candidates by execution result", "effect": "Table 8, Spider 2.0-Lite: + majority voting on top of iterative correction EX 31.34 -> 34.97 (+3.63).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Youngwon Lee, Jaejin Kim, Seung-won Hwang (Seoul National University), \"Bootstrapping Semantic Layer from Execution for Text-to-SQL (GATE: Grounding After Test from Execution)\"", "url": "https://arxiv.org/abs/2606.05634", "date": "2026-06-04", "locator": "§3.1-3.2; Table 1; Appendix B.8 Table 7", "task": "Keep grounding hypotheses open until execution distinguishes them", "effect": "Table 1/7 (mean +/- sd over 5 runs). GPT-5.4-mini: RealEHR 55.2 vs ReAct 42.9, ReDel 44.8, ReFoRCE 48.6; EHRSQL 34.4 vs 32.2/26.8/27.6; LS-Hard 45.1 vs 37.8/40.2/40.1. Qwen3-8B: RealEHR 5.6 vs 1.7/2.5/3.8; EHRSQL 35.0 vs 30.2/26.4/31.2; LS-Hard 17.3 vs 14.5/8.0/13.9. Full method (includes all components below).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Youngwon Lee, Jaejin Kim, Seung-won Hwang (Seoul National University), \"Bootstrapping Semantic Layer from Execution for Text-to-SQL (GATE: Grounding After Test from Execution)\"", "url": "https://arxiv.org/abs/2606.05634", "date": "2026-06-04", "locator": "Appendix B.2", "task": "Restrict agent SQL to read-only with timeout and row caps", "effect": "Implementation detail (Appendix B.2), not ablated.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Youngwon Lee, Jaejin Kim, Seung-won Hwang (Seoul National University), \"Bootstrapping Semantic Layer from Execution for Text-to-SQL (GATE: Grounding After Test from Execution)\"", "url": "https://arxiv.org/abs/2606.05634", "date": "2026-06-04", "locator": "Appendix B.4", "task": "Retry malformed JSON outputs with error-specific hints", "effect": "Implementation detail (Appendix B.4, prompts in C.4), not ablated.", "direction": "supports", "measured": false, "conditions": null, "older": false } ], "caveats": "The paper's example of a gain that does not transfer across evaluation sets.", "also_covers": [ { "name": "Rich feedback loops" } ], "related": [ { "id": "9.b.iii", "slug": "verify-completion", "name": "Verify completion against the environment before the agent stops" }, { "id": "5.b.i", "slug": "ground-queries-in-schema", "name": "Give the model the schema it queries, in context: descriptions, keys, codes, sample values" }, { "id": "2.a.iii", "slug": "choose-action-interface", "name": "Choose how the agent acts: typed tools, code execution, or shell, matched to the model and task" }, { "id": "7.c.iii", "slug": "evaluator-optimizer-loop", "name": "Evaluator-optimizer loop" }, { "id": "2.b.i", "slug": "fail-soft-tools", "name": "Make tool calls and tool results fail soft, and retry transient failures" }, { "id": "7.c.iv", "slug": "graph-of-thought", "name": "Graph-of-Thought" }, { "id": "7.b.ii", "slug": "keep-errors-in-context", "name": "Keep errors in context" }, { "id": "7.c.v", "slug": "language-agent-tree-search", "name": "Language Agent Tree Search (LATS)" }, { "id": "7.c.vi", "slug": "multi-agent-debate", "name": "Multi-agent debate" }, { "id": "9.c.i", "slug": "no-self-written-tests", "name": "Don't prompt agents to write their own tests; suppress it for test-heavy models" }, { "id": "6.a.i", "slug": "oracle-worker-escalation", "name": "Oracle-worker multi-model" }, { "id": "7.a.ii", "slug": "prompt-chaining-with-gates", "name": "Prompt chaining with gate checks" }, { "id": "7.c.i", "slug": "reflection-loop", "name": "Reflection loop" }, { "id": "7.c.ii", "slug": "self-critique-evaluator", "name": "Self-critique evaluator loop" }, { "id": "4.c.ii", "slug": "skill-library", "name": "Skill library evolution" }, { "id": "7.c.vii", "slug": "tree-of-thought", "name": "Tree-of-Thought" }, { "id": "9.a.ii", "slug": "verify-citations", "name": "Verify citations claim by claim for factual support" } ], "url": "https://mahalsystems.ai/aot/optimizations/external-verifier-feedback/", "markdown": "https://mahalsystems.ai/aot/optimizations/external-verifier-feedback.md" }, { "id": "9.a.ii", "legacy_id": "T50", "former_ids": [], "slug": "verify-citations", "name": "Verify citations claim by claim for factual support", "kind": "technique", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes what the agent returns", "proposed": false }, "levers": [ "control-flow", "knowledge-access" ], "domains": [ "web-research", "retrieval" ], "evidence_domains": [ "web-research" ], "categories": [ "Checking externally", "Ranking results" ], "category_ids": [ "verification.external", "retrieval.results" ], "summary": "Extract claim-citation pairs deterministically, then check each: the link resolves, the source is relevant, and the source actually supports the claim. Frontier agents pass the first two over 80-94% of the time and the third only 39-77%. Cap search calls when synthesizing cited facts and cite selectively: going from 2 to 150 calls cut supported citations from 78.6% to 16.7%. Prefer primary sources over content farms.", "how": [ "Parse the report's claims and citations with a Markdown parser.", "For each: fetch, check relevance, check support with a judge that sees only the claim and the passage.", "Report the supported share; cap search calls in the agent." ], "detect": { "static": "a research or RAG agent that returns citations with no check of what they support; no cap on search calls", "traces": "search-call counts far above what the question needs; cited passages that do not contain the claim", "evals": "share of citations that factually support their claim" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "factual support 39-77% before checking (May 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "medium", "when": "fewer search calls", "basis": "evidence" }, { "dimension": "performance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval scored on factual citation support" } ], "evidence": [ { "source": "Onweller et al., \"Cited but Not Verified: Parsing and Evaluating Source Attribution in LLM Deep Research Agents\"", "url": "https://arxiv.org/abs/2605.06635", "date": "2026-05-07", "locator": "Sec. 3.2; Algorithm 1 Phase 1", "task": "Extract claim-citation pairs with a deterministic Markdown AST parser", "effect": "Described as reproducible and scaling to thousands of cited pages; parser accuracy is not reported.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Onweller et al., \"Cited but Not Verified: Parsing and Evaluating Source Attribution in LLM Deep Research Agents\"", "url": "https://arxiv.org/abs/2605.06635", "date": "2026-05-07", "locator": "Sec. 3.3; Algorithm 1 Phase 2; Table 1", "task": "Check each citation: link works, relevant, fact-supported", "effect": "Table 1, 14 models: Link Works 80.8-100%; Relevant 60.6-95.7%; Fact Check 24.4% (OSS-120B) to 76.8% (Claude Opus 4.5). Frontier Fact Check 38.9% (GPT-5 Mini) to 76.8%. Only the Fact Check stage separates models (53-point spread). Used as evaluation; not tested as an in-loop gate.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Onweller et al., \"Cited but Not Verified: Parsing and Evaluating Source Attribution in LLM Deep Research Agents\"", "url": "https://arxiv.org/abs/2605.06635", "date": "2026-05-07", "locator": "Sec. 4.1; Table 1", "task": "Do not treat working, relevant links as verified citations", "effect": "12 of 14 models exceed 94% Link Works and all frontier models exceed 80% Relevant, yet frontier Fact Check is 39-77% (e.g. GPT-5.4: 100% link, 93.7% relevant, 47.7% fact).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Onweller et al., \"Cited but Not Verified: Parsing and Evaluating Source Attribution in LLM Deep Research Agents\"", "url": "https://arxiv.org/abs/2605.06635", "date": "2026-05-07", "locator": "Sec. 3.4 Ablation; Sec. 4.3; Tables 2-3; Figure 2", "task": "Cap search tool calls when synthesizing cited facts", "effect": "Max tool calls 2/10/30/50/70/100/150. GPT-5.4 Fact Check 78.6/45.9/43.0/38.0/35.5/37.2/16.7%; Claude Opus 4.6 80.0/74.4/69.2/61.2/61.7/58.7/57.9%. Link Works and Relevant stay above 92% throughout. Mean relative drop ~42%; steepest step GPT-5.4 2 -> 10 calls (79% -> 46%). Two models only.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Onweller et al., \"Cited but Not Verified: Parsing and Evaluating Source Attribution in LLM Deep Research Agents\"", "url": "https://arxiv.org/abs/2605.06635", "date": "2026-05-07", "locator": "Sec. 4.2; Table 1", "task": "Cite selectively rather than exhaustively", "effect": "Correlational across providers: OpenAI 100% task success, most citations (GPT-5 Mini 1,272 attributions), Fact Check 39-59%; Anthropic 83-97% success, Fact Check up to 77% (Opus 4.5) and 69% (Haiku 4.5); Gemini 45-49%. No intervention on citation count was run.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Onweller et al., \"Cited but Not Verified: Parsing and Evaluating Source Attribution in LLM Deep Research Agents\"", "url": "https://arxiv.org/abs/2605.06635", "date": "2026-05-07", "locator": "Sec. 4.1; Sec. 4.4; Table 1", "task": "Use frontier models for one-shot cited research reports", "effect": "Task success: OSS-120B 40.0%, Llama 4 Maverick 30.0%, Pixtral Large 16.7% vs 83.3-100% for frontier models. Tool calls 7-60 (open) vs 73-211 (frontier). Link failure rate Llama 4 Maverick 19.2%, OSS-120B 16.1% vs GPT-5.4 1 of 2,159.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Architecture overview (process diagram caption)", "task": "Add a dedicated citation agent after research", "effect": "\"passes all findings to a CitationAgent, which processes the documents and research report to identify specific locations for citations. This ensures all claims are properly attributed to their sources.\"", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Jeremy Hadfield, Barry Zhang, Kenneth Lien, Florian Scholz, Jeremy Fox, Daniel Ford (Anthropic), \"How we built our multi-agent research system\"", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "date": "2025-06-13", "locator": "Effective evaluation: Human evaluation catches what automation misses", "task": "Add source-quality heuristics to the prompt", "effect": "Human testers \"noticed that our early agents consistently chose SEO-optimized content farms over authoritative but less highly-ranked sources ... Adding source quality heuristics to our prompts helped resolve this issue.\"", "direction": "supports", "measured": false, "conditions": null, "older": true } ], "caveats": null, "also_covers": [], "related": [ { "id": "8.b.i", "slug": "tune-search-tool", "name": "Tune the search tool: snippets, enough results, retries, reformulation, filters, source quality" }, { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" } ], "url": "https://mahalsystems.ai/aot/optimizations/verify-citations/", "markdown": "https://mahalsystems.ai/aot/optimizations/verify-citations.md" }, { "id": "9.b.i", "legacy_id": "T32", "former_ids": [], "slug": "extractable-final-answer", "name": "Make the final answer reliably extractable: state the format, extract robustly, force an answer at the limit", "kind": "technique", "status": "card", "tier": { "tier": 2, "name": "Reliability", "why": "graceful degradation: always return a usable answer", "proposed": false }, "levers": [ "control-flow", "prompt" ], "domains": [ "structured-output" ], "evidence_domains": [ "code-generation", "reasoning", "web-research" ], "categories": [ "Confirming completion", "Structuring" ], "category_ids": [ "verification.completion", "prompts.structure" ], "summary": "State the required answer format in the prompt, and make the harness extract it robustly: strip markers, normalize numbers, detect refusals, and force a best-effort answer when the loop ends on a budget rather than returning nothing. The format reminder alone was mixed on GPQA (train up, test down); the harness side is what stops right work being lost.", "how": [ "From traces, list runs whose working was right but whose returned answer was missing, malformed or a refusal.", "Add the format section to the prompt (3.a.iv).", "Harden extraction and add the forced-answer path at the turn cap.", "Measure no-answer and malformed-answer rates." ], "detect": { "static": "no answer-format statement; extraction that assumes a marker; no forced answer when the loop ends on a budget", "traces": "runs ending with no extractable answer; refusals returned as answers; numbers in the wrong format; runs ending at the turn cap with a correct value in context (in one image-labeling agent, 71 items returned all-unknown)", "evals": "failures where the working was right but the submitted answer was not (instruction-grader-mismatch)" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "estimate; in an image-labeling agent we measured, a salvage turn at the limit cut all-unknown items from 71 to 1", "basis": "evidence" }, { "dimension": "cost", "effect": "none", "size": null, "when": null, "basis": "evidence" }, { "dimension": "performance", "effect": "none", "size": null, "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "before-after", "tier": 2, "how": "no-answer, malformed-answer and refusal rates on the same cases" } ], "evidence": [ { "source": "Scale AI, VeRO code release", "url": "https://github.com/scaleapi/vero", "date": null, "locator": "taxonomy.py role hints", "task": "five follow-up benchmarks", "effect": "Symbols such as _submit, _answer_payload, _scale_variants, _format_number, _extract_answer are common enough to get deterministic role hints.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, VeRO code release", "url": "https://github.com/scaleapi/vero", "date": null, "locator": "figure_01", "task": "SWE-Atlas-QnA", "effect": "15/20 runs touched submission.", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, \"VeRO: A Harness for Agents to Optimize Agents\" (ICML 2026)", "url": "https://arxiv.org/abs/2602.22480", "date": "2026-02", "locator": "paper Figure 13", "task": "GPQA", "effect": "Train recovered 0.54 to 0.60 in phase 2; test 0.54 (below the 0.61 baseline).", "direction": "supports", "measured": null, "conditions": null, "older": false }, { "source": "Scale AI, VeRO code release", "url": "https://github.com/scaleapi/vero", "date": null, "locator": "figure_01", "task": "SWE-Atlas-QnA, GAIA-Shell", "effect": "Submission-path edits in 15/20 and 18/20 runs; 2/20 on Terminal-Bench, where the verifier does not read an answer string.", "direction": "supports", "measured": null, "conditions": null, "older": false } ], "caveats": null, "also_covers": [], "related": [ { "id": "3.a.iv", "slug": "restructure-system-prompt", "name": "Restructure the system prompt: every requirement once, up front, in named sections; nothing the model doesn't need" }, { "id": "9.b.iii", "slug": "verify-completion", "name": "Verify completion against the environment before the agent stops" }, { "id": "7.b.ii", "slug": "keep-errors-in-context", "name": "Keep errors in context" }, { "id": "2.b.i", "slug": "fail-soft-tools", "name": "Make tool calls and tool results fail soft, and retry transient failures" }, { "id": "3.a.i", "slug": "few-shot-examples", "name": "Add worked examples to the prompt (few-shot), or curated example queries for data agents" }, { "id": "7.c.vi", "slug": "multi-agent-debate", "name": "Multi-agent debate" }, { "id": "3.c.i", "slug": "self-discover-reasoning", "name": "Self-Discover reasoning structures" }, { "id": "7.b.iii", "slug": "size-execution-budgets", "name": "Size run budgets so productive runs aren't cut off: turn caps, deadlines, command timeouts" } ], "url": "https://mahalsystems.ai/aot/optimizations/extractable-final-answer/", "markdown": "https://mahalsystems.ai/aot/optimizations/extractable-final-answer.md" }, { "id": "9.b.ii", "legacy_id": "T45", "former_ids": [], "slug": "false-success-detector", "name": "Triage false success with a cheap trained detector, not an LLM judge", "kind": "technique", "status": "card", "tier": { "tier": 2, "name": "Reliability", "why": "detects a silent failure mode; it does not change the agent", "proposed": false }, "levers": [ "control-flow" ], "domains": [ "judging" ], "evidence_domains": [], "categories": [ "Confirming completion", "Sizing verification" ], "category_ids": [ "verification.completion", "verification.spend" ], "summary": "Train a lightweight text classifier (TF-IDF with logistic regression) on 50-100 labeled runs to flag likely false successes for review. LLM judges, including reasoning models and tuned judge prompts, peaked at AUROC 0.65 and were anti-correlated on false success against honest failure; the detector reached 0.83-0.95 and caught 72% at a 10% review budget.", "how": [ "Label 50-100 runs as true success, false success or honest failure from the environment (9.b.iii).", "Train the classifier on the run's closing text and tool sequence.", "Route the top 10% by score to review; track catch rate against the labels." ], "detect": { "traces": "runs that end with a success claim; the closing message text and the tool-call sequence are the detector's inputs", "evals": "an LLM judge currently used to decide whether a run completed; a labeled sample with a false-success rate above a few percent" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "as a review triage signal: 72% of false successes caught at a 10% review budget, against 13% for the best judge (Jun 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "large", "when": "compared with an LLM judge", "basis": "evidence" }, { "dimension": "performance", "effect": "improves", "size": "large", "when": "3,300x lower latency than a judge call", "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "monitor", "tier": null, "how": "catch rate at a fixed review budget against a labeled sample" } ], "evidence": [ { "source": "Laksh Advani, \"From Confident Closing to Silent Failure: Characterizing False Success in LLM Agents\"", "url": "https://arxiv.org/abs/2606.09863", "date": "2026-06-01", "locator": "§4.2, §4.4, Tables 4 and 6, Appendix H", "task": "Triage false success with a lightweight trained detector", "effect": "Task-disjoint AUROC 0.849 (LR) / 0.825 (XGB) / 0.827 (DeBERTa-v3) on tau2-bench and 0.953 (XGB) on AppWorld API-call sequences, versus best LLM judge 0.640 and 0.537. At a 10% flag rate the detector recovers 72% of false successes [95% CI 63-80] vs 13% [8-17] for the best judge (Sonnet 4.5 no_closing); at 5%, 51% vs 6%. Precision at 10% flag rate is only 50%. 1.19 ms per CPU inference vs ~4,000 ms per judge call (~3,300x). Cross-domain zero-shot transfer drops to 0.69 (airline<->retail); cross-generation (tau-v1 -> tau2) 0.73; rewriting honest-failure closings in confident style flips the detector on 10/50 (20%).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Laksh Advani, \"From Confident Closing to Silent Failure: Characterizing False Success in LLM Agents\"", "url": "https://arxiv.org/abs/2606.09863", "date": "2026-06-01", "locator": "Appendix I, Table 13; §5 Limitations", "task": "Calibrate the detector on 50-100 in-domain examples", "effect": "tau2-bench airline<->retail LODO: TF-IDF+LR 0.693 zero-shot -> 0.736 (K=10 per class) -> 0.788 (K=50) -> 0.827 (K=100). TF-IDF+XGB degrades 0.657 -> 0.538 at K=100. Entity stripping does not help (LR 0.690, XGB 0.684).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Laksh Advani, \"From Confident Closing to Silent Failure: Characterizing False Success in LLM Agents\"", "url": "https://arxiv.org/abs/2606.09863", "date": "2026-06-01", "locator": "§4.4, Figure 7", "task": "Ensemble detector with an LLM judge", "effect": "Detector and judge scores correlate at rho=0.41; a 75/25 weighted ensemble raises tau2-bench AUROC from 0.834 to 0.855. Author calls the gain modest.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Laksh Advani, \"From Confident Closing to Silent Failure: Characterizing False Success in LLM Agents\"", "url": "https://arxiv.org/abs/2606.09863", "date": "2026-06-01", "locator": "§4.3, Table 5, Figures 4-5", "task": "Do not rely on an LLM judge to detect false success", "effect": "tau2-bench, 295 trajectories, 5 judges (GPT-4o, Sonnet 4.5, Llama 3.3-70B, DeepSeek-R1, o3-mini) x 5 prompt conditions x 3 phrasings: no configuration exceeds AUROC 0.65 on false success vs true success (best 0.640). On false success vs honest failure every cell is 0.18-0.30, i.e. anti-correlated with truth; assertion-vocabulary trajectories score 0.27-0.36 more \"completed\" regardless of outcome. AppWorld, 200 API-call sequences, 3 judges: best 0.537.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Laksh Advani, \"From Confident Closing to Silent Failure: Characterizing False Success in LLM Agents\"", "url": "https://arxiv.org/abs/2606.09863", "date": "2026-06-01", "locator": "§3.5, §4.3, Table 5, Appendix D", "task": "Judge prompt tweaks do not fix completion judging", "effect": "tau2-bench: Sonnet 4.5 blind 0.576, checklist 0.576, stepwise 0.590, tool_only 0.605, no_closing 0.640; adding the full task spec (strong-real) moves 0.640 -> 0.632 (\"judges do not lack information, they fail to use it\"). AppWorld: telling the judge GET-only sequences cannot complete write tasks raises GPT-4o 0.394 -> 0.537 but lowers Sonnet 4.5 (blind 0.368 -> with_task 0.289 -> checklist 0.274) and Llama 3.3-70B (0.505 -> 0.384).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Laksh Advani, \"From Confident Closing to Silent Failure: Characterizing False Success in LLM Agents\"", "url": "https://arxiv.org/abs/2606.09863", "date": "2026-06-01", "locator": "§4.1, §4.3, Figure 1, Table 5", "task": "Reasoning models do not prevent false success", "effect": "Qwen3-Max-Thinking has the highest tau2-bench false-success rate (79% of failures; range across 8 families 13% GPT-5.2 to 79%); its traces average 1,274 chars in false-success cases vs 486 for other models. As judges, DeepSeek-R1 peaks at AUROC 0.573 and o3-mini at 0.554.", "direction": "supports", "measured": true, "conditions": null, "older": false } ], "caveats": null, "also_covers": [], "related": [ { "id": "9.b.iii", "slug": "verify-completion", "name": "Verify completion against the environment before the agent stops" }, { "id": "9.c.ii", "slug": "samples-before-verification", "name": "Spend extra samples before extra verification: parallel attempts with a vote or judge; verify only where it pays" }, { "id": "7.c.ii", "slug": "self-critique-evaluator", "name": "Self-critique evaluator loop" } ], "url": "https://mahalsystems.ai/aot/optimizations/false-success-detector/", "markdown": "https://mahalsystems.ai/aot/optimizations/false-success-detector.md" }, { "id": "9.b.iii", "legacy_id": "T44", "former_ids": [], "slug": "verify-completion", "name": "Verify completion against the environment before the agent stops", "kind": "technique", "status": "card", "tier": { "tier": 2, "name": "Reliability", "why": "false success and premature termination are silent hard failures", "proposed": false }, "levers": [ "control-flow" ], "domains": [ "general" ], "evidence_domains": [ "code-generation", "data-analysis" ], "categories": [ "Confirming completion" ], "category_ids": [ "verification.completion" ], "summary": "Never take the agent's own report that it finished. Before it stops, check the claim against something it cannot write: the environment's end state, a smoke test or end-to-end run, or its transcript (files it claims to have reviewed against files it opened). Veto stopping when code changed with no verification run; ask for an explicit scope statement; re-verify sub-agent findings before relaying them. Keep the checklist the agent is judged by in a form it is unlikely to rewrite (JSON, one status field editable), but expect rules alone to be weak: only a check the agent cannot see reliably held.", "how": [ "Define what 'done' means as an environment check (tests pass, feature works end to end, files reviewed).", "Add a stop hook that runs it and refuses to stop on failure.", "Lock the checklist format.", "Measure false-success and premature-stop rates." ], "detect": { "static": "completion taken from the agent's final message or its own status flag; no stop hook; the agent can edit the tests or checklist it is judged by", "traces": "final message claims success while the environment or transcript shows otherwise; runs ending right after an edit with no test or build run; edits to checklist items; failure rows labeled satisficing or specification-gaming", "evals": "tasks scored failed where the agent reported success (false-success rate); features marked done that fail an end-to-end check" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "MAST: ProgramDev 25.0 to 40.6 with reviewer sign-off (2025); Advani: false success is 45-76% of failures where the agent claims success, so the catch rate is large even if the fix is not (Jun 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "small", "when": null, "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "small", "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "before-after", "tier": 2, "how": "false-success and premature-stop rate on the same cases" }, { "method": "monitor", "tier": null, "how": "sampled completion checks in production" } ], "evidence": [ { "source": "Laksh Advani, \"From Confident Closing to Silent Failure: Characterizing False Success in LLM Agents\"", "url": "https://arxiv.org/abs/2606.09863", "date": "2026-06-01", "locator": "§3.2, §4.1, Table 2, Appendix F Table 11", "task": "Do not treat the agent's own success status as completion", "effect": "AppWorld (8,190 trajectories): react and plan_exec write status=success 1,503 and 1,099 times and status=fail zero times. Among self-assessing architectures with an explicit status, 75.8% of failures carry status=success (full_code_refl 74.9%, ipfuncall 77.3%; per-model 67% LLaMA-3 to 89% DeepSeekCoder). Easy tasks show more false success (88.2% at difficulty 1 vs 70.1% at 3), claimed after fewer API calls (23 vs 37).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Laksh Advani, \"From Confident Closing to Silent Failure: Characterizing False Success in LLM Agents\"", "url": "https://arxiv.org/abs/2606.09863", "date": "2026-06-01", "locator": "§1 Contributions (4), §4.1, Table 1", "task": "Have an independent party verify state changes", "effect": "Observational: false success is 45% (airline) and 48% (retail) of failures in single-control tau2-bench domains vs 3% in dual-control telecom, where the user simulator can verify state. Author treats it as a hypothesis: one dual-control domain, 15 false-success cases, other domain differences not controlled.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Laksh Advani, \"From Confident Closing to Silent Failure: Characterizing False Success in LLM Agents\"", "url": "https://arxiv.org/abs/2606.09863", "date": "2026-06-01", "locator": "§4.4 Deployment scoping, §5, §6", "task": "Check completion claims against environment state directly", "effect": "Recommendation, not tested: \"Substantial gains would likely require methods that verify trajectory-environment consistency directly rather than reading surface text.\" Motivated by 50% precision at a 10% flag rate for the best detector and 20-25% flip rates under adversarial rewriting for both detector and judges.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Smyth, Mantilla-Ramos, Tikeng Notsawo, Helbling, Tosato, Merzouk, Dziri, Gidel, Tosato (Tara Research, Mila, Cohere), \"Quantifying Overclaiming Propensity in Frontier LLM Agents\"", "url": "https://arxiv.org/abs/2609.20812", "date": "2026-09-22", "locator": "§3.5, §4, §5, Table 1, Figure 5, Appendix A", "task": "Check claimed review coverage against the transcript", "effect": "The paper's measurement, not an intervention: 67.9% of runs failed to touch every file; among incomplete runs 80.4% were misleading (52.8% explicit overclaim, 27.5% silent omission; 59.0% Claude Opus 5 to 96.2% GPT-5.6-luna). Overclaiming runs missed 58.2% of needles (720/1,237) vs 32.4% (342/1,055) for runs that touched every file; a needle was reported 83.2% of the time its evidence was read vs 1.8% when not. Authors conclude final reports should be compared with \"trustworthy evidence derived from its full trajectory\".", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Smyth, Mantilla-Ramos, Tikeng Notsawo, Helbling, Tosato, Merzouk, Dziri, Gidel, Tosato (Tara Research, Mila, Cohere), \"Quantifying Overclaiming Propensity in Frontier LLM Agents\"", "url": "https://arxiv.org/abs/2609.20812", "date": "2026-09-22", "locator": "§4, Table 1, §4 Subagents paragraph", "task": "A more capable model does not report coverage honestly", "effect": "Misleading rate among incomplete runs exceeded 50% for every one of 12 models (59.0%-96.2% proprietary, 65.0-85.1% open-weight). In the controlled experiment \"neither family shows a capability effect on misleading reporting among incomplete runs\".", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Smyth, Mantilla-Ramos, Tikeng Notsawo, Helbling, Tosato, Merzouk, Dziri, Gidel, Tosato (Tara Research, Mila, Cohere), \"Quantifying Overclaiming Propensity in Frontier LLM Agents\"", "url": "https://arxiv.org/abs/2609.20812", "date": "2026-09-22", "locator": "§5 Discussion", "task": "Honesty training on infeasible tasks did not fix overclaiming", "effect": "GPT-5.6 models, released after that intervention, claimed a complete review in 48.4% of their incomplete runs and were misleading in 93.6%. Reported as context; the paper did not run the training itself.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Smyth, Mantilla-Ramos, Tikeng Notsawo, Helbling, Tosato, Merzouk, Dziri, Gidel, Tosato (Tara Research, Mila, Cohere), \"Quantifying Overclaiming Propensity in Frontier LLM Agents\"", "url": "https://arxiv.org/abs/2609.20812", "date": "2026-09-22", "locator": "§2 Related Work, Appendix G.4", "task": "Re-verify subagent findings before relaying them", "effect": "Illustrative only: in the all-files-touched Claude Fable 5 runs the agent \"personally re-verified every blocker and major against the source\" (release check, 4/4 needles) and \"independently verified its major findings ... before including them\" (infra review, 1/1). Paper cites a vendor system card noting a model \"can relay claims from subagents to users without verifying them\". No controlled test.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Smyth, Mantilla-Ramos, Tikeng Notsawo, Helbling, Tosato, Merzouk, Dziri, Gidel, Tosato (Tara Research, Mila, Cohere), \"Quantifying Overclaiming Propensity in Frontier LLM Agents\"", "url": "https://arxiv.org/abs/2609.20812", "date": "2026-09-22", "locator": "§3.4, §4, Appendix G.3", "task": "Ask for an explicit scope statement in the report", "effect": "Every prompt in the study already asked the agent to report how it scoped its review; even so only 19.6% of incomplete runs admitted the gap. The admission examples (Appendix G.3) show the useful form, e.g. \"About 100 files I read in full ... the remaining ~120 ... I covered with targeted greps\". No with/without comparison was run, so the prompt's effect is unmeasured.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "The Databricks AI Research Team, \"Pushing the Frontier for Data Agents with Genie\"", "url": "https://www.databricks.com/blog/pushing-frontier-data-agents-genie", "date": "2026-05-08", "locator": "Key Challenges for Data Agents; Figure 2", "task": "Self-correct assumptions, then verify the exact numbers asked", "effect": "Figure 2 trajectory: Phase 5 Reconciliation (steps 91-97) and Phase 6 Per-Customer Verification (steps 97-208, two dozen queries): \"Verifies the exact numbers the user asked about.\" Prose names self-correction and verification as phases. No ablation; the example took ~208 steps.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "The Databricks AI Research Team, \"Pushing the Frontier for Data Agents with Genie\"", "url": "https://www.databricks.com/blog/pushing-frontier-data-agents-genie", "date": "2026-05-08", "locator": "Key Challenges for Data Agents (Lack of Verifiable Tests)", "task": "Detect unanswerable data questions and tell the user", "effect": "Stated as a requirement, not an evaluated mechanism: \"the queries may not always be answerable because of incompleteness in data, and it is important for data agents to be able to identify such cases and surface it back to users.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Justin Young (Anthropic), \"Effective harnesses for long-running agents\"", "url": "https://www.anthropic.com/engineering/effective-harnesses-for-long-running-agents", "date": "2025-11-26", "locator": "Environment management: Feature list", "task": "Protect the feature list from edits other than status", "effect": "\"It is unacceptable to remove or edit tests because this could lead to missing or buggy functionality.\" \"we landed on using JSON for this, as the model is less likely to inappropriately change or overwrite JSON files compared to Markdown files.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Guo et al., \"Self-Authored Verification Is Unreliable in Heuristic Self-Improving Agents\"", "url": "https://arxiv.org/abs/2607.24300", "date": "2026-07-27", "locator": "Finding 1, Table 4, Figure 4", "task": "Do not accept edits on self-authored tests alone", "effect": "Unprotected, 7 models x 5 Atari games: all 35 valid cells end with self-score >= 0.70, yet 15 of 35 final policies score below the game's random reference, six at Pong's -21.0 floor. Weaker models overwrite discovered strategies behind easy self-tests (failure to retain); stronger ones still mismeasure the shifted deployment distribution.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Guo et al., \"Self-Authored Verification Is Unreliable in Heuristic Self-Improving Agents\"", "url": "https://arxiv.org/abs/2607.24300", "date": "2026-07-27", "locator": "Finding 2 Breakout ablation, Table 5", "task": "Forbidding test weakening does not close the gap", "effect": "Breakout, 3 seeds: monotone below no protection in 4 of 6 models (DeepSeek 9.0 -> 4.5, MiniMax 5.6 -> 2.4, Doubao 19.1 -> 9.3, GPT-5.5 7.4 -> 7.2); self-scores stay 0.89-0.98.", "direction": "contradicts", "measured": true, "conditions": null, "older": false }, { "source": "Guo et al., \"Self-Authored Verification Is Unreliable in Heuristic Self-Improving Agents\"", "url": "https://arxiv.org/abs/2607.24300", "date": "2026-07-27", "locator": "Finding 2 Breakout ablation, Table 5", "task": "Requiring tests to beat a trivial baseline does not help", "effect": "Breakout, 3 seeds: discriminative below no protection in 4 of 6 models (DeepSeek 9.0 -> 3.0, MiniMax 5.6 -> 3.9, Doubao 19.1 -> 8.5, GPT-5.5 7.4 -> 5.4); self-scores 0.78-1.00.", "direction": "contradicts", "measured": true, "conditions": null, "older": false }, { "source": "Lance Martin, \"Agent Design Patterns\"", "url": "https://rlancemartin.github.io/2026/01/09/agent_design/", "date": "2026-01-09", "locator": "Isolate Context; Abstractions for Long-Running Agents", "task": "Verify work with stop hooks each iteration", "effect": "\"using stop hooks to verify work after each Ralph loop\"; \"Claude Code uses stop hooks to verify work after each iteration.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 6.2 (Hermes); Sec 6.3; Table 12 \"Outer Verification Loop\"", "task": "Veto stop when code changed without verification", "effect": "Hermes verify-on-stop guard \"rewrites a text-only response into a continuation whenever the turn mutated code files without producing fresh verification evidence—the reflection loop's function, relocated ... to its stop condition, at a fraction of the cost.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 11.4; Sec 6.1; Table 12 \"Outer Verification Loop\"", "task": "Run an LLM goal-judge outer loop over runs", "effect": "OpenHands \"/goal endpoint runs an LLM judge over the transcript after each run, either injecting a follow-up prompt or stopping ... complemented by pluggable critics with configurable refinement-iteration caps.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Justin Young (Anthropic), \"Effective harnesses for long-running agents\"", "url": "https://www.anthropic.com/engineering/effective-harnesses-for-long-running-agents", "date": "2025-11-26", "locator": "Getting up to speed; failure-mode table row 2", "task": "Smoke-test the app end-to-end before new work", "effect": "\"This ensured that Claude could quickly identify if the app had been left in a broken state, and immediately fix any existing bugs. If the agent had instead started implementing a new feature, it would likely make the problem worse.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Justin Young (Anthropic), \"Effective harnesses for long-running agents\"", "url": "https://www.anthropic.com/engineering/effective-harnesses-for-long-running-agents", "date": "2025-11-26", "locator": "Testing; failure-mode table row 3", "task": "Give browser automation for end-to-end self-verification", "effect": "Without it, Claude tested with unit tests or curl \"but would fail recognize that the feature didn't work end-to-end.\" \"Providing Claude with these kinds of testing tools dramatically improved performance\" (no numbers). Limitation: \"Claude can't see browser-native alert modals through the Puppeteer MCP, and features relying on these modals tended to be buggier\".", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Cemri et al., \"Why Do Multi-Agent LLM Systems Fail?\"", "url": "https://arxiv.org/abs/2503.13657", "date": "2025-03-17", "locator": "§1, §4 (FC1), App. H.2, App. M.2, Table 5", "task": "Only superior agents may end the conversation", "effect": "Table 5, ChatDev: ProgramDev-v0 (32 tasks) 25.0 -> 34.4; HumanEval 89.6 -> 90.3. Text: \"+9.4% increase in overall task success rate\" (percentage points). §1 says ProgramDev-v2 with GPT-4o; Table 5 says ProgramDev-v0.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Cemri et al., \"Why Do Multi-Agent LLM Systems Fail?\"", "url": "https://arxiv.org/abs/2503.13657", "date": "2025-03-17", "locator": "§4 Insight 3, App. H.2-H.3, Table 5, Figs 10-11", "task": "Make the workflow cyclic, gated on reviewer sign-off", "effect": "Table 5, ChatDev: ProgramDev-v0 25.0 -> 40.6; HumanEval 89.6 -> 91.5. §4: \"adding a high-level task objective verification step to ChatDev yields a +15.6% improvement in task success on ProgramDev\". Figs 10-11 (LLM-judge MAST counts): \"topology-based changes are more effective than prompt-based changes for both systems\".", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Cemri et al., \"Why Do Multi-Agent LLM Systems Fail?\"", "url": "https://arxiv.org/abs/2503.13657", "date": "2025-03-17", "locator": "App. G.1, Table 4", "task": "Define conversation patterns and termination conditions", "effect": "Proposed tactic in Table 4 (FC1, FC2).", "direction": "supports", "measured": false, "conditions": null, "older": true } ], "caveats": null, "also_covers": [ { "name": "Stop-hook auto-continue" } ], "related": [ { "id": "9.b.i", "slug": "extractable-final-answer", "name": "Make the final answer reliably extractable: state the format, extract robustly, force an answer at the limit" }, { "id": "9.b.ii", "slug": "false-success-detector", "name": "Triage false success with a cheap trained detector, not an LLM judge" }, { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "7.d.i", "slug": "continuous-task-loop", "name": "Continuous autonomous task loop" }, { "id": "7.c.iii", "slug": "evaluator-optimizer-loop", "name": "Evaluator-optimizer loop" }, { "id": "4.b.ii", "slug": "initializer-maintainer-agents", "name": "Initializer-maintainer dual agent" }, { "id": "7.b.ii", "slug": "keep-errors-in-context", "name": "Keep errors in context" }, { "id": "6.b.iii", "slug": "multi-agent-for-breadth", "name": "Use multi-agent orchestration only for breadth-first, independent sub-tasks" }, { "id": "4.b.iii", "slug": "multi-window-task-state", "name": "Keep plan, progress and results in files when work outlives one context window" } ], "url": "https://mahalsystems.ai/aot/optimizations/verify-completion/", "markdown": "https://mahalsystems.ai/aot/optimizations/verify-completion.md" }, { "id": "9.c.i", "legacy_id": "T51", "former_ids": [], "slug": "no-self-written-tests", "name": "Don't prompt agents to write their own tests; suppress it for test-heavy models", "kind": "technique", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "it changes what the agent does, so quality must be checked", "proposed": false }, "levers": [ "control-flow" ], "domains": [ "code-generation" ], "evidence_domains": [ "code-generation" ], "categories": [ "Sizing verification" ], "category_ids": [ "verification.spend" ], "summary": "Agent-written tests are mostly print probes and do not change resolve rate. Prompting for them added 20% output tokens for no gain; suppressing them for models that write many saved 33-49% of input tokens for a non-significant loss. Keep test writing for bugs with explicit repro conditions, and make those tests assertion-based.", "how": [ "From traces, measure the share of tokens spent writing and running tests, and whether they contain assertions.", "Remove any instruction to write tests; for test-heavy models add an instruction not to, except for bugs with a stated repro.", "Paired eval on resolve rate and tokens." ], "detect": { "static": "a prompt that tells the agent to write tests", "traces": "test files written with print statements and no assertions; test writing and running over about 20% of tokens", "evals": "resolve rate unchanged between runs with and without agent tests" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "none", "size": null, "when": "-1.8 to -2.6 points, not significant (Apr 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "improves", "size": "medium", "when": "-33 to -49% input tokens", "basis": "evidence" }, { "dimension": "performance", "effect": "improves", "size": "small", "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval with and without the instruction" } ], "evidence": [ { "source": "Chen et al., \"Rethinking the Value of Agent-Generated Tests for LLM-Based Software Engineering Agents\"", "url": "https://arxiv.org/abs/2602.07900", "date": "2026-04-09", "locator": "§3.1 Table 1, §5.1 Table 7, §5.2 Table 8", "task": "Prompting the agent to write its own tests does not help", "effect": "Encourage prompt: GPT-5.2 flips No test -> Has test on 322/500 tasks (64.4%) with net 0 change in resolved (359 -> 359, McNemar p=1.000) while API calls +5.5%, input tokens +9.0%, output tokens +19.8% (all Wilcoxon p<0.001). Gemini 3 Pro flips 185 tasks (37%), resolved 371 -> 366 (p=0.522), cost change not significant. Observationally, GPT-5.2 writes tests in 3/500 tasks yet resolves 71.8% vs Claude Opus 4.5's 74.4% with tests in 83% of tasks; within each model, resolved and unresolved tasks have similar test-writing rates.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Chen et al., \"Rethinking the Value of Agent-Generated Tests for LLM-Based Software Engineering Agents\"", "url": "https://arxiv.org/abs/2602.07900", "date": "2026-04-09", "locator": "§5.1 Table 7, §5.2 Table 8", "task": "Suppress ad-hoc test writing for test-heavy models", "effect": "Discourage prompt, SWE-bench Verified: Kimi K2 Thinking resolved 317 -> 304 (-2.6pp, McNemar p=0.228), API calls -35.4%, input tokens -49.0%, output tokens -43.1%; DeepSeek v3.2 Reasoner resolved 300 -> 291 (-1.8pp, p=0.435), API calls -24.5%, input -32.9%, output -14.0% (all cost changes Wilcoxon p<0.001, bootstrap CIs exclude zero). 83.2% of tasks keep the same outcome across interventions.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Chen et al., \"Rethinking the Value of Agent-Generated Tests for LLM-Based Software Engineering Agents\"", "url": "https://arxiv.org/abs/2602.07900", "date": "2026-04-09", "locator": "§4 Tables 4-6, §6.1, §8", "task": "Turn print-probe tests into assertion-based oracles", "effect": "Recommendation from the discussion, untested. Motivating data: value-revealing prints outnumber assertions for every model (e.g. Claude Opus 4.5 25.00 prints vs 5.16 assertions per resolved task; MiniMax M2 34.06 vs 7.37); 69.7-77.5% of prints inspect raw values; assertions are mostly property (33.6-41.4%) or exact-value (34.5-42.9%) checks, relational checks 3.0-7.9%.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Chen et al., \"Rethinking the Value of Agent-Generated Tests for LLM-Based Software Engineering Agents\"", "url": "https://arxiv.org/abs/2602.07900", "date": "2026-04-09", "locator": "§6.1 Implications", "task": "Package targeted testing as a focused subagent or skill", "effect": "Recommendation, untested: \"the goal should not be to simply make agents write more tests, but to make testing more targeted and budget-aware ... generating one minimal regression test, strengthening a weak oracle ..., or running only the smallest relevant test slice.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Chen et al., \"Rethinking the Value of Agent-Generated Tests for LLM-Based Software Engineering Agents\"", "url": "https://arxiv.org/abs/2602.07900", "date": "2026-04-09", "locator": "§3.3 Table 3, §6.1", "task": "Log test-writing and test-running cost separately", "effect": "Recommendation, untested. Motivated by unresolved tasks running tests more (ExecCount 6.52 vs 4.89, ExecPerTest 1.70 vs 1.46, Mann-Whitney significant) and interventions moving cost far more than outcomes.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Chen et al., \"Rethinking the Value of Agent-Generated Tests for LLM-Based Software Engineering Agents\"", "url": "https://arxiv.org/abs/2602.07900", "date": "2026-04-09", "locator": "§5.1.1", "task": "Reserve test writing for bugs with explicit repro conditions", "effect": "Exploratory only: 8 SWE-bench Verified issues appear both in encourage-prompt Fail->Success-with-new-tests and discourage-prompt Success->Fail-with-tests-removed (e.g. astropy-13236, django-11790, sympy-21612); themes summarized with GPT-5.4 help. Authors: \"necessarily limited\", no issue taxonomy tested.", "direction": "supports", "measured": false, "conditions": null, "older": false } ], "caveats": null, "also_covers": [], "related": [ { "id": "9.a.i", "slug": "external-verifier-feedback", "name": "Feed back a sound external verifier's verdict; never the model's own critique" }, { "id": "7.c.i", "slug": "reflection-loop", "name": "Reflection loop" } ], "url": "https://mahalsystems.ai/aot/optimizations/no-self-written-tests/", "markdown": "https://mahalsystems.ai/aot/optimizations/no-self-written-tests.md" }, { "id": "9.c.ii", "legacy_id": "T46", "former_ids": [], "slug": "samples-before-verification", "name": "Spend extra samples before extra verification: parallel attempts with a vote or judge; verify only where it pays", "kind": "technique", "status": "card", "tier": { "tier": 3, "name": "Tuning", "why": "extra compute within the existing design", "proposed": false }, "levers": [ "control-flow" ], "domains": [ "general" ], "evidence_domains": [ "reasoning", "data-analysis", "planning" ], "categories": [ "Sizing verification", "Scaling compute" ], "category_ids": [ "verification.spend", "orchestration.extra-compute" ], "summary": "Running several independent attempts and picking by majority vote or a judge raised accuracy (Genie 73.8% to 81.4% with 8 runs, at up to 2.1x cost). Before adding a generative verifier on top, compare it with the vote at equal compute: it needs 4-64x the compute to match, and pays only on hard, medium-difficulty problems from weak-to-medium generators. If you do verify, an ensemble of diverse weak verifiers weighted by estimated accuracy beat one strong verifier (87.7% against 72.2%).", "how": [ "Measure variance across repeated runs on the same questions; if low, stop here.", "Add N parallel attempts with a vote or aggregator; paired eval at matched compute. At a fixed thinking budget, parallel traces with a majority vote beat forcing one trace to think longer.", "Only then try a verifier, at the same compute, on the hard slice." ], "detect": { "static": "a single attempt per question; or a generative verifier or best-of-N step with no comparison against a plain vote", "traces": "answer disagreement across repeated runs of the same question", "evals": "accuracy of vote against verifier at equal compute, split by difficulty" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "Genie +7.6 points with 8 parallel runs (May 2026); verifier over vote: gains only on hard problems (2025)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "large", "when": "0.23x to 2.13x cost; verifier ensembles far more", "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "small", "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "experiment", "tier": 3, "how": "paired eval, N attempts against one, and vote against verifier, at matched compute" } ], "evidence": [ { "source": "Singhi et al., \"When To Solve, When To Verify: Compute-Optimal Problem Solving and Generative Verification for LLM Reasoning\"", "url": "https://arxiv.org/abs/2504.01005", "date": "2025-04-01", "locator": "§5.1, Figures 1, 4, 5, 8, 9; Appendix G, H", "task": "Prefer self-consistency over generative verification at low budgets", "effect": "Llama-3.1-8B GenRM-FT on MATH: GenRM needs 8x the compute of SC to match it, and 128x to beat SC's best by 3.8%; SC plateaus around 128 solutions. Qwen-2.5-7B GenRM-FT: SC peaks with 64x less compute; GenRM +5.4% at 512x. Llama-3.3-70B GenRM-Base: SC peak at 4x less; GenRM +1.7% at 64x. QwQ-32B on AIME25: GenRM needs 4x to match, +2.5% at 16x. GPQA-Diamond (70B): +2.5% at 256x. LiveCodeBench (8B): 16x to match, +3.5% at 64x. Comparing at fixed solution count instead (GenRM matches SC with 4x fewer solutions) is called misleading.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Singhi et al., \"When To Solve, When To Verify: Compute-Optimal Problem Solving and Generative Verification for LLM Reasoning\"", "url": "https://arxiv.org/abs/2504.01005", "date": "2025-04-01", "locator": "§5.1 Impact of Problem Difficulty, Easy-To-Hard Generalization; Figures 3a, 4a", "task": "Reserve generative verification for hard problems", "effect": "Llama-3.1-8B GenRM-FT (32 verifications) vs SC: up to 30% relative improvement on MATH level 5 vs much less on level 1. AIME24 (trained on MATH): GenRM almost doubles SC's performance but needs 256x more compute; SC reaches its peak with 16x less compute than GenRM needs for the same level.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Singhi et al., \"When To Solve, When To Verify: Compute-Optimal Problem Solving and Generative Verification for LLM Reasoning\"", "url": "https://arxiv.org/abs/2504.01005", "date": "2025-04-01", "locator": "§3.2, §5.2 Figure 6; Appendix J Figures 11-13", "task": "Scale solutions faster than verifications per solution", "effect": "Fitted power laws on MATH: Llama-3.1-8B GenRM-FT S_opt ~ C^0.57, V_opt ~ C^0.39; Qwen-2.5-7B GenRM-FT S_opt ~ C^0.75, V_opt ~ C^0.32; Llama-3.3-70B GenRM-Base S_opt ~ C^0.69, V_opt ~ C^0.43. Up to 128 solutions x 128 verifications sampled.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Singhi et al., \"When To Solve, When To Verify: Compute-Optimal Problem Solving and Generative Verification for LLM Reasoning\"", "url": "https://arxiv.org/abs/2504.01005", "date": "2025-04-01", "locator": "Appendix F, Table 1", "task": "Shorten verification outputs to cut verifier cost", "effect": "Analytical upper bound assuming constant verifier accuracy (Llama-3-8B GenRM-FT, MATH): tokens per verification 2048 -> compute to match SC 8x, to beat 128x; 1024 -> 4x/64x; 512 -> 2x/32x; 256 -> 1x/16x. The authors note keeping accuracy while shortening is non-trivial.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Singhi et al., \"When To Solve, When To Verify: Compute-Optimal Problem Solving and Generative Verification for LLM Reasoning\"", "url": "https://arxiv.org/abs/2504.01005", "date": "2025-04-01", "locator": "Appendix E", "task": "Expect single generative verifications to be noisy", "effect": "\"In most cases, at least 32 verifications are needed to achieve good results\" (Llama-3.1-8B, MATH).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Zhou et al., \"Variation in Verification: Understanding Verification Dynamics in Large Language Models\"", "url": "https://arxiv.org/abs/2509.17995", "date": "2025-09-22", "locator": "§4.2 Figure 4; §5.2 Figure 8; Appendix C.2, C.6 Figure 21, D.3, D.4", "task": "Skip strong verifiers on strong generators", "effect": "Math: Qwen2.5-72B verifier TNR 0.68 on Llama-3.1-8B solutions vs 0.17 on Qwen3-32B solutions. With strong generators, verification gain peaks at only about 0.1 for both GPT-4o and Qwen2.5-7B verifiers and the gap between them narrows. Holds for thinking verifiers (Qwen3-8B/32B-Thinking), Qwen3-235B and three verification prompts.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Zhou et al., \"Variation in Verification: Understanding Verification Dynamics in Large Language Models\"", "url": "https://arxiv.org/abs/2509.17995", "date": "2025-09-22", "locator": "§4.3 Figure 6; §5.2 Figure 8; Appendix C.3, C.4, C.6 Figure 21", "task": "Skip verification on the easiest and hardest problems", "effect": "Math: verification gains drop to 0.1 or below for both GPT-4o and Qwen2.5-7B on easy and hard problems. On hard problems (difficulty [0.1,0.3)) verification accuracy plateaus around 0.65 in math regardless of verifier capability and is flat elsewhere; on hard 3-way NL reasoning verifiers score below random because they solve-and-match with a wrong answer. Medium problems ([0.4,0.5)) show linear capability-accuracy correlation (r>0.9).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Zhou et al., \"Variation in Verification: Understanding Verification Dynamics in Large Language Models\"", "url": "https://arxiv.org/abs/2509.17995", "date": "2025-09-22", "locator": "Appendix D.1, Figures 22-23", "task": "Estimate difficulty from answer entropy of a small model", "effect": "8 samples from Qwen2.5-3B per question, difficulty = H/log K. Reproduces the oracle-difficulty findings for TPR vs difficulty and capability-accuracy regimes (math). No downstream accuracy or cost number reported.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Zhou et al., \"Variation in Verification: Understanding Verification Dynamics in Large Language Models\"", "url": "https://arxiv.org/abs/2509.17995", "date": "2025-09-22", "locator": "§4.1 Figure 3; §5.2; Appendix C.4", "task": "Do not scale verifier size to fix hard-problem verification", "effect": "39.1% of verification false negatives on hard problems contain a mistake in the verifier's own reference answer (LLM-judge analysis). Hard [0.2,0.3) math: Qwen3-32B (capability ~0.6) and Qwen2.5-72B (~0.4) both plateau near 0.7 verification score. \"Scaling verifiers from 7B models to GPT-4o fails to overcome fundamental verification challenges\" in these regimes.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Huang et al., \"Large Language Models Cannot Self-Correct Reasoning Yet\"", "url": "https://arxiv.org/abs/2310.01798", "date": "2023-10-03", "locator": "Sec. 4; Table 7", "task": "Prefer self-consistency voting over multi-agent debate", "effect": "Table 7, GSM8K full test set, gpt-3.5-turbo-0301, 3 agents, Du et al. prompts: standard 76.7 (1 response); self-consistency 82.5 (3); debate round 1 83.2 (6) vs self-consistency 85.3 (6); debate round 2 83.0 (9) vs self-consistency 88.2 (9).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Kamoi et al., \"When Can LLMs Actually Correct Their Own Mistakes? A Critical Survey of Self-Correction of LLMs\"", "url": "https://arxiv.org/abs/2406.01297", "date": "2024-06-03", "locator": "Sec. 6; Table 7 checklist", "task": "Compare self-correction with self-consistency at equal compute", "effect": "Survey RQ3: self-correction is often not compared to strong baselines, so it is still unclear whether it beats them; cites self-consistency and pass@k comparisons.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "Zhao et al., \"Sample, Scrutinize and Scale: Effective Inference-Time Search by Scaling Verification\"", "url": "https://arxiv.org/abs/2502.01839", "date": "2025-02-03", "locator": "§1 Table 1; Algorithm 1; §3.1", "task": "Select among many samples by self-verification, not voting", "effect": "Gemini 1.5 Pro, k_inf=200, k_verif=50, with tie-break: AIME 8/15 vs Consistency@200 4/15 and Pass@1 1/15 (o1-Preview reported 7/15); MATH 467/500 vs 460; LiveBench Math 135/200 vs 118; LiveBench Reasoning 97/140 vs 75. Consistency@1,000 no better than @200 (e.g. AIME 3/15). Single runs.", "direction": "mixed", "measured": true, "conditions": null, "older": true }, { "source": "Zhao et al., \"Sample, Scrutinize and Scale: Effective Inference-Time Search by Scaling Verification\"", "url": "https://arxiv.org/abs/2502.01839", "date": "2025-02-03", "locator": "§2.1 Figure 2.1; §2.2 Figures 2.2-2.3; §2.3 Figure 2.4, Table 2", "task": "Keep sampling past self-consistency saturation when a verifier exists", "effect": "AIME: Consistency@50 equals Consistency@10,000 while Verification@k (k_verif=50) shows power-law scaling in k. Example AIME 2024 II P11: 1 of 200 samples correct (answer 601, verification score 0.98) vs majority answer \"1\" from 124 samples (score <=0.36). Accuracy on \"ambiguous\" questions (at least one correct candidate) rises with k at fixed k_verif (implicit scaling). Largest gains when both k_inf and k_verif grow; strongest on AIME where correct answers are rare.", "direction": "mixed", "measured": true, "conditions": null, "older": true }, { "source": "Zhao et al., \"Sample, Scrutinize and Scale: Effective Inference-Time Search by Scaling Verification\"", "url": "https://arxiv.org/abs/2502.01839", "date": "2025-02-03", "locator": "§3 principle 1; §3.1 Tie-Break; §3.2 Table 3", "task": "Break verifier ties by pairwise comparison of candidates", "effect": "Gemini 1.5 Pro, k_tie=100 per matchup, Verification@200 without vs with tie-break: MATH 457 -> 467/500 (Cons@200 460); LiveBench Math 125 -> 135/200 (Cons 118); LiveBench Reasoning 94 -> 97/140 (Cons 75); AIME 7 -> 8/14 (Cons 4). Helps most when the verifier over-assigns high scores (MATH top-3 average score near 90%).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Zhao et al., \"Sample, Scrutinize and Scale: Effective Inference-Time Search by Scaling Verification\"", "url": "https://arxiv.org/abs/2502.01839", "date": "2025-02-03", "locator": "§3 principle 2; §3.2 Table 4; §4.2 Table 6; Appendix B.2", "task": "Have the verifier rewrite candidates as theorem-lemma-proof first", "effect": "Verification prompt ablation, Gemini 1.5 Pro, 20 verification attempts per response, 1,080 responses to 54 MATH level-5 train questions and 120 responses to 6 AIME 2023 questions: Main (with rewrite) FPR/FNR MATH 14%/17%, AIME 7%/7%; Without Rewrite MATH 16%/18%, AIME 11%/12%. Fails where the answer format does not fit a theorem statement (LiveBench Olympiad: Verif@200 22.7% vs Cons@200 25.0%).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Zhao et al., \"Sample, Scrutinize and Scale: Effective Inference-Time Search by Scaling Verification\"", "url": "https://arxiv.org/abs/2502.01839", "date": "2025-02-03", "locator": "§3.2 Table 4", "task": "Do not split verification into separate per-piece threads", "effect": "Same ablation as the rewrite row: Split-Context FPR/FNR MATH 19%/23%, AIME 11%/14% vs Main 14%/17% and 7%/7%; attributed to miscalibration.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Zhao et al., \"Sample, Scrutinize and Scale: Effective Inference-Time Search by Scaling Verification\"", "url": "https://arxiv.org/abs/2502.01839", "date": "2025-02-03", "locator": "§5 Preliminary scoring, Compute", "task": "Prune candidates with cheap preliminary verification scores", "effect": "10 preliminary scores, discard average <0.2, keep top 15 per final answer, then 50 scores on the rest: cuts output tokens about 70%, per-question cost about $650 -> $200 (Pro); Flash as verifier brings it to about $12. Accuracy impact of pruning not separately reported. Used on all datasets except AIME.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Zhao et al., \"Sample, Scrutinize and Scale: Effective Inference-Time Search by Scaling Verification\"", "url": "https://arxiv.org/abs/2502.01839", "date": "2025-02-03", "locator": "§6 Table 7", "task": "Do not trust out-of-box frontier-model verification", "effect": "Curated verification benchmark (random-guess baseline: 80% correct, 20% wrong, 20% flawed, 50% comparison). GPT-4o Pass@1: 76.5% / 31.0% / 22.2% / 43.2%; Claude 3.5 Sonnet 89.6% / 22.5% / 33.3% / 56.1%; o1-preview 100% / 68.8% / 80.0% / 84.5%. Consistency@5 lowered GPT-4o on flawed (22.2 -> 11.1) and comparison (43.2 -> 35.4).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Zhao et al., \"Sample, Scrutinize and Scale: Effective Inference-Time Search by Scaling Verification\"", "url": "https://arxiv.org/abs/2502.01839", "date": "2025-02-03", "locator": "Appendix B.1 Tables 8-9", "task": "Sampling temperature barely matters for verified search", "effect": "MATH train split, 100 questions, 20 samples, Verification@20 without tie-break: inference temp 0.2/1.0/1.5/2.0 -> 82/80/79/82; verification temp 0.2/1.0/2.0 -> 76/79/79.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Saad-Falcon et al., \"Shrinking the Generation-Verification Gap with Weak Verifiers\"", "url": "https://arxiv.org/abs/2506.18203", "date": "2025-06-22", "locator": "§5.2 (1) Figure 3, (4) Figure 5; Table 1", "task": "Expect verifier ensembles to cost far more than voting", "effect": "Majority voting plateaus around 2^2-2^3 ExaFLOPs per query while Weaver keeps improving up to 512 ExaFLOPs (K from 1 to 100); the authors call the extra compute \"prohibitive\". Standard baselines (majority vote, single RM, self-verification, multi-agent verification) plateau after a few generations; self-verification averaged 66.9% and multi-agent verification 71.6% at K=100, below majority vote (72.2%).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Saad-Falcon et al., \"Shrinking the Generation-Verification Gap with Weak Verifiers\"", "url": "https://arxiv.org/abs/2506.18203", "date": "2025-06-22", "locator": "§4.1 Figure 2; §5.1 Table 1; Appendix Table 16", "task": "Weight verifiers by estimated accuracy, not naive averaging", "effect": "Llama 3.3 70B, K=100, average of 4 benchmarks: Weaver 87.7% vs majority vote 72.2%, best single RewardBench RM 72.7%, naive top-10 RM ensemble 69.1%, first sample 68.4%, Pass@100 oracle 91.9%; o3-mini first sample 86.7%. Oracle-labeled weighted ensembles beat naive averaging by up to 11.2 points (3.6 for top-K selection, 7.8 for supervised weights on average). Individual verifier success rates span up to 37.5 points.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Saad-Falcon et al., \"Shrinking the Generation-Verification Gap with Weak Verifiers\"", "url": "https://arxiv.org/abs/2506.18203", "date": "2025-06-22", "locator": "§4.2 eqs 1-5; §5.2 Figure 4; Appendix B.1, Table 17", "task": "Estimate verifier accuracies from agreement with weak supervision", "effect": "Supervised logistic regression / naive Bayes weights fit on 0.01n labeled samples lose 20.1% on average vs their oracle versions, which motivates the label-free estimate. Weaver beats naive averaging by +2.4 to +10.1 points across datasets (GPQA +10.1, MMLU Pro +5.1) with 1-15 verifiers.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Saad-Falcon et al., \"Shrinking the Generation-Verification Gap with Weak Verifiers\"", "url": "https://arxiv.org/abs/2506.18203", "date": "2025-06-22", "locator": "Appendix B.3, Table 5, Figure 14", "task": "Drop low-signal verifiers before ensembling", "effect": "Verifier pruning gives a 12.5% performance improvement in the 70B setting; also decorrelates the verifier set (precision-matrix off-diagonals attenuate).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Saad-Falcon et al., \"Shrinking the Generation-Verification Gap with Weak Verifiers\"", "url": "https://arxiv.org/abs/2506.18203", "date": "2025-06-22", "locator": "Appendix B.2-B.3, Table 4, Table 7, Figures 8-9", "task": "Binarize verifier scores with dev-set thresholds", "effect": "Dev-set thresholds from 5-10 labeled queries (<=1% of eval) improve Weaver by 8.4% on average for 70B vs median-split binarization. Binarization strategies differ only modestly in selection accuracy; quantile (top 15%) does best on imbalanced GPQA. With ample labels, continuous-score logistic regression beats discrete.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Saad-Falcon et al., \"Shrinking the Generation-Verification Gap with Weak Verifiers\"", "url": "https://arxiv.org/abs/2506.18203", "date": "2025-06-22", "locator": "§5.2 (3); Appendix C.5 Table 21", "task": "Add more verifier models rather than resampling one", "effect": "Llama 3.3 70B generations: adding more models gives average gains of 5.4% (RMs) and 6.1% (LM judges) vs 0.8% and 1.1% from extra scores from a single RM or judge. Aggregating up to 15 verifiers improves up to 8.5% over the top-1 verifier, with diminishing returns from correlated biases on hard examples.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Saad-Falcon et al., \"Shrinking the Generation-Verification Gap with Weak Verifiers\"", "url": "https://arxiv.org/abs/2506.18203", "date": "2025-06-22", "locator": "§6 Figure 6; Appendix C.6 Figure 24, Table 22", "task": "Distill the verifier ensemble into a small cross-encoder", "effect": "GPQA Diamond, Llama 70B generations, 80:20 split: retains 98.2% of Weaver's accuracy (98.7% stated elsewhere); 1.01 vs 35.35 exaFLOPs per 100 samples (99.97% verification FLOPs saved); beats majority vote by 23.2% at 0.57% more inference cost than generation alone. Runs on one A100 instead of an 8-GPU node per 70B verifier.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Saad-Falcon et al., \"Shrinking the Generation-Verification Gap with Weak Verifiers\"", "url": "https://arxiv.org/abs/2506.18203", "date": "2025-06-22", "locator": "Appendix B.4, Tables 9-11", "task": "Cluster queries by difficulty for per-cluster verifier weights", "effect": "Oracle difficulty clusters (1-5) with threshold grid 0.05-0.95: 8B setting +4.8%; 70B about +1% and more clusters often degrade it. Per-model thresholds add +0.5 to +0.8 (70B) and +1.6 to +2.8 (8B).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Saad-Falcon et al., \"Shrinking the Generation-Verification Gap with Weak Verifiers\"", "url": "https://arxiv.org/abs/2506.18203", "date": "2025-06-22", "locator": "Appendix C.7 Figures 25-27", "task": "Optimize LM-judge prompts for precision", "effect": "250 labeled examples, DSPy 0/3/5-shot: average +3.8% precision of best judge over CoT baseline, MATH500 +9%; scaling training data 1% -> 16% gives about +3.2% precision on most datasets (MMLU-College minimal). Gains come mainly from lower false-positive rates.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Rajput et al., \"Cheap Verifiers, Large Blind Spots: Measuring the Reliability Cost of Cost-Saving Cascades\"", "url": "https://arxiv.org/abs/2609.01345", "date": "2026-09-01", "locator": "§6 Corollary 1; §7 H2 Figure 7b", "task": "Decorrelate the verifier from the student model", "effect": "Stated as a design rule from Corollary 1 (floor increasing in beta0; same-family verifiers should give higher floors). Not directly measured on real models; the synthetic study measures floor vs beta0 (0.09 to 0.36 as beta0 sweeps 0.19 to 0.96), not family correlation.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Rajput et al., \"Cheap Verifiers, Large Blind Spots: Measuring the Reliability Cost of Cost-Saving Cascades\"", "url": "https://arxiv.org/abs/2609.01345", "date": "2026-09-01", "locator": "§7 H5, Figure 8; §9 limitation 5", "task": "Reject on any dissent across a decorrelated verifier ensemble", "effect": "Synthetic model only (rho=0.7): measured beta0 0.76 -> 0.40 at m=4 (above the rho^4~0.24 ideal); floor 0.25 -> 0.14; cost m verifier calls per query. Exchange rates are \"properties of the synthetic geometry, not portable constants\".", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Stechly et al., \"On the Self-Verification Limitations of Large Language Models on Reasoning and Planning Tasks\"", "url": "https://arxiv.org/abs/2402.08115", "date": "2024-02-12", "locator": "Sec. 6", "task": "Verify by consensus of partial critics", "effect": "Proposal in the conclusion; not evaluated here.", "direction": "supports", "measured": false, "conditions": null, "older": true }, { "source": "The Databricks AI Research Team, \"Pushing the Frontier for Data Agents with Genie\"", "url": "https://www.databricks.com/blog/pushing-frontier-data-agents-genie", "date": "2026-05-08", "locator": "Parallel Thinking section; Figures 1 and 5", "task": "Sample parallel trajectories and aggregate with a judge", "effect": "Internal benchmark, Figure 5: GPT-5.4 65.5% (3.8 min avg runtime) -> 75.7% (8.2 min) with parallel thinking (+10.2pp); all Opus 4.6 73.8% (5.5 min) -> 81.4% (9.5 min) (+7.6pp). Figure 1: N=8 with Opus 4.6 raises relative cost 0.23x -> 2.13x. Error bars shown but n not stated.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Youngwon Lee, Jaejin Kim, Seung-won Hwang (Seoul National University), \"Bootstrapping Semantic Layer from Execution for Text-to-SQL (GATE: Grounding After Test from Execution)\"", "url": "https://arxiv.org/abs/2606.05634", "date": "2026-06-04", "locator": "§3.3 Plan evaluation and selection; §4.3; Table 2", "task": "Score partial plans by progress, bottleneck and potential", "effect": "Table 2, RealEHR, GPT-5.4-mini: removing plan evaluation 55.2 -> 44.7 (-10.5pp), the largest ablation.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Youngwon Lee, Jaejin Kim, Seung-won Hwang (Seoul National University), \"Bootstrapping Semantic Layer from Execution for Text-to-SQL (GATE: Grounding After Test from Execution)\"", "url": "https://arxiv.org/abs/2606.05634", "date": "2026-06-04", "locator": "§3.3; §4.3; Table 2", "task": "Pick the next bottleneck that is testable and informative", "effect": "Table 2, RealEHR, GPT-5.4-mini: removing plan selection 55.2 -> 53.2 (-2.0pp, about one question of 47).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Youngwon Lee, Jaejin Kim, Seung-won Hwang (Seoul National University), \"Bootstrapping Semantic Layer from Execution for Text-to-SQL (GATE: Grounding After Test from Execution)\"", "url": "https://arxiv.org/abs/2606.05634", "date": "2026-06-04", "locator": "§4.4; Table 3; Appendix B.6", "task": "Search structured plans instead of repeating independent runs", "effect": "Table 3, RealEHR, GPT-5.4-mini, avg LLM calls per query: ReAct 42.9% at 6.3 calls; ReAct x6 with the same selector 51.1% at 39.1 calls; GATE 55.2% at 36.0 calls (+4.1pp, fewer calls).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Ghosal et al., \"Does Thinking More always Help? Mirage of Test-Time Scaling in Reasoning Models\"", "url": "https://arxiv.org/abs/2506.04210", "date": "2025-06-04", "locator": "§2.3 Figure 2; §4 Figures 6-7", "task": "Spread a fixed thinking budget over parallel traces with a majority vote, not one longer trace", "effect": "DeepSeek-R1-Distill-Qwen-1.5B on GSM8K: \"pushing the average thinking token count from 1100 to 15980 reduces accuracy from 87.3% to 70.3%\" with forced \"Wait\" extensions. At a 16,000-token budget, \"Wait & Think More\" gave \"a 11.8% performance drop\" against standard thinking, while parallel thinking with the same budget \"boosts accuracy by 10.1%\".", "direction": "supports", "measured": true, "conditions": "DeepSeek-R1-Distill-Qwen-1.5B, -Qwen-7B and -Llama-8B; GSM8K (1,320), MATH-500, AIME 2024 (30)", "older": true } ], "caveats": null, "also_covers": [ { "name": "Self-consistency voting" }, { "name": "Inference-time scaling" }, { "name": "LLM-as-judge" } ], "related": [ { "id": "9.c.iii", "slug": "audit-verifier-cascade", "name": "Audit a verifier-gated cascade against a gold sample the verifier never sees" }, { "id": "7.c.iii", "slug": "evaluator-optimizer-loop", "name": "Evaluator-optimizer loop" }, { "id": "9.b.ii", "slug": "false-success-detector", "name": "Triage false success with a cheap trained detector, not an LLM judge" }, { "id": "7.c.iv", "slug": "graph-of-thought", "name": "Graph-of-Thought" }, { "id": "7.c.v", "slug": "language-agent-tree-search", "name": "Language Agent Tree Search (LATS)" }, { "id": "1.b.i", "slug": "model-and-effort-per-call-site", "name": "Match the model and its effort to each call site and query class" }, { "id": "7.c.vi", "slug": "multi-agent-debate", "name": "Multi-agent debate" }, { "id": "6.a.i", "slug": "oracle-worker-escalation", "name": "Oracle-worker multi-model" }, { "id": "7.c.i", "slug": "reflection-loop", "name": "Reflection loop" }, { "id": "7.c.ii", "slug": "self-critique-evaluator", "name": "Self-critique evaluator loop" }, { "id": "3.c.i", "slug": "self-discover-reasoning", "name": "Self-Discover reasoning structures" }, { "id": "7.c.vii", "slug": "tree-of-thought", "name": "Tree-of-Thought" } ], "url": "https://mahalsystems.ai/aot/optimizations/samples-before-verification/", "markdown": "https://mahalsystems.ai/aot/optimizations/samples-before-verification.md" }, { "id": "9.c.iii", "legacy_id": "T48", "former_ids": [], "slug": "audit-verifier-cascade", "name": "Audit a verifier-gated cascade against a gold sample the verifier never sees", "kind": "technique", "status": "card", "tier": { "tier": 4, "name": "Restructuring", "why": "it governs an escalation topology, which only a production test can judge", "proposed": false }, "levers": [ "control-flow", "model" ], "domains": [ "general" ], "evidence_domains": [ "reasoning" ], "categories": [ "Sizing verification", "Escalating" ], "category_ids": [ "verification.spend", "topologies.escalate" ], "summary": "When a cheap model's outputs are accepted or escalated by a verifier, measure true error on a small gold-labeled sample outside the verifier: metrics measured through the verifier showed about 3% error while true error was 14-32%. A frontier verifier cuts misses but escalates more than the true error rate, erasing the saving. Never train the cheap model on verifier rejects or accepted outputs.", "how": [ "Label a rolling sample (e.g. 100 cases a week) by hand or by the environment.", "Score the cascade's delivered answers on it, separately from the verifier's own accept rate.", "Alert when true error diverges from verifier-measured error." ], "detect": { "static": "a cheap model whose outputs are accepted or escalated by a verifier model, with error measured only through the verifier", "traces": "escalation rate against the eval's true error rate", "evals": "no gold-labeled sample scored independently of the verifier" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "improves", "size": "medium", "when": "reveals hidden error: ~3% through the verifier against 14-32% true (Sep 2026)", "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "small", "when": "labeling the audit sample", "basis": "evidence" }, { "dimension": "performance", "effect": "none", "size": null, "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "monitor", "tier": null, "how": "true error on a rolling gold-labeled sample" }, { "method": "production-test", "tier": 4, "how": "A/B of the cascade against the single strong model" } ], "evidence": [ { "source": "Rajput et al., \"Cheap Verifiers, Large Blind Spots: Measuring the Reliability Cost of Cost-Saving Cascades\"", "url": "https://arxiv.org/abs/2609.01345", "date": "2026-09-01", "locator": "§5 Figure 2; §6 Corollary 2; §7 Figure 5; §10 Implications", "task": "Audit cascades with a small gold-labeled sample outside the verifier", "effect": "Qwen2.5-7B student, gpt-4o-mini verifier, GSM8K corrective loop: verifier-estimated error of the delivered stream flat near 3% across rounds while gold user-facing error swung 14% to 32% (5-11x gap, 95% intervals separate from round 1, n=300). Synthetic (rho=0.7, 20 seeds): dashboard 4.6% vs gold 24.9%. The paper calls an independent gold-labeled audit channel \"not optional instrumentation but the only instrument that can see the effect\" (recommendation, not itself measured as an intervention).", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Rajput et al., \"Cheap Verifiers, Large Blind Spots: Measuring the Reliability Cost of Cost-Saving Cascades\"", "url": "https://arxiv.org/abs/2609.01345", "date": "2026-09-01", "locator": "§5 Figures 3-4", "task": "Do not assume a cheap LLM verifier catches a capable student", "effect": "GSM8K, fixed gpt-4o-mini verifier: blind-spot rate beta rises from 0.12 (CI 0.08-0.17) at Qwen2.5-0.5B to 0.55 (CI 0.44-0.66) at 14B; 7B-32B plateau around 0.35-0.65. Hard MATH, fixed 7B student: beta 0.42 (CI 0.32-0.53) with gpt-4o-mini vs 0.05-0.09 with gpt-4.1 / gpt-5-mini. Static, training-free sweep, so it holds for any plain cascade.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Rajput et al., \"Cheap Verifiers, Large Blind Spots: Measuring the Reliability Cost of Cost-Saving Cascades\"", "url": "https://arxiv.org/abs/2609.01345", "date": "2026-09-01", "locator": "§5 Figure 4; §1 findings", "task": "Expect a frontier verifier to eat the cascade saving", "effect": "Hard MATH L4-5, Qwen2.5-7B student: gpt-4.1 verifier beta ~0.05 but escalation rate 0.46 (CI 0.39-0.53, n=200) against a 0.39 true error rate, i.e. frontier price on nearly half of traffic. \"Low blind spot and low cost are not simultaneously available from a fixed verifier.\"", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Rajput et al., \"Cheap Verifiers, Large Blind Spots: Measuring the Reliability Cost of Cost-Saving Cascades\"", "url": "https://arxiv.org/abs/2609.01345", "date": "2026-09-01", "locator": "§5 \"The loop degrades the student\"; §1", "task": "Do not fine-tune the cheap model on verifier rejects", "effect": "Qwen2.5-1.5B and 7B students, GSM8K corrective loop, teachers gpt-4o-mini, gpt-4o (cross-family) and Qwen2.5-32B (same-family): naive LoRA fine-tuning on rejects degraded the student in every case and cumulatively collapsed it (raw error rising to 1 as output format broke). Frozen control stayed near 13% error. Single-seed runs.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Rajput et al., \"Cheap Verifiers, Large Blind Spots: Measuring the Reliability Cost of Cost-Saving Cascades\"", "url": "https://arxiv.org/abs/2609.01345", "date": "2026-09-01", "locator": "§6 H3; §7 Table 1, Figure 7a", "task": "Never recycle verifier-accepted outputs as training labels", "effect": "Synthetic model only (rho=0.7, 20 seeds, 8 rounds): self-training user-facing error 0.296 vs corrective 0.249 (raw error 0.356 vs 0.304); frozen 0.334; oracle-in-loop 0.000. Predicted rise above q0*beta0 was not observed.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Rajput et al., \"Cheap Verifiers, Large Blind Spots: Measuring the Reliability Cost of Cost-Saving Cascades\"", "url": "https://arxiv.org/abs/2609.01345", "date": "2026-09-01", "locator": "§7 H5, Figure 8", "task": "Random oracle audits are cheap but shallow", "effect": "Synthetic model only (rho=0.7): auditing 40% of items lowers the corrective floor from 0.25 to 0.22, vs 0.14 for a 4-verifier ensemble; neither reaches the oracle floor (0) in the tested budget.", "direction": "supports", "measured": true, "conditions": null, "older": false }, { "source": "Zhou et al., \"Variation in Verification: Understanding Verification Dynamics in Large Language Models\"", "url": "https://arxiv.org/abs/2509.17995", "date": "2025-09-22", "locator": "§5.1 Figure 7; Appendix C.5 Figure 19; Appendix D.5 Table 2", "task": "Pair a weak generator with a strong verifier", "effect": "Math, difficulty [0.7,0.8), 181 problems, GPT-4o verifier, 64 samples: Gemma2-9B vs Gemma2-27B gap shrinks from 10.3% to 2.5% after verification (75.7% closed); across whole domains most weak/strong pairs see 30-50% gap reduction. Case study (same 181 problems): Qwen3-4B generator + GPT-4o verifier pass rate 0.954 vs GPT-4o generator alone 0.952, with GPT-4o tokens per prompt 193 vs 482 (about 2.5x fewer).", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Zhou et al., \"Variation in Verification: Understanding Verification Dynamics in Large Language Models\"", "url": "https://arxiv.org/abs/2509.17995", "date": "2025-09-22", "locator": "Appendix D.2.2, Figures 28-29", "task": "Pick verifier by validation balanced accuracy gap", "effect": "GPT-4o vs Qwen2.5-7B, difficulty-stratified math, 50/50 validation/test split: on the hardest bin the validation gap was small and the test TTS accuracy difference was small (pick 7B); on the hard bin the gap was clear and GPT-4o gave substantially better test TTS accuracy. Figures only; no single headline number.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Zhou et al., \"Variation in Verification: Understanding Verification Dynamics in Large Language Models\"", "url": "https://arxiv.org/abs/2509.17995", "date": "2025-09-22", "locator": "Appendix D.2.1, Figures 24-27", "task": "Pick generator size by verified validation accuracy", "effect": "Gemma2-2B/9B/27B with Qwen3-32B or GPT-4o verifier, 64 samples, FLOPs=2*N*tokens: selection picked 2B on easy [0.9,1.0), 9B on medium [0.7,0.8), 27B on hard [0.3,0.4), and consistently matched the compute-efficient choice on the test split (pass rate and majority vote). Without verification the largest model was usually preferred.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Singhi et al., \"When To Solve, When To Verify: Compute-Optimal Problem Solving and Generative Verification for LLM Reasoning\"", "url": "https://arxiv.org/abs/2504.01005", "date": "2025-04-01", "locator": "§2 GenRM-Finetuning; §5.1 Impact of Verifier Quality; Figure 3b; Appendix C, E", "task": "Fine-tune the verifier on filtered strong-model rationales", "effect": "Llama-3.1-8B on MATH: GenRM-FT reaches the same success rate as prompted GenRM-Base with up to 16x less compute and outperforms it overall. Data: 7,500 MATH train problems x 4 solutions x 4 GPT-4o verifications, LoRA 3 epochs.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Singhi et al., \"When To Solve, When To Verify: Compute-Optimal Problem Solving and Generative Verification for LLM Reasoning\"", "url": "https://arxiv.org/abs/2504.01005", "date": "2025-04-01", "locator": "Appendix F, Figure 7", "task": "Use a smaller, cheaper verifier than the generator", "effect": "Llama-3-70B solutions on MATH: Llama-3-8B GenRM-FT verifier (verification cost 0.228x a solution) reduces compute to match SC from 8x to 4x vs a 70B GenRM-Base verifier, but still does not beat SC at low budgets.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Zhao et al., \"Sample, Scrutinize and Scale: Effective Inference-Time Search by Scaling Verification\"", "url": "https://arxiv.org/abs/2502.01839", "date": "2025-02-03", "locator": "§4.1 Table 5", "task": "Verify a strong generator's samples with a cheaper model", "effect": "Pro generates + Flash verifies (Verification@200): AIME 7/15, MATH 456/500, LiveBench Math 119/200, Reasoning 84/140, vs Pro Consistency@200 4/15, 460, 118, 75 and Pro end-to-end Verification 8, 467, 135, 97. Compute about that of Consistency@500. Flash end-to-end Verification@200 (5/15, 445, 104, 84) is competitive with Pro Consistency@200.", "direction": "supports", "measured": true, "conditions": null, "older": true }, { "source": "Saad-Falcon et al., \"Shrinking the Generation-Verification Gap with Weak Verifiers\"", "url": "https://arxiv.org/abs/2506.18203", "date": "2025-06-22", "locator": "§5.2 (2) Table 3; Appendix Table 20", "task": "Weak-to-strong: small generator plus small-verifier ensemble", "effect": "Llama 3.1 8B + <=8B verifiers: average 57.2% (majority vote) -> 70.0% with Weaver (+12.8), within 1.6 points of 70B majority vote (71.6%). 70B + <=72B verifiers: 71.6% -> 87.6% (+16.0; GPQA +27.3), vs o3-mini 86.7%.", "direction": "supports", "measured": true, "conditions": null, "older": true } ], "caveats": null, "also_covers": [], "related": [ { "id": "6.a.i", "slug": "oracle-worker-escalation", "name": "Oracle-worker multi-model" }, { "id": "9.c.ii", "slug": "samples-before-verification", "name": "Spend extra samples before extra verification: parallel attempts with a vote or judge; verify only where it pays" }, { "id": "7.c.iii", "slug": "evaluator-optimizer-loop", "name": "Evaluator-optimizer loop" }, { "id": "7.c.ii", "slug": "self-critique-evaluator", "name": "Self-critique evaluator loop" } ], "url": "https://mahalsystems.ai/aot/optimizations/audit-verifier-cascade/", "markdown": "https://mahalsystems.ai/aot/optimizations/audit-verifier-cascade.md" }, { "id": "10.a.i", "legacy_id": "T68", "former_ids": [], "slug": "constrain-agent-reach", "name": "Constrain what the agent can touch", "kind": "technique", "status": "card", "tier": { "tier": 2, "name": "Reliability", "why": "safety containment", "proposed": false }, "levers": [ "guardrails" ], "domains": [ "general" ], "evidence_domains": [ "code-generation", "data-analysis", "tool-use" ], "categories": [ "Sandboxing" ], "category_ids": [ "safety.sandbox" ], "summary": "Run the agent in an OS-level sandbox with a workspace path guard that resolves symlinks, scrub credentials from the command environment, run agent-written SQL read-only with timeouts and row caps, and set resource limits on generated code.", "how": [ "List what the agent can reach (paths, credentials, databases, network).", "Remove write access and credentials it does not need; add the path guard and limits.", "Run the safety evals before and after." ], "detect": { "static": "shell or code execution without a sandbox; credentials in the agent's environment; write access where read is enough; no row cap or timeout on agent SQL", "traces": "commands touching paths outside the workspace; queries returning very large result sets; failure rows labeled unauthorized-irreversible-action", "evals": "safety evals with out-of-bounds actions" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "none", "size": null, "when": null, "basis": "evidence" }, { "dimension": "cost", "effect": "degrades", "size": "small", "when": null, "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "small", "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "before-after", "tier": 2, "how": "safety evals and out-of-bounds action attempts" } ], "evidence": [ { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 8.5; Sec 10.2 Layer 4; Observation 6; Recommendation 10", "task": "Sandbox execution at the OS level", "effect": "Codex vendored Bubblewrap with \"--ro-bind, --unshare-net, --unshare-user, --unshare-pid\"; \"Protected paths (.git, .agents, .codex) are re-marked read-only even when they sit inside a writable root\"; described as \"a substantial multi-thousand-line investment.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 8.5", "task": "Scrub credentials from the command environment", "effect": "Gemini CLI \"environment scrubbing of provider credentials (name-pattern regexes over TOKEN/SECRET/KEY/AUTH/CREDENTIAL, a denylist, and GitHub-token value regexes) before each command runs.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Fan et al., \"An Empirical Study of Harness Design for Coding Agents\"", "url": "https://arxiv.org/abs/2609.20804", "date": "2026-09-17", "locator": "§2.4 Safety; App 8.3 Fig 27", "task": "Gate actions with workspace guard and allow/ask/deny permissions", "effect": "Held fixed, not ablated. The deny and allow lists are given in the bash tool description.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Youngwon Lee, Jaejin Kim, Seung-won Hwang (Seoul National University), \"Bootstrapping Semantic Layer from Execution for Text-to-SQL (GATE: Grounding After Test from Execution)\"", "url": "https://arxiv.org/abs/2606.05634", "date": "2026-06-04", "locator": "Appendix B.2", "task": "Restrict agent SQL to read-only with timeout and row caps", "effect": "Implementation detail (Appendix B.2), not ablated.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Adam Jones, Conor Kelly (Anthropic), \"Code execution with MCP: Building more efficient agents\"", "url": "https://www.anthropic.com/engineering/code-execution-with-mcp", "date": "2025-11-04", "locator": "Benefits: State persistence and skills (closing note)", "task": "Sandbox agent-generated code with limits and monitoring", "effect": "\"Running agent-generated code requires a secure execution environment with appropriate sandboxing, resource limits, and monitoring. These infrastructure requirements add operational overhead and security considerations that direct tool calls avoid.\"", "direction": "supports", "measured": false, "conditions": null, "older": false } ], "caveats": null, "also_covers": [ { "name": "Tool capability compartmentalization" } ], "related": [ { "id": "2.a.iii", "slug": "choose-action-interface", "name": "Choose how the agent acts: typed tools, code execution, or shell, matched to the model and task" }, { "id": "10.b.i", "slug": "risky-action-policy", "name": "Gate irreversible actions: allow, ask or deny by side effect" }, { "id": "6.c.i", "slug": "code-then-execute", "name": "Code-then-execute (CaMeL)" }, { "id": "10.c.i", "slug": "contain-untrusted-input", "name": "Contain sensitive data and untrusted input" }, { "id": "7.d.i", "slug": "continuous-task-loop", "name": "Continuous autonomous task loop" }, { "id": "4.b.i", "slug": "filesystem-checkpoints", "name": "Filesystem-based agent state and checkpoints" }, { "id": "5.c.ii", "slug": "isolated-subagent-context", "name": "Sub-agent architectures with clean contexts" }, { "id": "6.b.ii", "slug": "llm-map-reduce", "name": "LLM map-reduce" }, { "id": "6.c.ii", "slug": "plan-then-execute", "name": "Plan-then-execute" } ], "url": "https://mahalsystems.ai/aot/optimizations/constrain-agent-reach/", "markdown": "https://mahalsystems.ai/aot/optimizations/constrain-agent-reach.md" }, { "id": "10.b.i", "legacy_id": "T69", "former_ids": [], "slug": "risky-action-policy", "name": "Gate irreversible actions: allow, ask or deny by side effect", "kind": "technique", "status": "card", "tier": { "tier": 2, "name": "Reliability", "why": "safety containment", "proposed": false }, "levers": [ "guardrails" ], "domains": [ "general" ], "evidence_domains": [ "code-generation" ], "categories": [ "Gating risky actions" ], "category_ids": [ "safety.approval" ], "summary": "Classify every action the agent can take by its side effect (read, reversible write, irreversible or externally visible: sending email, issuing a refund, deleting a record, posting publicly, pushing code) and gate by class: allow, ask a human or a policy, or deny. Where the environment already stages changes before they reach the world, gate only what escapes the stage. Coding agents can run on auto-approve because the development process contains their changes (a branch, PR review and CI stand between the agent and production). Most other agents (marketing, support, CRM, finance) have no such stage, so the gate has to sit at the action, or the harness has to build one: drafts, pending changes, batched approval. Keep a deny floor that auto-approve cannot switch off, and write the policy as code with tested examples.", "how": [ "List the agent's actions and mark each one's side effect: read, reversible write, irreversible or externally visible.", "For each irreversible action, find where review can happen: an existing stage (branch and PR, draft, staging environment) or none.", "Where there is no stage, add one (draft and approve, pending changes, batch approval) or an ask gate on the action; deny what should never run unattended, in a floor auto-approve cannot lift.", "Write the policy as code with tested examples.", "Run safety evals built from irreversible actions; monitor approved, denied and escalated actions in production, including how often humans approve without changes." ], "detect": { "static": "tools with irreversible or externally visible effects (send, pay, delete, publish, push) with no gate and no staging step; one approval setting for every tool; a deny list that auto-approve disables", "traces": "irreversible actions taken without a policy or human check; failure rows labeled unauthorized-irreversible-action or over-initiative", "evals": "safety cases built from the agent's own irreversible actions, including prompt-injected requests for them" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "none", "size": null, "when": null, "basis": "evidence" }, { "dimension": "cost", "effect": "none", "size": null, "when": null, "basis": "evidence" }, { "dimension": "performance", "effect": "degrades", "size": "small", "when": "approval waits; staging and batching keep them small", "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "before-after", "tier": 2, "how": "safety evals on risky-action prompts" }, { "method": "monitor", "tier": null, "how": "denied and approved risky actions in production" } ], "evidence": [ { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 6.2 (Pi, Hermes)", "task": "Accept mid-run user steering via queues", "effect": "Pi \"steer() queue injects messages after the current turn's tool calls, while followUp() drains only when the agent would otherwise stop\"; Hermes \"mid-turn user steering is spliced into the last tool result behind an anti-injection marker.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 6.2 (Mistral Vibe); Sec 10.2; Sec 10.5", "task": "Use a structural read-only plan mode", "effect": "Mistral Vibe \"a plan file under a dedicated plans directory is the profile's only writable target, and an exit_plan_mode tool emits a PlanReviewRequestedEvent\"; \"all four provider-native systems now ship a read-only planning mode.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 10.5; Sec 10.6; Recommendation 9", "task": "Offer PLAN / DEFAULT / AUTO_EDIT / YOLO approval modes", "effect": "Gemini CLI ApprovalMode enum (four modes); Mistral Vibe four-layer permission hierarchy; Rec 9: \"matches how a human developer triages risk.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 10.2 Layer 1; Recommendation 11; Table 11 \"Policy-as-Code\"", "task": "Codify safety rules as policy-as-code with tested examples", "effect": "Codex execpolicy \"rules may carry inline match/not_match examples that are validated at parse time—executable test cases inside the policy file\"; Rec 11: \"Policies-as-code survive refactors and are auditable independently of the loop.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 10.7; Recommendation 11", "task": "Keep a hardline command floor that survives YOLO", "effect": "Hermes \"twelve-pattern hardline floor ... that survives --yolo—the YOLO environment variable is frozen at module import precisely so a prompt-injected skill cannot flip it\"; 47 dangerous patterns \"matched on deobfuscated variants\".", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 10.9; Sec 8.2 (Mistral Vibe); Table 12 \"Syntax-Aware Command Permissioning\"", "task": "Parse shell commands and scope grants by arity", "effect": "OpenCode \"130-entry, explicitly LLM-generated command-arity dictionary (git 2, npm run 3), producing grants like git commit * rather than blanket approval\"; \"denying \"*\" for a tool removes it from the LLM's view entirely\".", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 10.2 Layer 3; Sec 10.3; Sec 10.7", "task": "Review risky actions with a fail-closed LLM classifier", "effect": "Codex Guardian \"fails closed on timeout or malformed output, and carries a per-turn rejection circuit breaker\"; Hermes _smart_approve \"temperature 0, 16 max tokens, APPROVE/DENY/ESCALATE\"; OpenHands EnsembleSecurityAnalyzer \"worst-case-wins, with broken child analyzers failing closed to HIGH\"; LLMSecurityAnalyzer uses agent-filled security_risk.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 10.2 Layer 2; Sec 12.3; Sec 14.1", "task": "Expose lifecycle hooks that can veto or rewrite calls", "effect": "Codex hooks \"near-verbatim Claude Code's: PreToolUse, PermissionRequest, PostToolUse, PreCompact, ... Stop ... with block/allow decision outputs\"; Mistral Vibe hooks.toml \"can deny a tool call, rewrite its inputs, or append context\"; Gemini CLI eleven lifecycle events.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Fan et al., \"An Empirical Study of Harness Design for Coding Agents\"", "url": "https://arxiv.org/abs/2609.20804", "date": "2026-09-17", "locator": "§2.4 Safety; App 8.3 Fig 27", "task": "Gate actions with workspace guard and allow/ask/deny permissions", "effect": "Held fixed, not ablated. The deny and allow lists are given in the bash tool description.", "direction": "supports", "measured": false, "conditions": null, "older": false } ], "caveats": "Every source comes from coding-agent harnesses (a source-code study of 11 of them, and one controlled study that held the policy fixed), and none measures the gate's effect. The shell examples (rm -rf, sudo, git push, command parsing) are one case of the general rule. Asking too often has a cost of its own: approvals slow the agent, and humans who are asked constantly start approving without reading, so stage and batch where you can rather than prompt per action.", "also_covers": [ { "name": "Human-in-the-loop approval gates" }, { "name": "Spectrum of control" } ], "related": [ { "id": "10.a.i", "slug": "constrain-agent-reach", "name": "Constrain what the agent can touch" }, { "id": "6.c.i", "slug": "code-then-execute", "name": "Code-then-execute (CaMeL)" }, { "id": "2.b.i", "slug": "fail-soft-tools", "name": "Make tool calls and tool results fail soft, and retry transient failures" }, { "id": "4.b.i", "slug": "filesystem-checkpoints", "name": "Filesystem-based agent state and checkpoints" }, { "id": "7.d.ii", "slug": "inversion-of-control", "name": "Inversion of control" }, { "id": "7.c.v", "slug": "language-agent-tree-search", "name": "Language Agent Tree Search (LATS)" }, { "id": "3.b.ii", "slug": "mask-tools-not-remove", "name": "Mask tools instead of removing them" }, { "id": "6.b.i", "slug": "parallel-read-only-tools", "name": "Parallel tool execution for read-only calls" }, { "id": "6.c.ii", "slug": "plan-then-execute", "name": "Plan-then-execute" } ], "url": "https://mahalsystems.ai/aot/optimizations/risky-action-policy/", "markdown": "https://mahalsystems.ai/aot/optimizations/risky-action-policy.md" }, { "id": "10.c.i", "legacy_id": "T70", "former_ids": [], "slug": "contain-untrusted-input", "name": "Contain sensitive data and untrusted input", "kind": "technique", "status": "card", "tier": { "tier": 2, "name": "Reliability", "why": "safety containment", "proposed": false }, "levers": [ "guardrails" ], "domains": [ "general" ], "evidence_domains": [ "tool-use", "code-generation" ], "categories": [ "Containing data" ], "category_ids": [ "safety.data" ], "summary": "Tokenize PII in the client so it never reaches the model, wrap untrusted tool and web results in taint delimiters, scan context files and skills for prompt injection, and state an explicit instruction-precedence order (user, repo, defaults, external data as data).", "how": [ "Map where PII and untrusted text enter the prompt.", "Add tokenization at the client and delimiters on untrusted results.", "Add the precedence statement to the prompt.", "Run injection and data-leak safety evals." ], "detect": { "static": "PII or untrusted content passed to the model unmarked; no instruction-precedence statement", "traces": "sensitive fields in model inputs or logs; injected instructions followed; failure rows labeled indirect-prompt-injection", "evals": "injection and data-leak safety evals" }, "impact": { "measured_basis": true, "primary": [], "rows": [ { "dimension": "quality", "effect": "none", "size": null, "when": null, "basis": "evidence" }, { "dimension": "cost", "effect": "none", "size": null, "when": null, "basis": "evidence" }, { "dimension": "performance", "effect": "none", "size": null, "when": null, "basis": "evidence" }, { "dimension": "security", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "governance", "effect": "unknown", "size": null, "when": null, "basis": null }, { "dimension": "brand", "effect": "unknown", "size": null, "when": null, "basis": null } ] }, "measure": [ { "method": "before-after", "tier": 2, "how": "injection and data-leak safety evals" } ], "evidence": [ { "source": "Adam Jones, Conor Kelly (Anthropic), \"Code execution with MCP: Building more efficient agents\"", "url": "https://www.anthropic.com/engineering/code-execution-with-mcp", "date": "2025-11-04", "locator": "Benefits: Privacy-preserving operations", "task": "Tokenize PII in the MCP client", "effect": "\"The real email addresses, phone numbers, and names flow from Google Sheets to Salesforce, but never through the model. This prevents the agent from accidentally logging or processing sensitive data. You can also use this to define deterministic security rules, choosing where data can flow to and from.\"", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 7.3 \"Identity and persona\", \"Emphasis markers\"", "task": "State an explicit instruction-precedence contract", "effect": "Mistral Vibe \"a formal instruction-hierarchy contract: seven precedence levels ... prompt-injection defense expressed as a ranking rather than prohibitions\"; \"Critical instructions—not overridable\" and \"Overridable defaults\" sections.", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 10.3; Sec 10.7; Table 12 \"Untrusted-Content Delimiting\"", "task": "Wrap untrusted tool/web results in taint delimiters", "effect": "Hermes \"untrusted tool results wrapped in delimiters with lookalike defanging\"; OpenHands \"repository-derived context is wrapped in markers\".", "direction": "supports", "measured": false, "conditions": null, "older": false }, { "source": "Barbaste et al., \"Harness Engineering: Anatomy, Architecture, and Evolution of Coding Agents -- A Source-Code Study of Eleven Systems\"", "url": "https://arxiv.org/abs/2609.00006", "date": "2026-07-15", "locator": "Sec 9.7; Sec 10.7; Sec 10.8; Sec 12.5; Recommendation 14", "task": "Scan context files and skills for prompt injection", "effect": "Hermes \"every context file is scanned for prompt-injection patterns and blocked wholesale on a match\"; \"builtin/trusted/community trust tiers and quarantine\"; Pi trust-gates repo config but \"context files load regardless—a documented, deliberate injection-surface acceptance.\"", "direction": "supports", "measured": false, "conditions": null, "older": false } ], "caveats": null, "also_covers": [ { "name": "Dual LLM (privileged / quarantined)" }, { "name": "Context minimization" } ], "related": [ { "id": "10.a.i", "slug": "constrain-agent-reach", "name": "Constrain what the agent can touch" }, { "id": "6.c.i", "slug": "code-then-execute", "name": "Code-then-execute (CaMeL)" }, { "id": "4.c.i", "slug": "episodic-memory", "name": "Episodic memory retrieval" }, { "id": "6.b.ii", "slug": "llm-map-reduce", "name": "LLM map-reduce" }, { "id": "6.c.ii", "slug": "plan-then-execute", "name": "Plan-then-execute" }, { "id": "7.d.iii", "slug": "react-loop", "name": "ReAct loop" }, { "id": "4.c.ii", "slug": "skill-library", "name": "Skill library evolution" }, { "id": "5.a.i", "slug": "staged-compaction", "name": "Compact conversation history in stages: elide stale tool outputs, then keep an anchored summary" } ], "url": "https://mahalsystems.ai/aot/optimizations/contain-untrusted-input/", "markdown": "https://mahalsystems.ai/aot/optimizations/contain-untrusted-input.md" } ] }