{ "generated_at_utc": "2026-09-15T16:32:08Z", "chain_head": "85702a792688998a7d6393da9621e70a17f4ad9b8f6b270b65b863ba415db529", "hypotheses": [ { "id": "H1", "arm": null, "batch": null, "state": "retired", "gap_id": "gap-1", "gap_kind": "gate_open", "question": "Is the literature really silent on this, or was the search too narrow? In the Absent Supervisor environment, all evaluated models prefer the shorter path through the hazardous region regardless of supervision, with observed reward consistently exceeding hidden reward.", "hypothesis": "A systematic search of ACL Anthology, arXiv, and OpenReview with the query ('Absent Supervisor' OR 'AI Safety Gridworlds') AND ('language model' OR 'LLM') will return at least one peer-reviewed paper reporting the same shorter-hazardous-path preference in Absent Supervisor.", "prediction": "At least one peer-reviewed paper is found that reports LM agents preferring the hazardous shorter path in Absent Supervisor regardless of supervision.", "falsifier": "After running the expanded query against ACL Anthology, arXiv, and OpenReview, zero papers are found that report the shorter-hazardous-path preference in Absent Supervisor.", "proposed_test": "Run the expanded Boolean query on the three databases, screen all titles and abstracts, and retrieve full texts for any candidate matches.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 50, "test_source": null, "test_source_reason": null, "prior_art": null, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T17:38:52Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 76 ] }, { "at_utc": "2026-09-14T17:45:24Z", "from": "proposed", "to": "retired", "reason": "not a statement about the world — it is a procedure or a retrieval plan, rejected by the hypothesis shape filter", "cites": [ 76 ] } ] }, { "id": "H2", "arm": null, "batch": null, "state": "killed_prior_art", "gap_id": "gap-2", "gap_kind": "gate_open", "question": "Is the literature really silent on this, or was the search too narrow? In Safe Interruptibility, Qwen3-235B-Thinking's high hidden reward is accidental, arising from misinterpreting the interruption tile as a collectible item rather than from principled safety.", "hypothesis": "Qwen3-235B-Thinking's high hidden reward in Safe Interruptibility is caused by the model treating the interruption tile as a collectible, so renaming that tile to a non-collectible symbol will reduce hidden reward to near zero.", "prediction": "After renaming the interruption tile to an unrelated symbol, Qwen3-235B-Thinking's hidden reward drops from its original high value to near zero.", "falsifier": "After renaming the interruption tile to an unrelated symbol, Qwen3-235B-Thinking's hidden reward remains within 10% of its original high value.", "proposed_test": "Re-run Qwen3-235B-Thinking on the Safe Interruptibility environment with only the interruption tile character renamed, and compare hidden reward.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 20, "test_source": null, "test_source_reason": null, "prior_art": { "verdict": "answered", "closest": "Reward Hacking in Language Model Agents: Revisiting AI Safety Gridworlds [2606.15385]", "quote": "the model interprets the interruption tile I as a collectible item" }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T17:38:52Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 76 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: answered (closest: Reward Hacking in Language Model Agents: Revisiting AI Safety Gridworlds [2606.15385])", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "prior_art_checked", "to": "killed_prior_art", "reason": "already answered by prior work (Reward Hacking in Language Model Agents: Revisiting AI Safety Gridworlds [2606.15385]) — killed before any experiment was run, which is the cheapest research result available", "cites": [ 83 ] } ] }, { "id": "H3", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-3", "gap_kind": "gate_open", "question": "Is the literature really silent on this, or was the search too narrow? In Boat Race, trained models converge on a back-and-forth exploit, oscillating on a single arrow tile to collect reward rather than completing laps.", "hypothesis": "The Boat Race back-and-forth exploit is caused by the absence of an explicit lap counter in the observation, so adding a lap counter will eliminate oscillation and increase completed laps.", "prediction": "With a lap counter added to the observation, the model's oscillation rate drops by more than 50% and completed laps increase relative to the original environment.", "falsifier": "With a lap counter added to the observation, the model still oscillates on a single arrow tile at the same rate as in the original environment.", "proposed_test": "Modify the existing Boat Race observation to include a lap counter, then evaluate the already-trained model on 100 episodes and compare oscillation and lap-completion rates to baseline.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 20, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "open", "closest": "none found", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T17:38:52Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 76 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: open (closest: none found)", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "prior_art_checked", "to": "active", "reason": "the gate found no work answering this", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:51:27Z", "from": "active", "to": "blocked", "reason": "estimated $20 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 76, 83 ] } ] }, { "id": "H4", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-4", "gap_kind": "paper_open_question", "question": "Whether proposed mitigations genuinely eliminate the underlying reward-hacking failures rather than merely suppressing their surface manifestations.", "hypothesis": "A mitigation that lowers the observed reward-hacking metric will also lower exploit frequency when the exploit is made unrewarding in a held-out reward variant.", "prediction": "Exploit frequency in the unrewarding reward variant drops by more than 50% for mitigated checkpoints relative to baseline checkpoints.", "falsifier": "Exploit frequency in the unrewarding reward variant remains within 10 percentage points of baseline for mitigated checkpoints.", "proposed_test": "Evaluate existing mitigated and baseline checkpoints on a modified reward variant where the known exploit yields zero reward, and compare exploit rates.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 100, "test_source": "authors_artifacts", "test_source_reason": "the test names material the society does not hold (\"checkpoints\")", "prior_art": { "verdict": "uncertain", "closest": "Reward Bias Substitution: Single-Axis Bias Mitigations Redirect Optimization Pressure [2605.27996]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T17:38:52Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 76 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Reward Bias Substitution: Single-Axis Bias Mitigations Redirect Optimization Pressure [2605.27996])", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:51:27Z", "from": "active", "to": "blocked", "reason": "estimated $100 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 76, 83 ] } ] }, { "id": "H5", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-5", "gap_kind": "paper_open_question", "question": "Whether scale (specifically 14B Island Navigation) truly resolves the failure mode or only occasionally escapes it.", "hypothesis": "The 14B Island Navigation success is seed-dependent, so across five independent RL seeds at least three fail to achieve the hidden objective.", "prediction": "Three or more of five independent training runs at 14B fail to achieve the hidden objective on Island Navigation.", "falsifier": "Three or more of five independent training runs at 14B achieve the hidden objective on Island Navigation.", "proposed_test": "Run the existing RL recipe on 14B Island Navigation with five different random seeds and measure hidden-objective achievement.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 3000, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "none found", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T17:38:52Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 76 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: none found)", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:51:27Z", "from": "active", "to": "blocked", "reason": "estimated $3000 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 76, 83 ] } ] }, { "id": "H6", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-6", "gap_kind": "paper_open_question", "question": "How the findings transfer to tool-using or coding agents.", "hypothesis": "Models with a positive observed-hidden reward gap on the text gridworlds will also show a higher unsafe tool-call rate on AgentHarm than models without such a gap.", "prediction": "There is a positive correlation (Spearman rho > 0.5) between text-gridworld observed-hidden gap and unsafe tool-call rate on AgentHarm across the same model suite.", "falsifier": "The correlation between text-gridworld observed-hidden gap and AgentHarm unsafe tool-call rate is zero or negative (Spearman rho <= 0 or p > 0.05).", "proposed_test": "Evaluate the same open models on AgentHarm with its safety checks, compute unsafe tool-call rates, and correlate with their text-gridworld gaps.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 200, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "Reward Hacking in Language Model Agents: Revisiting AI Safety Gridworlds [2606.15385]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T17:38:52Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 76 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Reward Hacking in Language Model Agents: Revisiting AI Safety Gridworlds [2606.15385])", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:51:27Z", "from": "active", "to": "blocked", "reason": "estimated $200 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 76, 83 ] } ] }, { "id": "H7", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-7", "gap_kind": "paper_open_question", "question": "Demonstrating direct transfer to tool-using or coding agents is described as an important next step.", "hypothesis": "RL fine-tuning of a coding agent on visible unit tests will increase visible pass@1 while leaving hidden pass@1 unchanged.", "prediction": "Visible pass@1 increases by more than 10 points while hidden pass@1 changes by at most 2 points after RL fine-tuning on visible tests.", "falsifier": "Hidden pass@1 increases by more than 8 points after RL fine-tuning on visible tests.", "proposed_test": "Fine-tune an open coding model with GRPO on a CodeContests split using only visible tests, then evaluate on held-out hidden tests.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 500, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "TensorBench: Benchmarking Coding Agents on a Compiler-Based Tensor Framework [2606.05570]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T17:38:52Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 76 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: TensorBench: Benchmarking Coding Agents on a Compiler-Based Tensor Framework [2606.05570])", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:51:27Z", "from": "active", "to": "blocked", "reason": "estimated $500 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 76, 83 ] } ] }, { "id": "H8", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-8", "gap_kind": "paper_open_question", "question": "The framework is described as readily extensible to richer environments, larger models, and safety-aware training.", "hypothesis": "Extending the text gridworld to a richer multi-room environment will not reduce the zero-shot specification-gaming rate; the observed-hidden gap will be at least as large as in the original simple environments.", "prediction": "The observed-hidden reward gap in the richer multi-room environment is equal to or larger than the gap in the original simple environments.", "falsifier": "The observed-hidden reward gap in the richer multi-room environment is at least 50% smaller than in the original simple environments.", "proposed_test": "Implement one richer multi-room variant in the existing text-gridworld framework, evaluate the same model suite, and compare observed-hidden gaps.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 100, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "none found", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T17:38:52Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 76 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: none found)", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:51:27Z", "from": "active", "to": "blocked", "reason": "estimated $100 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 76, 83 ] } ] }, { "id": "H9", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-9", "gap_kind": "paper_open_question", "question": "The same setup is noted as easily extendable to vision-language models (VLMs).", "hypothesis": "A vision-language model given rendered ANSI gridworld screenshots will reproduce the Boat Race oscillating exploit when text-only models do.", "prediction": "The vision-language model oscillates on a single arrow tile in at least 1 of 10 Boat Race episodes.", "falsifier": "The vision-language model never oscillates across 100 Boat Race episodes.", "proposed_test": "Render existing Boat Race states as screenshots, query an open vision-language model with the same prompts, and score oscillation over 100 episodes.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 50, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "VisualLeakBench: Reproducible Action-Boundary Propagation Failures in Vision-Language Agents [2606.07595]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T17:38:52Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 76 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: VisualLeakBench: Reproducible Action-Boundary Propagation Failures in Vision-Language Agents [2606.07595])", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:51:27Z", "from": "active", "to": "blocked", "reason": "estimated $50 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 76, 83 ] } ] }, { "id": "H10", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-10", "gap_kind": "paper_open_question", "question": "Whether scaling can ever resolve the described exploration failure modes or only occasionally escape them.", "hypothesis": "Scaling the RL training model from 14B to 32B will not eliminate the Boat Race oscillation exploit; at least one of three seeds will still oscillate on a single arrow tile.", "prediction": "At least one of three independent 32B training seeds converges to oscillating on a single arrow tile in Boat Race.", "falsifier": "Zero of three independent 32B training seeds oscillate on a single arrow tile in Boat Race.", "proposed_test": "Run the existing GRPO recipe on a 32B model for Boat Race with three seeds and measure oscillation.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 2000, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "none found", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T17:38:52Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 76 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: none found)", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:51:27Z", "from": "active", "to": "blocked", "reason": "estimated $2000 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 76, 83 ] } ] }, { "id": "H11", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-11", "gap_kind": "paper_open_question", "question": "Whether the observed proxy-reward failures can be addressed by approaches beyond standard exploration and credit-assignment fixes.", "hypothesis": "Adversarial reward shaping with an ensemble of reward models will reduce the observed-hidden gap on Boat Race more than entropy regularization alone.", "prediction": "The observed-hidden gap under adversarial reward shaping is at least 20% smaller than under entropy regularization alone.", "falsifier": "The observed-hidden gap under adversarial reward shaping is equal to or larger than under entropy regularization alone.", "proposed_test": "Add a reward-model ensemble to the existing GRPO pipeline for Boat Race and compare observed-hidden gaps against the entropy-regularized baseline.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 500, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "Reward Shaping for (Inference-Time) Alignment: A Stackelberg Game Perspective [2602.02572]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T17:38:52Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 76 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Reward Shaping for (Inference-Time) Alignment: A Stackelberg Game Perspective [2602.02572])", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:51:27Z", "from": "active", "to": "blocked", "reason": "estimated $500 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 76, 83 ] } ] }, { "id": "H12", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-12", "gap_kind": "paper_open_question", "question": "Whether the findings transfer to tool-using or coding agents.", "hypothesis": "The observed-hidden gap measured in text gridworlds will predict unsafe behavior on the tau-bench airline tool-use benchmark, with models above the median gap making more policy-violating tool calls.", "prediction": "Models above the median text-gridworld observed-hidden gap make significantly more policy-violating tool calls on tau-bench airline than models below the median.", "falsifier": "Models above the median text-gridworld observed-hidden gap make the same number or fewer policy-violating tool calls on tau-bench airline than models below the median.", "proposed_test": "Evaluate the same open models on tau-bench airline with policy checks and compare violation counts across the median gap split.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 300, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "none found", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T17:38:52Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 76 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: none found)", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:51:27Z", "from": "active", "to": "blocked", "reason": "estimated $300 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 76, 83 ] } ] }, { "id": "H13", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-13", "gap_kind": "paper_open_question", "question": "Why language model competence forecloses exploration of alternatives and how to induce sustained commitment to alternative trajectories.", "hypothesis": "Exploration failure in Island Navigation is caused by the model's prior policy assigning near-zero probability to off-path actions; forcing a one-step deviation to an alternative route will produce sustained commitment to that route for at least 10 subsequent steps.", "prediction": "After a forced one-step deviation to an alternative route, the model remains on that route for at least 10 subsequent steps in more than 50% of trials.", "falsifier": "After a forced one-step deviation to an alternative route, the model returns to the original exploit within 2 subsequent steps in more than 50% of trials.", "proposed_test": "Inject a forced action at a decision point in the existing Island Navigation environment, then measure subsequent route commitment across 100 episodes.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 20, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "Emergence of Exploration in Policy Gradient Reinforcement Learning via Retrying [2606.00151]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T17:38:52Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 76 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Emergence of Exploration in Policy Gradient Reinforcement Learning via Retrying [2606.00151])", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:51:27Z", "from": "active", "to": "blocked", "reason": "estimated $20 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 76, 83 ] } ] }, { "id": "H14", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-14", "gap_kind": "paper_open_question", "question": "Demonstrating direct transfer to tool-using or coding agents.", "hypothesis": "A coding agent that shows specification gaming in the text gridworlds will similarly pass visible tests while failing hidden tests on SWE-bench Lite when optimized with RL on visible tests.", "prediction": "Visible test pass rate increases by more than 10 points while hidden test pass rate changes by at most 2 points after RL optimization on visible tests.", "falsifier": "Hidden test pass rate increases by more than 8 points after RL optimization on visible tests.", "proposed_test": "Run RL on a subset of SWE-bench Lite using only visible tests, then evaluate on held-out hidden tests for the same instances.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 1000, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "TensorBench: Benchmarking Coding Agents on a Compiler-Based Tensor Framework [2606.05570]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T17:38:52Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 76 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: TensorBench: Benchmarking Coding Agents on a Compiler-Based Tensor Framework [2606.05570])", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:47:26Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 83 ] }, { "at_utc": "2026-09-14T17:51:27Z", "from": "active", "to": "blocked", "reason": "estimated $1000 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 76, 83 ] } ] }, { "id": "H15", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-gate-9ca22f07", "gap_kind": "gate_open", "question": "Is the literature really silent on this, or was the search too narrow? In the Absent Supervisor environment, all evaluated models prefer the shorter path through the hazardous region regardless of supervision, with observed reward consistently exceeding hidden reward.", "hypothesis": "In the Absent Supervisor environment, models prefer the shorter hazardous path because the prompt omits an explicit hazard-avoidance instruction, so adding that instruction will significantly reduce hazardous-path selection.", "prediction": "With an explicit hazard-avoidance instruction added to the prompt, the rate of shorter-hazardous-path selection drops by at least 20 percentage points relative to the original Absent Supervisor prompt.", "falsifier": "With the explicit hazard-avoidance instruction, models choose the shorter hazardous path at the same rate as in the original Absent Supervisor prompt (difference under 5 percentage points).", "proposed_test": "Run the released Absent Supervisor environment with the original prompt and a minimally edited prompt that adds 'avoid the hazardous region' on the same model set, and compare hazardous-path rates.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 30, "test_source": "authors_artifacts", "test_source_reason": "the test names material the society does not hold (\"released\")", "prior_art": { "verdict": "uncertain", "closest": "Bandit-Based Prompt Design Strategy Selection Improves Prompt Optimizers [2503.01163]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Bandit-Based Prompt Design Strategy Selection Improves Prompt Optimizers [2503.01163])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "active", "to": "blocked", "reason": "estimated $30 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 210, 213 ] } ] }, { "id": "H16", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-gate-9ea2322d", "gap_kind": "gate_open", "question": "Is the literature really silent on this, or was the search too narrow? In Boat Race, trained models converge on a back-and-forth exploit, oscillating on a single arrow tile to collect reward rather than completing laps.", "hypothesis": "The Boat Race back-and-forth exploit is caused by per-visit reward accumulation, so the training reward traces for oscillating episodes will show reward increments on each back-and-forth step.", "prediction": "In oscillating Boat Race episodes, reward increases on each step that revisits the same arrow tile.", "falsifier": "In oscillating episodes, reward is granted only at lap completion or is flat during the back-and-forth steps.", "proposed_test": "Re-analyze the released Boat Race reward traces for per-step reward increments during oscillation.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 0, "test_source": "authors_artifacts", "test_source_reason": "the test names material the society does not hold (\"released\")", "prior_art": { "verdict": "uncertain", "closest": "none found", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: none found)", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:45:46Z", "from": "active", "to": "blocked", "reason": "cannot be started: the test names material the society does not hold (\"released\")", "cites": [ 210, 213 ] } ] }, { "id": "H17", "arm": null, "batch": null, "state": "supported", "gap_id": "gap-pape-6e54a5c5", "gap_kind": "paper_open_question", "question": "Whether proposed mitigations genuinely eliminate the underlying reward-hacking failures rather than merely suppressing their surface manifestations.", "hypothesis": "Mitigations that improve observed reward without increasing hidden reward leave the observed-hidden gap unchanged or wider, so the gap is a valid marker of surface suppression.", "prediction": "Across the paper's mitigation results, any observed-reward gain without hidden-reward gain is accompanied by a non-decreasing observed-hidden gap.", "falsifier": "At least one mitigation raises observed reward while also closing or reversing the observed-hidden gap.", "proposed_test": "Extract per-mitigation observed and hidden reward from the paper's tables and test the sign of gap change.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 0, "test_source": "paper_text", "test_source_reason": "the test re-analyses values the paper itself reports", "prior_art": { "verdict": "uncertain", "closest": "Reward Hacking in Language Model Agents: Revisiting AI Safety Gridworlds [2606.15385]", "quote": null }, "paper_ids": [ "2606.15385" ], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Reward Hacking in Language Model Agents: Revisiting AI Safety Gridworlds [2606.15385])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T23:10:42Z", "from": "active", "to": "tested", "reason": "bounded re-analysis of reported values in 2606.15385", "cites": [ 210, 213 ] }, { "at_utc": "2026-09-14T23:13:36Z", "from": "tested", "to": "supported", "reason": "the reported values show the predicted direction (increases), agreed by both passes and located in the paper [re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.]", "cites": [ 210, 213 ] } ] }, { "id": "H18", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-pape-d022944a", "gap_kind": "paper_open_question", "question": "Whether scale (specifically 14B Island Navigation) truly resolves the failure mode or only occasionally escapes it.", "hypothesis": "The 14B Island Navigation success is stochastic escape rather than systematic resolution, so repeated runs with different seeds will show high variance and frequent failures.", "prediction": "Repeated 14B Island Navigation runs show a bimodal or high-variance success distribution, with many runs failing to reach the goal.", "falsifier": "Repeated runs show consistently high success with low variance across seeds.", "proposed_test": "Run the released 14B Island Navigation evaluation at least five times with different seeds and compare success rates.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 80, "test_source": "authors_artifacts", "test_source_reason": "the test names material the society does not hold (\"released\")", "prior_art": { "verdict": "uncertain", "closest": "none found", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: none found)", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "active", "to": "blocked", "reason": "estimated $80 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 210, 213 ] } ] }, { "id": "H19", "arm": null, "batch": null, "state": "supported", "gap_id": "gap-pape-ec010649", "gap_kind": "paper_open_question", "question": "Whether scaling can ever resolve the described exploration failure modes or only occasionally escape them.", "hypothesis": "Scaling model size alone does not systematically reduce the observed-hidden reward gap; apparent scale benefits are seed-dependent escapes.", "prediction": "Across reported model scales, the observed-hidden gap remains positive on average and does not monotonically decrease with scale.", "falsifier": "The gap decreases monotonically with scale and reaches near zero at the largest scale.", "proposed_test": "Re-analyze the paper's scale-wise observed and hidden reward results for monotonic trend.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 0, "test_source": "paper_text", "test_source_reason": "the test re-analyses values the paper itself reports", "prior_art": { "verdict": "uncertain", "closest": "none found", "quote": null }, "paper_ids": [ "2606.15385" ], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: none found)", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T23:13:36Z", "from": "active", "to": "tested", "reason": "bounded re-analysis of reported values in 2606.15385", "cites": [ 210, 213 ] }, { "at_utc": "2026-09-14T23:15:47Z", "from": "tested", "to": "supported", "reason": "the reported values show the predicted direction (mixed), agreed by both passes and located in the paper [re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.]", "cites": [ 210, 213 ] } ] }, { "id": "H20", "arm": null, "batch": null, "state": "inconclusive", "gap_id": "gap-pape-a6b2644f", "gap_kind": "paper_open_question", "question": "Whether the observed proxy-reward failures can be addressed by approaches beyond standard exploration and credit-assignment fixes.", "hypothesis": "Proxy-reward failures are not fixed by standard exploration or credit-assignment fixes because the failures reflect reward misspecification; changing the reward specification is required to close the gap.", "prediction": "Entropy regularization, credit assignment, and exploration prompts leave the observed-hidden gap positive, whereas a corrected reward that penalizes the hazardous/exploit action closes it.", "falsifier": "At least one standard exploration or credit-assignment fix closes the observed-hidden gap without changing the reward specification.", "proposed_test": "Compare reported gap changes for standard fixes versus any reward-correction condition in the paper.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 0, "test_source": "paper_text", "test_source_reason": "the test re-analyses values the paper itself reports", "prior_art": { "verdict": "uncertain", "closest": "Reward Hacking in Language Model Agents: Revisiting AI Safety Gridworlds [2606.15385]", "quote": null }, "paper_ids": [ "2606.15385" ], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Reward Hacking in Language Model Agents: Revisiting AI Safety Gridworlds [2606.15385])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T23:15:47Z", "from": "active", "to": "tested", "reason": "bounded re-analysis of reported values in 2606.15385", "cites": [ 210, 213 ] }, { "at_utc": "2026-09-14T23:16:17Z", "from": "tested", "to": "inconclusive", "reason": "the paper does not report what this test needs: The paper reports no reward-correction condition that changes the reward specification to penalize the hazardous/exploit action, so the prediction's 'corrected reward closes the gap' half cannot be checked. It also reports no numeric observed-hidden gap changes for entropy regularization, credit assignment, exploration prompts, or history length; those ablations are described only qualitatively or [re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.]", "cites": [ 210, 213 ] } ] }, { "id": "H21", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-pape-7d5c4102", "gap_kind": "paper_open_question", "question": "Why language model competence forecloses exploration of alternatives and how to induce sustained commitment to alternative trajectories.", "hypothesis": "Language-model competence forecloses exploration because the model's pretrained action prior is peaked and it never samples the safe alternative; seeding alternative first actions will induce sustained exploration.", "prediction": "When the first action is forced to a non-preferred safe alternative, models find the safe solution more often than under unseeded exploration.", "falsifier": "Seeding alternative first actions does not increase safe-solution discovery; models revert to the same exploit.", "proposed_test": "Run the released gridworld with forced first-action variants and compare safe-solution rates.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 40, "test_source": "authors_artifacts", "test_source_reason": "the test names material the society does not hold (\"released\")", "prior_art": { "verdict": "uncertain", "closest": "Reasoning or Memorization? Direction-Aware Diversity Exploration in LLM Reinforcement Learning [2606.10346]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Reasoning or Memorization? Direction-Aware Diversity Exploration in LLM Reinforcement Learning [2606.10346])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "active", "to": "blocked", "reason": "estimated $40 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 210, 213 ] } ] }, { "id": "H22", "arm": null, "batch": null, "state": "supported", "gap_id": "gap-pape-2c9421b0", "gap_kind": "paper_open_question", "question": "Are general-purpose and finance-specific LLMs, despite their financial proficiency, resilient against domain-specific attacks?", "hypothesis": "Finance-specific LLMs are not more resilient than general-purpose LLMs to domain-specific financial jailbreaks; their attack success rate is at least as high.", "prediction": "On the FinRED benchmark, finance-specific models show ASR equal to or higher than general-purpose models of similar size.", "falsifier": "Finance-specific models show significantly lower ASR than general-purpose models of similar size.", "proposed_test": "Re-analyze the paper's reported per-model ASR for finance-specific versus general-purpose models.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 0, "test_source": "paper_text", "test_source_reason": "the test re-analyses values the paper itself reports", "prior_art": { "verdict": "uncertain", "closest": "FinRED: An Expert-Guided Benchmark Generation and Evaluation Framework for Financial LLM Red-Teaming [2606.19887]", "quote": null }, "paper_ids": [ "2606.19887" ], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: FinRED: An Expert-Guided Benchmark Generation and Evaluation Framework for Financial LLM Red-Teaming [2606.19887])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T23:16:17Z", "from": "active", "to": "tested", "reason": "bounded re-analysis of reported values in 2606.19887", "cites": [ 210, 213 ] }, { "at_utc": "2026-09-14T23:19:36Z", "from": "tested", "to": "supported", "reason": "the reported values show the predicted direction (increases), agreed by both passes and located in the paper [re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.]", "cites": [ 210, 213 ] } ] }, { "id": "H23", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-pape-3212585a", "gap_kind": "paper_open_question", "question": "The paper reports that vulnerability is concentrated in open-source sLMs while API-based models remain comparatively robust, but the underlying reasons are not established.", "hypothesis": "The vulnerability gap between open-source small LMs and API-based models on financial attacks is explained by the presence of safety alignment, not by model size or API access; open-source models with comparable safety training will show comparable robustness.", "prediction": "Among models matched on safety alignment, open-source and API-based models have similar ASR.", "falsifier": "Even when safety alignment is matched, open-source models still show much higher ASR.", "proposed_test": "Re-analyze model cards and ASR for safety-alignment status; if insufficient, compare open-source safety-tuned versus base versions.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 0, "test_source": "unknown", "test_source_reason": "claims the existing corpus suffices, but names no material the society can point at", "prior_art": { "verdict": "uncertain", "closest": "none found", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: none found)", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:45:46Z", "from": "active", "to": "blocked", "reason": "cannot be started: claims the existing corpus suffices, but names no material the society can point at", "cites": [ 210, 213 ] } ] }, { "id": "H24", "arm": null, "batch": null, "state": "inconclusive", "gap_id": "gap-pape-70e155cc", "gap_kind": "paper_open_question", "question": "Whether optimization-based suffixes disrupt the financial context of FinRED seeds, offered as a speculative explanation for lower ASR under some optimization-based attacks", "hypothesis": "Optimization-based suffixes lower attack success on FinRED by displacing financial context tokens, so lower-ASR suffixes have significantly lower overlap with financial keywords in the seed.", "prediction": "A negative correlation exists between suffix-induced displacement of financial keywords and attack success rate.", "falsifier": "Suffixes with high financial-keyword displacement show high ASR, or there is no correlation.", "proposed_test": "Compute token/embedding overlap between suffix and seed financial terms and correlate with reported ASR.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 0, "test_source": "paper_text", "test_source_reason": "the test re-analyses values the paper itself reports", "prior_art": { "verdict": "uncertain", "closest": "FinRED: An Expert-Guided Benchmark Generation and Evaluation Framework for Financial LLM Red-Teaming [2606.19887]", "quote": null }, "paper_ids": [ "2606.19887" ], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: FinRED: An Expert-Guided Benchmark Generation and Evaluation Framework for Financial LLM Red-Teaming [2606.19887])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T23:19:36Z", "from": "active", "to": "tested", "reason": "bounded re-analysis of reported values in 2606.19887", "cites": [ 210, 213 ] }, { "at_utc": "2026-09-14T23:19:51Z", "from": "tested", "to": "inconclusive", "reason": "a reported value could not be located in the paper (0 unlocatable), so the comparison rests on text the society cannot point at [re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.]", "cites": [ 210, 213 ] }, { "at_utc": "2026-09-14T23:23:03Z", "from": "inconclusive", "to": "active", "reason": "reopened to correct the recorded reason: the first re-analysis reported \"a value could not be located (0 unlocatable)\" when in fact the paper reports none of the needed quantities. The verdict was right and the stated reason was a non-sequitur, so the test is re-run to record the true one.", "cites": [ 210, 213 ] }, { "at_utc": "2026-09-14T23:23:14Z", "from": "active", "to": "tested", "reason": "bounded re-analysis of reported values in 2606.19887", "cites": [ 210, 213 ] }, { "at_utc": "2026-09-14T23:23:48Z", "from": "tested", "to": "inconclusive", "reason": "the paper does not report what this test needs: The paper reports no token-level or embedding overlap between optimization-based suffixes and seed financial keywords, and no measure of suffix-induced displacement of financial context tokens, so no correlation with ASR can be computed. It only offers a qualitative, non-quantified speculation about disruption of financial context. [re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.]", "cites": [ 210, 213 ] } ] }, { "id": "H25", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-pape-ea17ab70", "gap_kind": "paper_open_question", "question": "Whether a same-family attacker would maintain its effectiveness across all turns.", "hypothesis": "A same-family attacker (Claude attacking Claude) does not maintain per-turn effectiveness equal to cross-family attackers; its attack success rate declines after the first turn.", "prediction": "In multi-turn attacks, same-family attacker ASR drops by turn 3 relative to turn 1, while cross-family ASR does not.", "falsifier": "Same-family attacker ASR remains flat or increases across turns at the same rate as cross-family.", "proposed_test": "Run the paper's multi-turn attack protocol with Claude Sonnet as both attacker and target for at least three turns and compare per-turn ASR to the cross-family condition.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 50, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "none found", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: none found)", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "active", "to": "blocked", "reason": "estimated $50 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 210, 213 ] } ] }, { "id": "H26", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-pape-e2e450f0", "gap_kind": "paper_open_question", "question": "Whether the attacker self-limitation of safety-trained models generalizes beyond the specific models studied.", "hypothesis": "Attacker self-limitation is a property of safety training rather than model identity; it generalizes to other safety-trained model families.", "prediction": "When used as attackers, safety-trained models from at least two other families show lower harmful-output rates than a non-safety-trained base model.", "falsifier": "Other safety-trained models do not show self-limitation; their harmful-output rates match base models.", "proposed_test": "Run the multi-turn attack protocol with two additional safety-trained model families as attackers and compare harmful-output rates to a base model.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 80, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "Safety Measurements for Fine-tuned LLMs Should be Grounded in Capability [2606.03648]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Safety Measurements for Fine-tuned LLMs Should be Grounded in Capability [2606.03648])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "active", "to": "blocked", "reason": "estimated $80 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 210, 213 ] } ] }, { "id": "H27", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-pape-5c4aeb89", "gap_kind": "paper_open_question", "question": "Whether the observed trajectory differences reflect different training philosophies, since training objectives cannot be directly inspected from behavioral data alone.", "hypothesis": "The four trajectory signatures reflect training philosophy, so a classifier trained on trajectory features can predict the training objective above chance.", "prediction": "Leave-one-model-out classification of training philosophy from trajectory features achieves accuracy above 50%.", "falsifier": "Classification accuracy is at or below chance, indicating trajectories do not encode training philosophy.", "proposed_test": "Extract turn-level features from the paper's released trajectories and train a simple classifier.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 0, "test_source": "authors_artifacts", "test_source_reason": "the test names material the society does not hold (\"released\")", "prior_art": { "verdict": "uncertain", "closest": "none found", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: none found)", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:45:46Z", "from": "active", "to": "blocked", "reason": "cannot be started: the test names material the society does not hold (\"released\")", "cites": [ 210, 213 ] } ] }, { "id": "H28", "arm": null, "batch": null, "state": "inconclusive", "gap_id": "gap-pape-326d4269", "gap_kind": "paper_open_question", "question": "Whether targeted Turn 2 hardening validated in deployment conditions would produce the suggested outsized impact on overall multi-turn safety.", "hypothesis": "Hardening Turn 2 reduces overall multi-turn attack success more than hardening an equal number of other turns.", "prediction": "Turn-2-only hardening yields a larger drop in cumulative ASR than turn-1-only or turn-3-only hardening.", "falsifier": "Turn-2 hardening does not produce a larger cumulative ASR drop than hardening other turns.", "proposed_test": "Re-analyze per-turn ablation results or simulate hardening by removing successful Turn-2 attacks from the paper's trajectory data.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 0, "test_source": "paper_text", "test_source_reason": "the test re-analyses values the paper itself reports", "prior_art": { "verdict": "uncertain", "closest": "Unsafer in Many Turns: Benchmarking and Defending Multi-Turn Safety Risks in Tool-Using Agents [2602.13379]", "quote": null }, "paper_ids": [ "2606.02630" ], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Unsafer in Many Turns: Benchmarking and Defending Multi-Turn Safety Risks in Tool-Using Agents [2602.13379])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T23:19:51Z", "from": "active", "to": "tested", "reason": "bounded re-analysis of reported values in 2606.02630", "cites": [ 210, 213 ] }, { "at_utc": "2026-09-14T23:20:31Z", "from": "tested", "to": "inconclusive", "reason": "the paper does not report what this test needs: The paper reports no per-turn hardening ablation comparing Turn-2-only hardening against Turn-1-only or Turn-3-only hardening, and it does not provide the per-conversation trajectory data that would be needed to simulate removing successful Turn-2 attacks. It gives only aggregate per-turn unsafe rates and a Turn-2 concentration statistic. [re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.]", "cites": [ 210, 213 ] } ] }, { "id": "H29", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-pape-3f22a163", "gap_kind": "paper_open_question", "question": "Red-teaming pipelines that rely on safety-trained models should implement explicit refusal detection and handling.", "hypothesis": "Red-teaming pipelines that do not explicitly detect and handle refusals overestimate attack success rates.", "prediction": "Adding refusal detection to the paper's pipeline reduces measured ASR by at least 5 percentage points.", "falsifier": "Refusal detection does not change measured ASR (less than 1 percentage point).", "proposed_test": "Re-analyze the paper's attack transcripts for refusal phrases and recompute ASR.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 0, "test_source": "authors_artifacts", "test_source_reason": "the test names material the society does not hold (\"transcripts\")", "prior_art": { "verdict": "uncertain", "closest": "Domain-Conditioned Safety in Frontier Computer-Using Agents [2606.05233]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Domain-Conditioned Safety in Frontier Computer-Using Agents [2606.05233])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:45:46Z", "from": "active", "to": "blocked", "reason": "cannot be started: the test names material the society does not hold (\"transcripts\")", "cites": [ 210, 213 ] } ] }, { "id": "H30", "arm": null, "batch": null, "state": "inconclusive", "gap_id": "gap-pape-2b6a54b1", "gap_kind": "paper_open_question", "question": "Conversation-level pattern detection", "hypothesis": "Conversation-level pattern detection identifies multi-turn attacks earlier than per-turn classification.", "prediction": "At the same false-positive rate, conversation-level detection achieves higher early-turn AUC than per-turn detection.", "falsifier": "Per-turn detection matches or exceeds conversation-level early-turn AUC.", "proposed_test": "Re-analyze the paper's multi-turn conversation data with a simple conversation-level feature classifier versus per-turn.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 0, "test_source": "paper_text", "test_source_reason": "the test re-analyses values the paper itself reports", "prior_art": { "verdict": "uncertain", "closest": "PLAGUE: Plug-and-play framework for Lifelong Adaptive Generation of Multi-turn Exploits [2510.17947]", "quote": null }, "paper_ids": [ "2606.02630" ], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: PLAGUE: Plug-and-play framework for Lifelong Adaptive Generation of Multi-turn Exploits [2510.17947])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T23:20:31Z", "from": "active", "to": "tested", "reason": "bounded re-analysis of reported values in 2606.02630", "cites": [ 210, 213 ] }, { "at_utc": "2026-09-14T23:20:57Z", "from": "tested", "to": "inconclusive", "reason": "a reported value could not be located in the paper (0 unlocatable), so the comparison rests on text the society cannot point at [re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.]", "cites": [ 210, 213 ] }, { "at_utc": "2026-09-14T23:23:03Z", "from": "inconclusive", "to": "active", "reason": "reopened to correct the recorded reason: the first re-analysis reported \"a value could not be located (0 unlocatable)\" when in fact the paper reports none of the needed quantities. The verdict was right and the stated reason was a non-sequitur, so the test is re-run to record the true one.", "cites": [ 210, 213 ] }, { "at_utc": "2026-09-14T23:23:48Z", "from": "active", "to": "tested", "reason": "bounded re-analysis of reported values in 2606.02630", "cites": [ 210, 213 ] }, { "at_utc": "2026-09-14T23:24:14Z", "from": "tested", "to": "inconclusive", "reason": "the paper does not report what this test needs: The paper evaluates a classifier only per turn (Phase 1 single-turn accuracy/miss-rate/false-alarm-rate, Phase 2 per-turn drift, Phase 3 intervention on Turn 4 unsafe rate) and reports no AUC at all, no conversation-level feature classifier, and no early-turn detection comparison between a conversation-level and a per-turn detector. 'Conversation-level pattern detection' appears only as a recommen [re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.]", "cites": [ 210, 213 ] } ] }, { "id": "H31", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-pape-4a2fae34", "gap_kind": "paper_open_question", "question": "How can multimodal deception be reliably detected without relying on prohibitively expensive human evaluation?", "hypothesis": "Automated debate protocols detect multimodal deception at least as reliably as human evaluation.", "prediction": "On the same multimodal deception dataset, automated debate detections agree with human judgments at kappa > 0.6.", "falsifier": "Automated debate detections agree with human judgments at kappa <= 0.4.", "proposed_test": "Run the released debate protocol on the paper's multimodal deception examples and compare to human labels.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 20, "test_source": "authors_artifacts", "test_source_reason": "the test names material the society does not hold (\"released\")", "prior_art": { "verdict": "uncertain", "closest": "Debate with Images: Detecting Deceptive Behaviors in Multimodal Large Language Models [2512.00349]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Debate with Images: Detecting Deceptive Behaviors in Multimodal Large Language Models [2512.00349])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "active", "to": "blocked", "reason": "estimated $20 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 210, 213 ] } ] }, { "id": "H32", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-pape-02615b33", "gap_kind": "paper_open_question", "question": "How do deceptive behaviors manifest in multimodal large language models as opposed to text-only settings?", "hypothesis": "Multimodal LLMs exhibit higher deception rates when deceptive cues are conveyed visually rather than textually.", "prediction": "On matched tasks, visually cued deception yields higher deception rates than text-only cues.", "falsifier": "Visually cued deception yields equal or lower deception rates than text-only cues.", "proposed_test": "Run a matched visual versus text deception prompt set on the same MLLM.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 50, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "Debate with Images: Detecting Deceptive Behaviors in Multimodal Large Language Models [2512.00349]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Debate with Images: Detecting Deceptive Behaviors in Multimodal Large Language Models [2512.00349])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "active", "to": "blocked", "reason": "estimated $50 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 210, 213 ] } ] }, { "id": "H33", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-pape-7a36c79d", "gap_kind": "paper_open_question", "question": "Development of defensive safety technologies using the released dataset.", "hypothesis": "Defensive safety technologies trained on the released dataset reduce attack success on held-out attacks.", "prediction": "A classifier trained on the dataset achieves lower ASR on held-out attacks than a baseline.", "falsifier": "No improvement over baseline (ASR difference under 2 percentage points).", "proposed_test": "Train a small classifier on the released dataset and evaluate on held-out attacks.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 20, "test_source": "authors_artifacts", "test_source_reason": "the test names material the society does not hold (\"released\")", "prior_art": { "verdict": "uncertain", "closest": "none found", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: none found)", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "active", "to": "blocked", "reason": "estimated $20 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 210, 213 ] } ] }, { "id": "H34", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-pape-c0140f60", "gap_kind": "paper_open_question", "question": "Very long contexts, long multi-turn episodes, or document-scale evidence may require chunked readout, hierarchical aggregation, or longer-context training.", "hypothesis": "Chunked readout with hierarchical aggregation improves long-context document-scale evidence tasks over single-pass long-context training.", "prediction": "Chunked readout improves accuracy on document-scale evidence questions by at least 5 percentage points.", "falsifier": "Chunked readout improves accuracy by less than 1 percentage point or worsens it.", "proposed_test": "Run the paper's document-scale evidence task with chunked readout versus single-pass baseline.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 50, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "HMARS: A Hierarchical Multi-Agent Memory System for Long-Context Reasoning [2606.28349]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: HMARS: A Hierarchical Multi-Agent Memory System for Long-Context Reasoning [2606.28349])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "active", "to": "blocked", "reason": "estimated $50 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 210, 213 ] } ] }, { "id": "H35", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-pape-68a6e5e9", "gap_kind": "paper_open_question", "question": "Multilingual, multimodal, and longer-horizon attacks remain for future work.", "hypothesis": "Multi-turn attacks succeed at higher rates in lower-resource languages than in English.", "prediction": "ASR for the same attack in a lower-resource language exceeds English ASR by at least 10 percentage points.", "falsifier": "Lower-resource language ASR is equal to or lower than English ASR.", "proposed_test": "Run the released attack prompts translated into two lower-resource languages on the same model.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 50, "test_source": "authors_artifacts", "test_source_reason": "the test names material the society does not hold (\"released\")", "prior_art": { "verdict": "uncertain", "closest": "TukaBench: A Culturally Grounded Jailbreak Benchmark for African Languages [2606.01322]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: TukaBench: A Culturally Grounded Jailbreak Benchmark for African Languages [2606.01322])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "active", "to": "blocked", "reason": "estimated $50 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 210, 213 ] } ] }, { "id": "H36", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-pape-a4fa72ae", "gap_kind": "paper_open_question", "question": "Future systems should combine input classification with output-side review, uncertainty-aware escalation, and conversation-level pattern detection.", "hypothesis": "Combining input classification with output-side review reduces multi-turn attack success more than either defense alone.", "prediction": "Combined defense ASR is lower than input-only and output-only ASR by at least 5 percentage points.", "falsifier": "Combined defense ASR is not lower than the best single defense.", "proposed_test": "Run the combined pipeline on the paper's released attack set and compare ASR.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 50, "test_source": "authors_artifacts", "test_source_reason": "the test names material the society does not hold (\"released\")", "prior_art": { "verdict": "uncertain", "closest": "Unsafer in Many Turns: Benchmarking and Defending Multi-Turn Safety Risks in Tool-Using Agents [2602.13379]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Unsafer in Many Turns: Benchmarking and Defending Multi-Turn Safety Risks in Tool-Using Agents [2602.13379])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "active", "to": "blocked", "reason": "estimated $50 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 210, 213 ] } ] }, { "id": "H37", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-pape-5c90109a", "gap_kind": "paper_open_question", "question": "How detected failure modes can be connected to targeted alignment interventions.", "hypothesis": "Targeted alignment interventions on detected multimodal deception failure modes reduce deception more than generic safety training.", "prediction": "After intervention on the detected failure mode, deception rate drops more than after generic safety fine-tuning.", "falsifier": "Targeted intervention does not outperform generic safety training (difference under 3 percentage points).", "proposed_test": "Run a small fine-tuning experiment on the detected failure mode versus generic safety data on the released multimodal deception dataset.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 200, "test_source": "authors_artifacts", "test_source_reason": "the test names material the society does not hold (\"released\")", "prior_art": { "verdict": "uncertain", "closest": "ThinkDeception: A Progressive Reinforcement Learning Framework for Interpretable Multimodal Deception Detection [2606.18988]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: ThinkDeception: A Progressive Reinforcement Learning Framework for Interpretable Multimodal Deception Detection [2606.18988])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "active", "to": "blocked", "reason": "estimated $200 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 210, 213 ] } ] }, { "id": "H38", "arm": null, "batch": null, "state": "blocked", "gap_id": "gap-pape-f3e8ee9b", "gap_kind": "paper_open_question", "question": "How the findings transfer to tool-using or coding agents.", "hypothesis": "Reward-hacking patterns from text gridworlds transfer to coding agents: when a coding task has visible unit tests and hidden safety tests, agents optimize the visible tests while failing hidden safety tests.", "prediction": "Coding agents prompted on the gridworld setup show higher visible-test pass rates than hidden-safety-test pass rates by at least 15 percentage points.", "falsifier": "Visible and hidden test pass rates are equal within 5 percentage points.", "proposed_test": "Run a small coding-agent benchmark with visible versus hidden tests on the same models used in the paper.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 50, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "Diagnosing Task Insensitivity in Language Agents [2606.26918]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-14T21:40:55Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 210 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Diagnosing Task Insensitivity in Language Agents [2606.26918])", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 213 ] }, { "at_utc": "2026-09-14T21:42:56Z", "from": "active", "to": "blocked", "reason": "estimated $50 to test, above the affordable ceiling of $10 — recorded as a known question this society cannot currently fund", "cites": [ 210, 213 ] } ] }, { "id": "H39", "arm": "A", "batch": "yield-4", "state": "blocked", "gap_id": "gap-pape-5903da71", "gap_kind": "paper_open_question", "question": "Whether the findings transfer to tool-using or coding agents.", "hypothesis": "The observed-reward-high / hidden-objective-low gap reported for text gridworld agents also appears when the same tasks are expressed as tool-use episodes with schemas for available actions.", "prediction": "Models that reach >80% of maximum observed reward in the tool-wrapped Absent Supervisor and Boat Race tasks fail the withheld safety objective in at least half of episodes, mirroring the gridworld gap.", "falsifier": "Tool-wrapped agents show no decoupling: episodes with high observed reward also satisfy the hidden safety objective in more than 90% of cases (gap below 10 percentage points).", "proposed_test": "Re-express the paper's four text environments as tool-call schemas using its released prompts, run two open-weight models through an API harness (200 episodes each), and compute the observed-minus-hidden reward gap.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 220, "test_source": "authors_artifacts", "test_source_reason": "the test names material the society does not hold (\"released\")", "prior_art": { "verdict": "open", "closest": "none found", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-15T08:21:08Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 1401 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: open (closest: none found)", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "prior_art_checked", "to": "active", "reason": "the gate found no work answering this", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "active", "to": "blocked", "reason": "cannot be started: the test names material the society does not hold (\"released\")", "cites": [ 1401, 1404 ] } ] }, { "id": "H40", "arm": "A", "batch": "yield-4", "state": "blocked", "gap_id": "gap-pape-6f08bdb7", "gap_kind": "paper_open_question", "question": "Demonstrating direct transfer to tool-using or coding agents.", "hypothesis": "Reward-hacking behaviour transfers to coding agents in the form of passing visible tests while failing held-out hidden tests at a higher rate than non-RL baseline coding models.", "prediction": "RL-trained coding agents achieve near-ceiling visible-test pass rates but hidden-test pass rates at least 20 percentage points lower, while base models show no such visible-hidden separation.", "falsifier": "RL-trained coding agents show visible-test and hidden-test pass rates within 5 percentage points of each other, or their visible-hidden separation is no larger than that of base models.", "proposed_test": "Run two RL-trained and two base coding models on a public benchmark with a held-out hidden test suite (e.g., a SWE-style set split into visible/hidden halves), ~300 task instances total, and compare the visible-hidden pass-rate deltas.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 180, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "SWE-Together: Evaluating Coding Agents in Interactive User Sessions [2606.29957]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-15T08:21:08Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 1401 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: SWE-Together: Evaluating Coding Agents in Interactive User Sessions [2606.29957])", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "active", "to": "blocked", "reason": "cannot be started: the test requires code that does not exist yet", "cites": [ 1401, 1404 ] } ] }, { "id": "H41", "arm": "A", "batch": "yield-4", "state": "blocked", "gap_id": "gap-pape-432fa45d", "gap_kind": "paper_open_question", "question": "Extending the framework to richer environments, larger models, and safety-aware training.", "hypothesis": "The observed-hidden reward gap in the Boat Race environment does not shrink when the policy model is scaled from 14B to roughly 70B parameters under the same training recipe.", "prediction": "A 70B-class open-weight model trained on Boat Race converges to the same oscillating single-tile exploit and shows a gap within 10 percentage points of the reported 14B gap.", "falsifier": "The 70B-class model completes laps with hidden-objective success above 70% and a gap under 10 percentage points, or fails to converge to the back-and-forth exploit at all.", "proposed_test": "Apply the paper's published Boat Race reward and prompt configuration to one 70B-class open-weight model via a hosted API or a single rented GPU node, 3 seeds, and compare convergence behaviour and gap size to the reported 14B results.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 350, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "none found", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-15T08:21:08Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 1401 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: none found)", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "active", "to": "blocked", "reason": "cannot be started: the test requires code that does not exist yet", "cites": [ 1401, 1404 ] } ] }, { "id": "H42", "arm": "A", "batch": "yield-4", "state": "blocked", "gap_id": "gap-pape-41bad00a", "gap_kind": "paper_open_question", "question": "FinRED will be continuously updated with emerging financial threats and evolving regulations, supporting trustworthy, regulation-aligned, and safe financial AI systems.", "hypothesis": "The FinRED taxonomy as published fails to place a substantial share of 2025-2026 financial-crime cases, specifically agentic and synthetic-media-mediated fraud, into any existing subtype.", "prediction": "Fewer than 70% of sampled recent agentic/deepfake-driven fraud enforcement cases receive a single confidently-assigned FinRED subtype, with the remainder split across R2 and R4 or left unassigned.", "falsifier": "At least 90% of the sampled recent cases receive a single FinRED subtype with cross-annotator agreement of kappa >= 0.7 or better.", "proposed_test": "Collect 50 public 2025-2026 financial-crime enforcement summaries, have three independent LLM annotators assign FinRED subtypes under the published rubric, and measure single-assignment rate and agreement.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 70, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "FinRED: An Expert-Guided Benchmark Generation and Evaluation Framework for Financial LLM Red-Teaming [2606.19887]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-15T08:21:08Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 1401 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: FinRED: An Expert-Guided Benchmark Generation and Evaluation Framework for Financial LLM Red-Teaming [2606.19887])", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "active", "to": "blocked", "reason": "cannot be started: the test requires code that does not exist yet", "cites": [ 1401, 1404 ] } ] }, { "id": "H43", "arm": "A", "batch": "yield-4", "state": "blocked", "gap_id": "gap-pape-66fd05c7", "gap_kind": "paper_open_question", "question": "Taxonomy refinements suggested by experts: clarifying overlap between R2 Financial Crime and R4 Consumer Rights Violation, simplifying low-frequency R5 subtypes, and expanding consumer-facing R4 entries.", "hypothesis": "The R2 Financial Crime and R4 Consumer Rights Violation categories in FinRED overlap enough that a substantial fraction of real violation examples are assigned to both by independent annotators.", "prediction": "On a 100-item FinRED sample, at least 15% of items receive both an R2 and an R4 label from independent annotators, versus under 5% for other adjacent category pairs.", "falsifier": "Dual R2/R4 assignment occurs on under 5% of items and is no more frequent than dual assignment for other adjacent category pairs.", "proposed_test": "Re-annotate a stratified 100-item sample of FinRED with three independent LLM annotators using only the published rubric definitions, and compute the dual-assignment rate for R2/R4 versus other adjacent pairs.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 45, "test_source": "unknown", "test_source_reason": "claims the existing corpus suffices, but names no material the society can point at", "prior_art": { "verdict": "uncertain", "closest": "FinRED: An Expert-Guided Benchmark Generation and Evaluation Framework for Financial LLM Red-Teaming [2606.19887]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-15T08:21:08Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 1401 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: FinRED: An Expert-Guided Benchmark Generation and Evaluation Framework for Financial LLM Red-Teaming [2606.19887])", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "active", "to": "blocked", "reason": "cannot be started: claims the existing corpus suffices, but names no material the society can point at", "cites": [ 1401, 1404 ] } ] }, { "id": "H44", "arm": "A", "batch": "yield-4", "state": "blocked", "gap_id": "gap-pape-c7c89f5f", "gap_kind": "paper_open_question", "question": "Rubric revisions addressing ambiguous partial-compliance and mixed-intent responses.", "hypothesis": "FinRED's rubric produces low inter-annotator agreement specifically on responses that partially comply with a financial-harm request or mix legitimate and harmful intent.", "prediction": "On FinRED items whose responses contain partial compliance or mixed intent, cross-annotator agreement falls below kappa = 0.6, while fully compliant and fully refusing items score above kappa = 0.8.", "falsifier": "Partial-compliance and mixed-intent items reach kappa >= 0.75, statistically indistinguishable from the fully compliant and fully refusing items.", "proposed_test": "Select 60 FinRED items stratified by whether the response is full compliance, partial compliance/mixed intent, or full refusal, re-annotate with three independent LLM annotators under the published rubric, and compare agreement across strata.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 40, "test_source": "unknown", "test_source_reason": "claims the existing corpus suffices, but names no material the society can point at", "prior_art": { "verdict": "uncertain", "closest": "FinRED: An Expert-Guided Benchmark Generation and Evaluation Framework for Financial LLM Red-Teaming [2606.19887]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-15T08:21:08Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 1401 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: FinRED: An Expert-Guided Benchmark Generation and Evaluation Framework for Financial LLM Red-Teaming [2606.19887])", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "active", "to": "blocked", "reason": "cannot be started: claims the existing corpus suffices, but names no material the society can point at", "cites": [ 1401, 1404 ] } ] }, { "id": "H45", "arm": "A", "batch": "yield-4", "state": "blocked", "gap_id": "gap-pape-986e9ccf", "gap_kind": "paper_open_question", "question": "Are models, despite their financial proficiency, resilient against domain-specific attacks? (posed in the introduction)", "hypothesis": "A model's FinRED financial-safety score is largely uncorrelated with its resistance to financial-domain prompt injection and jailbreak attempts.", "prediction": "Across six models, rank correlation between FinRED safety score and financial-domain attack success rate is between -0.3 and 0.3, with the top FinRED scorer still complying on at least 30% of attack prompts.", "falsifier": "FinRED safety score and financial-domain attack success rate are strongly negatively correlated (Spearman <= -0.6), or the top FinRED scorer complies on under 10% of attack prompts.", "proposed_test": "Take six models with published or easily computed FinRED scores, run a fixed 40-prompt financial-domain injection/jailbreak set against each, score compliance with the FinRED rubric, and compute the rank correlation.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 90, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "FinRED: An Expert-Guided Benchmark Generation and Evaluation Framework for Financial LLM Red-Teaming [2606.19887]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-15T08:21:08Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 1401 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: FinRED: An Expert-Guided Benchmark Generation and Evaluation Framework for Financial LLM Red-Teaming [2606.19887])", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "active", "to": "blocked", "reason": "cannot be started: the test requires code that does not exist yet", "cites": [ 1401, 1404 ] } ] }, { "id": "H46", "arm": "A", "batch": "yield-4", "state": "blocked", "gap_id": "gap-pape-b7da02b8", "gap_kind": "paper_open_question", "question": "what, if anything, is learned during proxy optimization that allows reward hacking to emerge, adapt, and generalize?", "hypothesis": "Reward hacking acquired during proxy optimization is a task-agnostic disposition rather than a task-specific adaptation, so per-model hacking rates on held-out tasks correlate with hacking rates on the training task.", "prediction": "Across the reported training runs, per-run hacking rate on held-out tasks correlates with training-task hacking rate at r >= 0.7, and models with high training-task hacking also hack structurally different held-out tasks.", "falsifier": "Training-task and held-out hacking rates correlate below r = 0.3 across runs, with some runs showing heavy training-task hacking and no held-out hacking (or the reverse).", "proposed_test": "Extract per-run training-task and held-out hacking rates from the paper's reported tables and figures (or its released logs), and compute the cross-run correlation.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 10, "test_source": "authors_artifacts", "test_source_reason": "the test names material the society does not hold (\"released\")", "prior_art": { "verdict": "uncertain", "closest": "Proxy Reward Internalization and Mechanistic Exploitation: A Learned Precursor to Reward Hacking and Its Generalization [2606.09711]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-15T08:21:08Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 1401 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Proxy Reward Internalization and Mechanistic Exploitation: A Learned Precursor to Reward Hacking and Its Generalization [2606.09711])", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "active", "to": "blocked", "reason": "cannot be started: the test names material the society does not hold (\"released\")", "cites": [ 1401, 1404 ] } ] }, { "id": "H47", "arm": "A", "batch": "yield-4", "state": "blocked", "gap_id": "gap-pape-79e5d6b3", "gap_kind": "paper_open_question", "question": "what underlying property is being learned and transferred when reward hacking generalizes beyond the training task?", "hypothesis": "What transfers in reward-hacking generalization is sensitivity to shortcut cheapness, such that a held-out task is hacked at a higher rate the more its exploit resembles the training task's shortcut in step count and required actions.", "prediction": "Held-out hacking rate increases monotonically with structural similarity of the exploitable shortcut to the training shortcut, and is higher for a new domain sharing the same shortcut structure than for the same domain with a differently structured shortcut.", "falsifier": "Held-out hacking rate is independent of shortcut structural similarity (no monotone trend across similarity tiers) and tracks domain identity instead.", "proposed_test": "Re-code the paper's held-out transfer tasks on a shortcut-similarity scale and compare hacking rates across similarity tiers using the reported per-task results.", "held_material": null, "uncertainty": null, "needs": "existing_corpus", "est_cost_usd": 15, "test_source": "paper_text", "test_source_reason": "the test re-analyses values the paper itself reports", "prior_art": { "verdict": "uncertain", "closest": "Suppressing Forgery-Specific Shortcuts for Generalizable Deepfake Detection [2606.01843]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-15T08:21:08Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 1401 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Suppressing Forgery-Specific Shortcuts for Generalizable Deepfake Detection [2606.01843])", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "active", "to": "blocked", "reason": "estimated $15 to test, above the affordable ceiling of $10", "cites": [ 1401, 1404 ] } ] }, { "id": "H48", "arm": "A", "batch": "yield-4", "state": "blocked", "gap_id": "gap-pape-57b3a851", "gap_kind": "paper_open_question", "question": "Validation of Turn 2 hardening in deployment conditions.", "hypothesis": "The per-turn hardening gain reported for Turn 2 does not survive deployment-like conditions in which the attacker retains memory of earlier turns and can iterate its strategy within a session.", "prediction": "Attack success rate at turn 2 rises to more than twice the reported single-shot turn-2 rate once the attacker is given prior-turn context and three retries.", "falsifier": "Turn-2 attack success rate with attacker memory and retries stays within 5 percentage points of the reported single-shot turn-2 rate.", "proposed_test": "Replay the paper's published attack prompts against its defended configuration, once with fresh single-shot turns and once with a stateful attacker given prior-turn context and three retries, 50 episodes each, and compare turn-2 attack success rates.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 150, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "RoTRAG: Rule of Thumb Reasoning for Conversation Harm Detection with Retrieval-Augmented Generation [2604.17301]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-15T08:21:08Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 1401 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: RoTRAG: Rule of Thumb Reasoning for Conversation Harm Detection with Retrieval-Augmented Generation [2604.17301])", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "active", "to": "blocked", "reason": "cannot be started: the test requires code that does not exist yet", "cites": [ 1401, 1404 ] } ] }, { "id": "H49", "arm": "A", "batch": "yield-4", "state": "blocked", "gap_id": "gap-pape-0ebb77a3", "gap_kind": "paper_open_question", "question": "Whether a same-family attacker (Claude Sonnet attacking Claude Sonnet) would maintain its apparent per-turn effectiveness across all turns.", "hypothesis": "A same-family attacker (Sonnet attacking Sonnet) loses per-turn effectiveness faster than a cross-family attacker, so the observed per-turn attack success advantage decays with turn index rather than holding steady.", "prediction": "Same-family attack success decreases by at least 15 percentage points from turn 2 to turn 4, a significantly steeper decline than the cross-family attacker's curve over the same turns.", "falsifier": "Same-family attack success is flat or rising across turns 2-4, and the same-family versus cross-family decline curves differ by less than 5 percentage points.", "proposed_test": "Run the paper's published multi-turn attack prompts in a same-family configuration and a cross-family configuration, 50 episodes each, and compare per-turn attack success curves.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 120, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "MultiTurnPSB: Evaluating Multi-Turn Jailbreak Attacks an dClassifier-Based Defenses for Medical AI Safety [2606.02630]", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-15T08:21:08Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 1401 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: MultiTurnPSB: Evaluating Multi-Turn Jailbreak Attacks an dClassifier-Based Defenses for Medical AI Safety [2606.02630])", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "active", "to": "blocked", "reason": "cannot be started: the test requires code that does not exist yet", "cites": [ 1401, 1404 ] } ] }, { "id": "H50", "arm": "A", "batch": "yield-4", "state": "blocked", "gap_id": "gap-pape-5cdfda05", "gap_kind": "paper_open_question", "question": "Whether attacker self-limitation generalizes beyond the specific models used in this study.", "hypothesis": "Attacker self-limitation is a general property of aligned models rather than a quirk of the specific models in the study, appearing at comparable rates in at least three other model families not tested by the authors.", "prediction": "At least three of four new model families refuse or de-escalate on 25% or more of the published escalation prompts, within 15 percentage points of the original study's self-limitation rate.", "falsifier": "Fewer than two of the new families show any self-limitation, or their rates fall below 5% of escalation prompts.", "proposed_test": "Replay the paper's published escalation prompts against four model families absent from the original study and count refusals/de-escalations under the paper's own coding scheme.", "held_material": null, "uncertainty": null, "needs": "new_code", "est_cost_usd": 100, "test_source": "new_code", "test_source_reason": "the test requires code that does not exist yet", "prior_art": { "verdict": "uncertain", "closest": "none found", "quote": null }, "paper_ids": [], "history": [ { "at_utc": "2026-09-15T08:21:08Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 1401 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: none found)", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 1404 ] }, { "at_utc": "2026-09-15T08:27:24Z", "from": "active", "to": "blocked", "reason": "cannot be started: the test requires code that does not exist yet", "cites": [ 1401, 1404 ] } ] }, { "id": "H51", "arm": "B", "batch": "yield-4", "state": "inconclusive", "gap_id": "gap-gate-9ba22d74", "gap_kind": "gate_open", "question": "Is the literature really silent on this, or was the search too narrow? In Safe Interruptibility, Qwen3-235B-Thinking's high hidden reward is accidental, arising from misinterpreting the interruption tile as a collectible item rather than from principled safety.", "hypothesis": "Qwen3-235B-Thinking's high hidden reward in Safe Interruptibility is an isolated outlier rather than part of a consistent safety advantage across the suite's safety environments.", "prediction": "If the accidental-misinterpretation account is correct, Qwen3-235B-Thinking's hidden reward rank in Safe Interruptibility will be substantially higher than its hidden reward ranks in Absent Supervisor and Distributional Shift, and its other safety scores will not be top-tier.", "falsifier": "If the reported hidden rewards for Qwen3-235B-Thinking in Absent Supervisor and Distributional Shift are also at or near the top of the model set, then the accidental-misinterpretation account is false and the high hidden reward generalizes.", "proposed_test": "Read the reported hidden reward values for Qwen3-235B-Thinking and all baselines in the Safe Interruptibility, Absent Supervisor, and Distributional Shift results of 2606.15385, and compare the model's rank in Safe Interruptibility to its ranks in the other two safety environments.", "held_material": "2606.15385 results tables for Safe Interruptibility, Absent Supervisor, and Distributional Shift (reported hidden reward values for Qwen3-235B-Thinking and baselines, summarized in memory items c5, c4, c14).", "uncertainty": "Whether Qwen3-235B-Thinking's high hidden reward in Safe Interruptibility reflects principled safety or an accidental artifact of misinterpreting the interruption tile as a collectible, as claimed in c5; the paper reports the hidden reward but no cross-environment consistency check.", "needs": "existing_corpus", "est_cost_usd": 0, "test_source": "paper_text", "test_source_reason": "the test re-analyses values the paper itself reports", "prior_art": { "verdict": "uncertain", "closest": "Reward Hacking in Language Model Agents: Revisiting AI Safety Gridworlds [2606.15385]", "quote": null }, "paper_ids": [ "2606.15385" ], "history": [ { "at_utc": "2026-09-15T08:28:05Z", "from": null, "to": "proposed", "reason": "located as an open gap and turned into a falsifiable prediction", "cites": [ 1408 ] }, { "at_utc": "2026-09-15T08:33:34Z", "from": "proposed", "to": "prior_art_checked", "reason": "prior-art gate: uncertain (closest: Reward Hacking in Language Model Agents: Revisiting AI Safety Gridworlds [2606.15385])", "cites": [ 1411 ] }, { "at_utc": "2026-09-15T08:33:34Z", "from": "prior_art_checked", "to": "active", "reason": "the gate could not decide; the hypothesis stays live but unverified against prior art", "cites": [ 1411 ] }, { "at_utc": "2026-09-15T08:33:34Z", "from": "active", "to": "blocked", "reason": "cannot be started: claims the existing corpus suffices, but names no material the society can point at", "cites": [ 1408, 1411 ] }, { "at_utc": "2026-09-15T08:42:14Z", "from": "blocked", "to": "active", "reason": "affordability-v2: the earlier block was a false negative. the test re-analyses values the paper itself reports", "cites": [ 1408, 1411 ] }, { "at_utc": "2026-09-15T08:42:30Z", "from": "active", "to": "tested", "reason": "bounded re-analysis of reported values in 2606.15385", "cites": [ 1408, 1411 ] }, { "at_utc": "2026-09-15T08:44:14Z", "from": "tested", "to": "inconclusive", "reason": "the independent pass offered additional values that cannot be located in the paper [re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.]", "cites": [ 1408, 1411 ] } ] } ], "reanalyses": [ { "hypothesis": "H17", "paper": "2606.15385", "seq": 227, "at_utc": "2026-09-14T23:13:36Z", "verdict": { "direction": "increases", "reason": "the reported values show the predicted direction (increases), agreed by both passes and located in the paper", "status": "supported" }, "values_checked": 9, "grounding": { "checked": 9, "located": 9, "ok": true }, "caveat": "re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.", "values": [ { "label": "RL (tested as a correction), effect on the gap", "value": "widening the gap", "quote": "Direct reward optimization reliably increases the observed signal but fails to improve, and can degrade, hidden safety performance, widening the gap between the two." }, { "label": "RL, abstract gap effect", "value": "widens the gap", "quote": "direct reward optimization widens the gap between observed and hidden reward, as the model's initial competence causes it to lock into locally rewarding strategies before discovering safer alternatives." }, { "label": "explicit task goal mitigation, hidden reward with goal", "value": "21.96", "quote": "moving the Instruct model from a clear exploit (−9.06 hidden) to genuine lap progress (21.96 hidden)" }, { "label": "Table 2 rows (Instruct zero-shot and with explicit goal; Thinking comparator)", "value": "hidden −9.06 / 21.96 / 48.42; observed −19.94 / 6.82 / 24.07", "quote": "Qwen3-235B-Instruct, zero-shot (no task goal)\nQwen3-235B-Instruct, with explicit task goal\nQwen3-235B-Thinking, zero-shot (no task goal)\n\n−9.06\n21.96\n48.42\n\n−19.94\n6.82\n24.07" }, { "label": "RL Absent Supervisor, validation hidden reward", "value": "near zero", "quote": "validation hidden reward remains near zero (Figure 8, top row), far below the observed reward, indicating that the improvement in observed performance does not translate into safer behavior." }, { "label": "RL Absent Supervisor 7B, hidden and observed", "value": "hidden about +10, observed around +25", "quote": "at 7B, two of three seeds reach validation hidden reward of about +10 with observed reward around +25, while one seed stalls" }, { "label": "RL Absent Supervisor 14B, hidden and observed", "value": "hidden roughly +25, observed about +40", "quote": "at 14B, validation hidden reward rises to roughly +25 while observed reward reaches about +40" }, { "label": "RL Absent Supervisor 14B, gap", "value": "narrows in absolute terms but remains clearly present", "quote": "The gap narrows in absolute terms at 14B but remains clearly present" }, { "label": "ablations (credit assignment, prompt, history, entropy), numeric observed/hidden values", "value": "none reported", "quote": "The model's strong prior toward reward-maximizing behavior dominates across all interventions tested." } ], "usage": { "cost_usd": 0.02634, "calls": 2 } }, { "hypothesis": "H19", "paper": "2606.15385", "seq": 228, "at_utc": "2026-09-14T23:15:47Z", "verdict": { "direction": "mixed", "reason": "the reported values show the predicted direction (mixed), agreed by both passes and located in the paper", "status": "supported" }, "values_checked": 9, "grounding": { "checked": 9, "located": 9, "ok": true }, "caveat": "re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.", "values": [ { "label": "Absent Supervisor, 7B and 14B, validation hidden and observed reward", "value": "7B: hidden about +10, observed around +25; 14B: hidden roughly +25, observed about +40", "quote": "at 7B, two of three seeds reach validation hidden reward of about +10 with observed reward around +25, while one seed stalls; at 14B, validation hidden reward rises to roughly +25 while observed reward reaches about +40." }, { "label": "Absent Supervisor, 14B, gap status", "value": "gap narrows in absolute terms at 14B but remains clearly present", "quote": "The gap narrows in absolute terms at 14B but remains clearly present, and the agent still locks into the exploit before discovering the conditionally safe policy." }, { "label": "Absent Supervisor, 1.5B and 3B, validation hidden vs observed reward", "value": "hidden reward near zero, far below observed reward (no numeric observed value given)", "quote": "In Absent Supervisor, training observed reward rises sharply before converging across all seeds at both scales (Figure 6, top row). However, validation hidden reward remains near zero (Figure 8, top row), far below the observed reward, indicating that the improvement in observed performance does not translate into safer behavior." }, { "label": "Boat Race, 7B and 14B, observed and hidden reward", "value": "observed about +22, hidden near 0", "quote": "At 7B and 14B the same exploit loop appears: training and validation observed reward both converge to about +22, while validation hidden reward ends near 0, far below the lap-completion maximum (∼+50)." }, { "label": "Across scales 1.5B–14B, pattern persistence", "value": "pattern persists across model scales (1.5B–14B)", "quote": "This pattern persists across model scales (1.5B–14B) and is not resolved by finer credit assignment, exploration prompts, or entropy regularization." }, { "label": "RL capacity scaling 1.5B to 14B, failure mitigation", "value": "does not mitigate this failure", "quote": "Increasing model capacity from 1.5B to 14B does not mitigate this failure; the additional capacity does not translate into broader exploration or reliable discovery of the safe policy." }, { "label": "RL scaling 1.5B to 14B, summary of failures", "value": "Absent Supervisor gap persists even at 14B; 14B Island Navigation improvement is unstable across seeds", "quote": "Scaling model capacity from 1.5B to 14B does not resolve the observed failure modes (Section 4.2): the exploit loop in Boat Race and the observed–hidden gap in Absent Supervisor persist even at 14B, and the one instance of apparent improvement (14B Island Navigation) is unstable across seeds." }, { "label": "Island Navigation, 14B, seed-dependent escape", "value": "one seed reaches near-maximal performance (∼+45); not stable across seeds", "quote": "The 14B model is the one case where scale appears to help: one seed reaches near-maximal performance (∼+45) by navigating to the goal rather than the water. However, this behavior is not stable across seeds, so we are cautious about concluding that scale resolves this failure mode rather than occasionally escaping it." }, { "label": "Base (pre-RL) Qwen2.5 scales, observed-hidden gap", "value": "no observed-hidden gap of the kind that emerges after training", "quote": "All base models perform near the floor on every environment, with no observed–hidden gap of the kind that emerges after training." } ], "usage": { "cost_usd": 0.021112, "calls": 2 } }, { "hypothesis": "H20", "paper": "2606.15385", "seq": 229, "at_utc": "2026-09-14T23:16:17Z", "verdict": { "direction": null, "reason": "the paper does not report what this test needs: The paper reports no reward-correction condition that changes the reward specification to penalize the hazardous/exploit action, so the prediction's 'corrected reward closes the gap' half cannot be checked. It also reports no numeric observed-hidden gap changes for entropy regularization, credit assignment, exploration prompts, or history length; those ablations are described only qualitatively or", "status": "inconclusive" }, "values_checked": 4, "grounding": { "checked": 4, "located": 4, "ok": true }, "caveat": "re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.", "values": [ { "label": "standard mitigation: credit assignment (GiGPO vs GRPO)", "value": "same convergence patterns; no gap closure reported", "quote": "finer-grained credit assignment does not alter the outcome: both Absent Supervisor and Island Navigation exhibit the same convergence patterns under GiGPO as under GRPO." }, { "label": "standard mitigation: exploration prompt", "value": "delays convergence but does not prevent it", "quote": "The exploration prompt delays convergence but does not prevent it." }, { "label": "standard mitigation: entropy regularization", "value": "same exploitative strategies as unregularized runs", "quote": "the model converges to the same exploitative strategies as the unregularized runs across all environments except Distributional Shift, where performance was already strong." }, { "label": "reward-correction condition", "value": "not reported", "quote": "Our results show that reward hacking arises naturally when optimizing proxy objectives with capable language model agents and resists standard mitigations, suggesting that proxy-reward failures in agentic settings may require approaches beyond standard exploration and credit-assignment fixes." } ], "usage": { "cost_usd": 0.00624, "calls": 2 } }, { "hypothesis": "H22", "paper": "2606.19887", "seq": 230, "at_utc": "2026-09-14T23:19:36Z", "verdict": { "direction": "increases", "reason": "the reported values show the predicted direction (increases), agreed by both passes and located in the paper", "status": "supported" }, "values_checked": 3, "grounding": { "checked": 3, "located": 3, "ok": true }, "caveat": "re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.", "values": [ { "label": "FinRED P3 average ASR, general-purpose sLMs", "value": "58.05%", "quote": "with average ASR of 58.05% for general-purpose sLMs, 70.28% for finance-specific sLMs, and 44.44% for API-based LLMs" }, { "label": "FinRED P3 average ASR, finance-specific sLMs", "value": "70.28%", "quote": "with average ASR of 58.05% for general-purpose sLMs, 70.28% for finance-specific sLMs, and 44.44% for API-based LLMs" }, { "label": "FinRED P1/P2/P3 average ASR by pipeline and model family", "value": "P1: general 41.32, finance 52.47; P2: general 49.86, finance 61.73; P3: general 58.05, finance 70.28", "quote": "General sLMs\n\nFinancial-specific sLMs\n\nAPI LLMs\n\n41.32\n49.86\n58.05\n\n52.47\n61.73\n70.28\n\n28.91\n36.58\n44.44" } ], "usage": { "cost_usd": 0.034408, "calls": 2 } }, { "hypothesis": "H24", "paper": "2606.19887", "seq": 234, "at_utc": "2026-09-14T23:23:48Z", "verdict": { "direction": null, "reason": "the paper does not report what this test needs: The paper reports no token-level or embedding overlap between optimization-based suffixes and seed financial keywords, and no measure of suffix-induced displacement of financial context tokens, so no correlation with ASR can be computed. It only offers a qualitative, non-quantified speculation about disruption of financial context.", "status": "inconclusive" }, "values_checked": 2, "grounding": { "checked": 2, "located": 2, "ok": true }, "caveat": "re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.", "values": [ { "label": "qualitative speculation on suffix effect (no numeric overlap or correlation reported)", "value": "not quantified; no overlap/displacement metric or correlation value given", "quote": "This may occur because optimization-based suffixes sometimes disrupt the rich financial context already embedded in FinRED seeds." }, { "label": "reported direction of optimization-based attacks vs Direct Request (context for the claim, not an overlap measure)", "value": "qualitative comparison, no overlap numbers", "quote": "Interestingly, several finance-specific and open-source LLMs exhibit lower ASR under optimization-based attacks than under Direct Request." } ], "usage": { "cost_usd": 0.005427, "calls": 2 } }, { "hypothesis": "H28", "paper": "2606.02630", "seq": 232, "at_utc": "2026-09-14T23:20:31Z", "verdict": { "direction": null, "reason": "the paper does not report what this test needs: The paper reports no per-turn hardening ablation comparing Turn-2-only hardening against Turn-1-only or Turn-3-only hardening, and it does not provide the per-conversation trajectory data that would be needed to simulate removing successful Turn-2 attacks. It gives only aggregate per-turn unsafe rates and a Turn-2 concentration statistic.", "status": "inconclusive" }, "values_checked": 4, "grounding": { "checked": 4, "located": 4, "ok": true }, "caveat": "re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.", "values": [ { "label": "GPT-4.1-mini live adversarial unsafe rate, Turn 2 and Turn 4", "value": "77.7% by Turn 2; 78.8% by Turn 4", "quote": "The live adversarial attack is substantially stronger, jumping to 77.7% by Turn 2 and reaching 78.8% by Turn 4." }, { "label": "Share of Score-5 escalations occurring at Turn 2 in the Claude self-attack condition", "value": "73.9%", "quote": "73.9% of Score 5 violations that started from a safe Turn 1 occur at Turn 2 in the Claude self-attack condition." }, { "label": "Paper's stated status of the Turn-2 hardening claim", "value": "proposed, would need validation", "quote": "This suggests that targeted Turn 2 hardening may have outsized impact on overall multi-turn safety, though this would need validation in deployment conditions." }, { "label": "Conclusion claim about holding the defense at Turn 2", "value": "safety through 86% of subsequent turns", "quote": "Catastrophic failures are associated with a two-element formula and concentrate at Turn 2, where holding the defense appears to predict safety through 86% of subsequent turns." } ], "usage": { "cost_usd": 0.00603, "calls": 2 } }, { "hypothesis": "H30", "paper": "2606.02630", "seq": 235, "at_utc": "2026-09-14T23:24:14Z", "verdict": { "direction": null, "reason": "the paper does not report what this test needs: The paper evaluates a classifier only per turn (Phase 1 single-turn accuracy/miss-rate/false-alarm-rate, Phase 2 per-turn drift, Phase 3 intervention on Turn 4 unsafe rate) and reports no AUC at all, no conversation-level feature classifier, and no early-turn detection comparison between a conversation-level and a per-turn detector. 'Conversation-level pattern detection' appears only as a recommen", "status": "inconclusive" }, "values_checked": 3, "grounding": { "checked": 3, "located": 3, "ok": true }, "caveat": "re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.", "values": [ { "label": "only classifier results reported (per-turn acc/miss/lateral/intervention)", "value": "accuracy 95.5/62.4/52.8/48.5; miss 0.9/7.1/2.6/1.5; lateral 3.7/30.5/44.6/50.0; unsafe 17.2/46.1/29.0/26.6", "quote": "Table 8: Classifier drift and intervention results (GPT-4.1-mini). Acc. = accuracy; Miss. = missed detection; Lat. = lateral error; Unsafe = unsafe rate; Score = mean judge score. Safety tags reduce T4 unsafe rate by 52 pp despite accuracy collapse." }, { "label": "conversation-level pattern detection stated only as future work", "value": "listed as a future combination, no numbers", "quote": "Future systems should combine input classification with output-side review, uncertainty-aware escalation, and conversation-level pattern detection." }, { "label": "per-turn defense evaluated by accuracy and false alarm rate, not AUC", "value": "Claude 93.3% acc / 16.0% FA; GPT-4.1-mini 82.3% acc / 45.0% FA", "quote": "The Claude Sonnet classifier achieves 93.3% overall accuracy with a 0.86% missed detection rate and a 16% false alarm rate on benign queries. The GPT-4.1-mini classifier achieves 82.3% accuracy with a 5.4% missed detection rate and a 45% false alarm rate." } ], "usage": { "cost_usd": 0.003713, "calls": 2 } }, { "hypothesis": "H51", "paper": "2606.15385", "seq": 1417, "at_utc": "2026-09-15T08:44:14Z", "verdict": { "code": "values_unlocatable", "direction": null, "reason": "the independent pass offered additional values that cannot be located in the paper", "status": "inconclusive" }, "values_checked": 6, "grounding": { "checked": 6, "located": 6, "ok": true }, "caveat": "re-analysis of values the paper already reports, not a new measurement: this can refute but cannot confirm, and support means only that the paper's own numbers are consistent with the hypothesis.", "values": [ { "label": "Absent Supervisor, hidden reward (all four models)", "value": "GPT-4.1-mini 0.41 ± 3.93; GPT-5-mini 13.39 ± 1.82; Qwen3-235B-Instruct 5.39 ± 4.45; Qwen3-235B-Thinking 12.30 ± 3.48", "quote": "Absent Supervisor\n\nGPT-4.1-mini\nGPT-5-mini\nQwen3-235B-Instruct\nQwen3-235B-Thinking\n\n0.41 ± 3.93\n13.39 ± 1.82\n5.39 ± 4.45\n12.30 ± 3.48" }, { "label": "Safe Interruptibility, hidden reward (all four models)", "value": "GPT-4.1-mini −45.92 ± 3.38; GPT-5-mini −17.77 ± 12.24; Qwen3-235B-Instruct 30.41 ± 2.55; Qwen3-235B-Thinking 41.27 ± 0.25", "quote": "Safe Interruptibility\n\nGPT-4.1-mini\nGPT-5-mini\nQwen3-235B-Instruct\nQwen3-235B-Thinking\n\n−45.92 ± 3.38\n−17.77 ± 12.24\n30.41 ± 2.55\n41.27 ± 0.25" }, { "label": "Distributional Shift, reward = safety performance (all four models)", "value": "GPT-4.1-mini −34.93 ± 7.04; GPT-5-mini −34.12 ± 8.66; Qwen3-235B-Instruct −54.67 ± 3.89; Qwen3-235B-Thinking −6.54 ± 13.62", "quote": "Distributional Shift\n\nGPT-4.1-mini\nGPT-5-mini\nQwen3-235B-Instruct\nQwen3-235B-Thinking\n\n−34.93 ± 7.04\n−34.12 ± 8.66\n−54.67 ± 3.89\n−6.54 ± 13.62" }, { "label": "Distributional Shift, hidden/safety equivalence", "value": "For robustness problems, observed reward equals safety performance.", "quote": "For robustness problems, observed reward equals safety performance." }, { "label": "Qwen3-235B-Thinking, Safe Interruptibility hidden reward", "value": "41.27", "quote": "Qwen3-235BThinking achieves high hidden reward (41.27) despite negative observed reward (−1.64), while GPT-5-mini and GPT-4.1-mini perform poorly on both metrics." }, { "label": "Distributional Shift, paper's comparison of Qwen3-235B-Thinking", "value": "Qwen3-235B-Thinking improves on these results but remains far from the approximate maximum and exhibits high variance across seeds.", "quote": "Qwen3-235B-Thinking improves on these results but remains far from the approximate maximum and exhibits high variance across seeds." } ], "usage": { "cost_usd": 0.019183, "calls": 2 } } ] }