Treat a reply cut off by the token limit as no answer

ask_about_branch warned about stop_reason max_tokens and then parsed the reply
anyway. The answer format asks for the verdict lines first with nothing before
them, so a truncated reply still carries a parseable verdict, and read_answer
reads it as a real one:

    - **Likely affected**: No
    - **Confidence**: high
    - **Reasoning**: The vulnerable memcpy in     <- cut off here

parses as (False, high), and decide_unsure turns that into NOT_AFFECTED. So a
branch could be cleared on an answer the model never finished, against this
module s own rule that no answer means AFFECTED. Thinking tokens share the
MAX_ANSWER_TOKENS budget, so the bigger the fix, the more likely it is

It now returns None, which decide_unsure already treats as no answer and
flags the branch for review. Three tests: the same text parses as a confident
no when the reply is complete, so the guard is what changes the outcome; the
truncated one is no answer; and the branch ends up AFFECTED

Reported by the security review on this pull request

Author: tianyiy-tim

util/backport/src/engine/consult_ai.py

@@ -333,12 +333,16 @@ def ask_about_branch(
         print(f"[ai] call failed for {branch}: {exc}", file=sys.stderr)
         return None
     if reply.stop_reason == "max_tokens":
-        # Thinking tokens ate the budget, so the answer may be cut off mid-sentence
+        # Thinking tokens ate the budget, so the answer is cut off. The verdict lines come
+        # first by design, so a truncated reply usually still carries a parseable "No" -
+        # and parsing it would clear a branch on an answer the model never finished.
+        # Treated as no answer instead, which leaves the branch flagged
         print(
-            f"[ai] reply for {branch} hit the token limit and may be cut short, "
-            "raise MAX_ANSWER_TOKENS in src/util/config.py",
+            f"[ai] reply for {branch} hit the token limit and was cut short, "
+            "leaving the branch flagged. Raise MAX_ANSWER_TOKENS in src/util/config.py",
             file=sys.stderr,
         )
+        return None
     raw = "".join(b.text for b in reply.content if hasattr(b, "text"))
     return read_answer(raw.strip())
 

util/backport/testing/test_engine.py

@@ -10,11 +10,13 @@
 """
 
 import datetime
+import io
 import subprocess
 import sys
 import unittest
+from contextlib import redirect_stderr
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Sequence
+from typing import Any, ClassVar, Dict, List, Optional, Sequence
 from unittest import mock
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
@@ -318,6 +320,74 @@ def test_confidence_matches_whole_words(self):
         self.assertEqual(consult_ai.read_answer("**Confidence**: high")[1], "high")
 
 
+class ATruncatedReplyIsNoAnswer(unittest.TestCase):
+    # The answer format puts the verdict first, so a reply cut off by the token limit
+    # still carries a parseable "No". Reading it would clear a branch on an answer the
+    # model never finished, which is the one failure that ships a vulnerability
+
+    # A real truncated reply: verdict present, reasoning stops mid-sentence
+    CUT_OFF = (
+        "- **Likely affected**: No\n"
+        "- **Confidence**: high\n"
+        "- **Reasoning**: The vulnerable memcpy in"
+    )
+
+    def ask(self, stop_reason):
+        """ask_about_branch against a stubbed Bedrock reply, as its return value"""
+
+        class Block:
+            text = self.CUT_OFF
+
+        class Reply:
+            content: ClassVar[list] = [Block()]
+
+        Reply.stop_reason = stop_reason
+
+        class Stream:
+            def __enter__(inner):
+                return inner
+
+            def __exit__(inner, *exc):
+                return False
+
+            def get_final_message(inner):
+                return Reply()
+
+        class Messages:
+            def stream(inner, **kwargs):
+                return Stream()
+
+        class Client:
+            messages = Messages()
+
+        with mock.patch.multiple(
+            consult_ai,
+            ai_client=lambda: Client(),
+            build_prompt=lambda *a, **k: "prompt",
+            load_model_config=lambda: {"opus": "a-model"},
+            branch_ref=lambda branch: f"upstream/{branch}",
+        ), redirect_stderr(io.StringIO()):
+            return consult_ai.ask_about_branch(FIX_SHA, BRANCH, SRC_FILES, BUG_COMMITS)
+
+    def test_the_same_text_parses_as_a_confident_no_when_complete(self):
+        # Establishes that the guard is what changes the outcome, not the text
+        self.assertEqual(self.ask("end_turn"), (False, "high"))
+
+    def test_hitting_the_token_limit_is_no_answer(self):
+        self.assertIsNone(self.ask("max_tokens"))
+
+    def test_so_the_branch_stays_flagged(self):
+        # No answer is what decide_unsure turns into AFFECTED, so check the whole path
+        verdicts = {BRANCH: config.UNSURE}
+        decided_by = {}
+        with mock.patch.object(consult_ai, "ask_about_branch", lambda *a, **k: None):
+            consult_ai.decide_unsure(
+                FIX_SHA, SRC_FILES, BUG_COMMITS, verdicts, decided_by
+            )
+        self.assertEqual(verdicts[BRANCH], config.AFFECTED)
+        self.assertIn("flagged for review", decided_by[BRANCH])
+
+
 # --- Test Doubles For The Verdict Layer ---
 
 # Stand-in names for one fix and one release branch. classify_branches only ever