Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 99 additions & 7 deletions bench/journeys_wave1.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,24 @@ type waveCorrectionStatus struct {
NextTransition *struct {
Kind string `json:"kind"`
ReasonCode string `json:"reason_code"`
Collect *struct {
Inputs []struct {
Name string `json:"name"`
Submission *waveSubmissionDescriptor `json:"submission"`
} `json:"inputs"`
} `json:"collect"`
} `json:"next_transition"`
}

type waveSubmissionDescriptor struct {
OperationToken string `json:"operation_token"`
ArgumentTokens []string `json:"argument_tokens"`
Value struct {
Slot string `json:"slot"`
SubstitutionLocation int `json:"substitution_location"`
} `json:"value"`
}

type waveRetryStatus struct {
Authority *struct {
LineageID string `json:"lineage_id"`
Expand Down Expand Up @@ -444,11 +459,71 @@ func readCorrectionStatus(r *journeyRun) (waveCorrectionStatus, error) {
}

func readCorrectionStatusFor(r *journeyRun, lineage string) (waveCorrectionStatus, error) {
observation := r.run(productArgsFor(r, "review", "status", "--contract", reviewContract, "--next-transition", "--lineage", lineage), false)
return readCorrectionStatusForContract(r, lineage, reviewContract)
}

func readCorrectionStatusForContract(r *journeyRun, lineage, contract string) (waveCorrectionStatus, error) {
arguments := []string{"review", "status", "--contract", contract, "--next-transition", "--lineage", lineage}
if contract == reviewContractV2 {
arguments = append(arguments, "--agent", "claude-code")
}
observation := r.run(productArgsFor(r, arguments...), false)
var status waveCorrectionStatus
return status, decodeWaveObservation(observation, &status, "corrected review status")
}

func correctionSubmissionArguments(r *journeyRun, status waveCorrectionStatus, reason, slot, value string) ([]string, error) {
if status.Authority == nil || status.NextTransition == nil || status.NextTransition.Kind != "collect" ||
status.NextTransition.ReasonCode != reason || status.NextTransition.Collect == nil || len(status.NextTransition.Collect.Inputs) != 1 {
return nil, fmt.Errorf("submission descriptor transition = %+v", status.NextTransition)
}
descriptor := status.NextTransition.Collect.Inputs[0].Submission
if descriptor == nil || descriptor.OperationToken != "finalize" || descriptor.Value.Slot != slot ||
descriptor.Value.SubstitutionLocation < 0 || descriptor.Value.SubstitutionLocation >= len(descriptor.ArgumentTokens) {
return nil, fmt.Errorf("submission descriptor = %+v", descriptor)
}
placeholders := 0
for index, token := range descriptor.ArgumentTokens {
if !strings.HasPrefix(token, "--") || strings.ContainsAny(token, " \t\r\n") || strings.HasPrefix(token, "--cwd=") || strings.Contains(token, r.sandbox.Root) {
return nil, fmt.Errorf("submission descriptor leaked a path or shell token: %q", token)
}
if strings.Contains(token, "{{value}}") {
placeholders++
if index != descriptor.Value.SubstitutionLocation || strings.Count(token, "{{value}}") != 1 {
return nil, fmt.Errorf("submission descriptor slot = %q at %d", token, index)
}
}
}
if len(descriptor.ArgumentTokens) < 5 {
return nil, fmt.Errorf("submission descriptor argv has %d tokens, need at least 5: %v", len(descriptor.ArgumentTokens), descriptor.ArgumentTokens)
}
if placeholders != 1 || !strings.HasPrefix(descriptor.ArgumentTokens[4], "--request-hash=") {
return nil, fmt.Errorf("submission descriptor argv = %v", descriptor.ArgumentTokens)
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
arguments := append([]string{"review", descriptor.OperationToken}, descriptor.ArgumentTokens...)
arguments[descriptor.Value.SubstitutionLocation+2] = strings.Replace(arguments[descriptor.Value.SubstitutionLocation+2], "{{value}}", value, 1)
if strings.Contains(arguments[descriptor.Value.SubstitutionLocation+2], "{{value}}") {
return nil, errors.New("submission descriptor did not replace its only value slot")
}
return arguments, nil
}

func submitCorrectionPlan(r *journeyRun) error {
status, err := readCorrectionStatusForContract(r, correctedDeliveryLineage, reviewContractV2)
if err != nil {
return err
}
arguments, err := correctionSubmissionArguments(r, status, "correction_plan_required", "correction_lines", "2")
if err != nil {
return err
}
result, err := decodeWaveOperation(r.runAt(r.sandbox.Root, arguments, false), "correction submission descriptor")
if err != nil || result.State != "correction_required" || result.LineageID != correctedDeliveryLineage {
return fmt.Errorf("correction submission descriptor result = %+v, %v", result, err)
}
return nil
}

func capturePassedCorrectionEvidence(r *journeyRun) error {
return capturePassedCorrectionEvidenceFor(r, correctedDeliveryLineage)
}
Expand Down Expand Up @@ -479,11 +554,15 @@ func capturePassedCorrectionEvidenceFor(r *journeyRun, lineage string) error {
}

func completeCorrectedReview(r *journeyRun) error {
return completeCorrectedReviewFor(r, correctedDeliveryLineage)
return completeCorrectedReviewForContract(r, correctedDeliveryLineage, reviewContractV2)
}

func completeCorrectedReviewFor(r *journeyRun, lineage string) error {
status, err := readCorrectionStatusFor(r, lineage)
return completeCorrectedReviewForContract(r, lineage, reviewContract)
}

func completeCorrectedReviewForContract(r *journeyRun, lineage, contract string) error {
status, err := readCorrectionStatusForContract(r, lineage, contract)
if err != nil {
return err
}
Expand All @@ -506,6 +585,20 @@ func completeCorrectedReviewFor(r *journeyRun, lineage string) error {
if err != nil {
return err
}
if contract == reviewContractV2 {
arguments, err := correctionSubmissionArguments(r, status, "targeted_validation_required", "validation", path)
if err != nil {
return err
}
result, err := decodeWaveOperation(r.runAt(r.sandbox.Root, arguments, false), "corrected review finalize")
if err != nil {
return err
}
if result.State != "approved" || result.LineageID != lineage {
return fmt.Errorf("corrected review finalized as %+v", result)
}
return nil
}
observation := r.run(productArgsFor(r, "review", "finalize", "--lineage", lineage,
"--validation", path, "--captured-evidence=true"), false)
result, err := decodeWaveOperation(observation, "corrected review finalize")
Expand Down Expand Up @@ -1126,13 +1219,12 @@ func waveOneJourneys() []Journey {
{Name: "finalize reviewer results into correction-required", Requires: finalizeResultsCapability,
Args: productArgs("review", "finalize", "--lineage", correctedDeliveryLineage, "--captured-results=true"),
After: requireReviewState("correction_required", correctedDeliveryLineage)},
{Name: "forecast the bounded correction", Requires: finalizeCorrectionCapability,
Args: productArgs("review", "finalize", "--lineage", correctedDeliveryLineage, "--correction-lines", "2")},
{Name: "derive and execute the correction submission descriptor", Requires: finalizeCorrectionCapability, Composite: submitCorrectionPlan},
{Name: "fixture: corrected candidate proven to change only the reviewed path", Fixture: writeCorrectedCandidate},
{Name: "capture passed repository evidence for the provider correction target", Requires: captureOutcomeEvidenceCapability, Composite: capturePassedCorrectionEvidence},
{Name: "complete targeted validation and approve the corrected receipt", Requires: finalizeValidationCapability, Composite: completeCorrectedReview},
{Name: "execute the targeted validation submission descriptor and approve the corrected receipt", Requires: finalizeValidationCapability, Composite: completeCorrectedReview},
{Name: "corrected receipt and advanced current snapshot are proven", Requires: statusCapability,
Args: productArgs("review", "status", "--contract", reviewContract, "--lineage", correctedDeliveryLineage), After: proveCorrectedReceipt},
Args: productArgs("review", "status", "--contract", reviewContractV2, "--agent", "claude-code", "--lineage", correctedDeliveryLineage), After: proveCorrectedReceipt},
{Name: "fixture: corrected candidate staged tree matches the receipt", Fixture: stageCorrectedTree},
{Name: "gate pre-commit on the corrected receipt", Requires: validateCapability,
Args: productArgs("review", "validate", "--gate", "pre-commit"),
Expand Down
31 changes: 31 additions & 0 deletions bench/journeys_wave1_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package main

import (
"encoding/json"
"strings"
"testing"
)
Expand All @@ -20,6 +21,36 @@ func TestWaveReviewInvocationArgs(t *testing.T) {
}
}

func TestCorrectionSubmissionArgumentsRejectsShortDescriptor(t *testing.T) {
var status waveCorrectionStatus
if err := json.Unmarshal([]byte(`{
"authority": {"lineage_id": "lineage"},
"next_transition": {
"kind": "collect",
"reason_code": "correction_plan_required",
"collect": {"inputs": [{
"name": "correction_lines",
"submission": {
"operation_token": "finalize",
"argument_tokens": [
"--contract=gentle-ai.review-integration/v2",
"--lineage=lineage",
"--expected-revision=sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"--target=sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
],
"value": {"slot": "correction_lines", "substitution_location": 3}
}
}]}
}
}`), &status); err != nil {
t.Fatal(err)
}
run := &journeyRun{sandbox: &Sandbox{Root: t.TempDir()}}
if _, err := correctionSubmissionArguments(run, status, "correction_plan_required", "correction_lines", "2"); err == nil || !strings.Contains(err.Error(), "has 4 tokens") {
t.Fatalf("short descriptor error = %v", err)
}
}

func TestRequireCandidateDeclinedGate(t *testing.T) {
sandbox := &Sandbox{Scratch: map[string]string{
"decline-base": "1111111111111111111111111111111111111111",
Expand Down
12 changes: 10 additions & 2 deletions bench/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,8 +151,12 @@ func (s *Sandbox) gitCallsSince() *int {

// invoke runs the product once and returns a full Observation.
func (s *Sandbox) invoke(args []string) Observation {
return s.invokeAt(s.Repo, args)
}

func (s *Sandbox) invokeAt(dir string, args []string) Observation {
cmd := exec.Command(s.Binary, args...)
cmd.Dir = s.Repo
cmd.Dir = dir
cmd.Env = s.env()
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
Expand Down Expand Up @@ -397,7 +401,11 @@ type journeyRun struct {
// run executes one product invocation inside a journey, folding it into the
// metrics. Composite steps call it directly.
func (r *journeyRun) run(args []string, modelRun bool) Observation {
observation := r.sandbox.invoke(args)
return r.runAt(r.sandbox.Repo, args, modelRun)
}

func (r *journeyRun) runAt(dir string, args []string, modelRun bool) Observation {
observation := r.sandbox.invokeAt(dir, args)
record := r.accumulator.observe(r.step, observation, r.sandbox.gitCallsSince(), modelRun)
r.accumulator.records = append(r.accumulator.records, record)
return observation
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
{
"schema": "gentle-ai.review-integration.capabilities/v2.2",
"contract": "gentle-ai.review-integration/v2",
"protocol": {"major": 2, "minor": 2},
"package": {"name": "gentle-ai", "version": "2.1.7-test", "release_channel": "prerelease"},
"build": {
"id": "sha256:444a9f0608eb22c4697fd2092f3afcfab040c0918343e7e04087730fd33fd5e1",
"go_version": "go1.25.10",
"module_version": "v2.1.7",
"vcs": "git",
"vcs_revision": "0123456789abcdef0123456789abcdef01234567",
"vcs_time": "2026-07-15T18:00:00Z",
"vcs_modified": "false"
},
"executable": {
"sha256": "sha256:dcc846103b16d365eaeeb9d7f289c23fc4f2897f23def1cb3fe7f05557b64705",
"evidence": "self-reported",
"verification": "compare-with-published-manifest"
},
"operations": [
"review.bind_sdd", "review.capabilities", "review.finalize", "review.repair",
"review.retry_final_verification", "review.start", "review.status", "review.validate"
],
"gates": ["post-apply", "pre-commit", "pre-push", "pre-pr", "release"],
"projections": ["staged", "workspace"],
"schemas": [
"gentle-ai.review-admitted-result/v2",
"gentle-ai.review-artifact-subject/v2",
"gentle-ai.review-authority-repair-assessment/v1",
"gentle-ai.review-authority-status/v1",
"gentle-ai.review-gate-request/v1",
"gentle-ai.review-integration.capabilities/v2.2",
"gentle-ai.review-integration.failure/v2",
"gentle-ai.review-final-verification-incident/v1",
"gentle-ai.review-integration.operation/v2",
"gentle-ai.review-integration.projection/v1",
"gentle-ai.review-integration.repair/v2",
"gentle-ai.review-integration.start/v3",
"gentle-ai.review-integration.status/v4",
"gentle-ai.review-receipt/v1",
"gentle-ai.review-receipt/v2",
"gentle-ai.review-result-artifact/v2",
"gentle-ai.review-targeted-validation-request/v1",
"gentle-ai.review-verification-evidence/v2",
"https://gentle-ai.dev/schema/review/refuter/v1",
"https://gentle-ai.dev/schema/review/reviewer/v1",
"https://gentle-ai.dev/schema/review/validator/v1",
"gentle-ai.review-integration.consent/v3"
],
"features": {
"mandatory": [
{"name": "compact_v2_authority", "supported": true, "requires": []},
{"name": "exact_receipt_replay", "supported": true, "requires": ["compact_v2_authority"]},
{"name": "five_delivery_gates", "supported": true, "requires": ["compact_v2_authority"]},
{"name": "immutable_snapshot", "supported": true, "requires": []},
{"name": "legacy_v1_target_scoped_read_only", "supported": true, "requires": ["target_scoped_status"]},
{"name": "repository_independent_capabilities", "supported": true, "requires": []},
{"name": "restart_safe_projection", "supported": true, "requires": ["target_scoped_status"]},
{"name": "sdd_receipt_binding", "supported": true, "requires": ["compact_v2_authority"]},
{"name": "target_scoped_status", "supported": true, "requires": ["repository_independent_capabilities"]},
{"name": "uniform_failure_envelope", "supported": true, "requires": ["repository_independent_capabilities"]}
],
"optional": [
{"name": "base_ref_workspace_overlay", "supported": true, "requires": ["immutable_snapshot", "restart_safe_projection"]},
{"name": "bounded_process_waits", "supported": true, "requires": ["uniform_failure_envelope"]},
{"name": "classified_authority_repair", "supported": true, "requires": ["native_next_transition", "uniform_failure_envelope"]},
{"name": "exact_gate_receipt_discovery", "supported": true, "requires": ["five_delivery_gates"]},
{"name": "native_frozen_candidate_context", "supported": true, "requires": ["immutable_snapshot"]},
{"name": "native_low_risk_verification", "supported": true, "requires": ["compact_v2_authority"]},
{"name": "native_next_transition", "supported": true, "requires": ["target_scoped_status"]},
{"name": "one_shot_final_verification_retry", "supported": true, "requires": ["compact_v2_authority", "exact_receipt_replay", "native_next_transition"]},
{"name": "opaque_repository_context", "supported": true, "requires": ["compact_v2_authority", "native_next_transition"]},
{"name": "outcome_bound_verification_evidence", "supported": true, "requires": ["compact_v2_authority", "native_next_transition"]},
{"name": "provider_artifact_admission", "supported": true, "requires": ["compact_v2_authority", "native_frozen_candidate_context", "opaque_repository_context"]},
{"name": "provider_targeted_validation_request", "supported": true, "requires": ["compact_v2_authority", "native_next_transition"]},
{"name": "recovered_correction_evidence", "supported": true, "requires": ["compact_v2_authority", "provider_targeted_validation_request"]},
{"name": "risk_reasons", "supported": true, "requires": ["repository_independent_capabilities"]},
{"name": "scope_change_diagnostics", "supported": true, "requires": ["uniform_failure_envelope"]},
{"name": "validating_result_reopen", "supported": true, "requires": ["compact_v2_authority", "provider_artifact_admission"]},
{"name": "provider_bound_native_git_context", "supported": true, "requires": ["native_frozen_candidate_context", "opaque_repository_context", "provider_artifact_admission"]},
{"name": "provider_submission_descriptors", "supported": true, "requires": ["native_next_transition", "opaque_repository_context", "provider_targeted_validation_request"]}
]
},
"bootstrap": {
"command": "gentle-ai review status --cwd \u003crepo\u003e --contract gentle-ai.review-integration/v2 --agent claude-code --next-transition",
"target_selector_variants": [
{"target_type": "staged", "arguments": ["--projection", "staged"]},
{"target_type": "base_ref", "arguments": ["--base-ref", "\u003cref\u003e"]},
{"target_type": "workspace_overlay_base_ref", "arguments": ["--workspace-overlay", "--base-ref", "\u003cref\u003e"]},
{"target_type": "workspace_overlay_base_tree", "arguments": ["--workspace-overlay", "--base-tree", "\u003ctree\u003e"]}
],
"required_feature": "native_next_transition",
"unsupported_outcome": "unsupported-capability",
"parent_only": true
},
"compatibility": {
"minimum_protocol_major": 2,
"maximum_protocol_major": 2,
"additive_minor_policy": "optional-fields-only",
"unknown_mandatory": "reject",
"unknown_optional": "ignore",
"modes": ["compact-v2", "legacy-v1"],
"legacy_window": {
"mode": "legacy-v1",
"state": "active",
"read_only": true,
"deprecation_started": true,
"removal": "not-scheduled",
"minimum_compatibility_releases": 1
}
}
}
Loading
Loading