Skip to content

docs: rename to Docker Agent Action and sync README #370

docs: rename to Docker Agent Action and sync README

docs: rename to Docker Agent Action and sync README #370

Workflow file for this run

name: Test Docker Agent Action
on:
pull_request:
types: [opened, synchronize, reopened]
branches: [main]
push:
branches: [main]
workflow_dispatch:
inputs:
pr_number:
description: 'PR number to run mention-reply E2E tests against'
required: true
type: string
permissions:
contents: read
jobs:
test-output-extraction:
name: Output Extraction Tests
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Run output extraction tests
run: |
cd tests
chmod +x test-output-extraction.sh
./test-output-extraction.sh
test-job-summary:
name: Job Summary Format Tests
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Run job summary tests
run: |
cd tests
chmod +x test-job-summary.sh
./test-job-summary.sh
test-pirate-agent:
name: Pirate Agent Test
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Check if fork PR
id: fork-check
run: |
# Use default empty string to handle edge cases (deleted branches, malformed events)
HEAD_REPO="${{ github.event.pull_request.head.repo.full_name || '' }}"
if [[ "${{ github.event_name }}" == "pull_request" && "$HEAD_REPO" != "${{ github.repository }}" && -n "$HEAD_REPO" ]]; then
echo "⏭️ Skipping - fork PR (secrets not available)"
echo "is_fork=true" >> $GITHUB_OUTPUT
else
echo "is_fork=false" >> $GITHUB_OUTPUT
fi
- name: Checkout code
if: steps.fork-check.outputs.is_fork != 'true'
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
if: steps.fork-check.outputs.is_fork != 'true'
uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5
with:
run_install: false
- name: Setup Node.js
if: steps.fork-check.outputs.is_fork != 'true'
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: 24
cache: pnpm
- name: Build action
if: steps.fork-check.outputs.is_fork != 'true'
run: pnpm install --frozen-lockfile && pnpm build
- name: Run test
if: steps.fork-check.outputs.is_fork != 'true'
id: pirate
uses: ./
with:
agent: agentcatalog/pirate
prompt: "What do we ship today?"
openai-api-key: ${{ secrets.OPENAI_API_KEY }}
- name: Validate output and exit code
if: steps.fork-check.outputs.is_fork != 'true'
run: |
OUTPUT_FILE="${{ steps.pirate.outputs.output-file }}"
# Check that exit code is 0 (success)
if [ "${{ steps.pirate.outputs.exit-code }}" != "0" ]; then
echo "❌ Agent failed with exit code: ${{ steps.pirate.outputs.exit-code }}"
exit 1
fi
echo "✅ Agent completed successfully with exit code 0"
# Check that output file exists
if [ ! -f "$OUTPUT_FILE" ]; then
echo "❌ Output file not found: $OUTPUT_FILE"
exit 1
fi
echo "✅ Output file found: $OUTPUT_FILE"
# Display the output for debugging
echo "--- Agent Output ---"
cat "$OUTPUT_FILE"
echo "--- End Output ---"
# Check that output is clean (no agent markers or metadata in output)
if grep -qF -- "--- Agent: root ---" "$OUTPUT_FILE"; then
echo "⚠️ Output still contains '--- Agent: root ---' marker (not fully cleaned)"
fi
# Check that output doesn't contain log metadata
if grep -qE "^(time=|level=)" "$OUTPUT_FILE"; then
echo "❌ Output contains log metadata (time= or level=) - cleaning failed"
exit 1
fi
echo "✅ Output is clean (no log metadata)"
# Check that there is actual content (non-empty, non-whitespace)
CONTENT=$(cat "$OUTPUT_FILE" | grep -v '^$' | head -n 5)
if [ -z "$CONTENT" ]; then
echo "❌ No content found in output file"
exit 1
fi
echo "✅ Found agent response content"
echo "Response preview: $(echo "$CONTENT" | head -n 1)"
test-invalid-agent:
name: Invalid Agent Test
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5
with:
run_install: false
- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: 24
cache: pnpm
- name: Build action
run: pnpm install --frozen-lockfile && pnpm build
- name: Test should fail on invalid agent
id: invalid-agent
continue-on-error: true
uses: ./
with:
agent: agentcatalog/nonexistent
prompt: "This should fail"
openai-api-key: ${{ secrets.OPENAI_API_KEY }}
- name: Verify invalid agent failed
run: |
OUTPUT_FILE="${{ steps.invalid-agent.outputs.output-file }}"
# Check exit code OR check for error in output (Docker Agent may exit 0 even on pull failure)
if [ "${{ steps.invalid-agent.outputs.exit-code }}" == "0" ]; then
# Exit code is 0, check if output contains error message
if [ -f "$OUTPUT_FILE" ] && grep -q "failed to pull" "$OUTPUT_FILE"; then
echo "✅ Invalid agent correctly failed (error in output)"
else
echo "❌ Invalid agent should have failed but succeeded with no error"
exit 1
fi
else
echo "✅ Invalid agent correctly failed (non-zero exit code)"
fi
test-mention-reply-toplevel:
name: Mention Reply (Top-Level) E2E Test
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch'
permissions:
contents: read
id-token: write
issues: write
env:
TEST_PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr_number }}
steps:
- name: Check if fork PR
id: fork-check
run: |
# Use default empty string to handle edge cases (deleted branches, malformed events)
HEAD_REPO="${{ github.event.pull_request.head.repo.full_name || '' }}"
if [[ "${{ github.event_name }}" == "pull_request" && "$HEAD_REPO" != "${{ github.repository }}" && -n "$HEAD_REPO" ]]; then
echo "⏭️ Skipping - fork PR (secrets not available)"
echo "is_fork=true" >> $GITHUB_OUTPUT
else
echo "is_fork=false" >> $GITHUB_OUTPUT
fi
- name: Checkout code
if: steps.fork-check.outputs.is_fork != 'true'
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
if: steps.fork-check.outputs.is_fork != 'true'
uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5
with:
run_install: false
- name: Setup Node.js
if: steps.fork-check.outputs.is_fork != 'true'
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: 24
cache: pnpm
- name: Build action
if: steps.fork-check.outputs.is_fork != 'true'
run: pnpm install --frozen-lockfile && pnpm build
- name: Setup credentials
if: steps.fork-check.outputs.is_fork != 'true'
uses: docker/cagent-action/setup-credentials@3f5dc9969f307d3c76acb7e9ccaefdd96bd62f4b # v1.5.4
- name: Create anchor issue comment on current PR
if: steps.fork-check.outputs.is_fork != 'true'
id: create-anchor
env:
GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }}
run: |
COMMENT_ID=$(gh api repos/docker/cagent-action/issues/$TEST_PR_NUMBER/comments \
--method POST \
--raw-field body="@docker-agent this is an automated e2e test — please reply with a brief acknowledgement." \
--jq .id)
echo "Created test anchor comment ID: $COMMENT_ID"
echo "test_comment_id=$COMMENT_ID" >> $GITHUB_OUTPUT
- name: Write synthetic issue_comment event
if: steps.fork-check.outputs.is_fork != 'true'
run: |
COMMENT_ID="${{ steps.create-anchor.outputs.test_comment_id }}"
jq -n \
--arg actor "${{ github.actor }}" \
--argjson comment_id "$COMMENT_ID" \
--argjson pr_number "$TEST_PR_NUMBER" \
'{
"action": "created",
"issue": {
"number": $pr_number,
"pull_request": { "url": ("https://api.github.com/repos/docker/cagent-action/pulls/" + ($pr_number | tostring)) }
},
"comment": {
"id": $comment_id,
"body": "@docker-agent this is an automated e2e test — please reply with a brief acknowledgement.",
"user": { "login": $actor, "type": "User" }
},
"repository": {
"owner": { "login": "docker" },
"name": "cagent-action"
},
"sender": { "login": $actor, "type": "User" }
}' > /tmp/test-event-toplevel.json
- name: Run mention-reply handler
if: steps.fork-check.outputs.is_fork != 'true'
id: mention-handler
env:
INPUT_GITHUB-TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }}
INPUT_ORG-MEMBERSHIP-TOKEN: ${{ env.ORG_MEMBERSHIP_TOKEN || github.token }}
run: |
export GITHUB_EVENT_PATH=/tmp/test-event-toplevel.json
export GITHUB_EVENT_NAME=issue_comment
node "$GITHUB_WORKSPACE/dist/mention-reply.js"
- name: Assert should-reply output
if: steps.fork-check.outputs.is_fork != 'true'
run: |
SHOULD_REPLY="${{ steps.mention-handler.outputs.should-reply }}"
echo "should-reply=$SHOULD_REPLY"
echo "owner=${{ steps.mention-handler.outputs.owner }}"
echo "repo=${{ steps.mention-handler.outputs.repo }}"
echo "pr-number=${{ steps.mention-handler.outputs.pr-number }}"
echo "is-inline=${{ steps.mention-handler.outputs.is-inline }}"
if [ "$SHOULD_REPLY" == 'false' ]; then
echo "⚠️ Warning: should-reply=false — ${{ github.actor }} may not be a docker org member (fork runners and external contributors are expected to see this). Skipping reply steps."
exit 0
fi
- name: Run mention reply
if: steps.fork-check.outputs.is_fork != 'true' && steps.mention-handler.outputs.should-reply == 'true'
id: run-reply
uses: ./review-pr/mention-reply
with:
mention-context: ${{ steps.mention-handler.outputs.prompt }}
owner: ${{ steps.mention-handler.outputs.owner }}
repo: ${{ steps.mention-handler.outputs.repo }}
pr-number: ${{ steps.mention-handler.outputs.pr-number }}
is-inline: ${{ steps.mention-handler.outputs.is-inline }}
in-reply-to-id: ${{ steps.mention-handler.outputs.in-reply-to-id }}
anthropic-api-key: ${{ env.ANTHROPIC_API_KEY_FROM_SSM || secrets.ANTHROPIC_API_KEY }}
openai-api-key: ${{ env.OPENAI_API_KEY_FROM_SSM || secrets.OPENAI_API_KEY }}
github-token: ${{ env.GITHUB_APP_TOKEN || github.token }}
skip-auth: "true"
- name: Verify reply was posted
if: steps.fork-check.outputs.is_fork != 'true' && steps.mention-handler.outputs.should-reply == 'true'
env:
GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }}
run: |
FOUND=$(gh api repos/docker/cagent-action/issues/$TEST_PR_NUMBER/comments \
--jq '[.[] | select(.body | contains("<!-- cagent-review-reply -->")) | select(.created_at > (now - 300 | todate))] | length')
if [ "$FOUND" -eq 0 ]; then
echo "❌ No reply comment found within the last 5 minutes"
exit 1
fi
echo "✅ Reply posted successfully ($FOUND comment(s) found)"
- name: Cleanup test comments
if: always() && steps.fork-check.outputs.is_fork != 'true'
continue-on-error: true
env:
GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }}
ANCHOR_ID: ${{ steps.create-anchor.outputs.test_comment_id }}
run: |
# Delete any test reply comments posted in the last 5 minutes
gh api repos/docker/cagent-action/issues/$TEST_PR_NUMBER/comments \
--jq '.[] | select(.body | contains("<!-- cagent-review-reply -->")) | select(.created_at > (now - 300 | todate)) | .id' | \
while read -r comment_id; do
gh api "repos/docker/cagent-action/issues/comments/$comment_id" -X DELETE || true
echo "Deleted comment $comment_id"
done
# Delete the anchor comment itself
if [ -n "$ANCHOR_ID" ]; then
gh api "repos/docker/cagent-action/issues/comments/$ANCHOR_ID" -X DELETE || true
echo "Deleted anchor comment $ANCHOR_ID"
fi
test-mention-reply-inline:
name: Mention Reply (Inline) E2E Test
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch'
permissions:
contents: read
id-token: write
pull-requests: write
env:
TEST_PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr_number }}
steps:
- name: Check if fork PR
id: fork-check
run: |
HEAD_REPO="${{ github.event.pull_request.head.repo.full_name || '' }}"
if [[ "${{ github.event_name }}" == "pull_request" && "$HEAD_REPO" != "${{ github.repository }}" && -n "$HEAD_REPO" ]]; then
echo "⏭️ Skipping - fork PR (secrets not available)"
echo "is_fork=true" >> $GITHUB_OUTPUT
else
echo "is_fork=false" >> $GITHUB_OUTPUT
fi
- name: Checkout code
if: steps.fork-check.outputs.is_fork != 'true'
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
if: steps.fork-check.outputs.is_fork != 'true'
uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5
with:
run_install: false
- name: Setup Node.js
if: steps.fork-check.outputs.is_fork != 'true'
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: 24
cache: pnpm
- name: Build action
if: steps.fork-check.outputs.is_fork != 'true'
run: pnpm install --frozen-lockfile && pnpm build
- name: Setup credentials
if: steps.fork-check.outputs.is_fork != 'true'
uses: docker/cagent-action/setup-credentials@3f5dc9969f307d3c76acb7e9ccaefdd96bd62f4b # v1.5.4
- name: Create anchor review comment on current PR
if: steps.fork-check.outputs.is_fork != 'true'
id: create-anchor
env:
GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }}
run: |
# Get the PR head SHA
HEAD_SHA=$(gh api repos/docker/cagent-action/pulls/$TEST_PR_NUMBER --jq '.head.sha')
echo "PR head SHA: $HEAD_SHA"
# Get first file in the diff to use as a safe anchor
DIFF_FILE=$(gh api repos/docker/cagent-action/pulls/$TEST_PR_NUMBER/files --jq '.[0].filename')
echo "Using diff file: $DIFF_FILE"
# Post a test inline comment to get a real comment ID
COMMENT_ID=$(gh api repos/docker/cagent-action/pulls/$TEST_PR_NUMBER/comments \
-X POST \
--input - <<< $(jq -n \
--arg sha "$HEAD_SHA" \
--arg path "$DIFF_FILE" \
'{"body": "e2e test anchor comment — safe to delete", "commit_id": $sha, "path": $path, "side": "RIGHT", "position": 1}') \
--jq '.id')
echo "Created test anchor comment ID: $COMMENT_ID"
echo "test_comment_id=$COMMENT_ID" >> $GITHUB_OUTPUT
- name: Write synthetic pull_request_review_comment event
if: steps.fork-check.outputs.is_fork != 'true'
run: |
COMMENT_ID="${{ steps.create-anchor.outputs.test_comment_id }}"
jq -n \
--arg actor "${{ github.actor }}" \
--argjson comment_id "$COMMENT_ID" \
--argjson pr_number "$TEST_PR_NUMBER" \
'{
"action": "created",
"pull_request": { "number": $pr_number },
"comment": {
"id": $comment_id,
"body": "@docker-agent this is an automated e2e test of the inline mention path.",
"path": "README.md",
"line": 1,
"original_line": 1,
"diff_hunk": "@@ -1,1 +1,1 @@\n-old\n+new",
"user": { "login": $actor, "type": "User" }
},
"repository": {
"owner": { "login": "docker" },
"name": "cagent-action"
},
"sender": { "login": $actor, "type": "User" }
}' > /tmp/test-event-inline.json
- name: Run mention-reply handler
if: steps.fork-check.outputs.is_fork != 'true'
id: mention-handler
env:
INPUT_GITHUB-TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }}
INPUT_ORG-MEMBERSHIP-TOKEN: ${{ env.ORG_MEMBERSHIP_TOKEN || github.token }}
run: |
export GITHUB_EVENT_PATH=/tmp/test-event-inline.json
export GITHUB_EVENT_NAME=pull_request_review_comment
node "$GITHUB_WORKSPACE/dist/mention-reply.js"
- name: Assert should-reply output
if: steps.fork-check.outputs.is_fork != 'true'
run: |
SHOULD_REPLY="${{ steps.mention-handler.outputs.should-reply }}"
echo "should-reply=$SHOULD_REPLY"
echo "is-inline=${{ steps.mention-handler.outputs.is-inline }}"
echo "in-reply-to-id=${{ steps.mention-handler.outputs.in-reply-to-id }}"
if [ "$SHOULD_REPLY" == 'false' ]; then
echo "⚠️ Warning: should-reply=false — ${{ github.actor }} may not be a docker org member. Skipping reply steps."
exit 0
fi
- name: Assert inline outputs
if: steps.fork-check.outputs.is_fork != 'true' && steps.mention-handler.outputs.should-reply == 'true'
run: |
IS_INLINE="${{ steps.mention-handler.outputs.is-inline }}"
IN_REPLY_TO_ID="${{ steps.mention-handler.outputs.in-reply-to-id }}"
EXPECTED_ID="${{ steps.create-anchor.outputs.test_comment_id }}"
if [ "$IS_INLINE" != 'true' ]; then
echo "❌ Expected is-inline=true, got: $IS_INLINE"
exit 1
fi
echo "✅ is-inline=true"
if [ "$IN_REPLY_TO_ID" != "$EXPECTED_ID" ]; then
echo "❌ Expected in-reply-to-id=$EXPECTED_ID, got: $IN_REPLY_TO_ID"
exit 1
fi
echo "✅ in-reply-to-id=$IN_REPLY_TO_ID (matches anchor comment)"
- name: Run mention reply
if: steps.fork-check.outputs.is_fork != 'true' && steps.mention-handler.outputs.should-reply == 'true'
id: run-reply
uses: ./review-pr/mention-reply
with:
mention-context: ${{ steps.mention-handler.outputs.prompt }}
owner: ${{ steps.mention-handler.outputs.owner }}
repo: ${{ steps.mention-handler.outputs.repo }}
pr-number: ${{ steps.mention-handler.outputs.pr-number }}
is-inline: ${{ steps.mention-handler.outputs.is-inline }}
in-reply-to-id: ${{ steps.mention-handler.outputs.in-reply-to-id }}
anthropic-api-key: ${{ env.ANTHROPIC_API_KEY_FROM_SSM || secrets.ANTHROPIC_API_KEY }}
openai-api-key: ${{ env.OPENAI_API_KEY_FROM_SSM || secrets.OPENAI_API_KEY }}
github-token: ${{ env.GITHUB_APP_TOKEN || github.token }}
skip-auth: "true"
- name: Verify inline reply was posted in thread
if: steps.fork-check.outputs.is_fork != 'true' && steps.mention-handler.outputs.should-reply == 'true'
env:
GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }}
ANCHOR_ID: ${{ steps.create-anchor.outputs.test_comment_id }}
run: |
FOUND=$(gh api repos/docker/cagent-action/pulls/$TEST_PR_NUMBER/comments \
| jq --argjson id "$ANCHOR_ID" \
'[.[] | select(.in_reply_to_id == $id and (.body | contains("<!-- cagent-review-reply -->"))) ] | length')
if [ "$FOUND" -eq 0 ]; then
echo "❌ No inline reply found in thread $ANCHOR_ID"
exit 1
fi
echo "✅ Inline reply posted successfully"
- name: Cleanup anchor comment and thread replies
if: always() && steps.fork-check.outputs.is_fork != 'true'
continue-on-error: true
env:
GH_TOKEN: ${{ env.GITHUB_APP_TOKEN || github.token }}
ANCHOR_ID: ${{ steps.create-anchor.outputs.test_comment_id }}
run: |
if [ -z "$ANCHOR_ID" ]; then
echo "No anchor comment ID — nothing to clean up"
exit 0
fi
# Delete any replies in the thread first
gh api repos/docker/cagent-action/pulls/$TEST_PR_NUMBER/comments \
| jq --argjson id "$ANCHOR_ID" \
'[.[] | select(.in_reply_to_id == $id)] | .[].id' | \
while read -r reply_id; do
gh api "repos/docker/cagent-action/pulls/comments/$reply_id" -X DELETE || true
echo "Deleted reply comment $reply_id"
done
# Delete the anchor comment itself
gh api "repos/docker/cagent-action/pulls/comments/$ANCHOR_ID" -X DELETE || true
echo "Deleted anchor comment $ANCHOR_ID"