diff --git a/README.md b/README.md index be1aa2bd..6e7f07f7 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,31 @@ bash /tmp/aidlc-install.sh install \ --admin ``` +For managed installs or updates behind an HTTP proxy, export the standard proxy +variables before downloading or running the installer: + +```bash +export HTTP_PROXY=http://proxy.example.com:8080 +export HTTPS_PROXY=http://proxy.example.com:8080 +export NO_PROXY=localhost,127.0.0.1 + +curl -fsSLo /tmp/aidlc-install.sh \ + https://raw.githubusercontent.com/aws-samples/sample-collaborative-ai-dlc/main/scripts/install.sh +bash /tmp/aidlc-install.sh install \ + --profile \ + --region \ + --environment dev \ + --admin +``` + +The installer inherits these settings and forwards non-empty `HTTP_PROXY`, +`HTTPS_PROXY`, `FTP_PROXY`, `NO_PROXY`, and `ALL_PROXY` variables (including +lowercase variants) to both Docker Buildx builds. Without those variables, no +proxy build arguments are added. Set `TF_VAR_docker_build_args` to a JSON object +to override auto-detection. Proxy values are hidden in Terraform CLI output but +remain in saved plans and Terraform state, so protect both and keep the state +backend encrypted and access-restricted. + The password prompt is silent. The permanent Cognito password is sent directly to Cognito and is never written to installer configuration. After installation, sign in at the URL reported by `status`: ```bash @@ -125,6 +150,9 @@ cp terraform/environments/dev.tfvars.example terraform/environments/dev.tfvars ./scripts/deploy-terraform.sh dev --phase apply --plan-file /tmp/aidlc-dev.tfplan ``` +Manual deployments honor the same proxy variables documented for the managed +installer. You can also define `docker_build_args` in the `.tfvars` file. + ### Post-install Configuration The installer creates the first Cognito user and grants `platform-admin` for v2 (`owner` for v1.1.0). Additional users and administrators are managed in **Admin → User Management**. diff --git a/docs/development/testing.md b/docs/development/testing.md index 087d6812..e82357f9 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -56,6 +56,22 @@ key presence, model syntax, and outbound connectivity. It supplies inert AWS credentials to DynamoDB Local and does not require an AWS profile, Terraform state, Cognito login, or deployed stack. +When running behind an HTTP proxy, export the standard proxy variables before +starting the E2E: + +```bash +export HTTP_PROXY=http://proxy.example.com:8080 +export HTTPS_PROXY=http://proxy.example.com:8080 +export NO_PROXY=localhost,127.0.0.1 +./scripts/agent-e2e-testing.sh +``` + +The harness forwards non-empty standard proxy variables, including lowercase +variants, to the Buildx build, outbound-connectivity check, and AgentCore test +containers. No proxy flags or container variables are added when none are set. +The Docker daemon still needs its own proxy configuration to pull base and test +images. + ### Run all CLIs To avoid placing credentials directly in shell history, enter them in a Bash diff --git a/docs/getting-started/setup.md b/docs/getting-started/setup.md index e0f7897f..f8ec4f86 100644 --- a/docs/getting-started/setup.md +++ b/docs/getting-started/setup.md @@ -21,6 +21,30 @@ bash /tmp/aidlc-install.sh install \ --admin ``` +For a managed install or update behind an HTTP proxy, export the proxy settings +before downloading or running the installer: + +```bash +export HTTP_PROXY=http://proxy.example.com:8080 +export HTTPS_PROXY=http://proxy.example.com:8080 +export NO_PROXY=localhost,127.0.0.1 + +curl -fsSLo /tmp/aidlc-install.sh \ + https://raw.githubusercontent.com/aws-samples/sample-collaborative-ai-dlc/main/scripts/install.sh +bash /tmp/aidlc-install.sh install \ + --profile \ + --region \ + --environment dev \ + --admin +``` + +The installer passes non-empty standard proxy variables, including lowercase +variants, to both Docker Buildx builds. It adds no proxy build arguments in a +non-proxy environment. Set `TF_VAR_docker_build_args` to a JSON object to +override auto-detection. These values are hidden in Terraform CLI output but +remain in saved plans and Terraform state, so protect both and keep the backend +encrypted and access-restricted. + The password prompt is silent and the permanent password is never stored. The installer keeps immutable tagged checkouts under the XDG data directory, keeps Terraform configuration under the XDG config directory, and only changes `current` after infrastructure, administrator setup, and frontend deployment all succeed. Use the same script to inspect or update the deployment: @@ -94,6 +118,9 @@ To review a saved plan before applying: ./scripts/deploy-terraform.sh dev --phase apply --plan-file /tmp/aidlc-dev.tfplan ``` +Manual deployments honor the same proxy variables documented for the managed +installer. You can also define `docker_build_args` in the `.tfvars` file. + The deployment takes 15-30 minutes. Neptune DB cluster creation takes the longest. ### Bootstrap the first platform administrator diff --git a/scripts/agent-e2e-testing.sh b/scripts/agent-e2e-testing.sh index 64fbd183..d696efb8 100755 --- a/scripts/agent-e2e-testing.sh +++ b/scripts/agent-e2e-testing.sh @@ -24,6 +24,7 @@ OUTPUT_DIR="${E2E_OUTPUT_DIR:-$ROOT/test/e2e/artifacts/agent-output/$RUN_ID}" SECRET_FILE="" OVERALL_FAIL=0 IMAGE="" +DOCKER_PROXY_VARIABLES="HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY ALL_PROXY http_proxy https_proxy ftp_proxy no_proxy all_proxy" declare -a SELECTED_CLIS=() SELECTED_CLI_COUNT=0 @@ -98,6 +99,9 @@ set_result() { } preflight() { + local proxy_name + local -a outbound_check=(docker run --rm --platform linux/arm64) + log "Preflight" command -v docker >/dev/null 2>&1 || fail "docker is required" command -v node >/dev/null 2>&1 || fail "node is required" @@ -116,7 +120,12 @@ preflight() { docker run --rm --platform linux/arm64 alpine:3.20 true >/dev/null 2>&1 || fail "Docker cannot execute linux/arm64 containers" - docker run --rm --platform linux/arm64 curlimages/curl:8.12.1 \ + + for proxy_name in $DOCKER_PROXY_VARIABLES; do + [ -n "${!proxy_name:-}" ] && outbound_check+=(--env "$proxy_name") + done + outbound_check+=(curlimages/curl:8.12.1) + "${outbound_check[@]}" \ -fsS --connect-timeout 10 --max-time 20 -o /dev/null https://aws.amazon.com/ || fail "containers do not have outbound HTTPS access" printf 'Preflight passed for %s\n' "${SELECTED_CLIS[*]}" @@ -124,6 +133,9 @@ preflight() { } build_image() { + local proxy_name + local -a build_command=(docker buildx build) + log "AgentCore image" if [ -n "${AGENTCORE_IMAGE:-}" ]; then IMAGE="$AGENTCORE_IMAGE" @@ -133,12 +145,17 @@ build_image() { return fi IMAGE="aidlc-agentcore-e2e:$RUN_ID" - docker buildx build \ - --platform linux/arm64 \ - --load \ - --tag "$IMAGE" \ - --file "$ROOT/lambda/agentcore/Dockerfile" \ + for proxy_name in $DOCKER_PROXY_VARIABLES; do + [ -n "${!proxy_name:-}" ] && build_command+=(--build-arg "$proxy_name") + done + build_command+=( + --platform linux/arm64 + --load + --tag "$IMAGE" + --file "$ROOT/lambda/agentcore/Dockerfile" "$ROOT/lambda" + ) + "${build_command[@]}" } write_secret_file() { @@ -152,6 +169,8 @@ write_secret_file() { } container_args() { + local proxy_name + printf '%s\n' \ --platform linux/arm64 \ --label "$LABEL" \ @@ -171,6 +190,9 @@ container_args() { --env "V2_QUESTION_PARK_GRACE_MS=300" \ --env "E2E_SECRET_FILE=/run/secrets/aidlc-e2e.env" \ --volume "$SECRET_FILE:/run/secrets/aidlc-e2e.env:ro" + for proxy_name in $DOCKER_PROXY_VARIABLES; do + [ -n "${!proxy_name:-}" ] && printf '%s\n' --env "$proxy_name" + done } start_services() { diff --git a/scripts/deploy-terraform.sh b/scripts/deploy-terraform.sh index 24a81a71..fea14422 100755 --- a/scripts/deploy-terraform.sh +++ b/scripts/deploy-terraform.sh @@ -44,6 +44,21 @@ if [[ "$PLAN_FILE" != /* ]]; then PLAN_FILE="$(pwd)/$PLAN_FILE" fi +configure_docker_build_args() { + if [[ -n "${TF_VAR_docker_build_args+x}" ]]; then + return + fi + + local detected_build_args + detected_build_args="$(node "$SCRIPT_DIR/docker-proxy-build-args.mjs")" + if [[ "$detected_build_args" == "{}" ]]; then + return + fi + + export TF_VAR_docker_build_args="$detected_build_args" + echo "Forwarding detected proxy settings to Docker image builds." +} + tfvar_string() { local name="$1" local file="$2" @@ -144,6 +159,8 @@ fi echo "Deploying environment: $ENVIRONMENT ($PHASE)" if [[ "$PHASE" == "plan" || "$PHASE" == "all" ]]; then + configure_docker_build_args + if [[ "${AIDLC_SKIP_NPM_CI:-0}" != "1" ]]; then echo "Installing root npm dependencies..." (cd "$SCRIPT_DIR/.." && npm ci) diff --git a/scripts/docker-proxy-build-args.mjs b/scripts/docker-proxy-build-args.mjs new file mode 100644 index 00000000..2388d8e5 --- /dev/null +++ b/scripts/docker-proxy-build-args.mjs @@ -0,0 +1,20 @@ +const proxyVariableNames = [ + 'HTTP_PROXY', + 'HTTPS_PROXY', + 'FTP_PROXY', + 'NO_PROXY', + 'ALL_PROXY', + 'http_proxy', + 'https_proxy', + 'ftp_proxy', + 'no_proxy', + 'all_proxy', +]; + +const buildArgs = Object.fromEntries( + proxyVariableNames + .filter((name) => typeof process.env[name] === 'string' && process.env[name].length > 0) + .map((name) => [name, process.env[name]]), +); + +process.stdout.write(JSON.stringify(buildArgs)); diff --git a/scripts/test/agent-e2e-proxy.test.mjs b/scripts/test/agent-e2e-proxy.test.mjs new file mode 100644 index 00000000..3d5cfac8 --- /dev/null +++ b/scripts/test/agent-e2e-proxy.test.mjs @@ -0,0 +1,91 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const e2eScript = join(root, 'scripts/agent-e2e-testing.sh'); +const proxyVariableNames = [ + 'HTTP_PROXY', + 'HTTPS_PROXY', + 'FTP_PROXY', + 'NO_PROXY', + 'ALL_PROXY', + 'http_proxy', + 'https_proxy', + 'ftp_proxy', + 'no_proxy', + 'all_proxy', +]; + +const runHarness = (proxyEnv) => { + const fixture = mkdtempSync(join(tmpdir(), 'aidlc-e2e-proxy-')); + const bin = join(fixture, 'bin'); + const dockerLog = join(fixture, 'docker.log'); + mkdirSync(bin); + + const docker = join(bin, 'docker'); + writeFileSync( + docker, + `#!/usr/bin/env bash +first="\${1:-}" +second="\${2:-}" +{ + printf '%s' "$first" + shift || true + for arg in "$@"; do printf '\\t%s' "$arg"; done + printf '\\n' +} >> "$DOCKER_LOG" +if [[ "$first" == "logs" && "$second" == *"-gremlin-"* ]]; then + printf 'Channel started at port 8182\\n' +fi +`, + ); + chmodSync(docker, 0o755); + + const node = join(bin, 'node'); + writeFileSync(node, '#!/usr/bin/env bash\nexit 0\n'); + chmodSync(node, 0o755); + + const env = { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + DOCKER_LOG: dockerLog, + E2E_CLIS: 'kiro', + E2E_OUTPUT_DIR: join(fixture, 'output'), + KIRO_API_KEY: 'test-key', + TMPDIR: fixture, + }; + for (const name of proxyVariableNames) delete env[name]; + Object.assign(env, proxyEnv); + + const result = spawnSync('bash', [e2eScript], { + cwd: root, + encoding: 'utf8', + env, + timeout: 30_000, + }); + assert.equal(result.status, 0, result.stderr); + return readFileSync(dockerLog, 'utf8'); +}; + +test('E2E harness forwards proxy names without putting values in Docker arguments', () => { + const log = runHarness({ + HTTP_PROXY: 'http://proxy.example.com:8080', + no_proxy: 'localhost,127.0.0.1', + }); + + assert.match(log, /^buildx\tbuild\t--build-arg\tHTTP_PROXY\t--build-arg\tno_proxy\t/m); + assert.match(log, /\t--env\tHTTP_PROXY(?:\t|\n)/); + assert.match(log, /\t--env\tno_proxy(?:\t|\n)/); + assert.doesNotMatch(log, /proxy\.example\.com|localhost,127\.0\.0\.1/); +}); + +test('E2E harness adds no proxy arguments without proxy variables', () => { + const log = runHarness({}); + assert.doesNotMatch(log, /--build-arg/); + assert.doesNotMatch(log, /\t--env\t(?:HTTP_PROXY|no_proxy)(?:\t|\n)/); +}); diff --git a/scripts/test/docker-proxy-build-args.test.mjs b/scripts/test/docker-proxy-build-args.test.mjs new file mode 100644 index 00000000..17fb1d15 --- /dev/null +++ b/scripts/test/docker-proxy-build-args.test.mjs @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { dirname, join } from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +const script = join(dirname(fileURLToPath(import.meta.url)), '..', 'docker-proxy-build-args.mjs'); + +const collectBuildArgs = (env) => + JSON.parse(execFileSync(process.execPath, [script], { encoding: 'utf8', env })); + +test('returns an empty map when no proxy variables are set', () => { + assert.deepEqual(collectBuildArgs({}), {}); +}); + +test('forwards non-empty uppercase and lowercase proxy variables', () => { + assert.deepEqual( + collectBuildArgs({ + HTTP_PROXY: 'http://proxy.example.com:8080', + HTTPS_PROXY: '', + NO_PROXY: 'localhost,127.0.0.1', + http_proxy: 'http://lower-proxy.example.com:8080', + npm_config_proxy: 'http://ignored.example.com:8080', + }), + { + HTTP_PROXY: 'http://proxy.example.com:8080', + NO_PROXY: 'localhost,127.0.0.1', + http_proxy: 'http://lower-proxy.example.com:8080', + }, + ); +}); + +test('emits JSON-safe proxy values without modification', () => { + const proxy = 'http://proxy.example.com/a"b\\c'; + assert.deepEqual(collectBuildArgs({ ALL_PROXY: proxy }), { ALL_PROXY: proxy }); +}); diff --git a/terraform/main.tf b/terraform/main.tf index 4fd273e2..cc5feae6 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -306,6 +306,7 @@ module "yjs_server" { project_name = var.project_name environment = var.environment aws_region = var.aws_region + docker_build_args = var.docker_build_args vpc_id = module.networking.vpc_id private_subnet_ids = module.networking.private_subnet_ids cognito_user_pool_id = module.auth.user_pool_id @@ -327,6 +328,7 @@ module "agentcore" { project_name = var.project_name environment = var.environment aws_region = var.aws_region + docker_build_args = var.docker_build_args neptune_endpoint = module.neptune.cluster_endpoint neptune_cluster_resource_id = module.neptune.cluster_resource_id artifacts_bucket_name = module.s3.artifacts_bucket_name diff --git a/terraform/modules/compute/agentcore/main.tf b/terraform/modules/compute/agentcore/main.tf index acf39f40..ac0ed28d 100644 --- a/terraform/modules/compute/agentcore/main.tf +++ b/terraform/modules/compute/agentcore/main.tf @@ -163,8 +163,9 @@ module "agentcore_docker_build" { source_path = local.agentcore_source_path docker_file_path = "${local.agentcore_source_path}/agentcore/Dockerfile" # AgentCore Runtime runs arm64 only. - platform = "linux/arm64" - builder = "default" + platform = "linux/arm64" + builder = "default" + build_args = var.docker_build_args triggers = { dir_sha = local.agentcore_files_sha diff --git a/terraform/modules/compute/agentcore/variables.tf b/terraform/modules/compute/agentcore/variables.tf index cb088337..1aa78faf 100644 --- a/terraform/modules/compute/agentcore/variables.tf +++ b/terraform/modules/compute/agentcore/variables.tf @@ -13,6 +13,13 @@ variable "aws_region" { type = string } +variable "docker_build_args" { + description = "Optional arguments passed to the AgentCore Docker build" + type = map(string) + default = {} + sensitive = true +} + variable "neptune_endpoint" { description = "Neptune cluster endpoint" type = string diff --git a/terraform/modules/realtime/yjs-server/main.tf b/terraform/modules/realtime/yjs-server/main.tf index f32a7268..c87dc67b 100644 --- a/terraform/modules/realtime/yjs-server/main.tf +++ b/terraform/modules/realtime/yjs-server/main.tf @@ -94,7 +94,8 @@ module "yjs_docker_build" { platform = "linux/amd64" # BuildKit session path instead of the provider's legacy tar.gz streaming — # see the agents module for rationale. - builder = "default" + builder = "default" + build_args = var.docker_build_args triggers = { dir_sha = local.yjs_files_sha diff --git a/terraform/modules/realtime/yjs-server/variables.tf b/terraform/modules/realtime/yjs-server/variables.tf index dee7a2b7..7e016678 100644 --- a/terraform/modules/realtime/yjs-server/variables.tf +++ b/terraform/modules/realtime/yjs-server/variables.tf @@ -13,6 +13,13 @@ variable "aws_region" { type = string } +variable "docker_build_args" { + description = "Optional arguments passed to the Yjs server Docker build" + type = map(string) + default = {} + sensitive = true +} + variable "vpc_id" { description = "VPC ID" type = string diff --git a/terraform/variables.tf b/terraform/variables.tf index 10771ee5..67b0c589 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -28,4 +28,9 @@ variable "aidlc_repo_ref" { default = "83ed7a812c4024904f2c5e4d744e28077e0a5acd" } - +variable "docker_build_args" { + description = "Optional arguments for local Docker image builds, such as HTTP_PROXY, HTTPS_PROXY, and NO_PROXY. Sensitive values are hidden in CLI output but remain stored in Terraform state." + type = map(string) + default = {} + sensitive = true +}