From ec559a931da801e86a626a8970806ea226c75493 Mon Sep 17 00:00:00 2001 From: skye rogers Date: Tue, 1 Jul 2025 16:18:35 -0700 Subject: [PATCH 01/10] environment security and more robust testing --- .github/scripts/clean_up_stream_table.sh | 17 +++ .github/scripts/create_stream.sh | 12 +++ .github/scripts/manipulate_properties.sh | 30 ++++++ .github/scripts/put_words_to_stream.sh | 16 +++ .github/scripts/start_kcl.sh | 48 +++++++++ .github/scripts/verify_kcl.sh | 20 ++++ .github/workflows/node.js.yml | 128 ++++++++++++++++++----- 7 files changed, 245 insertions(+), 26 deletions(-) create mode 100644 .github/scripts/clean_up_stream_table.sh create mode 100644 .github/scripts/create_stream.sh create mode 100644 .github/scripts/manipulate_properties.sh create mode 100644 .github/scripts/put_words_to_stream.sh create mode 100644 .github/scripts/start_kcl.sh create mode 100644 .github/scripts/verify_kcl.sh diff --git a/.github/scripts/clean_up_stream_table.sh b/.github/scripts/clean_up_stream_table.sh new file mode 100644 index 00000000..292566c3 --- /dev/null +++ b/.github/scripts/clean_up_stream_table.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +aws kinesis delete-stream --stream-name $STREAM_NAME || true + +# Reset the values of checkpoint, leaseCounter, ownerSwitchesSinceCheckpoint, and leaseOwner in DynamoDB table +echo "Resetting DDB table" +aws dynamodb update-item \ + --table-name $APP_NAME \ + --key '{"leaseKey": {"S": "shardId-000000000000"}}' \ + --update-expression "SET checkpoint = :checkpoint, leaseCounter = :counter, ownerSwitchesSinceCheckpoint = :switches, leaseOwner = :owner" \ + --expression-attribute-values '{ + ":checkpoint": {"S": "TRIM_HORIZON"}, + ":counter": {"N": "0"}, + ":switches": {"N": "0"}, + ":owner": {"S": "AVAILABLE"} + }' \ + --return-values NONE \ No newline at end of file diff --git a/.github/scripts/create_stream.sh b/.github/scripts/create_stream.sh new file mode 100644 index 00000000..f0bbb243 --- /dev/null +++ b/.github/scripts/create_stream.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +for i in {1..10}; do + if aws kinesis create-stream --stream-name $STREAM_NAME --shard-count 1; then + break + else + echo "Stream creation failed, attempt $i/10. Waiting $((i * 3)) seconds..." + sleep $((i * 3)) + fi +done +aws kinesis wait stream-exists --stream-name $STREAM_NAME \ No newline at end of file diff --git a/.github/scripts/manipulate_properties.sh b/.github/scripts/manipulate_properties.sh new file mode 100644 index 00000000..a7e016dd --- /dev/null +++ b/.github/scripts/manipulate_properties.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e + +# Manipulate sample.properties file that the KCL application pulls properties from (ex: streamName, applicationName) +# Depending on the OS, different properties need to be changed +if [[ "$RUNNER_OS" == "macOS" ]]; then + sed -i "" "s/streamName = .*/streamName = $STREAM_NAME/" samples/basic_sample/consumer/sample.properties + sed -i "" "s/applicationName = .*/applicationName = $APP_NAME/" samples/basic_sample/consumer/sample.properties + sed -i "" "s/#*idleTimeBetweenReadsInMillis.*/idleTimeBetweenReadsInMillis = 250/" samples/basic_sample/consumer/sample.properties + sed -i "" "s/stream : 'kclnodejssample'/stream : '$STREAM_NAME'/" samples/basic_sample/producer/config.js + sed -i "" "s/shards : 2/shards : 1/" samples/basic_sample/producer/config.js +elif [[ "$RUNNER_OS" == "Linux" ]]; then + sed -i "s/streamName = .*/streamName = $STREAM_NAME/" samples/basic_sample/consumer/sample.properties + sed -i "s/applicationName = .*/applicationName = $APP_NAME/" samples/basic_sample/consumer/sample.properties + sed -i "s/#*idleTimeBetweenReadsInMillis.*/idleTimeBetweenReadsInMillis = 250/" samples/basic_sample/consumer/sample.properties + sed -i "s/stream : 'kclnodejssample'/stream : '$STREAM_NAME'/" samples/basic_sample/producer/config.js + sed -i "s/shards : 2/shards : 1/" samples/basic_sample/producer/config.js +elif [[ "$RUNNER_OS" == "Windows" ]]; then + sed -i "s/streamName = .*/streamName = $STREAM_NAME/" samples/basic_sample/consumer/sample.properties + sed -i "s/applicationName = .*/applicationName = $APP_NAME/" samples/basic_sample/consumer/sample.properties + sed -i "s/#*idleTimeBetweenReadsInMillis.*/idleTimeBetweenReadsInMillis = 250/" samples/basic_sample/consumer/sample.properties + sed -i "s/stream : 'kclnodejssample'/stream : '$STREAM_NAME'/" samples/basic_sample/producer/config.js + sed -i "s/shards : 2/shards : 1/" samples/basic_sample/producer/config.js +else + echo "Unknown OS: $RUNNER_OS" + exit 1 +fi + +cat samples/basic_sample/consumer/sample.properties +cat samples/basic_sample/producer/config.js \ No newline at end of file diff --git a/.github/scripts/put_words_to_stream.sh b/.github/scripts/put_words_to_stream.sh new file mode 100644 index 00000000..70d2e371 --- /dev/null +++ b/.github/scripts/put_words_to_stream.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +cd samples/basic_sample/producer +node sample_kinesis_producer_app.js + +# Get records from stream to verify they exist before continuing +SHARD_ITERATOR=$(aws kinesis get-shard-iterator --stream-name $STREAM_NAME --shard-id shardId-000000000000 --shard-iterator-type TRIM_HORIZON --query 'ShardIterator' --output text) +INITIAL_RECORDS=$(aws kinesis get-records --shard-iterator $SHARD_ITERATOR) +RECORD_COUNT_BEFORE=$(echo $INITIAL_RECORDS | jq '.Records | length') + +if [ "$RECORD_COUNT_BEFORE" -eq 0 ]; then + echo "No records found in stream. Test cannot proceed." + exit 1 +fi +echo "Found $RECORD_COUNT_BEFORE records in stream before KCL start" \ No newline at end of file diff --git a/.github/scripts/start_kcl.sh b/.github/scripts/start_kcl.sh new file mode 100644 index 00000000..c4197a1a --- /dev/null +++ b/.github/scripts/start_kcl.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -e +set -o pipefail + +# Reset the values of checkpoint, leaseCounter, ownerSwitchesSinceCheckpoint, and leaseOwner in DynamoDB table +echo "Resetting checkpoint for shardId-000000000000" +aws dynamodb update-item \ + --table-name $APP_NAME \ + --key '{"leaseKey": {"S": "shardId-000000000000"}}' \ + --update-expression "SET checkpoint = :checkpoint, leaseCounter = :counter, ownerSwitchesSinceCheckpoint = :switches, leaseOwner = :owner" \ + --expression-attribute-values '{ + ":checkpoint": {"S": "TRIM_HORIZON"}, + ":counter": {"N": "0"}, + ":switches": {"N": "0"}, + ":owner": {"S": "AVAILABLE"} + }' \ + --return-values NONE + +# Get records from stream to verify they exist before continuing +SHARD_ITERATOR=$(aws kinesis get-shard-iterator --stream-name $STREAM_NAME --shard-id shardId-000000000000 --shard-iterator-type TRIM_HORIZON --query 'ShardIterator' --output text) +INITIAL_RECORDS=$(aws kinesis get-records --shard-iterator $SHARD_ITERATOR) +RECORD_COUNT_BEFORE=$(echo $INITIAL_RECORDS | jq '.Records | length') + +echo "Found $RECORD_COUNT_BEFORE records in stream before KCL start" + +if [[ "$RUNNER_OS" == "macOS" ]]; then + brew install coreutils + cd samples/basic_sample/consumer + KCL_COMMAND="../../../bin/kcl-bootstrap --java /usr/bin/java -e -p ./sample.properties" + gtimeout 300 $KCL_COMMAND 2>&1 | tee kcl_output.log || [ $? -eq 124 ] +elif [[ "$RUNNER_OS" == "Linux" ]]; then + cd samples/basic_sample/consumer + KCL_COMMAND="../../../bin/kcl-bootstrap -e -p ./sample.properties" + timeout 300 $KCL_COMMAND 2>&1 | tee kcl_output.log || [ $? -eq 124 ] +elif [[ "$RUNNER_OS" == "Windows" ]]; then + cd samples/basic_sample/consumer + KCL_COMMAND="../../../bin/kcl-bootstrap -e -p ./sample.properties" + timeout 300 $KCL_COMMAND 2>&1 | tee kcl_output.log || [ $? -eq 124 ] +else + echo "Unknown OS: $RUNNER_OS" + exit 1 +fi + +echo "---------ERROR LOGS HERE-------" +grep -i error kcl_output.log || echo "No errors found in logs" + + + diff --git a/.github/scripts/verify_kcl.sh b/.github/scripts/verify_kcl.sh new file mode 100644 index 00000000..a411a6a6 --- /dev/null +++ b/.github/scripts/verify_kcl.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +LEASE_EXISTS=$(aws dynamodb scan --table-name $APP_NAME --select "COUNT" --query "Count" --output text || echo "0") +CHECKPOINT_EXISTS=$(aws dynamodb scan --table-name $APP_NAME --select "COUNT" --filter-expression "attribute_exists(checkpoint) AND checkpoint <> :trim_horizon" --expression-attribute-values '{":trim_horizon": {"S": "TRIM_HORIZON"}}' --query "Count" --output text || echo "0") + +echo "Found $LEASE_EXISTS leases and $CHECKPOINT_EXISTS non-TRIM-HORIZON checkpoints in DynamoDB" + +echo "Printing checkpoint values" +aws dynamodb scan --table-name $APP_NAME --projection-expression "leaseKey,checkpoint" --output json + +if [ "$LEASE_EXISTS" -gt 0 ] && [ "$CHECKPOINT_EXISTS" -gt 0 ]; then + echo "Test passed: Found both leases and non-TRIM_HORIZON checkpoints in DDB (KCL is fully functional)" + exit 0 +else + echo "Test failed: KCL not fully functional" + echo "Lease(s) found: $LEASE_EXISTS" + echo "non-TRIM_HORIZON checkpoint(s) found: $CHECKPOINT_EXISTS" + exit 1 +fi \ No newline at end of file diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 5dd42317..b9189690 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -2,42 +2,76 @@ # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ -name: Sample Run and Dependabot Auto-merge +name: Sample Run Tests and Dependabot on: push: - branches: [ master ] + branches: [ master, sample-improve ] + pull_request_target: + branches: [ master, sample-improve ] + types: [opened, synchronize, reopened, labeled, unlabeled] -permissions: - id-token: write - contents: write - pull-requests: write - statuses: write +concurrency: + group: node.js.yml + cancel-in-progress: false jobs: + # Evaluates if the sample tests should run. If the workflow is triggered by a push OR + # is triggered by a pull request without the 'skip-sample-tests' label, the sample tests should run. + # Otherwise, the sample tests will be skipped + check-if-should-run: + runs-on: ubuntu-latest + if: ${{ github.event_name == 'push' || (github.event_name == 'pull_request_target' && !contains(github.event.pull_request.labels.*.name, 'skip-sample-tests')) }} + outputs: + should_run: 'true' + is_fork: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.fork }} + steps: + - run: echo "Evaluating workflow conditions" + + # Workflow will pause and wait here if it is triggered by a fork PR. The workflow will continue to wait until + # an approved member of the environment 'manual_approval' allows the workflow to run + wait-for-approval: + needs: [ check-if-should-run ] + if: ${{ needs.check-if-should-run.outputs.is_fork == 'true' }} + runs-on: ubuntu-latest + environment: manual-approval + steps: + - run: echo "Fork PR approved by a team member." + + # Sample run tests of the KCL + # Runs only if (check-if-should-run allows AND (the PR is not from a fork OR it has been approved to run)) sample-run: - timeout-minutes: 8 + needs: [ check-if-should-run, wait-for-approval ] + permissions: + id-token: write + if: ${{ always() && needs.check-if-should-run.outputs.should_run == 'true' && (needs.check-if-should-run.outputs.is_fork != 'true' || needs.wait-for-approval.result == 'success') }} + timeout-minutes: 20 runs-on: ${{ matrix.os }} defaults: run: shell: bash + # Initialize matrix based on PR labels (more-tests label runs more tests) strategy: fail-fast: false matrix: - node-version: [ '18.x', '20.x', '21.x' ] - jdk-version: [ "8", "11", "17", "21", "24" ] + node-version: ${{ github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'more-tests') && fromJSON('["18.x", "21.x"]') || fromJSON('["18.x"]') }} + jdk-version: ${{ github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'more-tests') && fromJSON('["8", "11", "17", "21", "24"]') || fromJSON('["8", "11"]') }} os: [ ubuntu-latest, macOS-latest, windows-latest ] steps: - - name: Checkout working directory + # For pull_request_target, checkout PR head instead of merge commit + - name: Checkout uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.ref }} + # Configure AWS Credentials. Role session name is unique to avoid OIDC errors when running multiple tests concurrently - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 - role-to-assume: arn:aws:iam::751999266872:role/GitHubNodejs - role-session-name: myGitHubActionsNodejs + role-to-assume: ${{ secrets.AWS_ARN_GHA }} + role-session-name: GHA-${{ github.run_id }}-${{ matrix.node-version }}-${{ matrix.jdk-version }}-${{ matrix.os }} - name: Set up JDK ${{ matrix.jdk-version }} uses: actions/setup-java@v4 @@ -49,7 +83,6 @@ jobs: uses: actions/setup-node@v3 with: node-version: ${{ matrix.node-version }} - cache: 'npm' - name: npm clean install, build, and test run: | @@ -61,23 +94,66 @@ jobs: run: | npm install - - name: Running data producer + # Create unique identifiers for the stream name and application name + - name: Set up unique identifiers + run: | + STREAM_NAME="kclnodesample-${{ matrix.os }}-node${{ matrix.node-version }}-jdk${{ matrix.jdk-version }}" + APP_NAME="NodeKCLSample-${{ matrix.os }}-node${{ matrix.node-version }}-jdk${{ matrix.jdk-version }}" + echo "STREAM_NAME=$STREAM_NAME" >> $GITHUB_ENV + echo "APP_NAME=$APP_NAME" >> $GITHUB_ENV + + # Manipulate sample.properties file to use unique stream name and application name + - name: Manipulate sample.properties file run: | - cd samples/basic_sample/producer - node sample_kinesis_producer_app.js + chmod +x .github/scripts/manipulate_properties.sh + .github/scripts/manipulate_properties.sh + env: + RUNNER_OS: ${{ matrix.os }} + STREAM_NAME: ${{ env.STREAM_NAME }} + APP_NAME: ${{ env.APP_NAME }} + + # Create kinesis stream with unique name and wait for it to exist + - name: Create and wait Kinesis stream + run: | + chmod +x .github/scripts/create_stream.sh + .github/scripts/create_stream.sh + env: + STREAM_NAME: ${{ env.STREAM_NAME }} - - name: Running KCL consumer (windows or ubuntu) - if: matrix.os != 'macOS-latest' + # Put words to sample stream with unique name based on run ID + - name: Put words to sample stream run: | - cd samples/basic_sample/consumer - timeout 45 ../../../bin/kcl-bootstrap -e -p ./sample.properties || status="$?"; if (( status == 124 )); then exit 0; else exit 1; fi; exit "$status" + chmod +x .github/scripts/put_words_to_stream.sh + .github/scripts/put_words_to_stream.sh + env: + STREAM_NAME: ${{ env.STREAM_NAME }} - - name: Running KCL consumer (macOS) - if: matrix.os == 'macOS-latest' + # Run sample KCL application + - name: Start KCL application + run: | + chmod +x .github/scripts/start_kcl.sh + .github/scripts/start_kcl.sh + env: + RUNNER_OS: ${{ matrix.os }} + STREAM_NAME: ${{ env.STREAM_NAME }} + + # Check and verify results of KCL test + - name: Verify KCL Functionality + run: | + chmod +x .github/scripts/verify_kcl.sh + .github/scripts/verify_kcl.sh + env: + APP_NAME: ${{ env.APP_NAME }} + + # Clean up all existing Streams and DDB tables + - name: Clean up Kinesis Stream and DynamoDB table + if: always() run: | - brew install coreutils - cd samples/basic_sample/consumer - gtimeout 45 ../../../bin/kcl-bootstrap --java /usr/bin/java -e -p ./sample.properties || status="$?"; if (( status == 124 )); then exit 0; else exit 1; fi; exit "$status" + chmod +x .github/scripts/clean_up_stream_table.sh + .github/scripts/clean_up_stream_table.sh + env: + STREAM_NAME: ${{ env.STREAM_NAME }} + APP_NAME: ${{ env.APP_NAME }} auto-merge-dependabot: needs: [ sample-run ] From 4769615cda6498547f0d7b7b2ff2ff3bc3882fc0 Mon Sep 17 00:00:00 2001 From: skye rogers Date: Wed, 2 Jul 2025 12:50:42 -0700 Subject: [PATCH 02/10] dependabot permissions and approvals --- .github/workflows/node.js.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index b9189690..55c36291 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -5,9 +5,9 @@ name: Sample Run Tests and Dependabot on: push: - branches: [ master, sample-improve ] + branches: [ master ] pull_request_target: - branches: [ master, sample-improve ] + branches: [ master ] types: [opened, synchronize, reopened, labeled, unlabeled] concurrency: @@ -159,6 +159,9 @@ jobs: needs: [ sample-run ] runs-on: ubuntu-latest if: github.actor == 'dependabot[bot]' && github.event.pull_request.user.login == 'dependabot[bot]' + permissions: + contents: read + pull-requests: write steps: - name: Fetch Dependabot metadata id: metadata @@ -167,12 +170,12 @@ jobs: alert-lookup: true github-token: "${{ secrets.GITHUB_TOKEN }}" -# - name: Approve PR -# if: steps.metadata.outputs.update-type != 'version-update:semver-major' -# run: gh pr review --approve "$PR_URL" -# env: -# PR_URL: ${{github.event.pull_request.html_url}} -# GH_TOKEN: ${{secrets.GITHUB_TOKEN}} + - name: Approve PR + if: steps.metadata.outputs.update-type != 'version-update:semver-major' + run: gh pr review --approve "$PR_URL" + env: + PR_URL: ${{github.event.pull_request.html_url}} + GH_TOKEN: ${{secrets.GITHUB_TOKEN}} # - name: Enable auto-merge for Dependabot PRs # if: steps.metadata.outputs.update-type != 'version-update:semver-major' From ec440439b419e46afaf6f91059b5867599e817f6 Mon Sep 17 00:00:00 2001 From: skye rogers Date: Wed, 2 Jul 2025 12:51:26 -0700 Subject: [PATCH 03/10] set dependabot frequency to weekly --- .github/dependabot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5e97d6e0..537a7c5f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,10 +4,10 @@ updates: directory: "/" open-pull-requests-limit: 4 schedule: - interval: "daily" + interval: "weekly" - package-ecosystem: "npm" directory: "/" open-pull-requests-limit: 4 schedule: - interval: "daily" + interval: "weekly" From 988ecca825eb8199d6f41dbc48bd26f85c5e80e6 Mon Sep 17 00:00:00 2001 From: skye rogers Date: Thu, 3 Jul 2025 09:52:15 -0700 Subject: [PATCH 04/10] fix dependabot trigger conditions --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 55c36291..c0edb61d 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -158,7 +158,7 @@ jobs: auto-merge-dependabot: needs: [ sample-run ] runs-on: ubuntu-latest - if: github.actor == 'dependabot[bot]' && github.event.pull_request.user.login == 'dependabot[bot]' + if: github.event.pull_request.user.login == 'dependabot[bot]' permissions: contents: read pull-requests: write From af198b86bee618acc732064e0a2201b1da1c7ff1 Mon Sep 17 00:00:00 2001 From: skye rogers Date: Thu, 3 Jul 2025 15:22:42 -0700 Subject: [PATCH 05/10] update is_fork and add validate step --- .github/workflows/node.js.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index c0edb61d..668186f7 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -23,7 +23,7 @@ jobs: if: ${{ github.event_name == 'push' || (github.event_name == 'pull_request_target' && !contains(github.event.pull_request.labels.*.name, 'skip-sample-tests')) }} outputs: should_run: 'true' - is_fork: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.fork }} + is_fork: ${{ github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.fork || github.event.pull_request.user.login == 'dependabot[bot]') }} steps: - run: echo "Evaluating workflow conditions" @@ -65,6 +65,12 @@ jobs: with: ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.ref }} + - name: Validate os, node-version, and jdk-version + run: | + [[ "${{ matrix.os }}" =~ ^(ubuntu-latest|macOS-latest|windows-latest)$ ]] || exit 1 + [[ "${{ matrix.node-version }}" =~ ^(18.x|21.x)$ ]] || exit 1 + [[ "${{ matrix.jdk-version }}" =~ ^(8|11|17|21|24)$ ]] || exit 1 + # Configure AWS Credentials. Role session name is unique to avoid OIDC errors when running multiple tests concurrently - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -176,10 +182,3 @@ jobs: env: PR_URL: ${{github.event.pull_request.html_url}} GH_TOKEN: ${{secrets.GITHUB_TOKEN}} - -# - name: Enable auto-merge for Dependabot PRs -# if: steps.metadata.outputs.update-type != 'version-update:semver-major' -# run: gh pr merge --auto --merge "$PR_URL" -# env: -# PR_URL: ${{github.event.pull_request.html_url}} -# GH_TOKEN: ${{secrets.GITHUB_TOKEN}} \ No newline at end of file From f382a609ec806c6d6cbb200c204bf2d6a5b23cf8 Mon Sep 17 00:00:00 2001 From: ehasah-aws Date: Tue, 8 Jul 2025 12:13:30 -0700 Subject: [PATCH 06/10] Added multi-lang support for leaseAssignmentIntervalMillis --- samples/basic_sample/consumer/sample.properties | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/samples/basic_sample/consumer/sample.properties b/samples/basic_sample/consumer/sample.properties index ce4fdf80..c1f138bc 100644 --- a/samples/basic_sample/consumer/sample.properties +++ b/samples/basic_sample/consumer/sample.properties @@ -175,3 +175,7 @@ regionName = us-east-1 # Duration format examples: PT15M (15 mins) PT10H (10 hours) P2D (2 days) # Refer to Duration.parse javadocs for more details # staleWorkerMetricsEntryCleanupDuration = + +# Interval in milliseconds at which Lease Assignment manager (LAM) runs +# If this value is not set then LAM will run at interval 2 * failoverTimeMillis. +# leaseAssignmentIntervalMillis = 20000 \ No newline at end of file From f3f852a7f9a2e99baad3562f323991b0289fc053 Mon Sep 17 00:00:00 2001 From: lucienlu-aws <132623944+lucienlu-aws@users.noreply.github.com> Date: Wed, 1 Oct 2025 13:26:48 -0700 Subject: [PATCH 07/10] Upgrade dependencies for 3.x (#441) --- pom.xml | 63 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/pom.xml b/pom.xml index 39d528e4..979bc99a 100644 --- a/pom.xml +++ b/pom.xml @@ -2,11 +2,11 @@ 4.0.0 - 2.25.64 - 3.0.0 - 4.1.118.Final + 2.33.0 + 3.1.3 + 4.2.4.Final 2.0.6 - 2.13.5 + 2.15.0 1.3.15 @@ -153,34 +153,44 @@ ${awssdk.version} - software.amazon.awssdk - http-auth-spi - ${awssdk.version} + software.amazon.awssdk + http-auth-spi + ${awssdk.version} + + + software.amazon.awssdk + http-auth + ${awssdk.version} - software.amazon.awssdk - http-auth - ${awssdk.version} + software.amazon.awssdk + http-auth-aws + ${awssdk.version} + + + software.amazon.awssdk + checksums-spi + ${awssdk.version} - software.amazon.awssdk - http-auth-aws - ${awssdk.version} + software.amazon.awssdk + checksums + ${awssdk.version} - software.amazon.awssdk - checksums-spi - ${awssdk.version} + software.amazon.awssdk + identity-spi + ${awssdk.version} - software.amazon.awssdk - checksums - ${awssdk.version} + software.amazon.awssdk + retries-spi + ${awssdk.version} - software.amazon.awssdk - identity-spi - ${awssdk.version} + software.amazon.awssdk + retries + ${awssdk.version} io.netty @@ -197,6 +207,11 @@ netty-codec ${netty.version} + + io.netty + netty-codec-base + ${netty.version} + io.netty netty-transport @@ -260,7 +275,7 @@ org.checkerframework checker-qual - 2.5.2 + 3.49.4 com.google.errorprone @@ -285,7 +300,7 @@ org.apache.commons commons-lang3 - 3.14.0 + 3.18.0 org.slf4j From ad5b692e9d3cd5c7f6c3520fa1af14e2ac481b9f Mon Sep 17 00:00:00 2001 From: lucienlu-aws <132623944+lucienlu-aws@users.noreply.github.com> Date: Wed, 1 Oct 2025 14:12:09 -0700 Subject: [PATCH 08/10] Prepare for release 3.1.0 (#443) --- README.md | 24 ++++++++++++++++-------- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index d3ffeedf..81821391 100644 --- a/README.md +++ b/README.md @@ -274,14 +274,22 @@ In this release, we have abstracted these implementation details away and expose ## Release Notes -### Release (v3.0.1 - May 28, 2025) -* [#414](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/414) Bump commons-beanutils:commons-beanutils from 1.9.4 to 1.11.0 -* [#413](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/413) Bump mocha from 10.4.0 to 11.5.0 -* [#410](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/410) Bump commander from 12.0.0 to 14.0.0 -* [#403](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/403) Bump io.netty:netty-handler from 4.1.115.Final to 4.1.118.Final -* [#384](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/384) Bump ch.qos.logback:logback-core from 1.3.14 to 1.3.15 -* [#368](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/368) Bump io.netty:netty-common from 4.1.108.Final to 4.1.115.Final -* [#367](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/367) Bump braces from 3.0.2 to 3.0.3 +### Release 3.1.0 (October 1, 2025) +* [PR #428](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/428) Add multi-lang support for leaseAssignmentIntervalMillis +* [PR #441](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/441) Upgrade amazon-kinesis-client from 3.1.1 to 3.1.3 +* [PR #441](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/441) Upgrade aws-sdk from 2.25.64 to 2.33.0 +* [PR #441](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/441) Upgrade org.apache.commons:commons-lang3 from 3.14.0 to 3.18.0 +* [PR #441](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/441) Upgrade netty.version from 4.1.125 to 4.2.4.Final +* [PR #441](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/441) Upgrade checker-qual from 2.5.2 to 3.49.4 + +### Release 3.0.1 (May 28, 2025) +* [PR #414](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/414) Bump commons-beanutils:commons-beanutils from 1.9.4 to 1.11.0 +* [PR #413](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/413) Bump mocha from 10.4.0 to 11.5.0 +* [PR #410](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/410) Bump commander from 12.0.0 to 14.0.0 +* [PR #403](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/403) Bump io.netty:netty-handler from 4.1.115.Final to 4.1.118.Final +* [PR #384](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/384) Bump ch.qos.logback:logback-core from 1.3.14 to 1.3.15 +* [PR #368](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/368) Bump io.netty:netty-common from 4.1.108.Final to 4.1.115.Final +* [PR #367](https://github.com/awslabs/amazon-kinesis-client-nodejs/pull/367) Bump braces from 3.0.2 to 3.0.3 ### Release 3.0.0 (November 6, 2024) * New lease assignment / load balancing algorithm diff --git a/package-lock.json b/package-lock.json index 4f511b9d..3fd9de17 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "aws-kcl", - "version": "3.0.1", + "version": "3.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "aws-kcl", - "version": "3.0.1", + "version": "3.1.0", "license": "Apache-2.0", "dependencies": { "commander": "~14.0.0", diff --git a/package.json b/package.json index 1dacabfd..2fbc1533 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "aws-kcl", "description": "Kinesis Client Libray (KCL) in Node.js.", - "version": "3.0.1", + "version": "3.1.0", "author": { "name": "Amazon Web Services", "url": "http://aws.amazon.com/" From f000fc3600cb8d9d6113781e582041c2320b8de3 Mon Sep 17 00:00:00 2001 From: skye rogers Date: Mon, 27 Oct 2025 16:17:43 -0700 Subject: [PATCH 09/10] bump netty.version from 4.2.4.Final to 4.2.7.Final --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 979bc99a..a3694f79 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 2.33.0 3.1.3 - 4.2.4.Final + 4.2.7.Final 2.0.6 2.15.0 1.3.15 From 908d329c7fb96fa733cb7bf9172f124f7aa7504b Mon Sep 17 00:00:00 2001 From: skye rogers Date: Mon, 27 Oct 2025 16:23:59 -0700 Subject: [PATCH 10/10] revert netty bump back to 4.2.4.Final --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a3694f79..979bc99a 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 2.33.0 3.1.3 - 4.2.7.Final + 4.2.4.Final 2.0.6 2.15.0 1.3.15