Backport CI & test output fixes to release-11.3

.github/actions/save_logs_and_results/action.yml

@@ -6,7 +6,7 @@ inputs:
 runs:
   using: composite
   steps:
-  - uses: actions/upload-artifact@v3.1.1
+  - uses: actions/upload-artifact@v4.6.0
     name: Upload logs
     with:
       name: ${{ inputs.folder }}

.github/actions/setup_extension/action.yml

@@ -17,7 +17,7 @@ runs:
           echo "PG_MAJOR=${{ inputs.pg_major }}" >> $GITHUB_ENV
         fi
     shell: bash
-  - uses: actions/download-artifact@v3.0.1
+  - uses: actions/download-artifact@v4.1.8
     with:
       name: build-${{ env.PG_MAJOR }}
   - name: Install Extension

.github/actions/upload_coverage/action.yml

@@ -21,7 +21,7 @@ runs:
       mkdir -p /tmp/codeclimate
       cc-test-reporter format-coverage -t lcov -o /tmp/codeclimate/${{ inputs.flags }}.json lcov.info
     shell: bash
-  - uses: actions/upload-artifact@v3.1.1
+  - uses: actions/upload-artifact@v4.6.0
     with:
       path: "/tmp/codeclimate/*.json"
-      name: codeclimate
+      name: codeclimate-${{ inputs.flags }}

.github/workflows/build_and_test.yml

@@ -40,8 +40,21 @@ jobs:
     runs-on: ubuntu-20.04
     container:
       image: ${{ needs.params.outputs.build_image_name }}:latest
+      volumes:
+        # see https://github.com/actions/checkout/issues/1474#issuecomment-2604308106
+        - /tmp:/__e/node20
       options: --user root
     steps:
+    # see https://github.com/actions/checkout/issues/1474#issuecomment-2604308106
+    - name: Install node.js for GitHub Actions
+      run: |
+        apt-get install -y curl ca-certificates &&
+        # Install a Node.js version that works in older Ubuntu containers (read: does not require very recent glibc)
+        NODE_VERSION=v20.18.1 &&
+        NODE_TAR_FILE=node-$NODE_VERSION-linux-x64-glibc-217.tar.gz &&
+        NODE_URL=https://unofficial-builds.nodejs.org/download/release/$NODE_VERSION/$NODE_TAR_FILE &&
+        curl -Lo /tmp/$NODE_TAR_FILE $NODE_URL &&
+        tar -C /__e/node20 -x --strip-components=1 -f /tmp/$NODE_TAR_FILE
     - uses: actions/[email protected]
     - name: Check Snapshots
       run: |
@@ -56,7 +69,7 @@ jobs:
     - name: Check Snapshots
       run: |
         git config --global --add safe.directory ${GITHUB_WORKSPACE}
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Check C Style
@@ -110,14 +123,14 @@ jobs:
       image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}"
       options: --user root
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - name: Expose $PG_MAJOR to Github Env
       run: echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV
       shell: bash
     - name: Build
       run: "./ci/build-citus.sh"
       shell: bash
-    - uses: actions/upload-artifact@v3.1.1
+    - uses: actions/upload-artifact@v4.6.0
       with:
         name: build-${{ env.PG_MAJOR }}
         path: |-
@@ -133,9 +146,9 @@ jobs:
         image_name:
           - ${{ needs.params.outputs.test_image_name }}
         pg_version:
-          - ${{ needs.params.outputs.pg13_version }}
           - ${{ needs.params.outputs.pg13_version }}
           - ${{ needs.params.outputs.pg14_version }}
+          - ${{ needs.params.outputs.pg15_version }}
         make:
           - check-split
           - check-multi
@@ -204,7 +217,7 @@ jobs:
     - params
     - build
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - uses: "./.github/actions/setup_extension"
     - name: Run Test
       run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }}
@@ -238,7 +251,7 @@ jobs:
           - ${{ needs.params.outputs.pg15_version }}
         parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - uses: "./.github/actions/setup_extension"
     - name: Test arbitrary configs
       run: |-
@@ -260,10 +273,12 @@ jobs:
             check-arbitrary-configs parallel=4 CONFIGS=$TESTS
     - uses: "./.github/actions/save_logs_and_results"
       if: always()
+      with:
+        folder: ${{ env.PG_MAJOR }}_arbitrary_configs_${{ matrix.parallel }}
     - uses: "./.github/actions/upload_coverage"
       if: always()
       with:
-        flags: ${{ env.pg_major }}_upgrade
+        flags: ${{ env.PG_MAJOR }}_arbitrary_configs_${{ matrix.parallel }}
         codecov_token: ${{ secrets.CODECOV_TOKEN }}
   test-pg-upgrade:
     name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade
@@ -286,7 +301,7 @@ jobs:
       old_pg_major: ${{ matrix.old_pg_major }}
       new_pg_major: ${{ matrix.new_pg_major }}
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - uses: "./.github/actions/setup_extension"
       with:
         pg_major: "${{ env.old_pg_major }}"
@@ -309,6 +324,8 @@ jobs:
       if: failure()
     - uses: "./.github/actions/save_logs_and_results"
       if: always()
+      with:
+        folder: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade
     - uses: "./.github/actions/upload_coverage"
       if: always()
       with:
@@ -324,7 +341,7 @@ jobs:
     - params
     - build
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - uses: "./.github/actions/setup_extension"
       with:
         skip_installation: true
@@ -354,10 +371,12 @@ jobs:
         done;
     - uses: "./.github/actions/save_logs_and_results"
       if: always()
+      with:
+        folder: ${{ env.PG_MAJOR }}_citus_upgrade
     - uses: "./.github/actions/upload_coverage"
       if: always()
       with:
-        flags: ${{ env.pg_major }}_upgrade
+        flags: ${{ env.PG_MAJOR }}_citus_upgrade
         codecov_token: ${{ secrets.CODECOV_TOKEN }}
   upload-coverage:
     if: always()
@@ -373,10 +392,11 @@ jobs:
       - test-citus-upgrade
       - test-pg-upgrade
     steps:
-      - uses: actions/download-artifact@v3.0.1
+      - uses: actions/download-artifact@v4.1.8
         with:
-          name: "codeclimate"
-          path: "codeclimate"
+          pattern: codeclimate*
+          path: codeclimate
+          merge-multiple: true
       - name: Upload coverage results to Code Climate
         run: |-
           cc-test-reporter sum-coverage codeclimate/*.json -o total.json
@@ -388,7 +408,7 @@ jobs:
     needs:
     - build
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - uses: azure/login@v1
       with:
         creds: ${{ secrets.AZURE_CREDENTIALS }}
@@ -406,7 +426,7 @@ jobs:
     needs:
     - build
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - uses: azure/login@v1
       with:
         creds: ${{ secrets.AZURE_CREDENTIALS }}
@@ -425,7 +445,7 @@ jobs:
     outputs:
       json: ${{ steps.parallelization.outputs.json }}
     steps:
-      - uses: actions/checkout@v3.5.0
+      - uses: actions/checkout@v4
       - uses: "./.github/actions/parallelization"
         id: parallelization
         with:
@@ -438,7 +458,7 @@ jobs:
     outputs:
       tests: ${{ steps.detect-regression-tests.outputs.tests }}
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Detect regression tests need to be ran
@@ -473,8 +493,8 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJson(needs.prepare_parallelization_matrix_32.outputs.json) }}
     steps:
-    - uses: actions/checkout@v3.5.0
-    - uses: actions/download-artifact@v3.0.1
+    - uses: actions/checkout@v4
+    - uses: actions/download-artifact@v4.1.8
     - uses: "./.github/actions/setup_extension"
     - name: Run minimal tests
       run: |-

.github/workflows/codeql.yml

@@ -21,7 +21,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2

.github/workflows/flaky_test_debugging.yml

@@ -28,13 +28,13 @@ jobs:
       image: ${{ vars.build_image_name }}:${{ vars.pg15_version  }}${{ vars.image_suffix }}
       options: --user root
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - name: Configure, Build, and Install
       run: |
         echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV
         ./ci/build-citus.sh
       shell: bash
-    - uses: actions/upload-artifact@v3.1.1
+    - uses: actions/upload-artifact@v4.6.0
       with:
         name: build-${{ env.PG_MAJOR }}
         path: |-
@@ -46,7 +46,7 @@ jobs:
     outputs:
       json: ${{ steps.parallelization.outputs.json }}
     steps:
-      - uses: actions/checkout@v3.5.0
+      - uses: actions/checkout@v4
       - uses: "./.github/actions/parallelization"
         id: parallelization
         with:
@@ -67,7 +67,7 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJson(needs.prepare_parallelization_matrix.outputs.json) }}
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - uses: "./.github/actions/setup_extension"
     - name: Run minimal tests
       run: |-

.github/workflows/packaging-test-pipelines.yml

@@ -6,6 +6,10 @@ on:
 
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
 
   get_postgres_versions_from_file:
@@ -14,7 +18,7 @@ jobs:
       pg_versions: ${{ steps.get-postgres-versions.outputs.pg_versions }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 2
       - name: Get Postgres Versions
@@ -42,9 +46,7 @@ jobs:
         # For this reason, we need to use a "matrix" to generate names of
         # rpm images, e.g. citus/packaging:centos-7-pg12
         packaging_docker_image:
-          - oraclelinux-7
           - oraclelinux-8
-          - centos-7
           - almalinux-8
           - almalinux-9
         POSTGRES_VERSION: ${{ fromJson(needs.get_postgres_versions_from_file.outputs.pg_versions) }}
@@ -55,7 +57,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set Postgres and python parameters for rpm based distros
         run: |
@@ -111,7 +113,6 @@ jobs:
           - debian-buster-all
           - debian-bookworm-all
           - debian-bullseye-all
-          - ubuntu-bionic-all
           - ubuntu-focal-all
           - ubuntu-jammy-all
 
@@ -123,7 +124,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set pg_config path and python parameters for deb based distros
         run: |

src/backend/distributed/utils/background_jobs.c

@@ -395,7 +395,7 @@
 
 		/* sleep for a while, before rechecking the task status */
 		CHECK_FOR_INTERRUPTS();
-		const long delay_ms = 1000;
+		const long delay_ms = 100;
 		(void) WaitLatch(MyLatch,
 						 WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
 						 delay_ms,

src/test/regress/expected/background_rebalance_parallel.out

@@ -513,20 +513,26 @@ FROM pg_dist_background_task WHERE job_id in (:job_id) ORDER BY task_id;
 (8 rows)
 
 -- increase citus.max_background_task_executors_per_node
+SELECT citus_task_wait(1013, desired_status => 'done');
+ citus_task_wait
+---------------------------------------------------------------------
+
+(1 row)
+
 ALTER SYSTEM SET citus.max_background_task_executors_per_node = 2;
 SELECT pg_reload_conf();
  pg_reload_conf
 ---------------------------------------------------------------------
  t
 (1 row)
 
-SELECT citus_task_wait(1015, desired_status => 'running');
+SELECT citus_task_wait(1014, desired_status => 'running');
  citus_task_wait
 ---------------------------------------------------------------------
 
 (1 row)
 
-SELECT citus_task_wait(1013, desired_status => 'done');
+SELECT citus_task_wait(1015, desired_status => 'running');
  citus_task_wait
 ---------------------------------------------------------------------
 

src/test/regress/expected/citus_non_blocking_split_shard_cleanup.out

@@ -107,6 +107,12 @@ SELECT pg_catalog.citus_split_shard_by_split_points(
 
 (1 row)
 
+SELECT public.wait_for_resource_cleanup();
+ wait_for_resource_cleanup
+---------------------------------------------------------------------
+
+(1 row)
+
 \c - - - :worker_2_port
 SET search_path TO "citus_split_test_schema";
 -- Replication slots should be cleaned up

src/test/regress/expected/foreign_key_to_reference_shard_rebalance.out

@@ -210,7 +210,7 @@ select create_distributed_table('partitioned_tbl_with_fkey','x');
 
 create table partition_1_with_fkey partition of partitioned_tbl_with_fkey for values from ('2022-01-01') to ('2022-12-31');
 create table partition_2_with_fkey partition of partitioned_tbl_with_fkey for values from ('2023-01-01') to ('2023-12-31');
-create table partition_3_with_fkey partition of partitioned_tbl_with_fkey for values from ('2024-01-01') to ('2024-12-31');
+create table partition_3_with_fkey partition of partitioned_tbl_with_fkey DEFAULT;
 insert into partitioned_tbl_with_fkey (x,y) select s,s%10 from generate_series(1,100) s;
 ALTER TABLE partitioned_tbl_with_fkey ADD CONSTRAINT fkey_to_ref_tbl FOREIGN KEY (y) REFERENCES ref_table_with_fkey(id);
 WITH shardid AS (SELECT shardid FROM pg_dist_shard where logicalrelid = 'partitioned_tbl_with_fkey'::regclass ORDER BY shardid LIMIT 1)