From af5fced935440089fe63cd3fe0f4758b4d24f422 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Fri, 31 Jan 2025 14:23:01 +0300 Subject: [PATCH 1/7] Upgrade upload-artifacts action to 4.6.0 (cherry picked from commit 398a2ea1978c7a0b00bd69600efa238b86d011ca) --- .github/actions/save_logs_and_results/action.yml | 2 +- .github/actions/upload_coverage/action.yml | 2 +- .github/workflows/build_and_test.yml | 2 +- .github/workflows/flaky_test_debugging.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/save_logs_and_results/action.yml b/.github/actions/save_logs_and_results/action.yml index 0f238835d19..b344c68f2ef 100644 --- a/.github/actions/save_logs_and_results/action.yml +++ b/.github/actions/save_logs_and_results/action.yml @@ -6,7 +6,7 @@ inputs: runs: using: composite steps: - - uses: actions/upload-artifact@v3.1.1 + - uses: actions/upload-artifact@v4.6.0 name: Upload logs with: name: ${{ inputs.folder }} diff --git a/.github/actions/upload_coverage/action.yml b/.github/actions/upload_coverage/action.yml index 0b5f581a6a4..abffa784a03 100644 --- a/.github/actions/upload_coverage/action.yml +++ b/.github/actions/upload_coverage/action.yml @@ -21,7 +21,7 @@ runs: mkdir -p /tmp/codeclimate cc-test-reporter format-coverage -t lcov -o /tmp/codeclimate/${{ inputs.flags }}.json lcov.info shell: bash - - uses: actions/upload-artifact@v3.1.1 + - uses: actions/upload-artifact@v4.6.0 with: path: "/tmp/codeclimate/*.json" name: codeclimate diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index d149ff650a8..4e3b9995838 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -125,7 +125,7 @@ jobs: - name: Build run: "./ci/build-citus.sh" shell: bash - - uses: actions/upload-artifact@v3.1.1 + - uses: actions/upload-artifact@v4.6.0 with: name: build-${{ env.PG_MAJOR }} path: |- diff --git a/.github/workflows/flaky_test_debugging.yml b/.github/workflows/flaky_test_debugging.yml index 7135f99fa02..812fbe2418a 100644 --- a/.github/workflows/flaky_test_debugging.yml +++ b/.github/workflows/flaky_test_debugging.yml @@ -34,7 +34,7 @@ jobs: echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV ./ci/build-citus.sh shell: bash - - uses: actions/upload-artifact@v3.1.1 + - uses: actions/upload-artifact@v4.6.0 with: name: build-${{ env.PG_MAJOR }} path: |- From a28f75cc778545aa5b8d589c46037ae192908f48 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Fri, 31 Jan 2025 14:39:55 +0300 Subject: [PATCH 2/7] Upgrade download-artifacts action to 4.1.8 (cherry picked from commit 5317cc7310d81eeab88fa5fca568817f80f5c1df) --- .github/actions/setup_extension/action.yml | 2 +- .github/workflows/build_and_test.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup_extension/action.yml b/.github/actions/setup_extension/action.yml index 96b408e7e43..33129f17de6 100644 --- a/.github/actions/setup_extension/action.yml +++ b/.github/actions/setup_extension/action.yml @@ -17,7 +17,7 @@ runs: echo "PG_MAJOR=${{ inputs.pg_major }}" >> $GITHUB_ENV fi shell: bash - - uses: actions/download-artifact@v3.0.1 + - uses: actions/download-artifact@v4.1.8 with: name: build-${{ env.PG_MAJOR }} - name: Install Extension diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 4e3b9995838..20f7377442f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -399,7 +399,7 @@ jobs: - test-citus-upgrade - test-pg-upgrade steps: - - uses: actions/download-artifact@v3.0.1 + - uses: actions/download-artifact@v4.1.8 with: name: "codeclimate" path: "codeclimate" @@ -516,6 +516,7 @@ jobs: matrix: ${{ fromJson(needs.prepare_parallelization_matrix_32.outputs.json) }} steps: - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4.1.8 - uses: "./.github/actions/setup_extension" - name: Run minimal tests run: |- From b6e3f395835b0acdd82f5396c84d42bb645326dd Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Fri, 31 Jan 2025 14:59:23 +0300 Subject: [PATCH 3/7] Fix flaky citus upgrade test (cherry picked from commit 4cad81d643df66e11c44e7b3e10edfc0748cd086) --- .../upgrade_pg_dist_cleanup_after_0.out | 9 +++++++++ .../upgrade_pg_dist_cleanup_before_0.out | 17 +++++++++++++++++ .../sql/upgrade_pg_dist_cleanup_after.sql | 5 +++++ .../sql/upgrade_pg_dist_cleanup_before.sql | 10 ++++++++++ 4 files changed, 41 insertions(+) diff --git a/src/test/regress/expected/upgrade_pg_dist_cleanup_after_0.out b/src/test/regress/expected/upgrade_pg_dist_cleanup_after_0.out index d71fad887c9..168c64ccaa3 100644 --- a/src/test/regress/expected/upgrade_pg_dist_cleanup_after_0.out +++ b/src/test/regress/expected/upgrade_pg_dist_cleanup_after_0.out @@ -28,3 +28,12 @@ SELECT * FROM pg_dist_cleanup; CALL citus_cleanup_orphaned_resources(); NOTICE: cleaned up 1 orphaned resources DROP TABLE table_with_orphaned_shards; +-- Re-enable automatic shard cleanup by maintenance daemon as +-- we have disabled it in upgrade_pg_dist_cleanup_before.sql +ALTER SYSTEM RESET citus.defer_shard_delete_interval; +SELECT pg_reload_conf(); + pg_reload_conf +--------------------------------------------------------------------- + t +(1 row) + diff --git a/src/test/regress/expected/upgrade_pg_dist_cleanup_before_0.out b/src/test/regress/expected/upgrade_pg_dist_cleanup_before_0.out index a0cf9ceb1ea..dd6c8868e32 100644 --- a/src/test/regress/expected/upgrade_pg_dist_cleanup_before_0.out +++ b/src/test/regress/expected/upgrade_pg_dist_cleanup_before_0.out @@ -30,6 +30,23 @@ SELECT COUNT(*) FROM pg_dist_placement WHERE shardstate = 1 AND shardid IN (SELE (1 row) -- create an orphaned placement based on an existing one +-- +-- But before doing that, first disable automatic shard cleanup +-- by maintenance daemon so that we can reliably test the cleanup +-- in upgrade_pg_dist_cleanup_after.sql. +ALTER SYSTEM SET citus.defer_shard_delete_interval TO -1; +SELECT pg_reload_conf(); + pg_reload_conf +--------------------------------------------------------------------- + t +(1 row) + +SELECT pg_sleep(0.1); + pg_sleep +--------------------------------------------------------------------- + +(1 row) + INSERT INTO pg_dist_placement(placementid, shardid, shardstate, shardlength, groupid) SELECT nextval('pg_dist_placement_placementid_seq'::regclass), shardid, 4, shardlength, 3-groupid FROM pg_dist_placement diff --git a/src/test/regress/sql/upgrade_pg_dist_cleanup_after.sql b/src/test/regress/sql/upgrade_pg_dist_cleanup_after.sql index e84c35b608c..333ac60ca93 100644 --- a/src/test/regress/sql/upgrade_pg_dist_cleanup_after.sql +++ b/src/test/regress/sql/upgrade_pg_dist_cleanup_after.sql @@ -13,3 +13,8 @@ SELECT COUNT(*) FROM pg_dist_placement WHERE shardid IN (SELECT shardid FROM pg_ SELECT * FROM pg_dist_cleanup; CALL citus_cleanup_orphaned_resources(); DROP TABLE table_with_orphaned_shards; + +-- Re-enable automatic shard cleanup by maintenance daemon as +-- we have disabled it in upgrade_pg_dist_cleanup_before.sql +ALTER SYSTEM RESET citus.defer_shard_delete_interval; +SELECT pg_reload_conf(); diff --git a/src/test/regress/sql/upgrade_pg_dist_cleanup_before.sql b/src/test/regress/sql/upgrade_pg_dist_cleanup_before.sql index 62ec8a1fb46..ec0eef353dc 100644 --- a/src/test/regress/sql/upgrade_pg_dist_cleanup_before.sql +++ b/src/test/regress/sql/upgrade_pg_dist_cleanup_before.sql @@ -16,6 +16,16 @@ SELECT create_distributed_table('table_with_orphaned_shards', 'a'); -- show all 32 placements are active SELECT COUNT(*) FROM pg_dist_placement WHERE shardstate = 1 AND shardid IN (SELECT shardid FROM pg_dist_shard WHERE logicalrelid='table_with_orphaned_shards'::regclass); -- create an orphaned placement based on an existing one +-- +-- But before doing that, first disable automatic shard cleanup +-- by maintenance daemon so that we can reliably test the cleanup +-- in upgrade_pg_dist_cleanup_after.sql. + +ALTER SYSTEM SET citus.defer_shard_delete_interval TO -1; +SELECT pg_reload_conf(); + +SELECT pg_sleep(0.1); + INSERT INTO pg_dist_placement(placementid, shardid, shardstate, shardlength, groupid) SELECT nextval('pg_dist_placement_placementid_seq'::regclass), shardid, 4, shardlength, 3-groupid FROM pg_dist_placement From 8783cae57f7dbe0422efea3d9b0af75af823c196 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Fri, 31 Jan 2025 15:46:42 +0300 Subject: [PATCH 4/7] Avoid publishing artifacts with conflicting names .. as documented in actions/upload-artifact#480. (cherry picked from commit 0d4c676b078c443b499797a1b4fcfaba3b3508b1) --- .github/actions/upload_coverage/action.yml | 2 +- .github/workflows/build_and_test.yml | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/actions/upload_coverage/action.yml b/.github/actions/upload_coverage/action.yml index abffa784a03..ba80ba63afa 100644 --- a/.github/actions/upload_coverage/action.yml +++ b/.github/actions/upload_coverage/action.yml @@ -24,4 +24,4 @@ runs: - uses: actions/upload-artifact@v4.6.0 with: path: "/tmp/codeclimate/*.json" - name: codeclimate + name: codeclimate-${{ inputs.flags }} diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 20f7377442f..db5ad0b316f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -284,10 +284,12 @@ jobs: check-arbitrary-configs parallel=4 CONFIGS=$TESTS - uses: "./.github/actions/save_logs_and_results" if: always() + with: + folder: ${{ env.PG_MAJOR }}_arbitrary_configs_${{ matrix.parallel }} - uses: "./.github/actions/upload_coverage" if: always() with: - flags: ${{ env.pg_major }}_upgrade + flags: ${{ env.PG_MAJOR }}_arbitrary_configs_${{ matrix.parallel }} codecov_token: ${{ secrets.CODECOV_TOKEN }} test-pg-upgrade: name: PG${{ matrix.old_pg_major }}-PG${{ matrix.new_pg_major }} - check-pg-upgrade @@ -335,6 +337,8 @@ jobs: if: failure() - uses: "./.github/actions/save_logs_and_results" if: always() + with: + folder: ${{ env.old_pg_major }}_${{ env.new_pg_major }}_upgrade - uses: "./.github/actions/upload_coverage" if: always() with: @@ -380,10 +384,12 @@ jobs: done; - uses: "./.github/actions/save_logs_and_results" if: always() + with: + folder: ${{ env.PG_MAJOR }}_citus_upgrade - uses: "./.github/actions/upload_coverage" if: always() with: - flags: ${{ env.pg_major }}_upgrade + flags: ${{ env.PG_MAJOR }}_citus_upgrade codecov_token: ${{ secrets.CODECOV_TOKEN }} upload-coverage: if: always() @@ -401,8 +407,9 @@ jobs: steps: - uses: actions/download-artifact@v4.1.8 with: - name: "codeclimate" - path: "codeclimate" + pattern: codeclimate* + path: codeclimate + merge-multiple: true - name: Upload coverage results to Code Climate run: |- cc-test-reporter sum-coverage codeclimate/*.json -o total.json From 7073f061537ebc5d63b7c063c9cb70d0986bb5cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Fri, 31 May 2024 20:52:17 +0300 Subject: [PATCH 5/7] Updates github checkout actions to v4 (#7611) (cherry picked from commit 3fe22406e62fb40da12a0d91f3ecc0cba81cdb24) --- .github/workflows/build_and_test.yml | 2 +- .github/workflows/packaging-test-pipelines.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index db5ad0b316f..32c761766d5 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -48,7 +48,7 @@ jobs: image: ${{ needs.params.outputs.build_image_name }}:${{ needs.params.outputs.sql_snapshot_pg_version }}${{ needs.params.outputs.image_suffix }} options: --user root steps: - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v4 - name: Check Snapshots run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} diff --git a/.github/workflows/packaging-test-pipelines.yml b/.github/workflows/packaging-test-pipelines.yml index 7f89b9f834b..db0fd08efd0 100644 --- a/.github/workflows/packaging-test-pipelines.yml +++ b/.github/workflows/packaging-test-pipelines.yml @@ -129,7 +129,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set pg_config path and python parameters for deb based distros run: | From 9a0cc282b7e26c29c69882cd9cb4dd709ef8ad0f Mon Sep 17 00:00:00 2001 From: Naisila Puka <37271756+naisila@users.noreply.github.com> Date: Tue, 4 Feb 2025 15:55:35 +0300 Subject: [PATCH 6/7] Changelog entries for v13.0.1 (#7873) (cherry picked from commit d28a5eae6c78935313824d319480632783d48d10) --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ebb6bec835..ee3f2d0a203 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### citus v13.0.1 (February 4th, 2025) ### + +* Drops support for PostgreSQL 14 (#7753) + ### citus v13.0.0 (January 17, 2025) ### * Adds support for PostgreSQL 17 (#7699, #7661) From 2d8be018530016228795ecaa1a9bc72c9e8328d4 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Tue, 4 Feb 2025 11:45:24 +0300 Subject: [PATCH 7/7] Disable 2PC recovery while executing ALTER EXTENSION cmd during Citus upgrade tests (cherry picked from commit b6b73e2f4c66211c23e167e094256264992c8bd4) --- .../citus_tests/upgrade/citus_upgrade_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py b/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py index 1ab44803117..c25a3448238 100755 --- a/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py +++ b/src/test/regress/citus_tests/upgrade/citus_upgrade_test.py @@ -62,10 +62,16 @@ def run_citus_upgrade_tests(config, before_upgrade_schedule, after_upgrade_sched install_citus(config.post_tar_path) + # disable 2pc recovery for all nodes to work around https://github.com/citusdata/citus/issues/7875 + disable_2pc_recovery_for_all_nodes(config.bindir, config) + restart_databases(config.bindir, config.datadir, config.mixed_mode, config) run_alter_citus(config.bindir, config.mixed_mode, config) verify_upgrade(config, config.mixed_mode, config.node_name_to_ports.values()) + # re-enable 2pc recovery for all nodes + enable_2pc_recovery_for_all_nodes(config.bindir, config) + run_test_on_coordinator(config, after_upgrade_schedule) remove_citus(config.post_tar_path) @@ -146,6 +152,18 @@ def restart_database(pg_path, abs_data_path, node_name, node_ports, logfile_pref subprocess.run(command, check=True) +def disable_2pc_recovery_for_all_nodes(pg_path, config): + for port in config.node_name_to_ports.values(): + utils.psql(pg_path, port, "ALTER SYSTEM SET citus.recover_2pc_interval TO -1;") + utils.psql(pg_path, port, "SELECT pg_reload_conf();") + + +def enable_2pc_recovery_for_all_nodes(pg_path, config): + for port in config.node_name_to_ports.values(): + utils.psql(pg_path, port, "ALTER SYSTEM RESET citus.recover_2pc_interval;") + utils.psql(pg_path, port, "SELECT pg_reload_conf();") + + def run_alter_citus(pg_path, mixed_mode, config): for port in config.node_name_to_ports.values(): if mixed_mode and port in (