Merge branch 'main' into contributing-dev

.devcontainer/.psqlrc

@@ -3,5 +3,5 @@
 \pset border 2
 \setenv PAGER 'pspg --no-mouse -bX --no-commandbar --no-topbar'
 \set HISTSIZE 100000
-\set PROMPT1 '\n%[%033[1m%]%M %n@%/:%>-%p%R%[%033[0m%]%# '
+\set PROMPT1 '\n%[%033[1m%]%M %n@%/:%> (PID: %p)%R%[%033[0m%]%# '
 \set PROMPT2 '  '

.devcontainer/.vscode/launch.json

@@ -16,5 +16,25 @@
                 }
             ],
         },
-    ]
+        {
+            "name": "Open core file",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "/home/citus/.pgenv/pgsql/bin/postgres",
+            "coreDumpPath": "${input:corefile}",
+            "cwd": "${workspaceFolder}",
+            "MIMode": "gdb",
+        }
+    ],
+    "inputs": [
+        {
+            "id": "corefile",
+            "type": "command",
+            "command": "extension.commandvariable.file.pickFile",
+            "args": {
+                "dialogTitle": "Select core file",
+                "include": "**/core*",
+            },
+        },
+    ],
 }

.devcontainer/Dockerfile

@@ -68,7 +68,7 @@ USER citus
 
 # build postgres versions separately for effective parrallelism and caching of already built versions when changing only certain versions
 FROM base AS pg14
-RUN MAKEFLAGS="-j $(nproc)" pgenv build 14.10
+RUN MAKEFLAGS="-j $(nproc)" pgenv build 14.15
 RUN rm .pgenv/src/*.tar*
 RUN make -C .pgenv/src/postgresql-*/ clean
 RUN make -C .pgenv/src/postgresql-*/src/include install
@@ -80,7 +80,7 @@ RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/
 RUN rm .pgenv-staging/config/default.conf
 
 FROM base AS pg15
-RUN MAKEFLAGS="-j $(nproc)" pgenv build 15.5
+RUN MAKEFLAGS="-j $(nproc)" pgenv build 15.10
 RUN rm .pgenv/src/*.tar*
 RUN make -C .pgenv/src/postgresql-*/ clean
 RUN make -C .pgenv/src/postgresql-*/src/include install
@@ -92,7 +92,7 @@ RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/
 RUN rm .pgenv-staging/config/default.conf
 
 FROM base AS pg16
-RUN MAKEFLAGS="-j $(nproc)" pgenv build 16.1
+RUN MAKEFLAGS="-j $(nproc)" pgenv build 16.6
 RUN rm .pgenv/src/*.tar*
 RUN make -C .pgenv/src/postgresql-*/ clean
 RUN make -C .pgenv/src/postgresql-*/src/include install
@@ -152,6 +152,7 @@ RUN sudo apt update \
     lsof \
     man \
     net-tools \
+    psmisc \
     pspg \
     tree \
     vim \
@@ -210,7 +211,7 @@ COPY --chown=citus:citus .psqlrc .
 RUN sudo chown --from=root:root citus:citus -R ~
 
 # sets default pg version
-RUN pgenv switch 16.1
+RUN pgenv switch 16.6
 
 # make connecting to the coordinator easy
 ENV PGPORT=9700

.devcontainer/devcontainer.json

@@ -2,8 +2,11 @@
     "image": "ghcr.io/citusdata/citus-devcontainer:main",
     "runArgs": [
         "--cap-add=SYS_PTRACE",
+        "--ulimit=core=-1",
+    ],
+    "forwardPorts": [
+        9700
     ],
-    "forwardPorts": [9700],
     "customizations": {
         "vscode": {
             "extensions": [
@@ -14,6 +17,7 @@
                 "github.vscode-pull-request-github",
                 "ms-vscode.cpptools-extension-pack",
                 "ms-vsliveshare.vsliveshare",
+                "rioj7.command-variable",
             ],
             "settings": {
                 "files.exclude": {
@@ -30,3 +34,4 @@
     "updateContentCommand": "./configure",
     "postCreateCommand": "make -C .devcontainer/",
 }
+

.devcontainer/src/test/regress/Pipfile

@@ -5,7 +5,7 @@ verify_ssl = true
 
 [packages]
 mitmproxy = {editable = true, ref = "main", git = "https://github.com/citusdata/mitmproxy.git"}
-construct = "==2.9.45"
+construct = "*"
 docopt = "==0.6.2"
 cryptography = ">=41.0.4"
 pytest = "*"
@@ -16,6 +16,7 @@ pytest-timeout = "*"
 pytest-xdist = "*"
 pytest-repeat = "*"
 pyyaml = "*"
+werkzeug = "==2.3.7"
 
 [dev-packages]
 black = "*"

.github/workflows/build_and_test.yml

@@ -24,27 +24,28 @@ jobs:
     runs-on: ubuntu-latest
     name: Initialize parameters
     outputs:
-      build_image_name: "citus/extbuilder"
-      test_image_name: "citus/exttester"
-      citusupgrade_image_name: "citus/citusupgradetester"
-      fail_test_image_name: "citus/failtester"
-      pgupgrade_image_name: "citus/pgupgradetester"
-      style_checker_image_name: "citus/stylechecker"
+      build_image_name: "ghcr.io/citusdata/extbuilder"
+      test_image_name: "ghcr.io/citusdata/exttester"
+      citusupgrade_image_name: "ghcr.io/citusdata/citusupgradetester"
+      fail_test_image_name: "ghcr.io/citusdata/failtester"
+      pgupgrade_image_name: "ghcr.io/citusdata/pgupgradetester"
+      style_checker_image_name: "ghcr.io/citusdata/stylechecker"
       style_checker_tools_version: "0.8.18"
-      image_suffix: "-v19b671f"
-      pg14_version: '{ "major": "14", "full": "14.10" }'
-      pg15_version: '{ "major": "15", "full": "15.5" }'
-      pg16_version: '{ "major": "16", "full": "16.1" }'
-      upgrade_pg_versions: "14.10-15.5-16.1"
+      sql_snapshot_pg_version: "16.6"
+      image_suffix: "-v5779674"
+      pg14_version: '{ "major": "14", "full": "14.15" }'
+      pg15_version: '{ "major": "15", "full": "15.10" }'
+      pg16_version: '{ "major": "16", "full": "16.6" }'
+      upgrade_pg_versions: "14.15-15.10-16.6"
     steps:
-      # Since GHA jobs needs at least one step we use a noop step here.
+      # Since GHA jobs need at least one step we use a noop step here.
       - name: Set up parameters
         run: echo 'noop'
   check-sql-snapshots:
     needs: params
     runs-on: ubuntu-20.04
     container:
-      image: ${{ needs.params.outputs.build_image_name }}:latest
+      image: ${{ needs.params.outputs.build_image_name }}:${{ needs.params.outputs.sql_snapshot_pg_version }}${{ needs.params.outputs.image_suffix }}
       options: --user root
     steps:
     - uses: actions/[email protected]
@@ -61,7 +62,7 @@ jobs:
     - name: Check Snapshots
       run: |
         git config --global --add safe.directory ${GITHUB_WORKSPACE}
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Check C Style
@@ -117,7 +118,7 @@ jobs:
       image: "${{ matrix.image_name }}:${{ fromJson(matrix.pg_version).full }}${{ matrix.image_suffix }}"
       options: --user root
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - name: Expose $PG_MAJOR to Github Env
       run: echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV
       shell: bash
@@ -227,7 +228,7 @@ jobs:
     - params
     - build
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - uses: "./.github/actions/setup_extension"
     - name: Run Test
       run: gosu circleci make -C src/test/${{ matrix.suite }} ${{ matrix.make }}
@@ -261,7 +262,7 @@ jobs:
           - ${{ needs.params.outputs.pg16_version }}
         parallel: [0,1,2,3,4,5] # workaround for running 6 parallel jobs
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - uses: "./.github/actions/setup_extension"
     - name: Test arbitrary configs
       run: |-
@@ -311,7 +312,7 @@ jobs:
       old_pg_major: ${{ matrix.old_pg_major }}
       new_pg_major: ${{ matrix.new_pg_major }}
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - uses: "./.github/actions/setup_extension"
       with:
         pg_major: "${{ env.old_pg_major }}"
@@ -349,7 +350,7 @@ jobs:
     - params
     - build
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - uses: "./.github/actions/setup_extension"
       with:
         skip_installation: true
@@ -413,7 +414,7 @@ jobs:
     needs:
     - build
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - uses: azure/login@v1
       with:
         creds: ${{ secrets.AZURE_CREDENTIALS }}
@@ -431,7 +432,7 @@ jobs:
     needs:
     - build
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - uses: azure/login@v1
       with:
         creds: ${{ secrets.AZURE_CREDENTIALS }}
@@ -450,7 +451,7 @@ jobs:
     outputs:
       json: ${{ steps.parallelization.outputs.json }}
     steps:
-      - uses: actions/checkout@v3.5.0
+      - uses: actions/checkout@v4
       - uses: "./.github/actions/parallelization"
         id: parallelization
         with:
@@ -463,7 +464,7 @@ jobs:
     outputs:
       tests: ${{ steps.detect-regression-tests.outputs.tests }}
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Detect regression tests need to be ran
@@ -514,7 +515,7 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJson(needs.prepare_parallelization_matrix_32.outputs.json) }}
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - uses: "./.github/actions/setup_extension"
     - name: Run minimal tests
       run: |-

.github/workflows/codeql.yml

@@ -21,7 +21,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2

.github/workflows/flaky_test_debugging.yml

@@ -28,7 +28,7 @@ jobs:
       image: ${{ vars.build_image_name }}:${{ vars.pg15_version  }}${{ vars.image_suffix }}
       options: --user root
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - name: Configure, Build, and Install
       run: |
         echo "PG_MAJOR=${PG_MAJOR}" >> $GITHUB_ENV
@@ -46,7 +46,7 @@ jobs:
     outputs:
       json: ${{ steps.parallelization.outputs.json }}
     steps:
-      - uses: actions/checkout@v3.5.0
+      - uses: actions/checkout@v4
       - uses: "./.github/actions/parallelization"
         id: parallelization
         with:
@@ -67,7 +67,7 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJson(needs.prepare_parallelization_matrix.outputs.json) }}
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
     - uses: "./.github/actions/setup_extension"
     - name: Run minimal tests
       run: |-

.github/workflows/packaging-test-pipelines.yml

@@ -19,7 +19,7 @@ jobs:
       pg_versions: ${{ steps.get-postgres-versions.outputs.pg_versions }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 2
       - name: Get Postgres Versions
@@ -51,26 +51,14 @@ jobs:
           - almalinux-8
           - almalinux-9
         POSTGRES_VERSION: ${{ fromJson(needs.get_postgres_versions_from_file.outputs.pg_versions) }}
-        # Postgres removed support for CentOS 7 in PG 16. Below block is needed to
-        # keep the build for CentOS 7 working for PG 14 and PG 15.
-        # Once dependent systems drop support for Centos 7, we can remove this block.
-        include:
-          - packaging_docker_image: centos-7
-            POSTGRES_VERSION: 14
-          - packaging_docker_image: centos-7
-            POSTGRES_VERSION: 15
-          - packaging_docker_image: oraclelinux-7
-            POSTGRES_VERSION: 14
-          - packaging_docker_image: oraclelinux-7
-            POSTGRES_VERSION: 15
 
     container:
       image: citus/packaging:${{ matrix.packaging_docker_image }}-pg${{ matrix.POSTGRES_VERSION }}
       options: --user root
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set Postgres and python parameters for rpm based distros
         run: |
@@ -128,7 +116,6 @@ jobs:
         # for each deb based image and we use POSTGRES_VERSION to set
         # PG_CONFIG variable in each of those runs.
         packaging_docker_image:
-          - debian-buster-all
           - debian-bookworm-all
           - debian-bullseye-all
           - ubuntu-focal-all