Merge pull request #281 from Maxxen/modernize

Simplify build, move to new extension template, fixup CI, apply patches

Author: Mytherin

.github/workflows/Linux.yml

@@ -1,4 +1,4 @@
-name: Linux
+name: Integration Tests
 on: [push, pull_request,repository_dispatch]
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
@@ -11,44 +11,42 @@ jobs:
   linux-tests-postgres:
     name: Run tests on Linux
     runs-on: ubuntu-latest
-    container: ${{ matrix.container }}
     strategy:
       matrix:
         # Add commits/tags to build against other DuckDB versions
         duckdb_version: [ '<submodule_version>' ]
-        arch: ['linux_amd64_gcc4']
+        arch: ['linux_amd64']
         vcpkg_version: [ '2023.04.15' ]
         include:
-          - arch: 'linux_amd64_gcc4'
-            container: 'quay.io/pypa/manylinux2014_x86_64'
+          - arch: 'linux_amd64'
             vcpkg_triplet: 'x64-linux'
 
     env:
       VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }}
       GEN: Ninja
       VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
     steps:
     - name: Install required ubuntu packages
-      if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }}
       run: |
-        apt-get update -y -qq
-        apt-get install -y -qq software-properties-common
-        add-apt-repository ppa:git-core/ppa
-        apt-get update -y -qq
-        apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client
-        apt-get install -y -qq tar pkg-config
+        sudo apt-get update -y -qq
+        sudo apt-get install -y -qq build-essential cmake ninja-build ccache python3
 
-    - name: Install Git 2.18.5
-      if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }}
+    - name: Setup PostgreSQL for Linux/macOS/Windows
+      uses: ikalnytskyi/action-setup-postgres@v7
+      with:
+        postgres-version: '15'
+        username: 'postgres'
+        password: 'postgres'
+        database: 'postgres'
+        port: '5432'
+        ssl: true
+
+    - name: Test Postgres Setup
       run: |
-        wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz
-        tar xvf v2.18.5.tar.gz
-        cd git-2.18.5
-        make
-        make prefix=/usr install
-        git --version
+        psql -c "SELECT 42"
+      env:
+        PGSERVICE: postgres
 
     - uses: actions/checkout@v3
       with:
@@ -61,60 +59,29 @@ jobs:
         cd duckdb
         git checkout ${{ matrix.duckdb_version }}
 
-    - name: Setup ManyLinux2014
-      if: ${{ matrix.arch == 'linux_amd64_gcc4' }}
-      run: |
-        ./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh openssl python_alias
-
-    - name: Setup Ubuntu
-      if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }}
-      uses: ./duckdb/.github/actions/ubuntu_18_setup
+    - name: Setup Ccache
+      uses: hendrikmuhs/ccache-action@main
       with:
-        aarch64_cross_compile: 1
+        key: ${{ github.job }}
+        save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb-postgres' }}
 
     - name: Setup vcpkg
       uses: lukka/[email protected]
       with:
         vcpkgGitCommitId: a42af01b72c28a8e1d7b48107b33e4f286a55ef6
 
-    - name: Install postgres (amd64_gcc4)
-      if: ${{ matrix.arch == 'linux_amd64_gcc4' }}
-      run: |
-        yum install -y sudo
-        yum install -y postgresql-server
-
-    - name: Setup Postgres
-      if: ${{ matrix.arch == 'linux_amd64_gcc4' }}
-      run: |
-        mkdir pgdata
-        chown postgres pgdata
-        sudo -u postgres initdb -D pgdata
-        sudo -u postgres pg_ctl -D pgdata start
-        sleep 5
-        sudo -u postgres createuser root -s
-        createdb
-        psql -c "SELECT 42"
-
     - name: Build extension
       env:
         GEN: ninja
         STATIC_LIBCPP: 1
-        CC: ${{ matrix.arch == 'linux_arm64' && 'aarch64-linux-gnu-gcc' || '' }}
-        CXX: ${{ matrix.arch == 'linux_arm64' && 'aarch64-linux-gnu-g++' || '' }}
       run: |
         make release
 
     - name: Test extension
-      if: ${{ matrix.arch == 'linux_amd64_gcc4' }}
       env:
         POSTGRES_TEST_DATABASE_AVAILABLE: 1
+        PGSERVICE: postgres
       run: |
         psql -c "SELECT 43"
         source ./create-postgres-tables.sh
-        make test_release
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{matrix.arch}}-extensions
-        path: |
-          build/release/extension/postgres_scanner/postgres_scanner.duckdb_extension
+        make test_release

.github/workflows/MainDistributionPipeline.yml

@@ -17,8 +17,9 @@ jobs:
     uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
     with:
       duckdb_version: main
+      ci_tools_version: main
       extension_name: postgres_scanner
-      exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'
+      exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_mingw'
 
   duckdb-stable-deploy:
     name: Deploy extension binaries
@@ -27,6 +28,7 @@ jobs:
     secrets: inherit
     with:
       duckdb_version: main
+      ci_tools_version: main
       extension_name: postgres_scanner
-      exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'
+      exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_mingw'
       deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "duckdb"]
 	path = duckdb
 	url = https://github.com/duckdb/duckdb.git
+[submodule "extension-ci-tools"]
+	path = extension-ci-tools
+	url = https://github.com/duckdb/extension-ci-tools.git

CMakeLists.txt

@@ -114,16 +114,88 @@ add_subdirectory(src)
 
 prepend(LIBPG_SOURCES_FULLPATH ${CMAKE_CURRENT_SOURCE_DIR} ${LIBPG_SOURCES})
 
-if(EXISTS "postgres")
-
-else()
-  execute_process(COMMAND sh pgconfigure ${LIBPG_SOURCES_FULLPATH}
-                  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/postgres)
+
+  # Download the PostgreSQL source code
+  message(STATUS "Downloading PostgreSQL source code")
+  file(DOWNLOAD
+    "https://github.com/postgres/postgres/archive/refs/tags/REL_15_2.tar.gz"
+    ${CMAKE_CURRENT_SOURCE_DIR}/pg.tar.gz
+    SHOW_PROGRESS
+    EXPECTED_MD5 615ef3fa75d19c46d2dafd6c66ddb5f6
+    STATUS PG_DOWNLOAD_RESULT)
+
+  if(NOT PG_DOWNLOAD_RESULT EQUAL 0)
+    file(REMOVE pg.tar.gz)
+    message(FATAL_ERROR "Failed to download PostgreSQL source code")
+  endif()
+
+  # Extract the PostgreSQL source code
+  message(STATUS "Extracting PostgreSQL source code")
+  file(ARCHIVE_EXTRACT INPUT ${CMAKE_CURRENT_SOURCE_DIR}/pg.tar.gz
+       DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/postgres_tmp)
+
+  # Move out of root directory
+  file(RENAME ${CMAKE_CURRENT_SOURCE_DIR}/postgres_tmp/postgres-REL_15_2
+            ${CMAKE_CURRENT_SOURCE_DIR}/postgres
+  )
+
+  # Remove the tmp directory
+  file(REMOVE_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/postgres_tmp)
+
+  # Remove the downloaded tarball
+  file(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/pg.tar.gz)
+
+  # Configure the PostgreSQL source code
+  message(STATUS "Configuring PostgreSQL source code")
+  if(WIN32)
+    # On windows, use the mkvcbuild.pl script to configure the source code
+    execute_process(
+      COMMAND perl mkvcbuild.pl
+      RESULT_VARIABLE PG_MKVCBUILD_RESULT
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/postgres/src/tools/msvc)
+
+    # Check if configuration was successful
+    if(NOT PG_MKVCBUILD_RESULT EQUAL 0)
+        file(REMOVE_RECURSE postgres)
+        message(FATAL_ERROR "Failed to configure PostgreSQL source code for windows")
+    endif()
+  else()
+    # On other platforms, use the configure script to configure the source code
+    set(ENV{CC} gcc)
+    set(ENV{CXX} g++)
+    execute_process(
+      COMMAND ./configure
+        --without-llvm
+        --without-icu
+        --without-tcl
+        --without-perl
+        --without-python
+        --without-gssapi
+        --without-pam
+        --without-bsd-auth
+        --without-ldap
+        --without-bonjour
+        --without-selinux
+        --without-systemd
+        --without-readline
+        --without-libxml
+        --without-libxslt
+        --without-zlib
+        --without-lz4
+        --without-openssl
+      RESULT_VARIABLE PG_CONFIGURE_RESULT
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/postgres
+    )
+    # Check if configuration was successful
+    if(NOT PG_CONFIGURE_RESULT EQUAL 0)
+      file(REMOVE_RECURSE postgres)
+      message(FATAL_ERROR "Failed to configure PostgreSQL source code")
+    endif ()
+  endif()
+  message(STATUS "Finished setting up PostgreSQL source code!")
 endif()
-add_custom_command(
-  OUTPUT ${LIBPG_SOURCES_FULLPATH}
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-  COMMAND sh pgconfigure)
+
 
 set(PARAMETERS "-no-warnings")
 build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${ALL_OBJECT_FILES}

Makefile

@@ -1,86 +1,8 @@
-.PHONY: all clean format debug release duckdb_debug duckdb_release pull update
+PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
-all: release
+# Configuration of extension
+EXT_NAME=postgres_scanner
+EXT_CONFIG=${PROJ_DIR}extension_config.cmake
 
-MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
-PROJ_DIR := $(dir $(MKFILE_PATH))
-
-ifeq ($(OS),Windows_NT)
-	TEST_PATH="/test/Release/unittest.exe"
-else
-	TEST_PATH="/test/unittest"
-endif
-
-#### OSX config
-OSX_BUILD_FLAG=
-ifneq (${OSX_BUILD_ARCH}, "")
-	OSX_BUILD_FLAG=-DOSX_BUILD_ARCH=${OSX_BUILD_ARCH}
-endif
-
-#### VCPKG config
-VCPKG_TOOLCHAIN_PATH?=
-ifneq ("${VCPKG_TOOLCHAIN_PATH}", "")
-	TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_MANIFEST_DIR='${PROJ_DIR}' -DVCPKG_BUILD=1 -DCMAKE_TOOLCHAIN_FILE='${VCPKG_TOOLCHAIN_PATH}'
-endif
-ifneq ("${VCPKG_TARGET_TRIPLET}", "")
-	TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}'
-endif
-
-#### Enable Ninja as generator
-ifeq ($(GEN),ninja)
-	GENERATOR=-G "Ninja" -DFORCE_COLORED_OUTPUT=1
-endif
-
-#### Configuration for this extension
-EXTENSION_NAME=POSTGRES_SCANNER
-EXTENSION_FLAGS=\
--DDUCKDB_EXTENSION_NAMES="postgres_scanner" \
--DDUCKDB_EXTENSION_${EXTENSION_NAME}_PATH="$(PROJ_DIR)" \
--DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=0 \
--DDUCKDB_EXTENSION_${EXTENSION_NAME}_LOAD_TESTS=1 \
--DDUCKDB_EXTENSION_${EXTENSION_NAME}_TEST_PATH="$(PROJ_DIR)test/sql"
-
-EXTRA_EXTENSIONS_FLAG=-DBUILD_EXTENSIONS="json;tpch;tpcds"
-
-BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 $(EXTENSION_FLAGS) ${EXTRA_EXTENSIONS_FLAG} $(OSX_BUILD_FLAG) $(TOOLCHAIN_FLAGS) -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}'
-
-
-pull:
-	git submodule init
-	git submodule update --recursive --remote
-
-clean:
-	rm -rf build
-	rm -rf testext
-	cd duckdb && make clean
-
-# Main build
-debug:
-	mkdir -p  build/debug && \
-	cmake $(GENERATOR) $(BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Debug -S ./duckdb/ -B build/debug && \
-	cmake --build build/debug --config Debug
-
-reldebug:
-	mkdir -p build/reldebug && \
-	cmake $(GENERATOR) $(BUILD_FLAGS) -DCMAKE_BUILD_TYPE=RelWithDebInfo -S ./duckdb/ -B build/reldebug && \
-	cmake --build build/reldebug --config RelWithDebInfo
-
-release:
-	mkdir -p build/release && \
-	cmake $(GENERATOR) $(BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Release -S ./duckdb/ -B build/release && \
-	cmake --build build/release --config Release
-
-test: test_release
-test_release: release
-	./build/release/$(TEST_PATH) --test-dir "$(PROJ_DIR)" "test/*"
-test_debug: debug
-	./build/debug/$(TEST_PATH) --test-dir "$(PROJ_DIR)" "test/*"
-
-format:
-	cp duckdb/.clang-format .
-	find src/ -iname "*.hpp" -o -iname "*.cpp" | xargs clang-format --sort-includes=0 -style=file -i
-	cmake-format -i CMakeLists.txt
-	rm .clang-format
-
-update:
-	git submodule update --remote --merge
+# Include the Makefile from extension-ci-tools
+include extension-ci-tools/makefiles/duckdb_extension.Makefile

duckdb

@@ -0,0 +1,15 @@
+# This file is included by DuckDB's build system. It specifies which extension to load
+
+# Extension from this repo
+duckdb_extension_load(postgres_scanner
+    SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}
+    DONT_LINK
+    LOAD_TESTS
+)
+
+duckdb_extension_load(tpch)
+duckdb_extension_load(tpcds)
+duckdb_extension_load(json)
+
+# Any extra extensions that should be built
+# e.g.: duckdb_extension_load(json)