Merge branch 'main' into mainschema

Author: Mytherin

.github/workflows/Linux.yml

@@ -75,7 +75,7 @@ jobs:
         GEN: ninja
         STATIC_LIBCPP: 1
       run: |
-        make release
+        make reldebug
 
     - name: Test extension
       env:
@@ -84,4 +84,4 @@ jobs:
       run: |
         psql -c "SELECT 43"
         source ./create-postgres-tables.sh
-        make test_release
+        make test_reldebug

.github/workflows/MainDistributionPipeline.yml

@@ -14,7 +14,7 @@ concurrency:
 jobs:
   duckdb-stable-build:
     name: Build extension binaries
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.2.2
     with:
       duckdb_version: main
       ci_tools_version: main
@@ -24,7 +24,7 @@ jobs:
   duckdb-stable-deploy:
     name: Deploy extension binaries
     needs: duckdb-stable-build
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.2.2
     secrets: inherit
     with:
       duckdb_version: main

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8.12...3.29)
 set(TARGET_NAME postgres_scanner)
 project(${TARGET_NAME})
 
@@ -118,7 +118,8 @@ if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/postgres)
 
   # Download the PostgreSQL source code
   message(STATUS "Downloading PostgreSQL source code")
-  file(DOWNLOAD
+  file(
+    DOWNLOAD
     "https://github.com/postgres/postgres/archive/refs/tags/REL_15_2.tar.gz"
     ${CMAKE_CURRENT_SOURCE_DIR}/pg.tar.gz
     SHOW_PROGRESS
@@ -132,13 +133,12 @@ if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/postgres)
 
   # Extract the PostgreSQL source code
   message(STATUS "Extracting PostgreSQL source code")
-  file(ARCHIVE_EXTRACT INPUT ${CMAKE_CURRENT_SOURCE_DIR}/pg.tar.gz
-       DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/postgres_tmp)
+  file(ARCHIVE_EXTRACT INPUT ${CMAKE_CURRENT_SOURCE_DIR}/pg.tar.gz DESTINATION
+       ${CMAKE_CURRENT_SOURCE_DIR}/postgres_tmp)
 
   # Move out of root directory
   file(RENAME ${CMAKE_CURRENT_SOURCE_DIR}/postgres_tmp/postgres-REL_15_2
-            ${CMAKE_CURRENT_SOURCE_DIR}/postgres
-  )
+       ${CMAKE_CURRENT_SOURCE_DIR}/postgres)
 
   # Remove the tmp directory
   file(REMOVE_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/postgres_tmp)
@@ -157,46 +157,32 @@ if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/postgres)
 
     # Check if configuration was successful
     if(NOT PG_MKVCBUILD_RESULT EQUAL 0)
-        file(REMOVE_RECURSE postgres)
-        message(FATAL_ERROR "Failed to configure PostgreSQL source code for windows")
+      file(REMOVE_RECURSE postgres)
+      message(
+        FATAL_ERROR "Failed to configure PostgreSQL source code for windows")
     endif()
   else()
     # On other platforms, use the configure script to configure the source code
     set(ENV{CC} gcc)
     set(ENV{CXX} g++)
     execute_process(
-      COMMAND ./configure
-        --without-llvm
-        --without-icu
-        --without-tcl
-        --without-perl
-        --without-python
-        --without-gssapi
-        --without-pam
-        --without-bsd-auth
-        --without-ldap
-        --without-bonjour
-        --without-selinux
-        --without-systemd
-        --without-readline
-        --without-libxml
-        --without-libxslt
-        --without-zlib
-        --without-lz4
-        --without-openssl
+      COMMAND
+        ./configure --without-llvm --without-icu --without-tcl --without-perl
+        --without-python --without-gssapi --without-pam --without-bsd-auth
+        --without-ldap --without-bonjour --without-selinux --without-systemd
+        --without-readline --without-libxml --without-libxslt --without-zlib
+        --without-lz4 --without-openssl
       RESULT_VARIABLE PG_CONFIGURE_RESULT
-      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/postgres
-    )
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/postgres)
     # Check if configuration was successful
     if(NOT PG_CONFIGURE_RESULT EQUAL 0)
       file(REMOVE_RECURSE postgres)
       message(FATAL_ERROR "Failed to configure PostgreSQL source code")
-    endif ()
+    endif()
   endif()
   message(STATUS "Finished setting up PostgreSQL source code!")
 endif()
 
-
 set(PARAMETERS "-no-warnings")
 build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${ALL_OBJECT_FILES}
                          ${LIBPG_SOURCES_FULLPATH})

create-postgres-tables.sh

@@ -1,12 +1,23 @@
 #!/bin/bash
+set -e
+
+DUCKDB_PATH=duckdb
+if test -f build/release/duckdb; then
+  DUCKDB_PATH=build/release/duckdb
+elif test -f build/reldebug/duckdb; then
+  DUCKDB_PATH=build/reldebug/duckdb
+elif test -f build/debug/duckdb; then
+  DUCKDB_PATH=build/debug/duckdb
+fi
+
 echo "
 CREATE SCHEMA tpch; 
 CREATE SCHEMA tpcds;
 CALL dbgen(sf=0.01, schema='tpch');
 CALL dsdgen(sf=0.01, schema='tpcds');
 EXPORT DATABASE '/tmp/postgresscannertmp';
 " | \
-./build/release/duckdb
+$DUCKDB_PATH
 
 dropdb --if-exists postgresscanner
 createdb postgresscanner

duckdb

@@ -20,6 +20,9 @@ struct PostgresBinaryReader {
 	~PostgresBinaryReader() {
 		Reset();
 	}
+	PostgresConnection &GetConn() {
+		return con;
+	}
 
 	bool Next() {
 		Reset();
@@ -29,6 +32,10 @@ struct PostgresBinaryReader {
 
 		// len -1 signals end
 		if (len == -1) {
+			auto final_result = PQgetResult(con.GetConn());
+			if (!final_result || PQresultStatus(final_result) != PGRES_COMMAND_OK) {
+				throw IOException("Failed to fetch header for COPY: %s", string(PQresultErrorMessage(final_result)));
+			}
 			return false;
 		}
 

extension-ci-tools

@@ -45,9 +45,13 @@ struct PostgresBindData : public FunctionData {
 	void SetTablePages(idx_t approx_num_pages);
 
 	void SetCatalog(PostgresCatalog &catalog);
+	void SetTable(PostgresTableEntry &table);
 	optional_ptr<PostgresCatalog> GetCatalog() const {
 		return pg_catalog;
 	}
+	optional_ptr<PostgresTableEntry> GetTable() const {
+		return pg_table;
+	}
 
 	unique_ptr<FunctionData> Copy() const override {
 		throw NotImplementedException("");
@@ -58,6 +62,7 @@ struct PostgresBindData : public FunctionData {
 
 private:
 	optional_ptr<PostgresCatalog> pg_catalog;
+	optional_ptr<PostgresTableEntry> pg_table;
 };
 
 class PostgresAttachFunction : public TableFunction {

src/include/postgres_binary_reader.hpp

@@ -68,6 +68,7 @@ class PostgresUtils {
 	static bool SupportedPostgresOid(const LogicalType &input);
 	static LogicalType RemoveAlias(const LogicalType &type);
 	static PostgresType CreateEmptyPostgresType(const LogicalType &type);
+	static string QuotePostgresIdentifier(const string &text);
 
 	static PostgresVersion ExtractPostgresVersion(const string &version);
 };

src/include/postgres_scanner.hpp

@@ -20,10 +20,12 @@ class PostgresSchemaEntry;
 
 class PostgresCatalog : public Catalog {
 public:
-	explicit PostgresCatalog(AttachedDatabase &db_p, const string &path, AccessMode access_mode, string schema_to_load);
+	explicit PostgresCatalog(AttachedDatabase &db_p, string connection_string, string attach_path,
+	                         AccessMode access_mode, string schema_to_load);
 	~PostgresCatalog();
 
-	string path;
+	string connection_string;
+	string attach_path;
 	AccessMode access_mode;
 
 public:
@@ -35,22 +37,24 @@ class PostgresCatalog : public Catalog {
 		return default_schema.empty() ? "public" : default_schema;
 	}
 
+	static string GetConnectionString(ClientContext &context, const string &attach_path, string secret_name);
+
 	optional_ptr<CatalogEntry> CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) override;
 
 	void ScanSchemas(ClientContext &context, std::function<void(SchemaCatalogEntry &)> callback) override;
 
-	optional_ptr<SchemaCatalogEntry> GetSchema(CatalogTransaction transaction, const string &schema_name,
-	                                           OnEntryNotFound if_not_found,
-	                                           QueryErrorContext error_context = QueryErrorContext()) override;
-
-	unique_ptr<PhysicalOperator> PlanInsert(ClientContext &context, LogicalInsert &op,
-	                                        unique_ptr<PhysicalOperator> plan) override;
-	unique_ptr<PhysicalOperator> PlanCreateTableAs(ClientContext &context, LogicalCreateTable &op,
-	                                               unique_ptr<PhysicalOperator> plan) override;
-	unique_ptr<PhysicalOperator> PlanDelete(ClientContext &context, LogicalDelete &op,
-	                                        unique_ptr<PhysicalOperator> plan) override;
-	unique_ptr<PhysicalOperator> PlanUpdate(ClientContext &context, LogicalUpdate &op,
-	                                        unique_ptr<PhysicalOperator> plan) override;
+	optional_ptr<SchemaCatalogEntry> LookupSchema(CatalogTransaction transaction, const EntryLookupInfo &schema_lookup,
+	                                              OnEntryNotFound if_not_found) override;
+
+	PhysicalOperator &PlanCreateTableAs(ClientContext &context, PhysicalPlanGenerator &planner, LogicalCreateTable &op,
+	                                    PhysicalOperator &plan) override;
+	PhysicalOperator &PlanInsert(ClientContext &context, PhysicalPlanGenerator &planner, LogicalInsert &op,
+	                             optional_ptr<PhysicalOperator> plan) override;
+	PhysicalOperator &PlanDelete(ClientContext &context, PhysicalPlanGenerator &planner, LogicalDelete &op,
+	                             PhysicalOperator &plan) override;
+	PhysicalOperator &PlanUpdate(ClientContext &context, PhysicalPlanGenerator &planner, LogicalUpdate &op,
+	                             PhysicalOperator &plan) override;
+
 	unique_ptr<LogicalOperator> BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table,
 	                                            unique_ptr<LogicalOperator> plan) override;