Merge remote-tracking branch 'upstream/main' into add_view_entry

Author: Tishj

README.md

@@ -8,7 +8,6 @@ To make a Postgres database accessible to DuckDB use the `ATTACH` command:
 
 ```sql
 ATTACH 'dbname=postgresscanner' AS postgres_db (TYPE postgres);
-USE postgres_db;
 ```
 
 The `ATTACH` command takes as input a [`libpq` connection string](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING) - which is a set of `key=value` pairs separated by spaces. Below are some example connection strings and commonly used parameters. A full list of available parameters can be found [in the Postgres documentation](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS).
@@ -32,14 +31,14 @@ host=localhost port=5432 dbname=mydb connect_timeout=10
 The tables in the file can be read as if they were normal DuckDB tables, but the underlying data is read directly from Postgres at query time.
 
 ```sql
-D SHOW TABLES;
+D SHOW ALL TABLES;
 ┌───────────────────────────────────────┐
 │                 name                  │
 │                varchar                │
 ├───────────────────────────────────────┤
 │ uuids                                 │
 └───────────────────────────────────────┘
-D SELECT * FROM uuids;
+D SELECT * FROM postgres_db.uuids;
 ┌──────────────────────────────────────┐
 │                  u                   │
 │                 uuid                 │
@@ -51,170 +50,7 @@ D SELECT * FROM uuids;
 └──────────────────────────────────────┘
 ```
 
-It might be desirable to create a copy of the Postgres databases in DuckDB to prevent the system from re-reading the tables from Postgres continuously, particularly for large tables.
-
-Data can be copied over from Postgres to DuckDB using standard SQL, for example:
-
-```sql
-CREATE TABLE duckdb_table AS FROM postgres_db.postgres_tbl;
-```
-
-## Writing Data to Postgres
-
-In addition to reading data from Postgres, the extension allows you to create tables, ingest data into Postgres and make other modifications to a Postgres database using standard SQL queries.
-
-This allows you to use DuckDB to, for example, export data that is stored in a Postgres database to Parquet, or read data from a Parquet file into Postgres.
-
-Below is a brief example of how to create a new table in Postgres and load data into it.
-
-```sql
-ATTACH 'dbname=postgresscanner' AS postgres_db (TYPE postgres);
-CREATE TABLE postgres_db.tbl(id INTEGER, name VARCHAR);
-INSERT INTO postgres_db.tbl VALUES (42, 'DuckDB');
-```
-Many operations on Postgres tables are supported. All these operations directly modify the Postgres database, and the result of subsequent operations can then be read using Postgres.
-Note that if modifications are not desired, `ATTACH` can be run with the `READ_ONLY` property which prevents making modifications to the underlying database. For example:
-
-```sql
-ATTACH 'dbname=postgresscanner' AS postgres_db (TYPE postgres, READ_ONLY);
-```
-
-Below is a list of supported operations.
-
-###### CREATE TABLE
-```sql
-CREATE TABLE postgres_db.tbl(id INTEGER, name VARCHAR);
-```
-
-###### INSERT INTO
-```sql
-INSERT INTO postgres_db.tbl VALUES (42, 'DuckDB');
-```
-
-###### SELECT
-```sql
-SELECT * FROM postgres_db.tbl;
-┌───────┬─────────┐
-│  id   │  name   │
-│ int64 │ varchar │
-├───────┼─────────┤
-│    42 │ DuckDB  │
-└───────┴─────────┘
-```
-
-###### COPY
-```sql
-COPY postgres_db.tbl TO 'data.parquet';
-COPY postgres_db.tbl FROM 'data.parquet';
-```
-
-###### UPDATE
-```sql
-UPDATE postgres_db.tbl SET name='Woohoo' WHERE id=42;
-```
-
-###### DELETE
-```sql
-DELETE FROM postgres_db.tbl WHERE id=42;
-```
-
-###### ALTER TABLE
-```sql
-ALTER TABLE postgres_db.tbl ADD COLUMN k INTEGER;
-```
-
-###### DROP TABLE
-```sql
-DROP TABLE postgres_db.tbl;
-```
-
-###### CREATE VIEW
-```sql
-CREATE VIEW postgres_db.v1 AS SELECT 42;
-```
-
-###### CREATE SCHEMA/DROP SCHEMA
-```sql
-CREATE SCHEMA postgres_db.s1;
-CREATE TABLE postgres_db.s1.integers(i int);
-INSERT INTO postgres_db.s1.integers VALUES (42);
-SELECT * FROM postgres_db.s1.integers;
-┌───────┐
-│   i   │
-│ int32 │
-├───────┤
-│    42 │
-└───────┘
-DROP SCHEMA postgres_db.s1;
-```
-
-###### Transactions
-```sql
-CREATE TABLE postgres_db.tmp(i INTEGER);
-BEGIN;
-INSERT INTO postgres_db.tmp VALUES (42);
-SELECT * FROM postgres_db.tmp;
-┌───────┐
-│   i   │
-│ int64 │
-├───────┤
-│    42 │
-└───────┘
-ROLLBACK;
-SELECT * FROM postgres_db.tmp;
-┌────────┐
-│   i    │
-│ int64  │
-├────────┤
-│ 0 rows │
-└────────┘
-```
-
-## Running SQL Queries in Postgres with postgres_query
-
-The postgres_query function allows you to run arbitrary SQL within an attached database. `postgres_query` takes the name of the attached Postgres database to execute the query in, as well as the SQL query to execute. The result of the query is returned.
-
-```sql
-postgres_query(attached_database::VARCHAR, query::VARCHAR)
-```
-
-###### Example
-
-```sql
-ATTACH 'dbname=postgresscanner' AS s (TYPE POSTGRES);
-SELECT * FROM postgres_query('s', 'SELECT * FROM cars LIMIT 3');
-┌──────────────┬───────────┬─────────┐
-│    brand     │   model   │  color  │
-│   varchar    │  varchar  │ varchar │
-├──────────────┼───────────┼─────────┤
-│ ferari       │ testarosa │ red     │
-│ aston martin │ db2       │ blue    │
-│ bentley      │ mulsanne  │ gray    │
-└──────────────┴───────────┴─────────┘
-```
-
-## Settings
-
-The extension exposes the following configuration parameters.
-
-|              name               |                                description                                 | default |
-|---------------------------------|----------------------------------------------------------------------------|---------|
-| pg_debug_show_queries           | DEBUG SETTING: print all queries sent to Postgres to stdout                | false   |
-| pg_experimental_filter_pushdown | Whether or not to use filter pushdown (currently experimental)             | false   |
-| pg_array_as_varchar             | Read Postgres arrays as varchar - enables reading mixed dimensional arrays | false   |
-| pg_connection_cache             | Whether or not to use the connection cache                                 | true    |
-| pg_connection_limit             | The maximum amount of concurrent Postgres connections                      | 64      |
-| pg_pages_per_task               | The amount of pages per task                                               | 1000    |
-| pg_use_binary_copy              | Whether or not to use BINARY copy to read data                             | true    |
-
-
-## Schema Cache
-
-To avoid having to continuously fetch schema data from Postgres, DuckDB keeps schema information - such as the names of tables, their columns, etc -  cached. If changes are made to the schema through a different connection to the Postgres instance, such as new columns being added to a table, the cached schema information might be outdated. In this case, the function `pg_clear_cache` can be executed to clear the internal caches.
-
-```sql
-CALL pg_clear_cache();
-```
+For more information on how to use the connector, refer to the [Postgres documentation on the website](https://duckdb.org/docs/extensions/postgres).
 
 ## Building & Loading the Extension
 

src/include/postgres_scanner.hpp

@@ -83,6 +83,7 @@ class PostgresClearCacheFunction : public TableFunction {
 	PostgresClearCacheFunction();
 
 	static void ClearCacheOnSetting(ClientContext &context, SetScope scope, Value &parameter);
+	static void ClearPostgresCaches(ClientContext &context);
 };
 
 class PostgresQueryFunction : public TableFunction {

src/include/storage/postgres_catalog.hpp

@@ -72,6 +72,20 @@ class PostgresCatalog : public Catalog {
 
 	void ClearCache();
 
+	//! Whether or not this catalog should search a specific type with the standard priority
+	CatalogLookupBehavior CatalogTypeLookupRule(CatalogType type) const override {
+		switch (type) {
+		case CatalogType::INDEX_ENTRY:
+		case CatalogType::TABLE_ENTRY:
+		case CatalogType::TYPE_ENTRY:
+		case CatalogType::VIEW_ENTRY:
+			return CatalogLookupBehavior::STANDARD;
+		default:
+			// unsupported type (e.g. scalar functions, aggregates, ...)
+			return CatalogLookupBehavior::NEVER_LOOKUP;
+		}
+	}
+
 private:
 	void DropSchema(ClientContext &context, DropInfo &info) override;
 

src/postgres_extension.cpp

@@ -12,9 +12,44 @@
 #include "duckdb/main/attached_database.hpp"
 #include "storage/postgres_catalog.hpp"
 #include "storage/postgres_optimizer.hpp"
+#include "duckdb/planner/extension_callback.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/main/client_context_state.hpp"
+#include "duckdb/main/connection_manager.hpp"
+#include "duckdb/common/error_data.hpp"
 
 using namespace duckdb;
 
+class PostgresExtensionState : public ClientContextState {
+public:
+	bool CanRequestRebind() override {
+		return true;
+	}
+	RebindQueryInfo OnPlanningError(ClientContext &context, SQLStatement &statement, ErrorData &error) override {
+		if (error.Type() != ExceptionType::BINDER) {
+			return RebindQueryInfo::DO_NOT_REBIND;
+		}
+		auto &extra_info = error.ExtraInfo();
+		auto entry = extra_info.find("error_subtype");
+		if (entry == extra_info.end()) {
+			return RebindQueryInfo::DO_NOT_REBIND;
+		}
+		if (entry->second != "COLUMN_NOT_FOUND") {
+			return RebindQueryInfo::DO_NOT_REBIND;
+		}
+		// clear caches and rebind
+		PostgresClearCacheFunction::ClearPostgresCaches(context);
+		return RebindQueryInfo::ATTEMPT_TO_REBIND;
+	}
+};
+
+class PostgresExtensionCallback : public ExtensionCallback {
+public:
+	void OnConnectionOpened(ClientContext &context) override {
+		context.registered_state.insert(make_pair("postgres_extension", make_shared<PostgresExtensionState>()));
+	}
+};
+
 static void SetPostgresConnectionLimit(ClientContext &context, SetScope scope, Value &parameter) {
 	if (scope == SetScope::LOCAL) {
 		throw InvalidInputException("pg_connection_limit can only be set globally");
@@ -76,6 +111,11 @@ static void LoadInternal(DatabaseInstance &db) {
 	OptimizerExtension postgres_optimizer;
 	postgres_optimizer.optimize_function = PostgresOptimizer::Optimize;
 	config.optimizer_extensions.push_back(std::move(postgres_optimizer));
+
+	config.extension_callbacks.push_back(make_uniq<PostgresExtensionCallback>());
+	for(auto &connection : ConnectionManager::Get(db).GetConnectionList()) {
+		connection->registered_state.insert(make_pair("postgres_extension", make_shared<PostgresExtensionState>()));
+	}
 }
 
 void PostgresScannerExtension::Load(DuckDB &db) {

src/storage/postgres_clear_cache.cpp

@@ -21,7 +21,7 @@ static unique_ptr<FunctionData> ClearCacheBind(ClientContext &context, TableFunc
 	return std::move(result);
 }
 
-static void ClearPostgresCaches(ClientContext &context) {
+void PostgresClearCacheFunction::ClearPostgresCaches(ClientContext &context) {
 	auto databases = DatabaseManager::Get(context).GetDatabases(context);
 	for (auto &db_ref : databases) {
 		auto &db = db_ref.get();
@@ -38,12 +38,12 @@ static void ClearCacheFunction(ClientContext &context, TableFunctionInput &data_
 	if (data.finished) {
 		return;
 	}
-	ClearPostgresCaches(context);
+	PostgresClearCacheFunction::ClearPostgresCaches(context);
 	data.finished = true;
 }
 
 void PostgresClearCacheFunction::ClearCacheOnSetting(ClientContext &context, SetScope scope, Value &parameter) {
-	ClearPostgresCaches(context);
+	PostgresClearCacheFunction::ClearPostgresCaches(context);
 }
 
 PostgresClearCacheFunction::PostgresClearCacheFunction()

test/sql/misc/autoload.test

@@ -0,0 +1,8 @@
+# name: test/sql/misc/autoloading.test
+# description: Test autoloading based on setting
+# group: [misc]
+
+require postgres_scanner
+
+statement ok
+SET pg_connection_cache=false