Merge pull request #261 from duckdb/copyfromdatabase

Add copy from database test - plus throw an error with unsupported CAST_FROM_VARCHAR composite column types

Author: Mytherin

.github/workflows/Linux.yml

@@ -113,7 +113,7 @@ jobs:
         source ./create-postgres-tables.sh
         make test_release
 
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v3
       with:
         name: ${{matrix.arch}}-extensions
         path: |

Makefile

@@ -72,9 +72,9 @@ release:
 
 test: test_release
 test_release: release
-	./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*"
+	./build/release/$(TEST_PATH) --test-dir "$(PROJ_DIR)" "test/*"
 test_debug: debug
-	./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*"
+	./build/debug/$(TEST_PATH) --test-dir "$(PROJ_DIR)" "test/*"
 
 format:
 	cp duckdb/.clang-format .

duckdb

@@ -177,6 +177,18 @@ static unique_ptr<FunctionData> PostgresBind(ClientContext &context, TableFuncti
 	return std::move(bind_data);
 }
 
+static bool ContainsCastToVarchar(const PostgresType &type) {
+	if (type.info == PostgresTypeAnnotation::CAST_TO_VARCHAR) {
+		return true;
+	}
+	for (auto &child : type.children) {
+		if (ContainsCastToVarchar(child)) {
+			return true;
+		}
+	}
+	return false;
+}
+
 static void PostgresInitInternal(ClientContext &context, const PostgresBindData *bind_data_p,
                                  PostgresLocalState &lstate, idx_t task_min, idx_t task_max) {
 	D_ASSERT(bind_data_p);
@@ -200,14 +212,20 @@ static void PostgresInitInternal(ClientContext &context, const PostgresBindData
 			col_names += KeywordHelper::WriteQuoted(bind_data->names[column_id], '"');
 			if (bind_data->postgres_types[column_id].info == PostgresTypeAnnotation::CAST_TO_VARCHAR) {
 				col_names += "::VARCHAR";
-			}
-			if (bind_data->types[column_id].id() == LogicalTypeId::LIST) {
+			} else if (bind_data->types[column_id].id() == LogicalTypeId::LIST) {
 				if (bind_data->postgres_types[column_id].info != PostgresTypeAnnotation::STANDARD) {
 					continue;
 				}
 				if (bind_data->postgres_types[column_id].children[0].info == PostgresTypeAnnotation::CAST_TO_VARCHAR) {
 					col_names += "::VARCHAR[]";
 				}
+			} else {
+				if (ContainsCastToVarchar(bind_data->postgres_types[column_id])) {
+					throw NotImplementedException("Error reading table \"%s\" - cast to varchar not implemented for "
+					                              "composite column \"%s\" (type %s)",
+					                              bind_data->table_name, bind_data->names[column_id],
+					                              bind_data->types[column_id].ToString());
+				}
 			}
 		}
 	}

src/postgres_scanner.cpp

@@ -253,7 +253,7 @@ LogicalType PostgresUtils::ToPostgresType(const LogicalType &input) {
 	case LogicalTypeId::UINTEGER:
 		return LogicalType::BIGINT;
 	case LogicalTypeId::UBIGINT:
-          return LogicalType::DECIMAL(20, 0);
+		return LogicalType::DECIMAL(20, 0);
 	case LogicalTypeId::HUGEINT:
 		return LogicalType::DOUBLE;
 	default:

src/postgres_utils.cpp

@@ -0,0 +1,65 @@
+# name: test/sql/storage/attach_copy_from_database.test
+# description: Test copy from database
+# group: [storage]
+
+require postgres_scanner
+
+require-env POSTGRES_TEST_DATABASE_AVAILABLE
+
+statement ok
+PRAGMA enable_verification
+
+statement ok
+ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES)
+
+statement ok
+DROP SCHEMA IF EXISTS s1.copy_schema CASCADE
+
+statement ok
+CREATE SCHEMA s1.copy_schema
+
+statement ok
+USE s1.copy_schema
+
+foreach table_name pg_numtypes pg_bytetypes pg_datetypes
+
+statement ok
+CREATE TABLE ${table_name} AS FROM public.${table_name}
+
+endloop
+
+statement ok
+USE memory
+
+statement ok
+DETACH s1
+
+statement ok
+ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES, SCHEMA 'copy_schema')
+
+statement ok
+USE s1.copy_schema
+
+statement ok
+create table big_tbl as from range(100000) t(id)
+
+statement ok
+create index i_index on big_tbl(id)
+
+statement ok
+create view my_view as select min(id) from copy_schema.big_tbl
+
+statement ok
+ATTACH '__TEST_DIR__/copy_database.db' AS new_db;
+
+statement ok
+COPY FROM DATABASE s1 TO new_db
+
+foreach table_name pg_numtypes pg_bytetypes pg_datetypes big_tbl my_view
+
+query I
+SELECT COUNT(*) FROM (FROM new_db.copy_schema.${table_name} EXCEPT FROM ${table_name})
+----
+0
+
+endloop