diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dd7ce0b9..133a2a9d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,90 @@ +### pgcopydb v0.12 (June 28, 2023) ### + +This is a bugfix release with a strong focus on our logical decoding client. +The support for test_decoding UPDATE messages is improved, and a lot of bugs +related to how we split JSON messages in separate files (same as Postgres +WAL naming) have now been found and fixed. + +An important fix has been implemented with respect to how Postgres snapshots +and logical decoding works. The replication command CREATE_REPLICATION_SLOT +is able to export a snapshot but can not import one, which means that +`pgcopydb snapshot` now has a new `--follow` command which creates a +replication slot and exports a snapshot to re-use in the rest of the +commands. + +### Added +* Add table column list to schema.json. (#315) +* Improve parsing of UPDATE messages from test_decoding. (#329) +* Implement pgcopydb list databases. (#270) +* Implement pgcopydb list progress --summary --json. (#235) +* Implement the feature to log to file, with support for JSON. (#234) +* Introduce new option --skip-vacuum. (#230) +* Implement a --fail-fast option. (#222) + +### Changed +* Set statement_timeout and lock_timeout to 0. (#344) +* Have --skip-extensions also skip pre-existing schemas on the target. (#341) +* Quote sequence names when checking for privileges. (#326) +* Use dynamic memory for connection strings handling. (#323) +* Introduce a new pgcopydb internal message: ENDPOS. (#321) +* Remove dead code (hostname_from_uri). (#325) +* Remove unnecessary reading for commit_lsn after reaching startpos. (#318) +* Switch from semaphores to message queue to share workload. (#305) +* Use JSON format for work-directory summary files. (#300) +* At follow mode switch, skip empty transform queues. (#301) +* When using --trace then enforce logging of the apply SQL statements. (#252) +* Use dynamic memory for variable length schema parts. (#249) +* Refrain from logging user data. (#227) +* Improve logs. (#215) +* Use OVERRIDING SYSTEM VALUE in INSERT statements in follow mode. (#214) +* Create new process group for pgcopydb initialization. (#211) +* Check that we have sequence privileges before selecting metadata. (#212) +* Only create pgcopydb schema and table_size table when --cache is used. (#210) +* Switch docs PDF building to xelatex engine. (#209) + +### Fixed +* Fix pgcopydb stream cleanup. (#351) +* Fix transform escape rules for SQL escape syntax, again. (#349) +* Fix resuming the transform process. (#348) +* Ensure startpos is updated when closing stream. (#345) +* Skip importing snapshot we won't use, fix pgsql error handling. (#340) +* Fix transform to SQL to escape string values. (#337) +* Fix transforming UPDATE messages WHERE/SET clauses. (#333) +* Fix schema queries for exclusion filters. (#314) +* When applying CDC (logical replication), set role to replica. (#308) +* Add unit testing coverage for "generated as identity" columns. (#309) +* Fix file rotation issue during streaming. (#298) +* Fix apply when file starts with non-begin/keepalive statements. (#304) +* transform: Fix transformation to empty txn when first line is COMMIT. (#303) +* Fix empty xid and timestamp for continued txn COMMIT messages. (#302) +* Fix missing data of txns whose BEGIN LSN is less than consistent_point. (#295) +* Filter out dropped columns (#294) +* Use column names in COPY statements. (#290) +* Fix how snapshot are exported when using CDC. (#279) +* Refrain from early exits on signals when sending messages to queues. (#285) +* Fix pgsql is_response_ok to accept also PGRES_COPY_BOTH. (#284) +* Fix off-by-one in size of transform messages array. (#283) +* Fix double precision out of range during COPY. (#281) +* Fix the transition between replay operating modes. (#277) +* Review and fix connection management for sentinel async updates. (#273) +* Update pgcopydb sentinel's replay_lsn asynchronously. (#267) +* Count pg_restore --list lines and dimension our array accordingly. (#268) +* Fix a typo in docs (#261) +* Assorted streaming fixes, including skipping of empty transactions. (#257) +* Fix wal2json bytea values. (#253) +* Transform empty transactions from the JSON to the SQL files. (#251) +* Fix special characters in SQL queries. (#248) +* Fix exclude-schema filtering to apply to pg_dump and pg_restore. (#247) +* Fix Postgres catalog queries that implement sequences filtering. (#246) +* Fix pgsql_state_is_connection_error. (#244) +* Fix parsing --fail-fast option, which requires no argument. (#243) +* Fix command line option log level increments (--verbose --notice). (#233) +* Fix when we VACUUM ANALYZE. (#228) +* Fix how we skip concurrent build of certain indexes. (#223) +* Ensure clean-up of System V resources. (#216) +* Fix the `pgcopydb list collations` SQL query. (#213) + + ### pgcopydb v0.11 (March 15, 2023) ### This release introduces support for the Postgres code logical decoding diff --git a/Dockerfile.debian b/Dockerfile.debian index ba0629c25..2f6f057b9 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -29,6 +29,7 @@ RUN apt-get update \ WORKDIR /usr/src/pgcopydb COPY ./Makefile ./ +COPY ./GIT-VERSION-GEN ./ COPY ./README.md ./ COPY ./CHANGELOG.md ./ COPY ./LICENSE ./ @@ -36,7 +37,7 @@ COPY ./src ./src/ COPY ./docs ./docs/ WORKDIR /usr/src -RUN tar czf pgcopydb_0.11.orig.tar.gz pgcopydb +RUN tar czf pgcopydb_0.12.orig.tar.gz pgcopydb WORKDIR /usr/src/pgcopydb diff --git a/Dockerfile.debian-qa b/Dockerfile.debian-qa index ade8f13c8..eb21eb57f 100644 --- a/Dockerfile.debian-qa +++ b/Dockerfile.debian-qa @@ -1,8 +1,8 @@ FROM debian:sid -ENV TAR v0.11.tar.gz -ENV ORIG pgcopydb_0.11.orig.tar.gz -ENV WORKDIR /usr/src/pgcopydb-0.11 +ENV TAR v0.12.tar.gz +ENV ORIG pgcopydb_0.12.orig.tar.gz +ENV WORKDIR /usr/src/pgcopydb-0.12 ENV ARCHIVE https://github.com/dimitri/pgcopydb/archive/refs/tags/ ENV RELEASE ${ARCHIVE}${TAR} diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index c7072ee24..5550539c3 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -1,7 +1,7 @@ #!/bin/sh GVF=GIT-VERSION-FILE -DEF_VER=0.11 +DEF_VER=0.12 LF=' ' diff --git a/README.md b/README.md index 1b1dc1f12..884b0ace0 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ $ pgcopydb help parse-list Parse pg_restore --list output from custom file pgcopydb list + databases List databases extensions List all the source extensions to copy collations List all the source collations to copy tables List all the source tables to copy data from @@ -102,21 +103,11 @@ $ pgcopydb help prefetch Stream JSON changes from the source database and transform them to SQL catchup Apply prefetched changes from SQL files to the target database replay Replay changes from the source to the target database, live - + create Create resources needed for pgcopydb - + drop Drop resources needed for pgcopydb + sentinel Maintain a sentinel table on the source database receive Stream changes from the source database transform Transform changes from the source database into SQL commands apply Apply changes from the source database into the target database - pgcopydb stream create - slot Create a replication slot in the source database - origin Create a replication origin in the target database - - pgcopydb stream drop - slot Drop a replication slot in the source database - origin Drop a replication origin in the target database - pgcopydb stream sentinel create Create the sentinel table on the source database drop Drop the sentinel table on the source database diff --git a/debian/changelog b/debian/changelog index 4aa12aca9..cd4c12c84 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +pgcopydb (0.12-1) unstable; urgency=medium + + * Official 0.12 release of pgcopydb + + -- Dimitri Fontaine Wed, 28 Jun 2023 12:23:11 +0200 + pgcopydb (0.11-1) unstable; urgency=medium * Official 0.11 release of pgcopydb diff --git a/docs/conf.py b/docs/conf.py index 188d918ff..563c30322 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,12 +18,12 @@ # -- Project information ----------------------------------------------------- project = 'pgcopydb' -copyright = '2022, Dimitri Fontaine' +copyright = '2022-2023, Dimitri Fontaine' author = 'Dimitri Fontaine' # The full version, including alpha/beta/rc tags -version = '0.11' -release = '0.11' +version = '0.12' +release = '0.12' # -- General configuration --------------------------------------------------- diff --git a/docs/install.rst b/docs/install.rst index 76e5c8f18..fe0aae60c 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -19,7 +19,23 @@ RPM packages ------------ The Postgres community repository for RPM packages is `yum.postgresql.org`__ -and does not include binary packages for pgcopydb at this time. +and includes binary packages for pgcopydb. The way packages are built for +RPM based systems means that the user needs to choose which version of +Postgres pgcopydb was built with. In practice this doesn't have much +importance, because ``libpq`` is meant to be compatible with many different +Postgres server versions. + +After following the instructions for installing the repository, in this +example in a Docker image for Rocky Linux (``docker run --rm -it +rockylinux:9``), then we get the following:: + + # dnf search pgcopydb + ... + pgcopydb_11.x86_64 : Automate pg_dump | pg_restore between two running Postgres servers + pgcopydb_12.x86_64 : Automate pg_dump | pg_restore between two running Postgres servers + pgcopydb_13.x86_64 : Automate pg_dump | pg_restore between two running Postgres servers + pgcopydb_14.x86_64 : Automate pg_dump | pg_restore between two running Postgres servers + pgcopydb_15.x86_64 : Automate pg_dump | pg_restore between two running Postgres servers __ https://yum.postgresql.org @@ -36,7 +52,7 @@ stable. To use this docker image:: - $ docker run --rm -it dimitri/pgcopydb:v0.11 pgcopydb --version + $ docker run --rm -it dimitri/pgcopydb:v0.12 pgcopydb --version __ https://hub.docker.com/r/dimitri/pgcopydb#! diff --git a/docs/ref/pgcopydb.rst b/docs/ref/pgcopydb.rst index e9262e707..9f3e11c7d 100644 --- a/docs/ref/pgcopydb.rst +++ b/docs/ref/pgcopydb.rst @@ -89,6 +89,7 @@ The ``pgcopydb help`` command lists all the supported sub-commands: parse-list Parse pg_restore --list output from custom file pgcopydb list + databases List databases extensions List all the source extensions to copy collations List all the source collations to copy tables List all the source tables to copy data from @@ -105,21 +106,11 @@ The ``pgcopydb help`` command lists all the supported sub-commands: prefetch Stream JSON changes from the source database and transform them to SQL catchup Apply prefetched changes from SQL files to the target database replay Replay changes from the source to the target database, live - + create Create resources needed for pgcopydb - + drop Drop resources needed for pgcopydb + sentinel Maintain a sentinel table on the source database receive Stream changes from the source database transform Transform changes from the source database into SQL commands apply Apply changes from the source database into the target database - pgcopydb stream create - slot Create a replication slot in the source database - origin Create a replication origin in the target database - - pgcopydb stream drop - slot Drop a replication slot in the source database - origin Drop a replication origin in the target database - pgcopydb stream sentinel create Create the sentinel table on the source database drop Drop the sentinel table on the source database diff --git a/src/bin/pgcopydb/defaults.h b/src/bin/pgcopydb/defaults.h index a38d9d572..6986669ce 100644 --- a/src/bin/pgcopydb/defaults.h +++ b/src/bin/pgcopydb/defaults.h @@ -9,7 +9,7 @@ #include "git-version.h" /* additional version information for printing version on CLI */ -#define PGCOPYDB_VERSION "0.11" +#define PGCOPYDB_VERSION "0.12" #ifdef GIT_VERSION #define VERSION_STRING GIT_VERSION diff --git a/src/bin/pgcopydb/pgsql.c b/src/bin/pgcopydb/pgsql.c index 20fe49a00..790b6d711 100644 --- a/src/bin/pgcopydb/pgsql.c +++ b/src/bin/pgcopydb/pgsql.c @@ -2429,15 +2429,15 @@ pg_copy_send_query(PGSQL *pgsql, PGresult *res = PQexec(pgsql->connection, sql); - free(sql); - if (PQresultStatus(res) != status) { pgcopy_log_error(pgsql, res, sql); + free(sql); return false; } + free(sql); return true; }