From 71431081e38ced73b7c13c704ce3a290be7d5ea9 Mon Sep 17 00:00:00 2001 From: Jelte Fennema Date: Wed, 6 May 2020 12:45:10 +0200 Subject: [PATCH 1/9] Support pgautofailover in update_os_package.pl --- automated_packaging/update_os_package.pl | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/automated_packaging/update_os_package.pl b/automated_packaging/update_os_package.pl index c3c0a091..33ab5606 100644 --- a/automated_packaging/update_os_package.pl +++ b/automated_packaging/update_os_package.pl @@ -13,6 +13,17 @@ $github_repo_name = "citus-enterprise"; $log_repo_name = "Citus Enterprise"; } +my $package_name = $github_repo_name; +if ( $PROJECT eq "pgautofailover" ) { + $github_repo_name = "pg_auto_failover"; + $package_name = "pg-auto-failover"; + $log_repo_name = "pg_auto_failover"; +} +if ( $PROJECT eq "pgautofailover-enterprise" ) { + $github_repo_name = "citus-ha"; + $package_name = "pg-auto-failover-enterprise"; + $log_repo_name = "pg_auto_failover enterprise"; +} my $github_token = get_and_verify_token(); @@ -70,9 +81,9 @@ sub get_changelog_for_debian { # Based on the repo, update the package related variables if ( $DISTRO_VERSION eq "redhat" || $DISTRO_VERSION eq "microsoft" || $DISTRO_VERSION eq "all") { - `sed -i 's|^Version:.*|Version: $VERSION.citus|g' $github_repo_name.spec`; - `sed -i 's|^Source0:.*|Source0: https:\/\/github.com\/citusdata\/$github_repo_name\/archive\/v$VERSION.tar.gz|g' $github_repo_name.spec`; - `sed -i 's|^%changelog|%changelog\\n* $abbr_day[$wday] $abbr_mon[$mon] $mday $year - $git_name <$microsoft_email> $VERSION.citus-1\\n- Update to $log_repo_name $VERSION\\n|g' $github_repo_name.spec`; + `sed -i 's|^Version:.*|Version: $VERSION.citus|g' $package_name.spec`; + `sed -i 's|^Source0:.*|Source0: https:\/\/github.com\/citusdata\/$package_name\/archive\/v$VERSION.tar.gz|g' $package_name.spec`; + `sed -i 's|^%changelog|%changelog\\n* $abbr_day[$wday] $abbr_mon[$mon] $mday $year - $git_name <$microsoft_email> $VERSION.citus-1\\n- Update to $log_repo_name $VERSION\\n|g' $package_name.spec`; } if ( $DISTRO_VERSION eq "debian" || $DISTRO_VERSION eq "microsoft" || $DISTRO_VERSION eq "all") { open( DEB_CLOG_FILE, "<./debian/changelog" ) || die "Debian changelog file not found"; @@ -85,7 +96,7 @@ sub get_changelog_for_debian { # Update the changelog file of the debian branch open( DEB_CLOG_FILE, ">./debian/changelog" ) || die "Debian changelog file not found"; - print DEB_CLOG_FILE "$github_repo_name ($VERSION.citus-1) stable; urgency=low\n"; + print DEB_CLOG_FILE "$package_name ($VERSION.citus-1) stable; urgency=low\n"; print DEB_CLOG_FILE @changelog_print; print DEB_CLOG_FILE " -- $git_name <$microsoft_email> $abbr_day[$wday], $mday $abbr_mon[$mon] $year $print_hour:$min:$sec +0000\n\n"; print DEB_CLOG_FILE @lines; From fcd0e22e186ab82b9be6fab9efa869a79c8a9c79 Mon Sep 17 00:00:00 2001 From: Jelte Fennema Date: Thu, 7 May 2020 13:21:18 +0200 Subject: [PATCH 2/9] Do not put the whole changelog into the package for pgautofailover --- automated_packaging/update_os_package.pl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/automated_packaging/update_os_package.pl b/automated_packaging/update_os_package.pl index 33ab5606..615c7928 100644 --- a/automated_packaging/update_os_package.pl +++ b/automated_packaging/update_os_package.pl @@ -83,7 +83,7 @@ sub get_changelog_for_debian { if ( $DISTRO_VERSION eq "redhat" || $DISTRO_VERSION eq "microsoft" || $DISTRO_VERSION eq "all") { `sed -i 's|^Version:.*|Version: $VERSION.citus|g' $package_name.spec`; `sed -i 's|^Source0:.*|Source0: https:\/\/github.com\/citusdata\/$package_name\/archive\/v$VERSION.tar.gz|g' $package_name.spec`; - `sed -i 's|^%changelog|%changelog\\n* $abbr_day[$wday] $abbr_mon[$mon] $mday $year - $git_name <$microsoft_email> $VERSION.citus-1\\n- Update to $log_repo_name $VERSION\\n|g' $package_name.spec`; + `sed -i 's|^%changelog|%changelog\\n* $abbr_day[$wday] $abbr_mon[$mon] $mday $year - $git_name <$microsoft_email> $VERSION.citus-1\\n- Official $VERSION release of $log_repo_name\\n|g' $package_name.spec`; } if ( $DISTRO_VERSION eq "debian" || $DISTRO_VERSION eq "microsoft" || $DISTRO_VERSION eq "all") { open( DEB_CLOG_FILE, "<./debian/changelog" ) || die "Debian changelog file not found"; @@ -92,12 +92,21 @@ sub get_changelog_for_debian { # Change hour and get changelog (TODO: may update it !) $print_hour = $hour - 3; - @changelog_print = get_changelog_for_debian(); + if ($PROJECT eq 'citus' || $PROJECT eq 'enterprise') { + @changelog_print = get_changelog_for_debian(); + } + # Update the changelog file of the debian branch open( DEB_CLOG_FILE, ">./debian/changelog" ) || die "Debian changelog file not found"; print DEB_CLOG_FILE "$package_name ($VERSION.citus-1) stable; urgency=low\n"; - print DEB_CLOG_FILE @changelog_print; + if ($PROJECT eq 'citus' || $PROJECT eq 'enterprise') { + print DEB_CLOG_FILE @changelog_print; + } + else + { + print DEB_CLOG_FILE "\n * Official $VERSION release of $log_repo_name\n\n"; + } print DEB_CLOG_FILE " -- $git_name <$microsoft_email> $abbr_day[$wday], $mday $abbr_mon[$mon] $year $print_hour:$min:$sec +0000\n\n"; print DEB_CLOG_FILE @lines; close(DEB_CLOG_FILE); From 95132b63efd47264bf7917b583b69c4ed706db9d Mon Sep 17 00:00:00 2001 From: Jelte Fennema Date: Thu, 7 May 2020 15:16:09 +0200 Subject: [PATCH 3/9] Some improvements of update_os_package.pl * Handles $DISTRO_VERSION "all" better * Branch name includes $VERSION --- automated_packaging/update_os_package.pl | 31 ++++++++++++++++-------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/automated_packaging/update_os_package.pl b/automated_packaging/update_os_package.pl index 615c7928..860cb75d 100644 --- a/automated_packaging/update_os_package.pl +++ b/automated_packaging/update_os_package.pl @@ -9,6 +9,7 @@ # Name of the repo is represented differently on logs and repos my $github_repo_name = "citus"; my $log_repo_name = "Citus"; +my $version_suffix = ".citus"; if ( $PROJECT eq "enterprise" ) { $github_repo_name = "citus-enterprise"; $log_repo_name = "Citus Enterprise"; @@ -18,11 +19,13 @@ $github_repo_name = "pg_auto_failover"; $package_name = "pg-auto-failover"; $log_repo_name = "pg_auto_failover"; + $version_suffix = ""; } if ( $PROJECT eq "pgautofailover-enterprise" ) { $github_repo_name = "citus-ha"; $package_name = "pg-auto-failover-enterprise"; $log_repo_name = "pg_auto_failover enterprise"; + $version_suffix = ""; } my $github_token = get_and_verify_token(); @@ -67,23 +70,25 @@ sub get_changelog_for_debian { # Necessary to create unique branch $curTime = time(); +my $main_branch = "$DISTRO_VERSION-$PROJECT"; +my $pr_branch = "$DISTRO_VERSION-$PROJECT-$VERSION-push-$curTime"; # Checkout the distro's branch -`git checkout $DISTRO_VERSION-$PROJECT`; +`git checkout $main_branch`; # Update distro's branch -`git pull origin $DISTRO_VERSION-$PROJECT`; +`git pull origin $main_branch`; # Create a new branch based on the distro's branch -`git checkout -b $DISTRO_VERSION-$PROJECT-push-$curTime`; +`git checkout -b $pr_branch`; # Update pkgvars -`sed -i 's/^pkglatest.*/pkglatest=$VERSION.citus-1/g' pkgvars`; +`sed -i 's/^pkglatest.*/pkglatest=$VERSION$version_suffix-1/g' pkgvars`; # Based on the repo, update the package related variables if ( $DISTRO_VERSION eq "redhat" || $DISTRO_VERSION eq "microsoft" || $DISTRO_VERSION eq "all") { - `sed -i 's|^Version:.*|Version: $VERSION.citus|g' $package_name.spec`; + `sed -i 's|^Version:.*|Version: $VERSION$version_suffix|g' $package_name.spec`; `sed -i 's|^Source0:.*|Source0: https:\/\/github.com\/citusdata\/$package_name\/archive\/v$VERSION.tar.gz|g' $package_name.spec`; - `sed -i 's|^%changelog|%changelog\\n* $abbr_day[$wday] $abbr_mon[$mon] $mday $year - $git_name <$microsoft_email> $VERSION.citus-1\\n- Official $VERSION release of $log_repo_name\\n|g' $package_name.spec`; + `sed -i 's|^%changelog|%changelog\\n* $abbr_day[$wday] $abbr_mon[$mon] $mday $year - $git_name <$microsoft_email> $VERSION$version_suffix-1\\n- Official $VERSION release of $log_repo_name\\n|g' $package_name.spec`; } if ( $DISTRO_VERSION eq "debian" || $DISTRO_VERSION eq "microsoft" || $DISTRO_VERSION eq "all") { open( DEB_CLOG_FILE, "<./debian/changelog" ) || die "Debian changelog file not found"; @@ -99,7 +104,7 @@ sub get_changelog_for_debian { # Update the changelog file of the debian branch open( DEB_CLOG_FILE, ">./debian/changelog" ) || die "Debian changelog file not found"; - print DEB_CLOG_FILE "$package_name ($VERSION.citus-1) stable; urgency=low\n"; + print DEB_CLOG_FILE "$package_name ($VERSION$version_suffix-1) stable; urgency=low\n"; if ($PROJECT eq 'citus' || $PROJECT eq 'enterprise') { print DEB_CLOG_FILE @changelog_print; } @@ -112,7 +117,13 @@ sub get_changelog_for_debian { close(DEB_CLOG_FILE); } + +my $commit_message_distro_version = "$DISTRO_VERSION "; +if ($DISTRO_VERSION eq "all") { + $commit_message_distro_version = ""; +} # Commit, push changes and open a pull request -`git commit -a -m "Bump $DISTRO_VERSION $log_repo_name $VERSION"`; -`git push origin $DISTRO_VERSION-$PROJECT-push-$curTime`; -`curl -g -H "Accept: application/vnd.github.v3.full+json" -X POST --user "$github_token:x-oauth-basic" -d '{\"title\":\"Bump $PROJECT $DISTRO_VERSION version to $VERSION\", \"head\":\"$DISTRO_VERSION-$PROJECT-push-$curTime\", \"base\":\"$DISTRO_VERSION-$PROJECT\"}' https://api.github.com/repos/citusdata/packaging/pulls`; +my $commit_message = "Bump $commit_message_distro_version$log_repo_name version to $VERSION"; +`git commit -a -m "$commit_message"`; +`git push origin $pr_branch`; +`curl -g -H "Accept: application/vnd.github.v3.full+json" -X POST --user "$github_token:x-oauth-basic" -d '{\"title\":\"$commit_message\", \"head\":\"$pr_branch\", \"base\":\"$main_branch\"}' https://api.github.com/repos/citusdata/packaging/pulls`; From 9214b6839f5a3cb1764e656ae062f410e93b8ad0 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Fri, 8 May 2020 11:45:42 +0300 Subject: [PATCH 4/9] Make the base branch master instead of develop (#72) We already open the pr's to update docker images against master, hence we should not use develop as the base branch. --- automated_packaging/update_docker.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated_packaging/update_docker.pl b/automated_packaging/update_docker.pl index 7d1fc46f..5df6bbcc 100644 --- a/automated_packaging/update_docker.pl +++ b/automated_packaging/update_docker.pl @@ -18,7 +18,7 @@ $curTime = time(); # Checkout to the release's branch -`git checkout develop`; +`git checkout master`; `git checkout -b release-$VERSION-$curTime`; # That means we want to update postgres version From 0dcc48245038f6ff6efd09d6f03bd54bf42335d9 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Fri, 8 May 2020 13:41:07 +0300 Subject: [PATCH 5/9] Small improvements in update_pgxn.pl (#73) * As we put README.md instead of the latest migration script into the pgxn package, remove related sed command. * Update pr title and commit message to be more explicit about we are actually updating pgxn package. --- automated_packaging/update_pgxn.pl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/automated_packaging/update_pgxn.pl b/automated_packaging/update_pgxn.pl index 58b0d680..6f2d7482 100644 --- a/automated_packaging/update_pgxn.pl +++ b/automated_packaging/update_pgxn.pl @@ -25,11 +25,10 @@ # Update META.json file `sed -i 's/$old_version_escape_dot/$NEW_VERSION/g' META.json`; -`sed -i 's/$old_minor_version-[[:digit:]]/$new_minor_version-$new_point_version/g' META.json`; # Commit changes to github -`git commit -a -m "Bump Citus to $NEW_VERSION"`; +`git commit -a -m "Bump Citus PGXN to $NEW_VERSION"`; `git push origin pgxn-citus-push-$curTime`; # Open a PR to the master -`curl -g -H "Accept: application/vnd.github.v3.full+json" -X POST --user "$github_token:x-oauth-basic" -d '{\"title\":\"Bump Citus to $NEW_VERSION\", \"base\":\"pgxn-citus\", \"head\":\"pgxn-citus-push-$curTime\"}' https://api.github.com/repos/citusdata/packaging/pulls`; +`curl -g -H "Accept: application/vnd.github.v3.full+json" -X POST --user "$github_token:x-oauth-basic" -d '{\"title\":\"Bump Citus PGXN to $NEW_VERSION\", \"base\":\"pgxn-citus\", \"head\":\"pgxn-citus-push-$curTime\"}' https://api.github.com/repos/citusdata/packaging/pulls`; From 8f78446d023a4e597b451a939d5d567aa81bbc4d Mon Sep 17 00:00:00 2001 From: SaitTalhaNisanci Date: Fri, 8 May 2020 14:03:49 +0300 Subject: [PATCH 6/9] remove unused variable DESTDIR from uncrustify makefile (#54) --- uncrustify/Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/uncrustify/Makefile b/uncrustify/Makefile index 42ba0121..90b8333e 100644 --- a/uncrustify/Makefile +++ b/uncrustify/Makefile @@ -18,14 +18,14 @@ clean: rm -f *.1 installdirs: - $(INSTALL) -d $(DESTDIR)$(bindir) $(DESTDIR)$(pkgsysconfdir) $(DESTDIR)$(mandir)/man1 + $(INSTALL) -d $(bindir) $(pkgsysconfdir) $(mandir)/man1 install: all installdirs - $(INSTALL_SCRIPT) citus_indent $(DESTDIR)$(bindir) - $(INSTALL_DATA) $(MANPAGES) $(DESTDIR)$(mandir)/man1 - $(INSTALL_DATA) citus-style.cfg $(DESTDIR)$(pkgsysconfdir) + $(INSTALL_SCRIPT) citus_indent $(bindir) + $(INSTALL_DATA) $(MANPAGES) $(mandir)/man1 + $(INSTALL_DATA) citus-style.cfg $(pkgsysconfdir) perl -pi -e 's,/usr/local/etc/citustools,$(pkgsysconfdir),g' \ - $(DESTDIR)$(bindir)/citus_indent \ - $(DESTDIR)$(mandir)/man1/citus_indent.1 + $(bindir)/citus_indent \ + $(mandir)/man1/citus_indent.1 .PHONY: all man clean installdirs install From e225ac1caf5adc8ac5b9f6f7c461dcbbb9720fcf Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Mon, 11 May 2020 12:48:52 +0300 Subject: [PATCH 7/9] Give warning for descriptions >78 instead of killing process (#74) --- automated_packaging/common_functions.pm | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/automated_packaging/common_functions.pm b/automated_packaging/common_functions.pm index 781f9fa9..0c96e0d7 100644 --- a/automated_packaging/common_functions.pm +++ b/automated_packaging/common_functions.pm @@ -150,11 +150,13 @@ sub create_release_changelog { foreach $line (@log_output) { if ($line =~ /^DESCRIPTION: */) { $description_part = substr($line, length($&), -1); + if (length($description_part) > 78) { - print("You have to shorten PR message $description_part of $pr_url"); - `git reset --hard`; - die "Can not add description longer than 78 charachters"; + print("You have to shorten PR message $description_part of $pr_url\n"); + print("Description should not be longer than 78 charachters, please manually shorten this description\n"); + push(@comment_lines, "TODO: " . "PLEASE SHORTEN THE NEXT LINE MANUALLY, IT SHOULD BE NO LONGER THAN 78 CHARS\n"); } + print("Description $description_part has been added ... \n"); push(@comment_lines, "* " . $description_part . "\n\n"); } From ed8e007a956d89e82a5e9908df110662b7aaf5bd Mon Sep 17 00:00:00 2001 From: Lukas Fittl Date: Sun, 14 Jun 2020 00:00:42 -0700 Subject: [PATCH 8/9] Add support for Postgres 13 This is modeled after d9eb817d1d66d22de561d6d78f389065517b3bde, with a few less changes because the logic was simplified back then. --- travis/install_custom_pg | 4 ++-- travis/setup_apt | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/travis/install_custom_pg b/travis/install_custom_pg index ee5689df..13dc8d81 100755 --- a/travis/install_custom_pg +++ b/travis/install_custom_pg @@ -17,8 +17,8 @@ if [ "$(ls -A postgresql)" ]; then git -C postgresql pull else pgmajornum="${PGVERSION%%.*}" - if [ "${pgmajornum}" -gt '12' ]; then - # twelve is highest stable build; use master for higher + if [ "${pgmajornum}" -gt '13' ]; then + # 13 is highest stable build; use master for higher gitref="master" elif [ "${pgmajornum}" -gt '9' ]; then gitref="REL_${PGVERSION}_STABLE" diff --git a/travis/setup_apt b/travis/setup_apt index e3ffbcb1..b79c9a8d 100755 --- a/travis/setup_apt +++ b/travis/setup_apt @@ -12,11 +12,11 @@ sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 7FCC7D46ACCC4CF8 sudo rm -f /etc/apt/sources.list.d/google-chrome* sudo rm -f /etc/apt/sources.list.d/pgdg.list -# add the PostgreSQL 12 repository -sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main 12" >> /etc/apt/sources.list.d/postgresql.list' +# add the PostgreSQL 13 repository +sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main 13" >> /etc/apt/sources.list.d/postgresql.list' -# need testing repository after version 12 -if [ "${PGVERSION%%.*}" -gt '12' ]; then +# need testing repository after version 13 +if [ "${PGVERSION%%.*}" -gt '13' ]; then # add a PostgreSQL testing repository sudo sh -Ec 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg-testing main ${PGVERSION}" >> /etc/apt/sources.list.d/postgresql.list' From 5555ed461cb52c99ea77a51ffe9a2c6002d670e2 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Fri, 24 Jul 2020 11:28:27 +0300 Subject: [PATCH 9/9] Hint for man citus_package (#79) --- packaging/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/README.md b/packaging/README.md index 0eb00814..08b775ad 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -10,6 +10,8 @@ ## Usage +First, please read `man citus_package`, we have a man page for it :) + Ensure your `GITHUB_TOKEN` environment variable is properly set (see the man page if you're not sure how to do that). Make sure Docker is running, then you're off to the races! For example, to build a `citus` community "release" on Debian Jessie and Ubuntu Xenial, first change your directory into "citusdata/packaging" repo directory and then checkout the `all-citus` (would be `all-enterprise` for enterprise) branch as this branch has the specific `pkgvars` for community packages. Then execute the following: `citus_package -p debian/jessie -p ubuntu/xenial local release`