Merge branch 'release/0.3.0'

CHANGELOG.md

@@ -1,3 +1,17 @@
+### citustools v0.3.0 (June 16, 2016) ###
+
+* Adds support for building PGXN releases
+
+* Adds support for signing generated deb and rpm files
+
+* Adds Travis CI scripts to build nightlies and releases
+
+* Adds Travis CI script to push new OSS commits to Enterprise
+
+* Adds Travis CI script to trigger Docker Hub nightly image build
+
+* Copies several scripts from the Citus packaging repo
+
 ### citustools v0.2.0 (May 13, 2016) ###
 
 * Adds wrapper to simplify generating OS packages for Citus projects

HomebrewFormula/citustools.rb

@@ -12,13 +12,16 @@ def message
 class Citustools < Formula
   desc "Tools and config used in Citus Data projects."
   homepage "https://github.com/citusdata/tools"
-  url "https://github.com/citusdata/tools/archive/v0.1.0.tar.gz"
-  sha256 "dc773c21989aa4d716b653ed7542d333f63f14a10d470f9a24fe12fac836b262"
+  url "https://github.com/citusdata/tools/archive/v0.2.0.tar.gz"
+  sha256 "605bea4cd59cf93cda2e53a4d9a42ba452960cbd7d6b2bac6e66102bf7f3b827"
 
   depends_on "uncrustify"
   depends_on Docker
 
   def install
+    # FIXME: ensure installdirs runs exactly once
+    ENV.deparallelize
+
     system "make", "install", "prefix=#{prefix}", "sysconfdir=#{etc}"
   end
 

packaging/citus_package

@@ -1,4 +1,4 @@
-#!/usr/bin/perl -w
+#!/usr/bin/env perl
 
 use strict;
 use warnings;
@@ -19,13 +19,16 @@ use constant BAD_USAGE  => 64;    ## no critic (ProhibitConstantPragma)
 use constant BAD_INPUT  => 65;    ## no critic (ProhibitConstantPragma)
 use constant NO_SERVICE => 69;    ## no critic (ProhibitConstantPragma)
 use constant BAD_CONFIG => 78;    ## no critic (ProhibitConstantPragma)
+use constant FINGERPRINT =>       ## no critic (ProhibitConstantPragma)
+  '47EA 3DE1 08AB EA75 0F81  E34B 4BD9 69CA 3F95 D6C6';
 
 my %supported_platforms = (
     debian => [ "jessie", "wheezy" ],
     el     => [ "7",      "6" ],
     fedora => [ "23",     "22" ],
     ol     => [ "7",      "6" ],
-    ubuntu => [ "xenial", "wily", "trusty", "precise" ]
+    ubuntu => [ "xenial", "wily", "trusty", "precise" ],
+    pgxn => [""]
 );
 
 my @rh_flavors = qw(el fedora ol);
@@ -35,7 +38,8 @@ my %docker_names = (
     el     => "centos",
     fedora => "fedora",
     ol     => "oraclelinux",
-    ubuntu => "ubuntu"
+    ubuntu => "ubuntu",
+    pgxn   => "pgxn"
 );
 
 sub verify_platforms {
@@ -51,6 +55,7 @@ sub verify_platforms {
 
     foreach my $platform (@platforms) {
         my ( $os, $release ) = split( '/', $platform, 2 );
+        $release //= '';
 
         if ( exists $supported_platforms{$os} ) {
             my @releases = @{ $supported_platforms{$os} };
@@ -87,7 +92,7 @@ sub get_and_verify_token {
     }
 
     my $github_token = $ENV{GITHUB_TOKEN};
-    if ( $ENV{GITHUB_TOKEN} =~ /^(\w+)$/ ) {
+    if ( $ENV{GITHUB_TOKEN} =~ /^(\w+)$/x ) {
         $github_token = $1;
     }
     else {
@@ -128,6 +133,93 @@ sub verify_docker_running {
     return;
 }
 
+sub get_signing_creds {
+    my ( $secret_key, $passphrase );
+
+    if ( exists $ENV{PACKAGING_SECRET_KEY} ) {
+        $secret_key = $ENV{PACKAGING_SECRET_KEY};
+    }
+    else {
+        my $result =
+          `gpg --batch --fingerprint packaging\@citusdata.com 2> /dev/null`;
+        my $exit_code   = $? >> 8;
+        my $fingerprint = FINGERPRINT;
+
+        unless ( $exit_code == 0 ) {
+            warn "Could not find signing key. Is gpg installed?\n";
+        }
+
+        if ( $result =~ /fingerprint.*\Q$fingerprint\E/x ) {
+
+            # remove spaces
+            $fingerprint =~ s/\s+//gx;
+
+            $secret_key =
+              `gpg --batch --export-secret-keys -a $fingerprint | base64`;
+        }
+    }
+
+    if ( exists $ENV{PACKAGING_PASSPHRASE} ) {
+        $passphrase = $ENV{PACKAGING_PASSPHRASE};
+    }
+
+    return ( $secret_key, $passphrase );
+}
+
+sub sign_packages {
+    my ( $tempdir, $secret_key, $passphrase ) = @_;
+    my @debfiles  = glob("$tempdir/*/*.deb");
+    my @rpmfiles  = glob("$tempdir/*/*.rpm");
+    my @base_args = (
+        qw(run --rm -v), "$tempdir:/packages",
+        '-e',            "PACKAGING_SECRET_KEY",
+        '-e',            'PACKAGING_PASSPHRASE',
+    );
+
+    local $ENV{PACKAGING_SECRET_KEY} = $secret_key;
+    local $ENV{PACKAGING_PASSPHRASE} = $passphrase;
+
+    if ( @debfiles > 0 ) {
+        my @deb_args = @base_args;
+        push @deb_args, 'citusdata/packaging:debsigner';
+
+        system( 'docker', @deb_args );
+
+        if ( $? == -1 ) {
+            die "failed to execute: $!\n";
+        }
+        elsif ( $? & 127 ) {
+            die "child died with signal %d, %s coredump\n",
+              ( $? & 127 ), ( $? & 128 ) ? 'with' : 'without';
+        }
+        else {
+            my $exit_code = $? >> 8;
+            die "docker run failed. see output for details.\n" if $exit_code;
+        }
+    }
+
+    if ( @rpmfiles > 0 ) {
+        my @rpm_args = @base_args;
+        push @rpm_args, 'citusdata/packaging:rpmsigner';
+
+        system( 'docker', @rpm_args );
+
+        if ( $? == -1 ) {
+            die "failed to execute: $!\n";
+        }
+        elsif ( $? & 127 ) {
+            die "child died with signal %d, %s coredump\n",
+              ( $? & 127 ), ( $? & 128 ) ? 'with' : 'without';
+        }
+        else {
+            my $exit_code = $? >> 8;
+            die "docker run failed. see output for details.\n" if $exit_code;
+        }
+    }
+
+    return;
+}
+
 my ( @platforms, $project, $build_type, $opt_help );
 
 GetOptions( 'p|platform=s' => \@platforms, 'help!' => \$opt_help )
@@ -149,7 +241,7 @@ if ( @ARGV != 2 ) {
 
 ( $project, $build_type ) = @ARGV;
 
-if ( $project =~ /^(citus|enterprise|rebalancer)$/ ) {
+if ( $project =~ /^(citus|enterprise|rebalancer)$/x ) {
     $project = $1;
 }
 else {
@@ -166,29 +258,48 @@ my $homedir      = ( getpwuid($<) )[7];
 my $tempdir = tempdir( ".citus_package.XXXXX", DIR => $homedir, CLEANUP => 1 );
 my $currentdir = getcwd();
 
+local $ENV{GITHUB_TOKEN} = $github_token;
+
+if ( any { $_ eq 'pgxn' } @platforms ) {
+    pod2usage(
+        -msg     => "PGXN unsupported for project: $project",
+        -exitval => BAD_INPUT
+    ) unless ( $project eq 'citus' );
+
+    pod2usage(
+        -msg     => "PGXN unsupported for build type: $build_type",
+        -exitval => BAD_INPUT
+    ) unless ( $build_type eq 'release' );
+}
+
 foreach my $platform (@platforms) {
     my ( $os, $release );
 
-    if ( $platform =~ /^(\w+)\/(\w+)$/ ) {
+    if ( $platform =~ /^(\w+)\/(\w+)$/x ) {
         $os      = $1;
         $release = $2;
     }
+    else {
+        $os      = 'pgxn';
+        $release = '';
+    }
+
+    my $docker_platform = $docker_names{$os};
+    $docker_platform .= "-$release" if $release;
 
-    my $docker_name     = $docker_names{$os};
-    my $docker_platform = "$docker_name-$release";
-    my $outputdir       = $tempdir . '/' . $docker_platform;
+    my $outputdir = $tempdir . '/' . $docker_platform;
     my @pg_versions =
       ( any { $_ eq $os } @rh_flavors ) ? qw (pg94 pg95) : qw (all);
 
+    # create output directory to ensure we, and not the docker user, own it
+    mkdir $outputdir;
+
     foreach my $pg (@pg_versions) {
         my @docker_args = (
             qw(run --rm -v),
-            "$outputdir:/packages",
-            '-e',
-            "GITHUB_TOKEN=$github_token",
+            "$outputdir:/packages", '-e', "GITHUB_TOKEN",
             "citusdata/packaging:$docker_platform-$pg",
-            $project,
-            $build_type
+            $project, $build_type
         );
 
         system( 'docker', @docker_args );
@@ -207,6 +318,15 @@ foreach my $platform (@platforms) {
     }
 }
 
+my ( $secret_key, $passphrase ) = get_signing_creds();
+
+if ( $secret_key and $passphrase ) {
+    sign_packages( $tempdir, $secret_key, $passphrase );
+}
+else {
+    warn "Could not get signing credentials. Skipping signing...\n";
+}
+
 system( 'mv', ( ( glob "$tempdir/*" ), $currentdir ) );
 
 __END__
@@ -273,6 +393,16 @@ B<Do not leave the next page until you've copied your new token!> Paste it
 into your e.g. C<.bash_profile> or C<.zshrc> to ensure your shells will have
 access to your new token.
 
+B<citus_package> will attempt to sign any Linux packages it builds if the
+C<PACKAGING_SECRET_KEY> and C<PACKAGING_PASSPHRASE> environment variables are
+set. C<PACKAGING_SECRET_KEY> must be a base64-encoded PGP private key. The
+passphrase needed to decrypt this key must be specified in the other variable,
+C<PACKAGING_PASSPHRASE>.
+
+If L<gpg(1)> is installed, B<citus_package> will search its keychain for the
+Citus Data private key so that only the C<PACKAGING_PASSPHRASE> will be needed
+to sign packages.
+
 =head1 SUPPORTED PROJECTS
 
 =over 4
@@ -313,12 +443,19 @@ access to your new token.
 
 =item I<ubuntu/precise> Ubuntu 12.04 LTS (Precise Pangolin)
 
+=item I<pgxn>           PostgreSQL Extension Network (PGXN)
+
 =back
 
 =head1 TODO
 
 Eventually support a different output folder.
 
+=head1 CAVEATS
+
+Projects other than Citus may not use the PGXN platform. When building a PGXN
+Citus package, only the I<release> build type may be used.
+
 =head1 SEE ALSO
 
 =over 4

travis/README.md

@@ -46,3 +46,38 @@ Runs regression tests against a more "complex" extension, usually one that:
   * Starts its own PostgreSQL instances within a `check` Make target.
 
 If this script is used, it is likely that `config_and_start_cluster.sh` is unnecessary.
+
+### `build_new_nightly`
+
+Checks packagecloud.io for the last nightly for the project being built. If any commits have been made to that project's GitHub development branch since the last nightly upload, this script builds a new nightly release (using `citus_package`).
+
+If no nightly is needed, exits immediately.
+
+### `build_new_release`
+
+Downloads the packaging files for the project being built. If the packaging files specify a version that is not yet in packagecloud.io, this script builds a new official release (using `citus_package`).
+
+If no new release is needed, exits immediately.
+
+### `release_pgxn`
+
+Downloads the PGXN `META.json` file for the project build built, produces a new PGXN-compatible archive, and uploads that archive to pgxn.org.
+
+Does not presently check ahead of time if PGXN already contains the archive; instead the script will exit successfully with a message indicating that the server already contained the version it uploaded.
+
+### `sync_to_enterprise`
+
+Pushes branches from the open-source Citus GitHub repository to the closed-source Citus Enterprise repository. Intended for use with the `master` branch and any branches beginning with `release-`.
+
+### `trigger_docker_nightly`
+
+Pairs with `build_new_nightly` to trigger a new Docker Hub nightly image build. Only runs if the following conditions are met:
+
+  * Project is `citus`
+  * OS is `debian`
+  * Release is `jessie`
+  * New nightly was produced
+
+### `fetch_build_files` and `parse_latest_release`
+
+Needed by packaging-related scripts. Copy-pasted from the `citusdata/packaging` repository. See that project for details.