citus_package

#!/usr/bin/env perl

use strict;
use warnings;
use Getopt::Long qw(:config no_auto_abbrev no_ignore_case);
use POSIX qw(setlocale LC_ALL);
use File::Temp qw(tempdir);
use List::Util qw(any none);
use Cwd qw(getcwd);
BEGIN { $Pod::Usage::Formatter = 'Pod::Text::Termcap'; }
use Pod::Usage qw(pod2usage);

# untaint environment
local $ENV{'PATH'} =
  '/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin';
delete @ENV{ 'IFS', 'CDPATH', 'ENV', 'BASH_ENV' };

use constant BAD_USAGE  => 64;    ## no critic (ProhibitConstantPragma)
use constant BAD_INPUT  => 65;    ## no critic (ProhibitConstantPragma)
use constant NO_SERVICE => 69;    ## no critic (ProhibitConstantPragma)
use constant BAD_CONFIG => 78;    ## no critic (ProhibitConstantPragma)
use constant FINGERPRINT =>       ## no critic (ProhibitConstantPragma)
  '47EA 3DE1 08AB EA75 0F81  E34B 4BD9 69CA 3F95 D6C6';
use constant PACKAGE_URL =>       ## no critic (ProhibitConstantPragma)
  'https://github.com/citusdata/packaging/archive';

my %supported_platforms = (
    debian => [ "buster", "stretch", "jessie", "wheezy"],
    el     => [ "8",      "7",      "6" ],
    fedora => [ "28",     "27",      "26"],
    ol     => [ "8",      "7",      "6" ],
    ubuntu => [ "focal",  "bionic", "xenial", "trusty", "noble" ],
    pgxn => [""]
);

my @rh_flavors = qw(el fedora ol);

my %docker_names = (
    debian => "debian",
    el     => "centos",
    fedora => "fedora",
    ol     => "oraclelinux",
    ubuntu => "ubuntu",
    pgxn   => "pgxn"
);

sub verify_platforms {
    my (@platforms) = @_;

    if ( @platforms == 0 ) {
        pod2usage(
            -msg     => "You must specify at least one platform.",
            -exitval => BAD_USAGE,
            -verbose => 1
        );
    }

    foreach my $platform (@platforms) {
        my ( $os, $release ) = split( '/', $platform, 2 );
        $release //= '';

        if ( exists $supported_platforms{$os} ) {
            my @releases = @{ $supported_platforms{$os} };
            if ( none { $_ eq $release } @releases ) {
                pod2usage(
                    -msg      => "Unrecognized $os release: $release",
                    -exitval  => BAD_INPUT,
                    -verbose  => 99,
                    -sections => "SYNOPSIS|OPTIONS|SUPPORTED PLATFORMS"
                );
            }
        }
        else {
            pod2usage(
                -msg      => "Unrecognized OS: $os",
                -exitval  => BAD_INPUT,
                -verbose  => 99,
                -sections => "SYNOPSIS|OPTIONS|SUPPORTED PLATFORMS"
            );
        }
    }

    return;
}

sub get_and_verify_token {
    unless ( exists $ENV{GITHUB_TOKEN} ) {
        pod2usage(
            -msg      => "You must have a GITHUB_TOKEN set.",
            -exitval  => BAD_CONFIG,
            -verbose  => 99,
            -sections => "ENVIRONMENT"
        );
    }

    my $github_token = $ENV{GITHUB_TOKEN};
    if ( $ENV{GITHUB_TOKEN} =~ /^(\w+)$/x ) {
        $github_token = $1;
    }
    else {
        pod2usage(
            -msg      => "Malformed GITHUB_TOKEN: $github_token",
            -exitval  => BAD_INPUT,
            -verbose  => 99,
            -sections => "ENVIRONMENT"
        );
    }

    my $cmd = "curl -sf -H 'Authorization: token $github_token' "
      . 'https://api.github.com/';
    my $result    = `$cmd > /dev/null 2>&1`;
    my $exit_code = $? >> 8;

    if ( $exit_code == 22 ) {
        pod2usage(
            -msg      => "Your token was rejected by GitHub.",
            -exitval  => BAD_INPUT,
            -verbose  => 99,
            -sections => "ENVIRONMENT"
        );
    }

    return $github_token;
}

sub verify_docker_running {
    my $result    = `docker info > /dev/null 2>&1`;
    my $exit_code = $? >> 8;

    unless ( $exit_code == 0 ) {
        warn "Cannot connect to the Docker daemon. Is Docker running?\n";
        exit NO_SERVICE;
    }

    return;
}

sub get_signing_creds {
    my ( $secret_key, $passphrase );

    if ( exists $ENV{PACKAGING_SECRET_KEY} ) {
        $secret_key = $ENV{PACKAGING_SECRET_KEY};
    }
    else {
        my $result =
          `gpg --batch --fingerprint packaging\@citusdata.com 2> /dev/null`;
        my $exit_code   = $? >> 8;
        my $fingerprint = FINGERPRINT;

        unless ( $exit_code == 0 ) {
            warn "Could not find signing key. Is gpg installed?\n";
        }

        if ( $result =~ /fingerprint.*\Q$fingerprint\E/x ) {

            # remove spaces
            $fingerprint =~ s/\s+//gx;

            $secret_key =
              `gpg --batch --export-secret-keys -a $fingerprint | base64`;
        }
    }

    if ( exists $ENV{PACKAGING_PASSPHRASE} ) {
        $passphrase = $ENV{PACKAGING_PASSPHRASE};
    }

    return ( $secret_key, $passphrase );
}

sub sign_packages {
    my ( $pkgsdir, $secret_key, $passphrase ) = @_;
    my @debfiles  = glob("$pkgsdir/*/*.deb");
    my @rpmfiles  = glob("$pkgsdir/*/*.rpm");
    my @base_args = (
        qw(run --rm -v), "$pkgsdir:/packages",
        '-e',            "PACKAGING_SECRET_KEY",
        '-e',            'PACKAGING_PASSPHRASE',
    );

    local $ENV{PACKAGING_SECRET_KEY} = $secret_key;
    local $ENV{PACKAGING_PASSPHRASE} = $passphrase;

    if ( @debfiles > 0 ) {
        my @deb_args = @base_args;
        push @deb_args, 'citusdata/packaging:debsigner';

        system( 'docker', @deb_args );

        if ( $? == -1 ) {
            die "failed to execute: $!\n";
        }
        elsif ( $? & 127 ) {
            die "child died with signal %d, %s coredump\n",
              ( $? & 127 ), ( $? & 128 ) ? 'with' : 'without';
        }
        else {
            my $exit_code = $? >> 8;
            die "docker run failed. see output for details.\n" if $exit_code;
        }
    }

    if ( @rpmfiles > 0 ) {
        my @rpm_args = @base_args;
        push @rpm_args, 'citusdata/packaging:rpmsigner';

        system( 'docker', @rpm_args );

        if ( $? == -1 ) {
            die "failed to execute: $!\n";
        }
        elsif ( $? & 127 ) {
            die "child died with signal %d, %s coredump\n",
              ( $? & 127 ), ( $? & 128 ) ? 'with' : 'without';
        }
        else {
            my $exit_code = $? >> 8;
            die "docker run failed. see output for details.\n" if $exit_code;
        }
    }

    return;
}

my ( @platforms, $project, $build_type, $opt_help );

GetOptions( 'p|platform=s' => \@platforms, 'help!' => \$opt_help )
  or pod2usage(
    -msg     => "See '$0 --help' for more information.",
    -exitval => BAD_USAGE
  );

pod2usage( -verbose => 1 ) if $opt_help;

verify_platforms(@platforms);

if ( @ARGV != 2 ) {
    pod2usage(
        -msg     => "You must specify a project and build type.",
        -exitval => BAD_USAGE
    );
}

( $project, $build_type ) = @ARGV;

verify_docker_running();

my $github_token = get_and_verify_token();
my $homedir      = ( getpwuid($<) )[7];
my $tempdir = tempdir( ".citus_package.XXXXX", DIR => $homedir, CLEANUP => 1 );
my $pkgsdir = $tempdir . '/packages';
my $filedir = $tempdir . '/buildfiles';
mkdir $pkgsdir;
mkdir $filedir;
my $currentdir = getcwd();

local $ENV{GITHUB_TOKEN} = $github_token;

if ( any { $_ eq 'pgxn' } @platforms ) {
    pod2usage(
        -msg     => "PGXN unsupported for project: $project",
        -exitval => BAD_INPUT
    ) unless ( $project eq 'citus' or $project eq 'local' );

    pod2usage(
        -msg     => "PGXN unsupported for build type: $build_type",
        -exitval => BAD_INPUT
    ) unless ( $build_type eq 'release' );
}

foreach my $platform (@platforms) {
    my ( $os, $release, $format );

    if ( $platform =~ /^(\w+)\/(\w+)$/x ) {
        $os      = $1;
        $release = $2;
        $format  = ( any { $_ eq $os } @rh_flavors ) ? 'redhat' : 'debian';
    }
    else {
        $os      = 'pgxn';
        $release = '';
        $format  = 'pgxn';
    }

    my $filesubdir =
      ( $project eq 'local' ) ? $currentdir : $filedir . '/' . $format;

    # if this format's build files don't exist yet, fetch them
    unless ( -d $filesubdir ) {
        mkdir $filesubdir;
        my $downloadurl = PACKAGE_URL . "/$format-$project.tar.gz";
        my $download    = $filedir . '/' . "/$format-$project.tar.gz";
        my $httpcode = `curl -sL '$downloadurl' -w '%{http_code}' -o $download`;

        unless ( $httpcode == 200 ) {
            pod2usage(
                -msg =>
                  "Could not retrieve $format build files for '$project'.\n"
                  . "Received a $httpcode response from:\n"
                  . "  $downloadurl",
                -exitval => NO_SERVICE
            );
        }

        # expand them directly into the target directory
        my $untar  = "tar xf '$download' -C '$filesubdir' --strip-components 1";
        my $result = `$untar > /dev/null 2>&1`;

        if ( $? == -1 ) {
            die "failed to execute: $!\n";
        }
        elsif ( $? & 127 ) {
            die "child died with signal %d, %s coredump\n",
              ( $? & 127 ), ( $? & 128 ) ? 'with' : 'without';
        }
        else {
            my $exit_code = $? >> 8;
            die "build files tarball expansion failed!\n" if $exit_code;
        }
    }

    # read release and nightly PG versions from pkgvars file
    my @pg_versions;
    if ( $format eq 'redhat' ) {
        my $releasepg = '9.6,10';
        my $nightlypg;
        my $pkgvars = "$filesubdir/pkgvars";

        open( my $fh, '<', $pkgvars )    ## no critic (RequireBriefOpen)
          or die "opening pkgvars file failed!\n";
        while (<$fh>) {
            chomp;
            my ( $key, $value ) = split /\s*=\s*/x;
            if ( $key eq 'releasepg' ) {
                $releasepg = $value;
            }
            elsif ( $key eq 'nightlypg' ) {
                $nightlypg = $value;
            }
        }
        close($fh);

        $nightlypg //= $releasepg;

        my $finalpg = ( $build_type eq 'release' ) ? $releasepg : $nightlypg;
        $finalpg =~ s/(\d)\.?(\d)/pg$1$2/gx;

        @pg_versions = split( /,/x, $finalpg );
    }
    else {
        @pg_versions = qw (all);
    }

    my $docker_platform = $docker_names{$os};
    $docker_platform .= "-$release" if $release;

    my $outputdir = $pkgsdir . '/' . $docker_platform;

    # create output directory to ensure we, and not the docker user, own it
    mkdir $outputdir;

    foreach my $pg (@pg_versions) {
        if (($os eq "el" or $os eq "ol") and $release eq "6" and $pg eq "pg13")
        {
            # CentOS and OracleLinux 6 doesn't have pg13 packages yet
            # so we dont't have package builder docker images yet.
            # So skip building distro packages for them.
            next;
        }

        my @docker_args = (
            'run',
            '--rm',
            '-v',
            "$outputdir:/packages",
            '-v',
            "$filesubdir:/buildfiles:ro",
            '-e',
            "GITHUB_TOKEN",
            '-e',
            "PACKAGE_ENCRYPTION_KEY",
            '-e',
            "UNENCRYPTED_PACKAGE",
            "citus/packaging:$docker_platform-$pg",
            $build_type
        );

        system( 'docker', @docker_args );

        if ( $? == -1 ) {
            die "failed to execute: $!\n";
        }
        elsif ( $? & 127 ) {
            die "child died with signal %d, %s coredump\n",
              ( $? & 127 ), ( $? & 128 ) ? 'with' : 'without';
        }
        else {
            my $exit_code = $? >> 8;
            die "docker run failed. see output for details.\n" if $exit_code;
        }
    }
}

my ( $secret_key, $passphrase ) = get_signing_creds();

if ( $secret_key and $passphrase ) {
    sign_packages( $pkgsdir, $secret_key, $passphrase );
}
else {
    warn "Could not get signing credentials. Skipping signing...\n";
}

system( 'mv', ( ( glob "$pkgsdir/*" ), $currentdir ) );

__END__

=head1 NAME

citus_package - easily create OS packages for Citus projects

=head1 SYNOPSIS

B<citus_package> [I<options>] I<project> I<build_type>

=head1 DESCRIPTION


Packages a Citus project for one or more platforms and places the results in
platform-specific directories within the working directory. B<citus_package>
uses Docker under the hood to ensure repeatable builds, so a working Docker
installation is the only prerequisite.

Given a Citus I<project> and I<build_type>, B<citus_package> will build one
package for a single platform, specified using the B<--platform> option. This
option can be provided multiple times in order to build a package for many
platforms at once. In addition, if the special value I<local> is provided as
the project name, B<citus_package> will use packaging files from the current
directory rather than fetching any particular project's files from GitHub.

The I<build_type> argument has two special values: I<release> and I<nightly>. A
release build is based on the latest release version (extracted from the build
files contained within the C<citusdata/packaging> GitHub repository), pulling
code from the corresponding git tag, which must be have a GitHub-verified
signature. A nightly build is based on the latest commit to the "active" branch
for a given project, which is usually C<master>, but can differ by project.

All other I<build_type> values are passed directly to GitHub, which is free to
interpret them how it sees fit, e.g. branch names, tags, or commit identifiers.

B<citus_package> uses the GitHub API to gather information about the project
it is building. As such, a valid C<GITHUB_TOKEN> environment variable must be
set. See the L</ENVIRONMENT> section for details.

=head1 OPTIONS

=over 4

=item B<-p> I<os/release>, B<--platform=>I<os/release>

Platform: required. Provide more than once for multi-platform builds

=back

=head1 ENVIRONMENT

For B<citus_package> to do its job, the C<GITHUB_TOKEN> environment variable
must be populated with a valid GitHub personal access token. It is recommended
that you add a line to your shell profile to ensure this variable is always
correctly set.

To generate a new access token, ensure you're logged into GitHub, then navigate
to your account settings. Choose "Personal access tokens" from the sidebar,
press the "Generate new token" button and name your token (like "packaging").
Ensure the top-level C<repo> and C<user> boxes are checked and press the
"Generate token" button.

B<Do not leave the next page until you've copied your new token!> Paste it
into your e.g. C<.bash_profile> or C<.zshrc> to ensure your shells will have
access to your new token.

B<citus_package> will attempt to sign any Linux packages it builds if the
C<PACKAGING_SECRET_KEY> and C<PACKAGING_PASSPHRASE> environment variables are
set. C<PACKAGING_SECRET_KEY> must be a base64-encoded PGP private key. The
passphrase needed to decrypt this key must be specified in the other variable,
C<PACKAGING_PASSPHRASE>.

If L<gpg(1)> is installed, B<citus_package> will search its keychain for the
Citus Data private key so that only the C<PACKAGING_PASSPHRASE> will be needed
to sign packages.

=head1 COMMON PROJECTS

=over 4

=item I<citus>          Citus (Open-Source)

=item I<enterprise>     Citus Enterprise

=item I<hll>            HyperLogLog

=item I<rebalancer>     Shard Rebalancer

=back

=head1 SUPPORTED PLATFORMS

=over 4

=item I<debian/buster> Debian 10 "Buster"

=item I<debian/stretch> Debian 9 "Stretch"

=item I<debian/jessie>  Debian 8 "Jessie"

=item I<debian/wheezy>  Debian 7 "Wheezy"

=item I<el/8>           Enterprise Linux 8.0 (CentOS, RedHat, Amazon Linux)

=item I<el/7>           Enterprise Linux 7.0 (CentOS, RedHat, Amazon Linux)

=item I<el/6>           Enterprise Linux 6.0 (CentOS, RedHat, Amazon Linux)

=item I<fedora/28>      Fedora 28

=item I<fedora/27>      Fedora 27

=item I<fedora/26>      Fedora 26

=item I<ol/8>           Oracle Linux 8.0

=item I<ol/7>           Oracle Linux 7.0

=item I<ol/6>           Oracle Linux 6.0

=item I<ubuntu/focal>   Ubuntu 20.04 LTS (Focal Fossa)

=item I<ubuntu/bionic>  Ubuntu 18.04 LTS (Bionic Beaver)

=item I<ubuntu/xenial>  Ubuntu 16.04 LTS (Xenial Xerus)

=item I<ubuntu/trusty>  Ubuntu 14.04 LTS (Trusty Tahr)

=item I<pgxn>           PostgreSQL Extension Network (PGXN)

=back

=head1 TODO

Eventually support a different output folder.

=head1 CAVEATS

Projects other than Citus may not use the PGXN platform. When building a PGXN
Citus package, only the I<release> build type may be used.

=head1 SEE ALSO

=over 4

=item L<GitHub Personal Access Tokens|https://github.com/settings/tokens>

=item L<Citus Packaging on GitHub|https://github.com/citusdata/packaging>

=item L<Citus Packaging on Docker Hub|https://hub.docker.com/r/citus/packaging>

=item L<Docker|https://www.docker.com>

=back

=head1 AUTHOR

Jason Petersen L<E<lt>jason@citusdata.comE<gt>>