diff --git a/CHANGELOG.md b/CHANGELOG.md index eaa79564..e8b61a85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,16 @@ -### citustools v0.7.11 (Aughust 18, 2019) ### +### citustools v0.7.13 (October 09, 2019) ### + +* Introduces `citus_dev` + +* Adds Debian Bionic and EL/OL 8 to supported packaging versions + +* Changes the docker hub org from CitusData to Citus for packaging + +### citustools v0.7.12 (September 02, 2019) ### + +* Add support for testing against PostgreSQL 12 + +### citustools v0.7.11 (August 18, 2019) ### * Bump Homebrew formula diff --git a/citus_dev/README.md b/citus_dev/README.md new file mode 100644 index 00000000..3c5a861e --- /dev/null +++ b/citus_dev/README.md @@ -0,0 +1,26 @@ +# Setup +To install dependencies run: +```bash +pip install -r requirements.txt +``` + +Add `citus_dev` to your PATH: +```bash +export PATH=$PATH: +``` + +You can also add this to your profile: +```bash +echo 'export PATH=$PATH:' >>~/.profile +``` + +After that, you can use the citus dev tool: + +```bash +citus_dev make clusterName +``` + +For the full command list: +```bash +citus_dev --help +``` diff --git a/citus_dev/bash_completion b/citus_dev/bash_completion new file mode 100644 index 00000000..b8c02fc6 --- /dev/null +++ b/citus_dev/bash_completion @@ -0,0 +1,52 @@ +# citus_dev completion + +_citus_dev() +{ + local cmd cur prev cmds opts clusters + COMPREPLY=() + _get_comp_words_by_ref cur prev + cmd="${COMP_WORDS[1]}" + cmds="make start stop restart" + + case $prev in + make) + return 0 + ;; + + start|stop|restart) + clusters=$(for x in `find . -type d -name 'coordinator' | awk -F '/' '{print $2}'`; do echo ${x} ; done ); + COMPREPLY=($( compgen -W "${clusters}" -- "${cur}" )); + return 0 + ;; + esac + + case $cmd in + make) + opts="--size --port --use-ssl --no-extension --mx" + if [[ ${cur} == -* ]] ; then + COMPREPLY=($( compgen -W "${opts}" -- "${cur}")) + fi + return 0 + ;; + + start|stop) + opts="--port" + if [[ ${cur} == -* ]] ; then + COMPREPLY=($( compgen -W "${opts}" -- "${cur}")) + fi + return 0 + ;; + + restart) + opts="--port --watch" + if [[ ${cur} == -* ]] ; then + COMPREPLY=($( compgen -W "${opts}" -- "${cur}")) + fi + return 0 + ;; + esac + + COMPREPLY=($( compgen -W "${cmds}" -- "${cur}")) + return 0 +} +complete -F _citus_dev citus_dev diff --git a/citus_dev/citus_dev b/citus_dev/citus_dev new file mode 100755 index 00000000..4479a7ff --- /dev/null +++ b/citus_dev/citus_dev @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""citus_dev + +Usage: + citus_dev make [--size=] [--port=] [--use-ssl] [--no-extension] [--mx] [--destroy] [--init-with=] + citus_dev restart [--watch] [--port=] + citus_dev (start|stop) [--port=] + +Options: + --size= Number of workers to create when 0 the coordinator will be added as a worker [default: 2] + --port= Port number to use for the coordinator. All workers take subsequent numbers [default: 9700] + --watch Watch for changes to the citus plugin and restart the cluster when the plugin updates + --use-ssl Create the cluster with ssl enabled + --no-extension Do not create the extension while creating the nodes + --mx Enable metadata sync for all workers + --destroy Destroy any old cluster with the same name + --init-with= A SQL script to run after creation of the cluster to set up any necessary tables and data + +""" +from docopt import docopt +from subprocess import call +from subprocess import Popen, PIPE +import os +import sys +import getpass + + +def createNodeCommands(clustername, role, index=None, usessl=False, mx=False): + cs = [] + + nodename = role + if index != None: + nodename += "%d" % index + + dir = "%s/%s" % (clustername, nodename) + cs.append("initdb -D %s" % dir) + cs.append("echo \"shared_preload_libraries = 'citus'\" >> %s/postgresql.conf" % dir) + cs.append('echo "wal_level = logical" >> %s/postgresql.conf' % dir) + + if usessl: + cs.append('echo "ssl = on" >> %s/postgresql.conf' % dir) + cs.append( + "echo \"citus.node_conninfo = 'sslmode=require'\" >> %s/postgresql.conf" + % dir + ) + cs.append( + "openssl req -new -x509 -days 365 -nodes -text -out %s/server.crt -keyout %s/server.key -subj '/CN=%s'" + % (dir, dir, nodename) + ) + cs.append("chmod 0600 %s/server.key" % dir) + + if mx: + cs.append( + "echo \"citus.replication_model = 'streaming'\" >> %s/postgresql.conf" % dir + ) + + return cs + + +def main(arguments): + print(arguments) + if arguments["make"]: + cs = [] + if arguments['--destroy']: + name = arguments[""] + for role in getRoles(name): + cs.append("pg_ctl stop -D %s/%s" % (name, role)) + cs.append('rm -rf %s' % (name)) + + cs += createNodeCommands( + arguments[""], + "coordinator", + usessl=arguments["--use-ssl"], + mx=arguments["--mx"], + ) + + size = int(arguments["--size"]) + + for i in range(size): + cs += createNodeCommands( + arguments[""], + "worker", + i, + usessl=arguments["--use-ssl"], + mx=arguments["--mx"], + ) + + port = int(arguments["--port"]) + + cport = port + role = "coordinator" + cs.append( + 'pg_ctl -D %s/%s -o "-p %d" -l %s_logfile start' + % (arguments[""], role, cport, role) + ) + port += 1 + + for i in range(size): + role = "worker%d" % i + cs.append( + 'pg_ctl start -D %s/%s -o "-p %d" -l %s_logfile' + % (arguments[""], role, port, role) + ) + port += 1 + port = cport + + if getpass.getuser() != 'postgres' and not os.getenv('PGDATABASE'): + for i in range(size + 1): + cs.append('createdb -p %d' % (port + i)) + + if not arguments["--no-extension"]: + for i in range(size + 1): + cs.append('psql -p %d -c "CREATE EXTENSION citus;"' % (port + i)) + + # If the cluster size is 0 we add the coordinator as the only node, otherwise we will add all other nodes + if size == 0: + cs.append( + "psql -p %d -c \"SELECT * from master_add_node('localhost', %d);\"" + % (port, port) + ) + else: + for i in range(size): + cs.append( + "psql -p %d -c \"SELECT * from master_add_node('localhost', %d);\"" + % (port, port + 1 + i) + ) + if arguments["--mx"]: + cs.append( + "psql -p %d -c \"SELECT start_metadata_sync_to_node('localhost', %d);\"" + % (port, port + 1 + i) + ) + + cs.append( + 'psql -p %d -c "SELECT * from master_get_active_worker_nodes();"' + % (port) + ) + if arguments['--init-with']: + cs.append('psql -p %d -f %s' % (cport, arguments['--init-with'])) + + for c in cs: + print(c) + os.system(c) + print("") + + elif arguments["stop"]: + cs = [] + name = arguments[""] + for role in getRoles(name): + cs.append("pg_ctl stop -D %s/%s" % (name, role)) + + for c in cs: + print(c) + os.system(c) + print("") + + elif arguments["start"]: + cs = [] + name = arguments[""] + port = int(arguments["--port"]) + cport = port + for role in getRoles(name): + cs.append( + 'pg_ctl start -D %s/%s -o "-p %d" -l %s_logfile' + % (name, role, cport, role) + ) + cport += 1 + + for c in cs: + print(c) + os.system(c) + print("") + + elif arguments["restart"]: + cs = [] + name = arguments[""] + port = int(arguments["--port"]) + if arguments["--watch"]: + cs.append( + "fswatch -0 '%s' | xargs -0 -n 1 -I{} citus_dev restart %s --port=%d" + % (citus_so(), name, port) + ) + + else: + cport = port + for role in getRoles(name): + cs.append( + 'pg_ctl restart -D %s/%s -o "-p %d" -l %s_logfile' + % (name, role, cport, role) + ) + cport += 1 + + for c in cs: + print(c) + os.system(c) + print("") + + else: + print("unknown command") + exit(1) + + +def getRoles(name): + roles = [f.name for f in os.scandir(name) if f.is_dir()] + roles.sort() + return roles + + +def pg_libdir(): + process = Popen(["pg_config"], stdout=PIPE) + (output, err) = process.communicate() + exit_code = process.wait() + + output = str(output) + + for line in output.split("\\n"): + if line.startswith("LIBDIR"): + return line.split("=", 1)[1].strip() + + raise Exception("can't find postgres lib dir") + + +def citus_so(): + return pg_libdir() + "/citus.so" + + +if __name__ == "__main__": + print(sys.argv) + main(docopt(__doc__, version="citus_dev")) diff --git a/citus_dev/requirements.txt b/citus_dev/requirements.txt new file mode 100644 index 00000000..1d9dff81 --- /dev/null +++ b/citus_dev/requirements.txt @@ -0,0 +1 @@ +docopt==0.6.2 diff --git a/packaging/citus_package b/packaging/citus_package index 04a97961..572dc917 100755 --- a/packaging/citus_package +++ b/packaging/citus_package @@ -25,10 +25,10 @@ use constant PACKAGE_URL => ## no critic (ProhibitConstantPragma) 'https://github.com/citusdata/packaging/archive'; my %supported_platforms = ( - debian => [ "stretch", "jessie", "wheezy"], - el => [ "7", "6" ], + debian => [ "buster", "stretch", "jessie", "wheezy"], + el => [ "8", "7", "6" ], fedora => [ "28", "27", "26"], - ol => [ "7", "6" ], + ol => [ "8", "7", "6" ], ubuntu => [ "bionic", "xenial", "trusty" ], pgxn => [""] ); @@ -369,7 +369,7 @@ foreach my $platform (@platforms) { "$filesubdir:/buildfiles:ro", '-e', "GITHUB_TOKEN", - "citusdata/packaging:$docker_platform-$pg", + "citus/packaging:$docker_platform-$pg", $build_type ); @@ -494,12 +494,16 @@ to sign packages. =over 4 +=item I Debian 10 "Buster" + =item I Debian 9 "Stretch" =item I Debian 8 "Jessie" =item I Debian 7 "Wheezy" +=item I Enterprise Linux 8.0 (CentOS, RedHat, Amazon Linux) + =item I Enterprise Linux 7.0 (CentOS, RedHat, Amazon Linux) =item I Enterprise Linux 6.0 (CentOS, RedHat, Amazon Linux) @@ -510,6 +514,8 @@ to sign packages. =item I Fedora 26 +=item I
    Oracle Linux 8.0 + =item I
      Oracle Linux 7.0 =item I
        Oracle Linux 6.0 @@ -541,7 +547,7 @@ Citus package, only the I build type may be used. =item L -=item L +=item L =item L