From 9420ff671c4b87b26cf67c56f71ebcbbf5e15528 Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Mon, 29 Oct 2018 14:51:22 +0100 Subject: [PATCH 01/14] add simple tool to create and manage local citus clusters --- citus_dev/citus_dev | 135 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100755 citus_dev/citus_dev diff --git a/citus_dev/citus_dev b/citus_dev/citus_dev new file mode 100755 index 00000000..4643e851 --- /dev/null +++ b/citus_dev/citus_dev @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""citus_dev + +Usage: + citus_dev make [--size=] [--port=] + citus_dev restart [--watch] [--port=] + citus_dev (start|stop) [--port=] + +Options: + --size= Number of workers to create when 0 the coordinator will be added as a worker [default: 2] + --port= Port number to use for the coordinator. All workers take subsequent numbers [default: 9700] + --watch Watch for changes to the citus plugin and restart the cluster when the plugin updates + +""" +from docopt import docopt +from subprocess import call +from subprocess import Popen, PIPE +import os +import sys + +def main(arguments): + print(arguments) + if arguments["make"]: + cs = [] + dir = "%s/coordinator" % arguments[""] + cs.append("initdb -D %s" % dir) + cs.append("echo \"shared_preload_libraries = 'citus'\" >> %s/postgresql.conf" % dir) + + size = int(arguments['--size']) + + for i in range(size): + dir = "%s/worker%d" % (arguments[""],i) + cs.append("initdb -D %s" % dir) + cs.append("echo \"shared_preload_libraries = 'citus'\" >> %s/postgresql.conf" % dir) + cs.append("echo \"wal_level = logical\" >> %s/postgresql.conf" % dir) + + port = int(arguments['--port']) + + + cport = port + role = "coordinator" + cs.append('pg_ctl -D %s/%s -o "-p %d" -l %s_logfile start' % (arguments[""], role, cport, role)) + cport += 1 + + for i in range(size): + role = "worker%d" % i + cs.append('pg_ctl start -D %s/%s -o "-p %d" -l %s_logfile' % (arguments[""], role, cport, role)) + cport += 1 + + for i in range(size+1): + cs.append('psql -p %d -c "CREATE EXTENSION citus;"'%(port+i)) + + # If the cluster size is 0 we add the coordinator as the only node, otherwise we will add all other nodes + if size == 0: + cs.append("psql -p %d -c \"SELECT * from master_add_node('localhost', %d);\"" % (port, port)) + else: + for i in range(size): + cs.append("psql -p %d -c \"SELECT * from master_add_node('localhost', %d);\"" % (port, port + 1 + i)) + + cs.append("psql -p %d -c \"SELECT * from master_get_active_worker_nodes();\"" % (port)) + + for c in cs: + print(c) + os.system(c) + print("") + + elif arguments["stop"]: + cs = [] + name = arguments[""] + for role in getRoles(name): + cs.append("pg_ctl stop -D %s/%s"%(name, role)) + + for c in cs: + print(c) + os.system(c) + print("") + + elif arguments["start"]: + cs = [] + name = arguments[""] + port = int(arguments["--port"]) + cport = port + for role in getRoles(name): + cs.append('pg_ctl start -D %s/%s -o "-p %d" -l %s_logfile' % (name, role, cport, role)) + cport += 1 + + for c in cs: + print(c) + os.system(c) + print("") + + elif arguments["restart"]: + cs = [] + name = arguments[""] + port = int(arguments["--port"]) + if arguments["--watch"]: + cs.append("fswatch -0 '%s' | xargs -0 -n 1 -I{} citus_dev restart %s --port=%d"%(citus_so(),name,port)) + + else: + cport = port + for role in getRoles(name): + cs.append('pg_ctl restart -D %s/%s -o "-p %d" -l %s_logfile' % (name, role, cport, role)) + cport += 1 + + for c in cs: + print(c) + os.system(c) + print("") + + else: + print("unknown command") + exit(1) + +def getRoles(name): + return [f.name for f in os.scandir(name) if f.is_dir() ] + +def pg_libdir(): + process = Popen(["pg_config"], stdout=PIPE) + (output, err) = process.communicate() + exit_code = process.wait() + + output = str(output) + + for line in output.split('\\n'): + if line.startswith('LIBDIR'): + return line.split('=',1)[1].strip() + + raise Exception("can't find postgres lib dir") + +def citus_so(): + return pg_libdir() + "/citus.so" + +if __name__ == '__main__': + print(sys.argv) + main(docopt(__doc__, version='citus_dev')) \ No newline at end of file From f5562998b2d49fffac09d4f9857cc49a2d03e585 Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Wed, 3 Jul 2019 12:57:18 +0200 Subject: [PATCH 02/14] add extra options --use-ssl and --no-extension --- citus_dev/citus_dev | 57 +++++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/citus_dev/citus_dev b/citus_dev/citus_dev index 4643e851..3b0e8420 100755 --- a/citus_dev/citus_dev +++ b/citus_dev/citus_dev @@ -2,7 +2,7 @@ """citus_dev Usage: - citus_dev make [--size=] [--port=] + citus_dev make [--size=] [--port=] [--use-ssl] [--no-extension] citus_dev restart [--watch] [--port=] citus_dev (start|stop) [--port=] @@ -10,6 +10,8 @@ Options: --size= Number of workers to create when 0 the coordinator will be added as a worker [default: 2] --port= Port number to use for the coordinator. All workers take subsequent numbers [default: 9700] --watch Watch for changes to the citus plugin and restart the cluster when the plugin updates + --use-ssl Create the cluster with ssl enabled + --no-extension Do not create the extension while creating the nodes """ from docopt import docopt @@ -18,21 +20,37 @@ from subprocess import Popen, PIPE import os import sys +def createNodeCommands(clustername, role, index=None, usessl=False): + cs = [] + + nodename = role + if index != None: + nodename += "%d" % index + + dir = "%s/%s" % (clustername, nodename) + cs.append("initdb -D %s" % dir) + cs.append("echo \"shared_preload_libraries = 'citus'\" >> %s/postgresql.conf" % dir) + cs.append("echo \"wal_level = logical\" >> %s/postgresql.conf" % dir) + + if usessl: + cs.append("echo \"ssl = on\" >> %s/postgresql.conf" % dir) + cs.append("echo \"citus.node_conninfo = 'sslmode=require'\" >> %s/postgresql.conf" % dir) + cs.append("openssl req -new -x509 -days 365 -nodes -text -out %s/server.crt -keyout %s/server.key -subj '/CN=%s'" % (dir, dir, nodename)) + cs.append("chmod 0600 %s/server.key" % dir) + + return cs + def main(arguments): print(arguments) if arguments["make"]: cs = [] - dir = "%s/coordinator" % arguments[""] - cs.append("initdb -D %s" % dir) - cs.append("echo \"shared_preload_libraries = 'citus'\" >> %s/postgresql.conf" % dir) + + cs += createNodeCommands(arguments[""], "coordinator", usessl=arguments["--use-ssl"]) size = int(arguments['--size']) for i in range(size): - dir = "%s/worker%d" % (arguments[""],i) - cs.append("initdb -D %s" % dir) - cs.append("echo \"shared_preload_libraries = 'citus'\" >> %s/postgresql.conf" % dir) - cs.append("echo \"wal_level = logical\" >> %s/postgresql.conf" % dir) + cs += createNodeCommands(arguments[""], "worker", i, usessl=arguments["--use-ssl"]) port = int(arguments['--port']) @@ -47,17 +65,18 @@ def main(arguments): cs.append('pg_ctl start -D %s/%s -o "-p %d" -l %s_logfile' % (arguments[""], role, cport, role)) cport += 1 - for i in range(size+1): - cs.append('psql -p %d -c "CREATE EXTENSION citus;"'%(port+i)) + if not arguments["--no-extension"]: + for i in range(size+1): + cs.append('psql -p %d -c "CREATE EXTENSION citus;"'%(port+i)) - # If the cluster size is 0 we add the coordinator as the only node, otherwise we will add all other nodes - if size == 0: - cs.append("psql -p %d -c \"SELECT * from master_add_node('localhost', %d);\"" % (port, port)) - else: - for i in range(size): - cs.append("psql -p %d -c \"SELECT * from master_add_node('localhost', %d);\"" % (port, port + 1 + i)) + # If the cluster size is 0 we add the coordinator as the only node, otherwise we will add all other nodes + if size == 0: + cs.append("psql -p %d -c \"SELECT * from master_add_node('localhost', %d);\"" % (port, port)) + else: + for i in range(size): + cs.append("psql -p %d -c \"SELECT * from master_add_node('localhost', %d);\"" % (port, port + 1 + i)) - cs.append("psql -p %d -c \"SELECT * from master_get_active_worker_nodes();\"" % (port)) + cs.append("psql -p %d -c \"SELECT * from master_get_active_worker_nodes();\"" % (port)) for c in cs: print(c) @@ -112,7 +131,9 @@ def main(arguments): exit(1) def getRoles(name): - return [f.name for f in os.scandir(name) if f.is_dir() ] + roles = [ f.name for f in os.scandir(name) if f.is_dir() ] + roles.sort() + return roles def pg_libdir(): process = Popen(["pg_config"], stdout=PIPE) From 05af47c91ea7786929be00d5514e8b28e27c5e36 Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Wed, 3 Jul 2019 13:02:00 +0200 Subject: [PATCH 03/14] add mx support to citus_dev --- citus_dev/citus_dev | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/citus_dev/citus_dev b/citus_dev/citus_dev index 3b0e8420..e615c5e0 100755 --- a/citus_dev/citus_dev +++ b/citus_dev/citus_dev @@ -2,7 +2,7 @@ """citus_dev Usage: - citus_dev make [--size=] [--port=] [--use-ssl] [--no-extension] + citus_dev make [--size=] [--port=] [--use-ssl] [--no-extension] [--mx] citus_dev restart [--watch] [--port=] citus_dev (start|stop) [--port=] @@ -12,6 +12,7 @@ Options: --watch Watch for changes to the citus plugin and restart the cluster when the plugin updates --use-ssl Create the cluster with ssl enabled --no-extension Do not create the extension while creating the nodes + --mx Enable metadata sync for all workers """ from docopt import docopt @@ -75,6 +76,8 @@ def main(arguments): else: for i in range(size): cs.append("psql -p %d -c \"SELECT * from master_add_node('localhost', %d);\"" % (port, port + 1 + i)) + if arguments['--mx']: + cs.append("psql -p %d -c \"SELECT start_metadata_sync_to_node('localhost', %d);\"" % (port, port + 1 + i)) cs.append("psql -p %d -c \"SELECT * from master_get_active_worker_nodes();\"" % (port)) From 973e53d1b9ccac7944392f59adbd0b55b6485d18 Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Wed, 3 Jul 2019 15:45:01 +0200 Subject: [PATCH 04/14] add bash completion script --- citus_dev/bash_completion | 52 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 citus_dev/bash_completion diff --git a/citus_dev/bash_completion b/citus_dev/bash_completion new file mode 100644 index 00000000..b8c02fc6 --- /dev/null +++ b/citus_dev/bash_completion @@ -0,0 +1,52 @@ +# citus_dev completion + +_citus_dev() +{ + local cmd cur prev cmds opts clusters + COMPREPLY=() + _get_comp_words_by_ref cur prev + cmd="${COMP_WORDS[1]}" + cmds="make start stop restart" + + case $prev in + make) + return 0 + ;; + + start|stop|restart) + clusters=$(for x in `find . -type d -name 'coordinator' | awk -F '/' '{print $2}'`; do echo ${x} ; done ); + COMPREPLY=($( compgen -W "${clusters}" -- "${cur}" )); + return 0 + ;; + esac + + case $cmd in + make) + opts="--size --port --use-ssl --no-extension --mx" + if [[ ${cur} == -* ]] ; then + COMPREPLY=($( compgen -W "${opts}" -- "${cur}")) + fi + return 0 + ;; + + start|stop) + opts="--port" + if [[ ${cur} == -* ]] ; then + COMPREPLY=($( compgen -W "${opts}" -- "${cur}")) + fi + return 0 + ;; + + restart) + opts="--port --watch" + if [[ ${cur} == -* ]] ; then + COMPREPLY=($( compgen -W "${opts}" -- "${cur}")) + fi + return 0 + ;; + esac + + COMPREPLY=($( compgen -W "${cmds}" -- "${cur}")) + return 0 +} +complete -F _citus_dev citus_dev From 38579459547dae351a254017d469813b813908ee Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Thu, 4 Jul 2019 13:03:36 +0200 Subject: [PATCH 05/14] set replication mode to streaming for mx --- citus_dev/citus_dev | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/citus_dev/citus_dev b/citus_dev/citus_dev index e615c5e0..770df120 100755 --- a/citus_dev/citus_dev +++ b/citus_dev/citus_dev @@ -21,7 +21,7 @@ from subprocess import Popen, PIPE import os import sys -def createNodeCommands(clustername, role, index=None, usessl=False): +def createNodeCommands(clustername, role, index=None, usessl=False, mx=False): cs = [] nodename = role @@ -39,6 +39,9 @@ def createNodeCommands(clustername, role, index=None, usessl=False): cs.append("openssl req -new -x509 -days 365 -nodes -text -out %s/server.crt -keyout %s/server.key -subj '/CN=%s'" % (dir, dir, nodename)) cs.append("chmod 0600 %s/server.key" % dir) + if mx: + cs.append("echo \"citus.replication_model = 'streaming'\" >> %s/postgresql.conf" % dir) + return cs def main(arguments): @@ -46,12 +49,12 @@ def main(arguments): if arguments["make"]: cs = [] - cs += createNodeCommands(arguments[""], "coordinator", usessl=arguments["--use-ssl"]) + cs += createNodeCommands(arguments[""], "coordinator", usessl=arguments["--use-ssl"], mx=arguments["--mx"]) size = int(arguments['--size']) for i in range(size): - cs += createNodeCommands(arguments[""], "worker", i, usessl=arguments["--use-ssl"]) + cs += createNodeCommands(arguments[""], "worker", i, usessl=arguments["--use-ssl"], mx=arguments["--mx"]) port = int(arguments['--port']) From 1157a3666ebc843457bef05dd1bd85923c8dfe7d Mon Sep 17 00:00:00 2001 From: Jelte Fennema Date: Mon, 12 Aug 2019 16:20:13 +0200 Subject: [PATCH 06/14] Autoformat citus_dev with black --- citus_dev/citus_dev | 293 ++++++++++++++++++++++++++------------------ 1 file changed, 172 insertions(+), 121 deletions(-) diff --git a/citus_dev/citus_dev b/citus_dev/citus_dev index 770df120..18fee5e6 100755 --- a/citus_dev/citus_dev +++ b/citus_dev/citus_dev @@ -21,142 +21,193 @@ from subprocess import Popen, PIPE import os import sys + def createNodeCommands(clustername, role, index=None, usessl=False, mx=False): - cs = [] + cs = [] + + nodename = role + if index != None: + nodename += "%d" % index - nodename = role - if index != None: - nodename += "%d" % index + dir = "%s/%s" % (clustername, nodename) + cs.append("initdb -D %s" % dir) + cs.append("echo \"shared_preload_libraries = 'citus'\" >> %s/postgresql.conf" % dir) + cs.append('echo "wal_level = logical" >> %s/postgresql.conf' % dir) - dir = "%s/%s" % (clustername, nodename) - cs.append("initdb -D %s" % dir) - cs.append("echo \"shared_preload_libraries = 'citus'\" >> %s/postgresql.conf" % dir) - cs.append("echo \"wal_level = logical\" >> %s/postgresql.conf" % dir) + if usessl: + cs.append('echo "ssl = on" >> %s/postgresql.conf' % dir) + cs.append( + "echo \"citus.node_conninfo = 'sslmode=require'\" >> %s/postgresql.conf" + % dir + ) + cs.append( + "openssl req -new -x509 -days 365 -nodes -text -out %s/server.crt -keyout %s/server.key -subj '/CN=%s'" + % (dir, dir, nodename) + ) + cs.append("chmod 0600 %s/server.key" % dir) - if usessl: - cs.append("echo \"ssl = on\" >> %s/postgresql.conf" % dir) - cs.append("echo \"citus.node_conninfo = 'sslmode=require'\" >> %s/postgresql.conf" % dir) - cs.append("openssl req -new -x509 -days 365 -nodes -text -out %s/server.crt -keyout %s/server.key -subj '/CN=%s'" % (dir, dir, nodename)) - cs.append("chmod 0600 %s/server.key" % dir) + if mx: + cs.append( + "echo \"citus.replication_model = 'streaming'\" >> %s/postgresql.conf" % dir + ) - if mx: - cs.append("echo \"citus.replication_model = 'streaming'\" >> %s/postgresql.conf" % dir) + return cs - return cs def main(arguments): - print(arguments) - if arguments["make"]: - cs = [] - - cs += createNodeCommands(arguments[""], "coordinator", usessl=arguments["--use-ssl"], mx=arguments["--mx"]) - - size = int(arguments['--size']) - - for i in range(size): - cs += createNodeCommands(arguments[""], "worker", i, usessl=arguments["--use-ssl"], mx=arguments["--mx"]) - - port = int(arguments['--port']) - - - cport = port - role = "coordinator" - cs.append('pg_ctl -D %s/%s -o "-p %d" -l %s_logfile start' % (arguments[""], role, cport, role)) - cport += 1 - - for i in range(size): - role = "worker%d" % i - cs.append('pg_ctl start -D %s/%s -o "-p %d" -l %s_logfile' % (arguments[""], role, cport, role)) - cport += 1 - - if not arguments["--no-extension"]: - for i in range(size+1): - cs.append('psql -p %d -c "CREATE EXTENSION citus;"'%(port+i)) - - # If the cluster size is 0 we add the coordinator as the only node, otherwise we will add all other nodes - if size == 0: - cs.append("psql -p %d -c \"SELECT * from master_add_node('localhost', %d);\"" % (port, port)) - else: - for i in range(size): - cs.append("psql -p %d -c \"SELECT * from master_add_node('localhost', %d);\"" % (port, port + 1 + i)) - if arguments['--mx']: - cs.append("psql -p %d -c \"SELECT start_metadata_sync_to_node('localhost', %d);\"" % (port, port + 1 + i)) - - cs.append("psql -p %d -c \"SELECT * from master_get_active_worker_nodes();\"" % (port)) - - for c in cs: - print(c) - os.system(c) - print("") - - elif arguments["stop"]: - cs = [] - name = arguments[""] - for role in getRoles(name): - cs.append("pg_ctl stop -D %s/%s"%(name, role)) - - for c in cs: - print(c) - os.system(c) - print("") - - elif arguments["start"]: - cs = [] - name = arguments[""] - port = int(arguments["--port"]) - cport = port - for role in getRoles(name): - cs.append('pg_ctl start -D %s/%s -o "-p %d" -l %s_logfile' % (name, role, cport, role)) - cport += 1 - - for c in cs: - print(c) - os.system(c) - print("") - - elif arguments["restart"]: - cs = [] - name = arguments[""] - port = int(arguments["--port"]) - if arguments["--watch"]: - cs.append("fswatch -0 '%s' | xargs -0 -n 1 -I{} citus_dev restart %s --port=%d"%(citus_so(),name,port)) - - else: - cport = port - for role in getRoles(name): - cs.append('pg_ctl restart -D %s/%s -o "-p %d" -l %s_logfile' % (name, role, cport, role)) - cport += 1 - - for c in cs: - print(c) - os.system(c) - print("") - - else: - print("unknown command") - exit(1) + print(arguments) + if arguments["make"]: + cs = [] + + cs += createNodeCommands( + arguments[""], + "coordinator", + usessl=arguments["--use-ssl"], + mx=arguments["--mx"], + ) + + size = int(arguments["--size"]) + + for i in range(size): + cs += createNodeCommands( + arguments[""], + "worker", + i, + usessl=arguments["--use-ssl"], + mx=arguments["--mx"], + ) + + port = int(arguments["--port"]) + + cport = port + role = "coordinator" + cs.append( + 'pg_ctl -D %s/%s -o "-p %d" -l %s_logfile start' + % (arguments[""], role, cport, role) + ) + cport += 1 + + for i in range(size): + role = "worker%d" % i + cs.append( + 'pg_ctl start -D %s/%s -o "-p %d" -l %s_logfile' + % (arguments[""], role, cport, role) + ) + cport += 1 + + if not arguments["--no-extension"]: + for i in range(size + 1): + cs.append('psql -p %d -c "CREATE EXTENSION citus;"' % (port + i)) + + # If the cluster size is 0 we add the coordinator as the only node, otherwise we will add all other nodes + if size == 0: + cs.append( + "psql -p %d -c \"SELECT * from master_add_node('localhost', %d);\"" + % (port, port) + ) + else: + for i in range(size): + cs.append( + "psql -p %d -c \"SELECT * from master_add_node('localhost', %d);\"" + % (port, port + 1 + i) + ) + if arguments["--mx"]: + cs.append( + "psql -p %d -c \"SELECT start_metadata_sync_to_node('localhost', %d);\"" + % (port, port + 1 + i) + ) + + cs.append( + 'psql -p %d -c "SELECT * from master_get_active_worker_nodes();"' + % (port) + ) + + for c in cs: + print(c) + os.system(c) + print("") + + elif arguments["stop"]: + cs = [] + name = arguments[""] + for role in getRoles(name): + cs.append("pg_ctl stop -D %s/%s" % (name, role)) + + for c in cs: + print(c) + os.system(c) + print("") + + elif arguments["start"]: + cs = [] + name = arguments[""] + port = int(arguments["--port"]) + cport = port + for role in getRoles(name): + cs.append( + 'pg_ctl start -D %s/%s -o "-p %d" -l %s_logfile' + % (name, role, cport, role) + ) + cport += 1 + + for c in cs: + print(c) + os.system(c) + print("") + + elif arguments["restart"]: + cs = [] + name = arguments[""] + port = int(arguments["--port"]) + if arguments["--watch"]: + cs.append( + "fswatch -0 '%s' | xargs -0 -n 1 -I{} citus_dev restart %s --port=%d" + % (citus_so(), name, port) + ) + + else: + cport = port + for role in getRoles(name): + cs.append( + 'pg_ctl restart -D %s/%s -o "-p %d" -l %s_logfile' + % (name, role, cport, role) + ) + cport += 1 + + for c in cs: + print(c) + os.system(c) + print("") + + else: + print("unknown command") + exit(1) + def getRoles(name): - roles = [ f.name for f in os.scandir(name) if f.is_dir() ] - roles.sort() - return roles + roles = [f.name for f in os.scandir(name) if f.is_dir()] + roles.sort() + return roles + def pg_libdir(): - process = Popen(["pg_config"], stdout=PIPE) - (output, err) = process.communicate() - exit_code = process.wait() + process = Popen(["pg_config"], stdout=PIPE) + (output, err) = process.communicate() + exit_code = process.wait() + + output = str(output) - output = str(output) + for line in output.split("\\n"): + if line.startswith("LIBDIR"): + return line.split("=", 1)[1].strip() - for line in output.split('\\n'): - if line.startswith('LIBDIR'): - return line.split('=',1)[1].strip() + raise Exception("can't find postgres lib dir") - raise Exception("can't find postgres lib dir") def citus_so(): - return pg_libdir() + "/citus.so" + return pg_libdir() + "/citus.so" + -if __name__ == '__main__': - print(sys.argv) - main(docopt(__doc__, version='citus_dev')) \ No newline at end of file +if __name__ == "__main__": + print(sys.argv) + main(docopt(__doc__, version="citus_dev")) From cac7c7aa30a36b24dd7e87b45862ff16ea01eeb7 Mon Sep 17 00:00:00 2001 From: Jelte Fennema Date: Mon, 19 Aug 2019 14:20:05 +0200 Subject: [PATCH 07/14] citus_dev: Add a --destroy option to make --- citus_dev/citus_dev | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/citus_dev/citus_dev b/citus_dev/citus_dev index 18fee5e6..67fb5574 100755 --- a/citus_dev/citus_dev +++ b/citus_dev/citus_dev @@ -2,7 +2,7 @@ """citus_dev Usage: - citus_dev make [--size=] [--port=] [--use-ssl] [--no-extension] [--mx] + citus_dev make [--size=] [--port=] [--use-ssl] [--no-extension] [--mx] [--destroy] citus_dev restart [--watch] [--port=] citus_dev (start|stop) [--port=] @@ -13,6 +13,7 @@ Options: --use-ssl Create the cluster with ssl enabled --no-extension Do not create the extension while creating the nodes --mx Enable metadata sync for all workers + --destroy Destroy any old cluster with the same name """ from docopt import docopt @@ -58,6 +59,11 @@ def main(arguments): print(arguments) if arguments["make"]: cs = [] + if arguments['--destroy']: + name = arguments[""] + for role in getRoles(name): + cs.append("pg_ctl stop -D %s/%s" % (name, role)) + cs.append('rm -rf %s' % (name)) cs += createNodeCommands( arguments[""], From cf6b00b956faf02f73e274c8dfc64408f13a9dc3 Mon Sep 17 00:00:00 2001 From: Jelte Fennema Date: Mon, 19 Aug 2019 16:04:40 +0200 Subject: [PATCH 08/14] citus_dev: Add an --init-with flag --- citus_dev/citus_dev | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/citus_dev/citus_dev b/citus_dev/citus_dev index 67fb5574..eabfa7b0 100755 --- a/citus_dev/citus_dev +++ b/citus_dev/citus_dev @@ -2,18 +2,19 @@ """citus_dev Usage: - citus_dev make [--size=] [--port=] [--use-ssl] [--no-extension] [--mx] [--destroy] + citus_dev make [--size=] [--port=] [--use-ssl] [--no-extension] [--mx] [--destroy] [--init-with=] citus_dev restart [--watch] [--port=] citus_dev (start|stop) [--port=] Options: - --size= Number of workers to create when 0 the coordinator will be added as a worker [default: 2] - --port= Port number to use for the coordinator. All workers take subsequent numbers [default: 9700] - --watch Watch for changes to the citus plugin and restart the cluster when the plugin updates - --use-ssl Create the cluster with ssl enabled - --no-extension Do not create the extension while creating the nodes - --mx Enable metadata sync for all workers - --destroy Destroy any old cluster with the same name + --size= Number of workers to create when 0 the coordinator will be added as a worker [default: 2] + --port= Port number to use for the coordinator. All workers take subsequent numbers [default: 9700] + --watch Watch for changes to the citus plugin and restart the cluster when the plugin updates + --use-ssl Create the cluster with ssl enabled + --no-extension Do not create the extension while creating the nodes + --mx Enable metadata sync for all workers + --destroy Destroy any old cluster with the same name + --init-with= A SQL script to run after creation of the cluster to set up any necessary tables and data """ from docopt import docopt @@ -91,15 +92,17 @@ def main(arguments): 'pg_ctl -D %s/%s -o "-p %d" -l %s_logfile start' % (arguments[""], role, cport, role) ) - cport += 1 + port += 1 for i in range(size): role = "worker%d" % i cs.append( 'pg_ctl start -D %s/%s -o "-p %d" -l %s_logfile' - % (arguments[""], role, cport, role) + % (arguments[""], role, port, role) ) - cport += 1 + port += 1 + port = cport + if not arguments["--no-extension"]: for i in range(size + 1): @@ -127,6 +130,8 @@ def main(arguments): 'psql -p %d -c "SELECT * from master_get_active_worker_nodes();"' % (port) ) + if arguments['--init-with']: + cs.append('psql -p %d -f %s' % (cport, arguments['--init-with'])) for c in cs: print(c) From 77f9214d864a2b9a9cf80e0f6e44495835de6a94 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 2 Sep 2019 22:36:04 +0300 Subject: [PATCH 09/14] Add requirements file for dependencies in citus_dev tool --- citus_dev/requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 citus_dev/requirements.txt diff --git a/citus_dev/requirements.txt b/citus_dev/requirements.txt new file mode 100644 index 00000000..1d9dff81 --- /dev/null +++ b/citus_dev/requirements.txt @@ -0,0 +1 @@ +docopt==0.6.2 From d01bf87f4a04e4531f30bcd046e136811ec41aa7 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 2 Sep 2019 23:34:55 +0300 Subject: [PATCH 10/14] Add readme for citus_dev tool --- citus_dev/README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 citus_dev/README.md diff --git a/citus_dev/README.md b/citus_dev/README.md new file mode 100644 index 00000000..3c5a861e --- /dev/null +++ b/citus_dev/README.md @@ -0,0 +1,26 @@ +# Setup +To install dependencies run: +```bash +pip install -r requirements.txt +``` + +Add `citus_dev` to your PATH: +```bash +export PATH=$PATH: +``` + +You can also add this to your profile: +```bash +echo 'export PATH=$PATH:' >>~/.profile +``` + +After that, you can use the citus dev tool: + +```bash +citus_dev make clusterName +``` + +For the full command list: +```bash +citus_dev --help +``` From 3bf9b42cc54a424834e30afb51da8c3153105202 Mon Sep 17 00:00:00 2001 From: Ekin Dursun Date: Thu, 10 Oct 2019 13:16:38 +0300 Subject: [PATCH 11/14] Make citus_dev compatible with any user (#53) If the user is not postgres, a new db created with user's name on every node. This allows that citus_dev can be used with any user. --- citus_dev/citus_dev | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/citus_dev/citus_dev b/citus_dev/citus_dev index eabfa7b0..4479a7ff 100755 --- a/citus_dev/citus_dev +++ b/citus_dev/citus_dev @@ -22,6 +22,7 @@ from subprocess import call from subprocess import Popen, PIPE import os import sys +import getpass def createNodeCommands(clustername, role, index=None, usessl=False, mx=False): @@ -103,6 +104,9 @@ def main(arguments): port += 1 port = cport + if getpass.getuser() != 'postgres' and not os.getenv('PGDATABASE'): + for i in range(size + 1): + cs.append('createdb -p %d' % (port + i)) if not arguments["--no-extension"]: for i in range(size + 1): From 45cb9a45e491b7449d4a44a94e002d0ed165e963 Mon Sep 17 00:00:00 2001 From: Hanefi Onaldi Date: Wed, 9 Oct 2019 13:46:34 +0300 Subject: [PATCH 12/14] Add new distro support to citus_package --- packaging/citus_package | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packaging/citus_package b/packaging/citus_package index 04a97961..309d86ce 100755 --- a/packaging/citus_package +++ b/packaging/citus_package @@ -25,10 +25,10 @@ use constant PACKAGE_URL => ## no critic (ProhibitConstantPragma) 'https://github.com/citusdata/packaging/archive'; my %supported_platforms = ( - debian => [ "stretch", "jessie", "wheezy"], - el => [ "7", "6" ], + debian => [ "buster", "stretch", "jessie", "wheezy"], + el => [ "8", "7", "6" ], fedora => [ "28", "27", "26"], - ol => [ "7", "6" ], + ol => [ "8", "7", "6" ], ubuntu => [ "bionic", "xenial", "trusty" ], pgxn => [""] ); @@ -494,12 +494,16 @@ to sign packages. =over 4 +=item I Debian 10 "Buster" + =item I Debian 9 "Stretch" =item I Debian 8 "Jessie" =item I Debian 7 "Wheezy" +=item I Enterprise Linux 8.0 (CentOS, RedHat, Amazon Linux) + =item I Enterprise Linux 7.0 (CentOS, RedHat, Amazon Linux) =item I Enterprise Linux 6.0 (CentOS, RedHat, Amazon Linux) @@ -510,6 +514,8 @@ to sign packages. =item I Fedora 26 +=item I
    Oracle Linux 8.0 + =item I
      Oracle Linux 7.0 =item I
        Oracle Linux 6.0 From 22e3b990e622684f7e5d319524d0044676ebc69a Mon Sep 17 00:00:00 2001 From: Hanefi Onaldi Date: Wed, 9 Oct 2019 13:47:15 +0300 Subject: [PATCH 13/14] Update changelog for 0.7.12 and 0.7.13 --- CHANGELOG.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eaa79564..e8b61a85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,16 @@ -### citustools v0.7.11 (Aughust 18, 2019) ### +### citustools v0.7.13 (October 09, 2019) ### + +* Introduces `citus_dev` + +* Adds Debian Bionic and EL/OL 8 to supported packaging versions + +* Changes the docker hub org from CitusData to Citus for packaging + +### citustools v0.7.12 (September 02, 2019) ### + +* Add support for testing against PostgreSQL 12 + +### citustools v0.7.11 (August 18, 2019) ### * Bump Homebrew formula From 5b24139220e2f83c9548ec10797cf7bb16c74902 Mon Sep 17 00:00:00 2001 From: Hanefi Onaldi Date: Wed, 9 Oct 2019 13:48:05 +0300 Subject: [PATCH 14/14] Change docker hub org to citus --- packaging/citus_package | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/citus_package b/packaging/citus_package index 309d86ce..572dc917 100755 --- a/packaging/citus_package +++ b/packaging/citus_package @@ -369,7 +369,7 @@ foreach my $platform (@platforms) { "$filesubdir:/buildfiles:ro", '-e', "GITHUB_TOKEN", - "citusdata/packaging:$docker_platform-$pg", + "citus/packaging:$docker_platform-$pg", $build_type ); @@ -547,7 +547,7 @@ Citus package, only the I build type may be used. =item L -=item L +=item L =item L