Merge branch 'release/0.6.4'

CHANGELOG.md

@@ -1,3 +1,11 @@
+### citustools v0.6.4 (August 15, 2017) ###
+
+* Bumps Homebrew formula
+
+* Adds automation script for valgrind tests
+
+* Starts to use REL_10_STABLE branch for PostgreSQL tests
+
 ### citustools v0.6.3 (July 11, 2017) ###
 
 * Addresses citus-style compatibility issues with uncrustify 0.65

HomebrewFormula/citustools.rb

@@ -12,8 +12,8 @@ def message
 class Citustools < Formula
   desc "Tools and config used in Citus Data projects."
   homepage "https://github.com/citusdata/tools"
-  url "https://github.com/citusdata/tools/archive/v0.6.0.tar.gz"
-  sha256 "85644f4910e17ed378748d930cf86e98b9316243467c8a6f009de8001a5bdbff"
+  url "https://github.com/citusdata/tools/archive/v0.6.3.tar.gz"
+  sha256 "5e2dc61136be940f436911ad17908bb48d8c6362ed8407d32c405b5a8e982d72"
 
   depends_on "uncrustify"
   depends_on Docker

Makefile

@@ -7,7 +7,7 @@ export mandir := $(datarootdir)/man
 export sysconfdir := $(prefix)/etc
 export pkgsysconfdir := $(sysconfdir)/$(PACKAGE_NAME)
 
-DIRNAMES = packaging uncrustify
+DIRNAMES = packaging uncrustify valgrind
 ifeq ($(TRAVIS), true)
 	DIRNAMES += travis
 endif

travis/install_custom_pg

@@ -16,10 +16,12 @@ cd ~
 if [ "$(ls -A postgresql)" ]; then
   git -C postgresql pull
 else
-  if [ "${PGVERSION}" -eq '10' ]; then
-    gitref='master'
+  if [ "${PGVERSION}" == '11' ]; then
+    gitref="master"
+  elif [ "${PGVERSION}" == '9.6' ]; then
+    gitref="REL9_6_STABLE"
   else
-    gitref="REL${PGVERSION//./_}_STABLE"
+    gitref="REL_${PGVERSION}_STABLE"
   fi
 
   git clone -b "${gitref}" --depth 1 git://git.postgresql.org/git/postgresql.git

valgrind/Makefile

@@ -0,0 +1,15 @@
+# needed variables will be passed in via top-level Makefile
+
+INSTALL := install -c
+INSTALL_SCRIPT := $(INSTALL) -m 755
+VALGRIND_SCRIPTS := $(filter-out README.md Makefile,$(wildcard *))
+
+all:
+
+clean:
+	rm -f $(VALGRIND_SCRIPTS)
+
+install: all
+	$(INSTALL_SCRIPT) $(VALGRIND_SCRIPTS) $(DESTDIR)$(bindir)
+
+.PHONY: clean installdirs install

valgrind/README.md

@@ -0,0 +1,13 @@
+# valgrind-test-automation
+
+Tools for automating Valgrind tests with Citus. This automation suite will open an instance in AWS and runs Citus with Valgrind. When tests are completed, it will send a mail to [email protected] and [email protected] with the valgrind logs. Logs will contain memory related problems and the call stack where each problem is found. If there are no problems, no report will be attached. Only a success message will be sent.
+
+# Usage
+
+You first need to install and configure aws command line client. Refer [here](http://docs.aws.amazon.com/cli/latest/userguide/installing.html) to see how to install aws command line client. After installation, configure it with `aws configure` command. Then to use aws instance for valgrind tests, run;
+
+```sh
+./launch-test-instance.sh
+```
+
+This command will create a special key pair and start an m3.xlarge instance with that key pair. Created instance will be tagged with Name=ValgrindTest. Then, scripts will build PostgreSQL and Citus from source and runs Citus' regression tests with Valgrind. After tests are completed, valgrind logs will be sent to [email protected] and [email protected] for now and instance will terminate itself. It is expected that the tests will take about 5 hours to complete.

valgrind/download-test-scripts

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# we send this script to aws while creating the instance, this script will run
+# as final step of the instance creation and download necessary scripts to run
+# valgrind tests from our repository.
+
+set -euo pipefail
+
+# download and install required packages
+apt-get update
+apt-get install git make -y
+
+# download the test scripts
+git clone https://github.com/citusdata/tools.git
+cd tools
+make install

valgrind/launch-test-instance

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# create a key pair just for valgrind tests and store it in valgrind-test.pem
+echo "Creating key pair..."
+key_name=valgrind_$RANDOM
+aws ec2 create-key-pair --key-name $key_name --query 'KeyMaterial' --output text > $key_name.pem
+chmod 600 $key_name.pem
+
+# start an instance with ami-f4cc1de2 image(i.e. Ubuntu Xenial 16.04)
+echo "Starting an instance..."
+valgrind_instance_id=$(aws ec2 run-instances \
+    --image-id ami-f4cc1de2 \
+    --count 1 \
+    --instance-type m3.xlarge \
+    --key-name $key_name \
+    --instance-initiated-shutdown-behavior terminate \
+    --user-data file://download-test-scripts \
+    --query 'Instances[0].InstanceId' \
+    --output text)
+
+# tag the instance as ValgrindTest
+echo "Tagging the instance..."
+aws ec2 create-tags \
+    --resources "${valgrind_instance_id}" \
+    --tags Key=Name,Value=ValgrindTest
+
+# wait for instance creation complete
+echo "Waiting for completion of instance creation... (This may take a several minutes)"
+aws ec2 wait instance-status-ok \
+    --instance-id "${valgrind_instance_id}"
+
+# get the instance ip address
+echo "Getting the ip address of the instance..."
+valgrind_instance_ip=$(aws ec2 describe-instances \
+    --instance-ids "${valgrind_instance_id}" \
+    --query 'Reservations[0].Instances[0].PublicIpAddress' \
+    --output text)
+
+# run valgrind tests
+echo "Running the valgrind tests..."
+echo "This will take hours, test results will be sent via e-mail."
+ssh -o StrictHostKeyChecking=no -i $key_name.pem ubuntu@$valgrind_instance_ip "screen -d -m run-valgrind-tests"
+
+#delete the key pair after we are done with tests
+aws ec2 delete-key-pair --key-name $key_name

valgrind/run-valgrind-tests

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# download and install required packages
+sudo apt-get update
+sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq \
+    build-essential \
+    libreadline6 \
+    libreadline6-dev \
+    zlib1g-dev \
+    flex \
+    bison \
+    libssl-dev \
+    valgrind \
+    mailutils
+
+# set environment variables
+export LC_ALL=en_US.UTF-8
+export LANG=en_US.UTF-8
+export LANGUAGE=en_US.UTF-8
+export PG_CONFIG=/usr/local/pgsql/bin/pg_config
+
+# download and install PostgreSQL
+git clone -b "REL9_6_STABLE" --depth 1 git://git.postgresql.org/git/postgresql.git
+cd postgresql/
+./configure --enable-cassert --enable-debug CFLAGS="-ggdb -Og -DUSE_VALGRIND"
+
+# we will use this to parallelize PostgreSQL compilation
+procs="$(nproc)"
+mjobs="$((procs + 1))"
+make -j "${mjobs}" -s
+sudo make install
+export PATH=/usr/local/pgsql/bin:$PATH
+
+# download and install Citus
+cd ..
+git clone https://github.com/citusdata/citus.git
+cd citus/
+./configure
+make clean
+make -j8 -s
+sudo make install
+
+# this is necessary to start tests
+sudo chown ubuntu /usr/local/pgsql/bin/ -R
+
+# run valgrind tests
+cd src/test/regress
+make check-multi-vg VALGRIND_LOG_FILE=logs.txt || true
+
+# surprisingly this hits inbox
+if [ -s logs.txt ]; then
+	mail -aFrom:[email protected] -s "[Valgrind Test Results] - Failure" -A logs.txt [email protected] [email protected] < /dev/null
+else
+	mail -aFrom:[email protected] -s "[Valgrind Test Results] - Success" [email protected] [email protected] < /dev/null
+fi
+
+# just to ensure everything is completed in the test instance
+sleep 30
+
+# shut-down the instance, this will also terminate the instance because we set instance-initiated-shutdown-behavior to terminate
+sudo shutdown -h now