From 0773382bc7068701772f7eb2badcaedeca9c9f46 Mon Sep 17 00:00:00 2001 From: Phillip Gomez Date: Tue, 27 Sep 2016 21:11:13 -0700 Subject: [PATCH 1/2] etcd: Add static cluster configuration option Currently the only way to configure an etcd cluster is by using the discovery service. This may not always be an option especially for offline scenarios. An option 'enable_discovery_service' is added to allow for the enabling/disabling of using the discovery service versus using a static cluster configuration. This option is defaulted to 'true' to maintain the use of the discovery service as the default. When the discovery service option is disabled the etcd cluster will be statically configured and will be comprised of each instance. Additionally static configuration requires that each etcd member be named explicitly. This means that each node requires a 'user-data' file that has the etcd member name set. When the discovery service option is disabled a 'user-data' file for each instance will be created. The name of each of these files will be 'user-data' appended with the instance id (ex. user-data-01) and will have the etcd member name set to the instance name (ex. core-01). --- .gitignore | 1 + Vagrantfile | 10 ++++-- config.rb.sample | 79 ++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 72 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 54c68c85..ea092aba 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .vagrant/ log/ user-data +user-data-* config.rb diff --git a/Vagrantfile b/Vagrantfile index 9948ebdc..97b104bd 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -21,6 +21,7 @@ $vm_cpus = 1 $vb_cpuexecutioncap = 100 $shared_folders = {} $forwarded_ports = {} +$enable_discovery_service = true # Attempt to apply the deprecated environment variable NUM_INSTANCES to # $num_instances while allowing config.rb to override it @@ -137,8 +138,13 @@ Vagrant.configure("2") do |config| config.vm.synced_folder ENV['HOME'], ENV['HOME'], id: "home", :nfs => true, :mount_options => ['nolock,vers=3,udp'] end - if File.exist?(CLOUD_CONFIG_PATH) - config.vm.provision :file, :source => "#{CLOUD_CONFIG_PATH}", :destination => "/tmp/vagrantfile-user-data" + # A single user-data file is used when using a discovery service. When statically configuring + # the cluster there will be a user-data file for each node with the node instance id appended (ex. user-data-01) + if $enable_discovery_service && File.exist?(CLOUD_CONFIG_PATH) + config.vm.provision :file, :source => CLOUD_CONFIG_PATH, :destination => "/tmp/vagrantfile-user-data" + config.vm.provision :shell, :inline => "mv /tmp/vagrantfile-user-data /var/lib/coreos-vagrant/", :privileged => true + elsif File.exist?("#{CLOUD_CONFIG_PATH}-%02d" % i) + config.vm.provision :file, :source => "#{CLOUD_CONFIG_PATH}-%02d" % i, :destination => "/tmp/vagrantfile-user-data" config.vm.provision :shell, :inline => "mv /tmp/vagrantfile-user-data /var/lib/coreos-vagrant/", :privileged => true end diff --git a/config.rb.sample b/config.rb.sample index b5173bc9..07517c3c 100644 --- a/config.rb.sample +++ b/config.rb.sample @@ -1,25 +1,59 @@ # Size of the CoreOS cluster created by Vagrant $num_instances=1 +# Change basename of the VM +# The default value is "core", which results in VMs named starting with +# "core-01" through to "core-${num_instances}". +#$instance_name_prefix="core" + +# Enables the use of the etcd discovery service. The default is enabled. +# When disabled the cluster will be statically defined with each instance +# as a member. +#$enable_discovery_service=true + # Used to fetch a new discovery token for a cluster of size $num_instances $new_discovery_url="https://discovery.etcd.io/new?size=#{$num_instances}" -# Automatically replace the discovery token on 'vagrant up' - if File.exists?('user-data') && ARGV[0].eql?('up') require 'open-uri' require 'yaml' - token = open($new_discovery_url).read - data = YAML.load(IO.readlines('user-data')[1..-1].join) - if data.key? 'coreos' and data['coreos'].key? 'etcd' - data['coreos']['etcd']['discovery'] = token - end + # Automatically replace the discovery token on 'vagrant up' + if $enable_discovery_service + token = open($new_discovery_url).read + + if data.key? 'coreos' and data['coreos'].key? 'etcd' + data['coreos']['etcd']['discovery'] = token + end + + if data.key? 'coreos' and data['coreos'].key? 'etcd2' + data['coreos']['etcd2']['discovery'] = token + end + else + # Static cluster configuration explicitly configures the initial cluster members + initial_cluster = (1..$num_instances).map{|i| + ip = "172.17.8.#{i+100}" + "%s-%02d=http://%s:2380" % [$instance_name_prefix, i, ip] + }.join(",") + + if data.key? 'coreos' and data['coreos'].key? 'etcd' + data['coreos']['etcd']['initial-cluster'] = initial_cluster + end + + if data.key? 'coreos' and data['coreos'].key? 'etcd2' + data['coreos']['etcd2']['initial-cluster'] = initial_cluster + end - if data.key? 'coreos' and data['coreos'].key? 'etcd2' - data['coreos']['etcd2']['discovery'] = token + # Remove discovery token if it exists + if data.key? 'coreos' and data['coreos'].key? 'etcd' and data['coreos']['etcd'].key? 'discovery' + data['coreos']['etcd'].delete 'discovery' + end + + if data.key? 'coreos' and data['coreos'].key? 'etcd2' and data['coreos']['etcd2'].key? 'discovery' + data['coreos']['etcd2'].delete 'discovery' + end end # Fix for YAML.load() converting reboot-strategy from 'off' to `false` @@ -29,8 +63,26 @@ if File.exists?('user-data') && ARGV[0].eql?('up') end end - yaml = YAML.dump(data) - File.open('user-data', 'w') { |file| file.write("#cloud-config\n\n#{yaml}") } + if $enable_discovery_service + yaml = YAML.dump(data) + File.open('user-data', 'w') { |file| file.write("#cloud-config\n\n#{yaml}") } + else + # When static configuration is used a name must be specified. + # This requires a unique user-data file for each node. The user-data files + # will have the node instance id appended. (ex. user-data-01) + (1..$num_instances).each do |i| + if data.key? 'coreos' and data['coreos'].key? 'etcd' + data['coreos']['etcd']['name'] = "%s-%02d" % [$instance_name_prefix, i] + end + + if data.key? 'coreos' and data['coreos'].key? 'etcd2' + data['coreos']['etcd2']['name'] = "%s-%02d" % [$instance_name_prefix, i] + end + + yaml = YAML.dump(data) + File.open("user-data-%02d" % i, 'w') { |file| file.write("#cloud-config\n\n#{yaml}") } + end + end end # @@ -40,11 +92,6 @@ end # uncomment the necessary lines, leaving the $, and replace everything # after the equals sign.. -# Change basename of the VM -# The default value is "core", which results in VMs named starting with -# "core-01" through to "core-${num_instances}". -#$instance_name_prefix="core" - # Change the version of CoreOS to be installed # To deploy a specific version, simply set $image_version accordingly. # For example, to deploy version 709.0.0, set $image_version="709.0.0". From 2e70734c8778f0b007b507470533dcff5c227463 Mon Sep 17 00:00:00 2001 From: Phillip Gomez Date: Wed, 28 Sep 2016 13:11:51 -0700 Subject: [PATCH 2/2] etcd: Fix etcd v1 static configuration Incorrectly assumed that etcd v1 and v2 static configuration was the same. In v1 a peers list should be provided that is just a list of the peers ip and port. In addition to a peers list an initial leader must be selected by not having a peers list. This was accomplished by deleting the peers list for the last instance of the cluster. --- config.rb.sample | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/config.rb.sample b/config.rb.sample index 07517c3c..c4db9403 100644 --- a/config.rb.sample +++ b/config.rb.sample @@ -33,15 +33,23 @@ if File.exists?('user-data') && ARGV[0].eql?('up') end else # Static cluster configuration explicitly configures the initial cluster members - initial_cluster = (1..$num_instances).map{|i| + + # etcd v1 uses peers list + peers = (1..$num_instances).map{|i| ip = "172.17.8.#{i+100}" - "%s-%02d=http://%s:2380" % [$instance_name_prefix, i, ip] + "%s:7001" % ip }.join(",") if data.key? 'coreos' and data['coreos'].key? 'etcd' - data['coreos']['etcd']['initial-cluster'] = initial_cluster + data['coreos']['etcd']['peers'] = peers end + # etcd2 uses initial cluster key-value list + initial_cluster = (1..$num_instances).map{|i| + ip = "172.17.8.#{i+100}" + "%s-%02d=http://%s:2380" % [$instance_name_prefix, i, ip] + }.join(",") + if data.key? 'coreos' and data['coreos'].key? 'etcd2' data['coreos']['etcd2']['initial-cluster'] = initial_cluster end @@ -79,6 +87,14 @@ if File.exists?('user-data') && ARGV[0].eql?('up') data['coreos']['etcd2']['name'] = "%s-%02d" % [$instance_name_prefix, i] end + # etcd v1 requires that a single node to not have a peer list. + # Choosing last instance to be initial leader. + if (i == $num_instances) + if data.key? 'coreos' and data['coreos'].key? 'etcd' and data['coreos']['etcd'].key? 'peers' + data['coreos']['etcd'].delete 'peers' + end + end + yaml = YAML.dump(data) File.open("user-data-%02d" % i, 'w') { |file| file.write("#cloud-config\n\n#{yaml}") } end