1. added bug fixes

2. added mirror maker configs
3. add splunk and newrelic configs

Author: Davinder Pal

NewRelic-Infra.md

@@ -0,0 +1,15 @@
+# NewRelic Infra Configuration
+
+* **Agent Version:** 1.30.0 or higher
+
+**Example**
+```
+license_key: 1cfxxxxxxxxxxxxxxxxxxxxxxxxxxx72cd
+collector_url: https://infra-api.newrelic.com
+display_name: xxxxxxxxxxxxxxxxx
+
+event_queue_depth: 5000
+
+custom_attributes:
+  label.env: xxx-kafka
+```

Readme.md

@@ -1,12 +1,12 @@
 # Apache Kafka Ansible
 
-It is group of playbooks to manage apache kafka. It is also 100% compliant with ansible-lint rules.
+It is group of playbooks to manage apache kafka in paysafe group. It is also 100% compliant with ansible-lint rules.
 
 ## **Requirements**
 * Download Apache Kafka Tar on Ansible Server ( Mandatory )
 * vagrant (optional)
 * Any OS with SystemD
-* ansible
+* Ansible
 
 ## **Notes***
 ```
@@ -21,6 +21,8 @@ https://github.com/116davinder/zookeeper-cluster-ansible
 
 # **Production Environment Setup**
 
+## Apache Kafka Playbooks
+
 ### **To start new cluster**
 * Update Required vars in ```inventory/<environment>/group_vars/all.yml``` .
 * Update Required vars in ```inventory/<environment>/cluster.ini``` .
@@ -79,13 +81,32 @@ https://github.com/116davinder/zookeeper-cluster-ansible
 
 ```ansible-playbook -i inventory/<environment>/cluster.ini clusterRemoveNodes.yml```
 
+## Apache Kafka Mirror Maker Playbooks
+
+### **To start new cluster**
+* Update Required vars in ```inventory/<environment>/group_vars/kafka-mirror-maker.yml``` .
+* Update Required vars in ```inventory/<environment>/mirror-maker.ini``` .
+
+```ansible-playbook -i inventory/<environment>/mirror-maker.ini clusterKafkaMirrorMaker.yml```
+
+### **To Remove nodes from cluster**
+* Update Required vars in ```inventory/<environment>/group_vars/kafka-mirror-maker.yml``` .
+* Update Required vars in ```inventory/<environment>/mirror-maker.ini``` .
+
+```ansible-playbook -i inventory/<environment>/mirror-maker.ini clusterKafkaMirrorMakerRemoveNodes.yml```
+
+### **Rolling restart cluster**
+
+```ansible-playbook -i inventory/<environment>/mirror-maker.ini clusterKafkaMirrorMakerRollingRestart.yml```
+
+
 ### **Tested OS**
 * CentOS 7
 * RedHat 7
 
 ### **Tested Ansible Version**
 ```
-ansible 2.7.10
+ansible 2.8.1
   config file = None
   configured module search path = ['/home/davinderpal/.ansible/plugins/modules', '/usr/share/ansible/plugins/modules']
   ansible python module location = /usr/local/lib/python3.6/dist-packages/ansible

Splunk-Config.md

@@ -0,0 +1,13 @@
+# Splunk Logging Configuration
+
+**Example**
+```
+[default]
+host = $HOSTNAME
+
+[monitor:///kafka/kafka-logs/*.log]
+disabled = false
+index = kafka
+sourcetype = kafka
+crcSalt = <SOURCE>
+```

clusterAddNodes.yml

@@ -23,8 +23,8 @@
       include_role:
         name: portCheck
       vars:
-        PortNumber: "{{ item }}"
-        PortStatus: started
+        portNumber: "{{ item }}"
+        portStatus: started
       loop:
         - "{{ kafkaPort }}"
         - "{{ kafkaJmxPort }}"

clusterBrokerProperties.yml

@@ -22,8 +22,8 @@
       include_role:
         name: portCheck
       vars:
-        PortNumber: "{{ item }}"
-        PortStatus: started
+        portNumber: "{{ item }}"
+        portStatus: started
       loop:
         - "{{ kafkaPort }}"
         - "{{ kafkaJmxPort }}"

clusterDownGradeVersion.yml

@@ -38,8 +38,8 @@
       include_role:
         name: portCheck
       vars:
-        PortNumber: "{{ item }}"
-        PortStatus: started
+        portNumber: "{{ item }}"
+        portStatus: started
       loop:
         - "{{ kafkaPort }}"
         - "{{ kafkaJmxPort }}"

clusterJvmConfigs.yml

@@ -20,8 +20,8 @@
       include_role:
         name: portCheck
       vars:
-        PortNumber: "{{ item }}"
-        PortStatus: started
+        portNumber: "{{ item }}"
+        portStatus: started
       loop:
         - "{{ kafkaPort }}"
         - "{{ kafkaJmxPort }}"

clusterKafkaMirrorMaker.yml

@@ -0,0 +1,40 @@
+---
+
+- hosts: kafka-mirror-maker
+  gather_facts: true
+  roles:
+    - java
+    - common
+    - install
+
+- hosts: kafka-mirror-maker
+  gather_facts: true
+  tasks:
+    - include_role:
+        name: configure
+        tasks_from: mirrorMaker.yml
+
+    - include_role:
+        name: service
+        tasks_from: kafka-mirror-maker.yml
+
+    - include_role:
+        name: serviceState
+      vars:
+        serviceName: "kafka-mirror-maker-{{ item }}"
+        serviceState: started
+      loop: "{{ range(1, kafkaMirrorMakerProcessCountPerNode + 1, 1) | list }}"
+
+    - include_role:
+        name: portCheck
+      vars:
+        portNumber: "{{ kafkaMirrorMakerJmxInitialPort + item }}"
+        portStatus: started
+      loop: "{{ range(1, kafkaMirrorMakerProcessCountPerNode + 1, 1) | list }}"
+
+- hosts: kafka-mirror-maker
+  gather_facts: true
+  tasks:
+    - include_role:
+        name: nri-kafka
+        tasks_from: kafka-mirror-maker.yml

clusterKafkaMirrorMakerRemoveNodes.yml

@@ -0,0 +1,10 @@
+---
+
+- hosts: kafka-mirror-maker-remove-nodes
+  gather_facts: true
+  any_errors_fatal: true
+  serial: 1
+  tasks:
+    - include_role:
+        name: decomissionNodes
+        tasks_from: kafka-mirror-maker.yml

clusterKafkaMirrorMakerRollingRestart.yml

@@ -0,0 +1,19 @@
+---
+
+- hosts: kafka-mirror-maker
+  gather_facts: true
+  serial: 1
+  tasks:
+    - include_role:
+        name: serviceState
+      vars:
+        serviceName: "kafka-mirror-maker-{{ item }}"
+        serviceState: restarted
+      loop: "{{ range(1, kafkaMirrorMakerProcessCountPerNode + 1, 1) | list }}"
+
+    - include_role:
+        name: portCheck
+      vars:
+        portNumber: "{{ kafkaMirrorMakerJmxInitialPort + item }}"
+        portStatus: started
+      loop: "{{ range(1, kafkaMirrorMakerProcessCountPerNode + 1, 1) | list }}"

clusterLogging.yml

@@ -22,8 +22,8 @@
       include_role:
         name: portCheck
       vars:
-        PortNumber: "{{ item }}"
-        PortStatus: started
+        portNumber: "{{ item }}"
+        portStatus: started
       loop:
         - "{{ kafkaPort }}"
         - "{{ kafkaJmxPort }}"

clusterRollingRestart.yml

@@ -15,8 +15,8 @@
       include_role:
         name: portCheck
       vars:
-        PortNumber: "{{ item }}"
-        PortStatus: started
+        portNumber: "{{ item }}"
+        portStatus: started
       loop:
         - "{{ kafkaPort }}"
         - "{{ kafkaJmxPort }}"

clusterSetup.yml

@@ -25,8 +25,8 @@
       include_role:
         name: portCheck
       vars:
-        PortNumber: "{{ item }}"
-        PortStatus: started
+        portNumber: "{{ item }}"
+        portStatus: started
       loop:
         - "{{ kafkaPort }}"
         - "{{ kafkaJmxPort }}"

clusterUpgrade.yml

@@ -29,11 +29,6 @@
 - hosts: clusterNodes:clusterAddNodes
   gather_facts: false
   serial: 1
-  pre_tasks:
-    - name: Switching Kafka Version to {{ kafkaVersion }} on all nodes (Rolling fashion)
-      pause:
-        prompt: "press enter to confirm or ctrl-c to cancel"
-      run_once: true
   tasks:
     - name: switching kafka version to latest
       import_role:
@@ -51,8 +46,8 @@
       include_role:
         name: portCheck
       vars:
-        PortNumber: "{{ item }}"
-        PortStatus: started
+        portNumber: "{{ item }}"
+        portStatus: started
       loop:
         - "{{ kafkaPort }}"
         - "{{ kafkaJmxPort }}"

inventory/development/group_vars/kafka-mirror-maker.yml

@@ -0,0 +1,30 @@
+---
+
+# Common vars
+kafkaMirrorMakerReplicateTopicWhiteListPattern: ".*kafka-perf-test.*"   # java regex based pattern
+kafkaMirrorMakerHeapSize: 2G
+kafkaMirrorMakerProcessCountPerNode: 2
+kafkaMirrorMakerJmxInitialPort: 9980        # Port will be used like 9981 & 9982 if kafkaMirrorMakerProcessCountPerNode is set 2
+
+# Kafka Mirror Maker Consumer Settings
+kafkaMirrorMakerSourceDcClusterAddressString: 192.168.56.100:9092,192.168.56.101:9092,192.168.56.102:9092           #no need to put all the hosts here
+kafkaMirrorMakerGroupId: "kafka-mirror-maker-consumer-{{ kafkaClusterName }}"
+kafkaMirrorMakerAutoOffsetReset: "earliest"
+kafkaMirrorMakerPartitionAssignmentStrategy: "org.apache.kafka.clients.consumer.RoundRobinAssignor"
+kafkaMirrorMakerFetchMaxBytes: 52428800
+kafkaMirrorMakerFetchMinBytes: 1000012
+kafkaMirrorMakerFetchMaxWaitMs: 500
+
+# Kafka Mirror Maker Producer Settings
+kafkaMirrorMakerDestinationDcClusterAddressString: 192.168.56.100:9092,192.168.56.101:9092,192.168.56.102:9092     #no need to put all the hosts here
+kafkaMirrorMakerCompressionType: "none"
+kafkaMirrorMakerPartitionerClass: "org.apache.kafka.clients.producer.internals.DefaultPartitioner"
+kafkaMirrorMakerRequestTimeoutMs: 30000
+kafkaMirrorMakerMaxBlockMs: 60000
+kafkaMirrorMakerLingerMs: 15000
+kafkaMirrorMakerMaxRequestSize: 1000000
+kafkaMirrorMakerBatchSize: 500000
+kafkaMirrorMakerBufferMemory: 200000000 
+kafkaMirrorMakerMaxInFlightRequestsPerConnection: 5
+kafkaMirrorMakerClientId: "kafka-mirror-maker-producer-{{ kafkaClusterName }}"
+kafkaMirrorMakerAcks: "all"

inventory/development/mirror-maker.ini

@@ -0,0 +1,10 @@
+[all:vars]
+ansible_connection=ssh 
+ansible_become_method=sudo 
+ansible_become=true
+
+[kafka-mirror-maker]
+10.0.0.[1:3]
+
+[kafka-mirror-maker-remove-nodes]
+10.0.0.[1:3]

roles/configure/tasks/mirrorMaker.yml

@@ -0,0 +1,30 @@
+---
+
+- name: Creating kafka Mirror Maker configurations | properties files
+  template:
+    src: "{{ item }}"
+    dest: "{{ kafkaInstallDir }}/kafka_{{ kafkaScalaVersion }}-{{ kafkaVersion }}/config/{{ item }}"
+    owner: "{{ kafkaUser }}"
+    group: "{{ kafkaGroup }}"
+    mode: 0775
+  loop:
+    - kafka-mirror-maker-consumer.properties
+    - kafka-mirror-maker-producer.properties
+
+- name: Creating kafka Mirror Maker configurations | properties files
+  template:
+    src: "tools-log4j.properties"
+    dest: "{{ kafkaInstallDir }}/kafka_{{ kafkaScalaVersion }}-{{ kafkaVersion }}/config/kafka-mirror-log4j-{{ item }}.properties"
+    owner: "{{ kafkaUser }}"
+    group: "{{ kafkaGroup }}"
+    mode: 0775
+  loop: "{{ range(1, kafkaMirrorMakerProcessCountPerNode + 1, 1) | list }}"
+
+- name: Creating kafka Mirror Maker configurations | kafka-mirror-maker.sh
+  template:
+    src: kafka-mirror-maker.sh
+    dest: "{{ kafkaInstallDir }}/kafka_{{ kafkaScalaVersion }}-{{ kafkaVersion }}/bin/kafka-mirror-maker-{{ item }}.sh"
+    owner: "{{ kafkaUser }}"
+    group: "{{ kafkaGroup }}"
+    mode: 0775
+  loop: "{{ range(1, kafkaMirrorMakerProcessCountPerNode + 1, 1) | list }}"

roles/configure/templates/kafka-mirror-maker-consumer.properties

@@ -0,0 +1,14 @@
+bootstrap.servers={{ kafkaMirrorMakerSourceDcClusterAddressString }}
+
+# consumer group id
+group.id={{ kafkaMirrorMakerGroupId }}
+
+# What to do when there is no initial offset in Kafka or if the current
+# offset does not exist any more on the server: latest, earliest, none
+auto.offset.reset={{ kafkaMirrorMakerAutoOffsetReset }}
+partition.assignment.strategy={{ kafkaMirrorMakerPartitionAssignmentStrategy }}
+fetch.max.bytes={{ kafkaMirrorMakerFetchMaxBytes }}
+fetch.min.bytes={{ kafkaMirrorMakerFetchMinBytes }}
+fetch.max.wait.ms={{ kafkaMirrorMakerFetchMaxWaitMs }}
+exclude.internal.topics=true
+allow.auto.create.topics=false

roles/configure/templates/kafka-mirror-maker-producer.properties

@@ -0,0 +1,33 @@
+############################# Producer Basics #############################
+
+bootstrap.servers={{ kafkaMirrorMakerDestinationDcClusterAddressString }}
+
+# specify the compression codec for all data generated: none, gzip, snappy, lz4, zstd
+compression.type={{ kafkaMirrorMakerCompressionType }}
+
+# name of the partitioner class for partitioning events; default partition spreads data randomly
+partitioner.class={{ kafkaMirrorMakerPartitionerClass }}
+
+# the maximum amount of time the client will wait for the response of a request
+request.timeout.ms={{ kafkaMirrorMakerRequestTimeoutMs }}
+
+# how long `KafkaProducer.send` and `KafkaProducer.partitionsFor` will block for
+max.block.ms={{ kafkaMirrorMakerMaxBlockMs }}
+
+# the producer will wait for up to the given delay to allow other records to be sent so that the sends can be batched together
+linger.ms={{ kafkaMirrorMakerLingerMs }}
+
+# the maximum size of a request in bytes
+max.request.size={{ kafkaMirrorMakerMaxRequestSize }}
+
+# the default batch size in bytes when batching multiple records sent to a partition
+batch.size={{ kafkaMirrorMakerBatchSize }}
+
+# the total bytes of memory the producer can use to buffer records waiting to be sent to the server
+buffer.memory={{ kafkaMirrorMakerBufferMemory }} 
+
+max.in.flight.requests.per.connection={{ kafkaMirrorMakerMaxInFlightRequestsPerConnection }}
+
+client.id={{ kafkaMirrorMakerClientId }}
+
+acks={{ kafkaMirrorMakerAcks }}