[kafka-metrics] Update dependencies, align with OTel/DD best practices

apps/kafka-metrics/.env.example

@@ -0,0 +1,6 @@
+# Required: Your Datadog API key
+DD_API_KEY=your_api_key_here
+
+# Required: Your Datadog site (default: datadoghq.com)
+# Options: datadoghq.com, datadoghq.eu, us3.datadoghq.com, us5.datadoghq.com, ap1.datadoghq.com
+DD_SITE=datadoghq.com

apps/kafka-metrics/Dockerfile.collector

@@ -1,13 +1,13 @@
-FROM alpine:latest as prep
+FROM alpine:latest AS prep
 
 # OpenTelemetry Collector Binary
-ARG OTEL_VERSION=0.93.0
+ARG OTEL_VERSION=0.115.0
 ARG TARGETARCH=linux_amd64
 ADD "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${OTEL_VERSION}/otelcol-contrib_${OTEL_VERSION}_${TARGETARCH}.tar.gz" /otelcontribcol
 RUN tar -zxvf /otelcontribcol
 
-# JMX Metrics Gatherer Jar
-ARG JMX_GATHERER_JAR_VERSION=1.27.0
+# JMX Metrics Jar - used by the JMX receiver to collect Kafka broker JMX metrics
+ARG JMX_GATHERER_JAR_VERSION=1.42.0
 ADD https://github.com/open-telemetry/opentelemetry-java-contrib/releases/download/v${JMX_GATHERER_JAR_VERSION}/opentelemetry-jmx-metrics.jar /opt/opentelemetry-jmx-metrics.jar
 # nonroot user id (https://groups.google.com/g/distroless-users/c/-DpzCr7xRDY/m/eQqJmJroCgAJ)
 ARG USER_UID=65532
@@ -19,6 +19,6 @@ FROM gcr.io/distroless/java17-debian11:nonroot
 COPY --from=prep /opt/opentelemetry-jmx-metrics.jar /opt/opentelemetry-jmx-metrics.jar
 COPY --from=prep /otelcol-contrib /otelcol-contrib
 
-EXPOSE 4317 55680 55679
+EXPOSE 4317 4318
 ENTRYPOINT ["/otelcol-contrib"]
 CMD ["--config", "/etc/otelcol-contrib/config.yaml"]

apps/kafka-metrics/Dockerfile.consumer

@@ -1,17 +1,17 @@
-FROM openjdk:17-buster
+FROM eclipse-temurin:17-jdk AS build
 
-RUN apt-get update -y; apt-get install curl -y
+RUN apt-get update -y && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /home/otel
 COPY ./consumer consumer/
 WORKDIR /home/otel/consumer
 RUN ./gradlew shadowJar
 
-ARG OTEL_VERSION=v2.2.0
+# OpenTelemetry Java auto-instrumentation agent
+ARG OTEL_VERSION=v2.11.0
 ADD "https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/${OTEL_VERSION}/opentelemetry-javaagent.jar" opentelemetry-javaagent.jar
 
-
-ENV JAVA_TOOL_OPTIONS="-javaagent:opentelemetry-javaagent.jar  -Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=consumer -Dcom.sun.management.jmxremote.port=1097 -Dcom.sun.management.jmxremote.rmi.port=1097"
+ENV JAVA_TOOL_OPTIONS="-javaagent:opentelemetry-javaagent.jar -Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=consumer -Dcom.sun.management.jmxremote.port=1097 -Dcom.sun.management.jmxremote.rmi.port=1097"
 ENV JMX_PORT=1097
 
 ENTRYPOINT ["java", "-jar" , "app/build/libs/app-all.jar"]

apps/kafka-metrics/Dockerfile.consumer-gatherer

@@ -1,17 +1,17 @@
-FROM alpine:latest as prep
+FROM alpine:latest AS prep
 
-# JMX Metrics Gatherer Jar
-ARG JMX_GATHERER_JAR_VERSION=1.27.0
+# JMX Metrics Gatherer Jar - collects Kafka consumer metrics via JMX
+ARG JMX_GATHERER_JAR_VERSION=1.42.0
 ADD https://github.com/open-telemetry/opentelemetry-java-contrib/releases/download/v${JMX_GATHERER_JAR_VERSION}/opentelemetry-jmx-metrics.jar /opt/opentelemetry-jmx-metrics.jar
 # nonroot user id (https://groups.google.com/g/distroless-users/c/-DpzCr7xRDY/m/eQqJmJroCgAJ)
 ARG USER_UID=65532
 RUN chown ${USER_UID} /opt/opentelemetry-jmx-metrics.jar
 
-FROM gcr.io/distroless/java11-debian11:nonroot
+FROM gcr.io/distroless/java17-debian11:nonroot
 
 COPY --from=prep /opt/opentelemetry-jmx-metrics.jar /opt/opentelemetry-jmx-metrics.jar
 
-EXPOSE 4317 55680 55679
+EXPOSE 4317
 ENTRYPOINT ["java"]
 CMD ["-Dotel.jmx.service.url=service:jmx:rmi:///jndi/rmi://consumer:1097/jmxrmi", \
 "-Dotel.jmx.target.system=kafka-consumer", \

apps/kafka-metrics/Dockerfile.kafka-gatherer

@@ -1,17 +1,17 @@
-FROM alpine:latest as prep
+FROM alpine:latest AS prep
 
-# JMX Metrics Gatherer Jar
-ARG JMX_GATHERER_JAR_VERSION=1.27.0
+# JMX Metrics Gatherer Jar - collects Kafka broker and JVM metrics via JMX
+ARG JMX_GATHERER_JAR_VERSION=1.42.0
 ADD https://github.com/open-telemetry/opentelemetry-java-contrib/releases/download/v${JMX_GATHERER_JAR_VERSION}/opentelemetry-jmx-metrics.jar /opt/opentelemetry-jmx-metrics.jar
 # nonroot user id (https://groups.google.com/g/distroless-users/c/-DpzCr7xRDY/m/eQqJmJroCgAJ)
 ARG USER_UID=65532
 RUN chown ${USER_UID} /opt/opentelemetry-jmx-metrics.jar
 
-FROM gcr.io/distroless/java11-debian11:nonroot
+FROM gcr.io/distroless/java17-debian11:nonroot
 
 COPY --from=prep /opt/opentelemetry-jmx-metrics.jar /opt/opentelemetry-jmx-metrics.jar
 
-EXPOSE 4317 55680 55679
+EXPOSE 4317
 ENTRYPOINT ["java"]
 CMD ["-Dotel.jmx.service.url=service:jmx:rmi:///jndi/rmi://kafka:1099/jmxrmi", \
 "-Dotel.jmx.target.system=kafka,jvm", \

apps/kafka-metrics/Dockerfile.producer

@@ -1,16 +1,16 @@
-FROM openjdk:17-buster
+FROM eclipse-temurin:17-jdk AS build
 
-RUN apt-get update -y; apt-get install curl -y
+RUN apt-get update -y && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /home/otel
 COPY ./producer producer/
 WORKDIR /home/otel/producer
 RUN ./gradlew shadowJar
 
-ARG OTEL_VERSION=v2.2.0
+# OpenTelemetry Java auto-instrumentation agent
+ARG OTEL_VERSION=v2.11.0
 ADD "https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/${OTEL_VERSION}/opentelemetry-javaagent.jar" opentelemetry-javaagent.jar
 
-
 ENV JAVA_TOOL_OPTIONS="-javaagent:opentelemetry-javaagent.jar -Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=producer -Dcom.sun.management.jmxremote.port=1098 -Dcom.sun.management.jmxremote.rmi.port=1098"
 ENV JMX_PORT=1098
 

apps/kafka-metrics/Dockerfile.producer-gatherer

@@ -1,17 +1,17 @@
-FROM alpine:latest as prep
+FROM alpine:latest AS prep
 
-# JMX Metrics Gatherer Jar
-ARG JMX_GATHERER_JAR_VERSION=1.27.0
+# JMX Metrics Gatherer Jar - collects Kafka producer metrics via JMX
+ARG JMX_GATHERER_JAR_VERSION=1.42.0
 ADD https://github.com/open-telemetry/opentelemetry-java-contrib/releases/download/v${JMX_GATHERER_JAR_VERSION}/opentelemetry-jmx-metrics.jar /opt/opentelemetry-jmx-metrics.jar
 # nonroot user id (https://groups.google.com/g/distroless-users/c/-DpzCr7xRDY/m/eQqJmJroCgAJ)
 ARG USER_UID=65532
 RUN chown ${USER_UID} /opt/opentelemetry-jmx-metrics.jar
 
-FROM gcr.io/distroless/java11-debian11:nonroot
+FROM gcr.io/distroless/java17-debian11:nonroot
 
 COPY --from=prep /opt/opentelemetry-jmx-metrics.jar /opt/opentelemetry-jmx-metrics.jar
 
-EXPOSE 4317 55680 55679
+EXPOSE 4317
 ENTRYPOINT ["java"]
 CMD ["-Dotel.jmx.service.url=service:jmx:rmi:///jndi/rmi://producer:1098/jmxrmi", \
 "-Dotel.jmx.target.system=kafka-producer", \

apps/kafka-metrics/Makefile

@@ -0,0 +1,28 @@
+.PHONY: help build-jmxreceiver up-jmxreceiver down-jmxreceiver build-gatherer up-gatherer down-gatherer clean
+
+help: ## Show this help message
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}'
+
+# JMX Receiver mode - collector uses built-in JMX receiver
+build-jmxreceiver: ## Build images for JMX Receiver mode
+	docker compose -f docker-compose.jmxreceiver.yaml build
+
+up-jmxreceiver: ## Start services in JMX Receiver mode
+	docker compose -f docker-compose.jmxreceiver.yaml up
+
+down-jmxreceiver: ## Stop services in JMX Receiver mode
+	docker compose -f docker-compose.jmxreceiver.yaml down
+
+# JMX Metrics Gatherer mode - uses sidecar JMX Metrics Gatherer containers
+build-gatherer: ## Build images for JMX Metrics Gatherer mode
+	docker compose -f docker-compose.jmxmetricsgatherer.yaml build
+
+up-gatherer: ## Start services in JMX Metrics Gatherer mode
+	docker compose -f docker-compose.jmxmetricsgatherer.yaml up
+
+down-gatherer: ## Stop services in JMX Metrics Gatherer mode
+	docker compose -f docker-compose.jmxmetricsgatherer.yaml down
+
+clean: ## Remove all containers, images, and volumes
+	docker compose -f docker-compose.jmxreceiver.yaml down --rmi all --volumes --remove-orphans 2>/dev/null || true
+	docker compose -f docker-compose.jmxmetricsgatherer.yaml down --rmi all --volumes --remove-orphans 2>/dev/null || true

apps/kafka-metrics/consumer/app/build.gradle

@@ -18,15 +18,15 @@ java {
 }
 
 dependencies {
-    // kafka
-    implementation 'org.apache.kafka:kafka_2.13:3.5.1'
-    implementation 'org.apache.kafka:kafka-clients:2.3.0'
+    // kafka - aligned producer and client versions
+    implementation 'org.apache.kafka:kafka_2.13:3.7.1'
+    implementation 'org.apache.kafka:kafka-clients:3.7.1'
 
     // logger
-    implementation(platform("org.apache.logging.log4j:log4j-bom:2.22.1"))
+    implementation(platform("org.apache.logging.log4j:log4j-bom:2.24.3"))
     implementation("org.apache.logging.log4j:log4j-api")
     implementation("org.apache.logging.log4j:log4j-core")
-    implementation("org.apache.logging.log4j:log4j-slf4j2-impl:2.22.1")
+    implementation("org.apache.logging.log4j:log4j-slf4j2-impl:2.24.3")
 }
 
 application {

apps/kafka-metrics/consumer/app/src/main/java/consumer/App.java

@@ -23,7 +23,7 @@ public class App {
     public static void main(String[] args) {
         String kafkaAddr = System.getenv("KAFKA_SERVICE_ADDR");
         if (kafkaAddr != null) {
-            log4jLogger.info("Using Kafka Broker Address: " + kafkaAddr);
+            log4jLogger.info("Using Kafka Broker Address: {}", kafkaAddr);
         } else {
             throw new RuntimeException("Environment variable KAFKA_SERVICE_ADDR is not set.");
         }
@@ -34,14 +34,22 @@ public static void main(String[] args) {
         props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, IntegerDeserializer.class.getName());
         props.put(ConsumerConfig.GROUP_ID_CONFIG, "App");
 
+        // Consumer resilience configuration
+        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
+        props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
+        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000");
+        props.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, "10000");
+
         KafkaConsumer<String, Integer> consumer = new KafkaConsumer<>(props);
         consumer.subscribe(Collections.singletonList("orders"));
 
         while (true) {
             ConsumerRecords<String, Integer> records = consumer.poll(Duration.ofMillis(100));
-            for (ConsumerRecord<String, Integer> record : records)
-            log4jLogger.info("Consumed Message. Received Order # {}", record.value());
-           }
+            for (ConsumerRecord<String, Integer> record : records) {
+                log4jLogger.info("Consumed Message. Received Order # {}", record.value());
+            }
+        }
         // consumer.close();
     }
 }