Skip to content

Commit fc10dc9

Browse files
sophokles73sophokles73
committed
[eclipse-hono#3558] Use Kafka in Raft mode for running integration tests
Also updated Kafka container image used for ITs to version 3.5.0. Fixes eclipse-hono#3558
1 parent 6fc41a1 commit fc10dc9

File tree

5 files changed

+22
-43
lines changed

5 files changed

+22
-43
lines changed

Diff for: ‎bom/pom.xml

+1-2
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
<javax.transaction-api.version>1.3</javax.transaction-api.version>
4141
<java-base-image.name>docker.io/library/eclipse-temurin:17-jre-jammy</java-base-image.name>
4242
<jjwt.version>0.11.5</jjwt.version>
43-
<kafka.image.name>docker.io/confluentinc/cp-kafka:7.3.5</kafka.image.name>
43+
<kafka.image.name>docker.io/confluentinc/cp-kafka:7.5.0</kafka.image.name>
4444
<logback.version>1.4.11</logback.version>
4545
<mongodb-image.name>docker.io/library/mongo:6.0</mongodb-image.name>
4646
<native.image.name>quay.io/quarkus/quarkus-micro-image:2.0</native.image.name>
@@ -51,7 +51,6 @@
5151
<slf4j.version>2.0.6</slf4j.version>
5252
<spring-security-crypto.version>6.1.4</spring-security-crypto.version>
5353
<truth.version>1.1.3</truth.version>
54-
<zookeeper.image.name>docker.io/confluentinc/cp-zookeeper:7.3.5</zookeeper.image.name>
5554

5655
<!-- The port at which the health check server should expose its resources -->
5756
<health.check.port>8088</health.check.port>

Diff for: ‎site/homepage/content/release-notes.md

+2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ description = "Information about changes in recent Hono releases. Includes new f
1414
* When using Pub/Sub messaging, there were potentially issues concerning the AMQP connection between protocol adapter
1515
and command router, leading for example to timeouts when MQTT devices subscribed/unsubscribed to the command topic.
1616
This has been fixed.
17+
* The integration tests now use Apache Kafka 3.5.0 in Raft mode which no longer requires running a separate Apache Zookeeper
18+
instance and thus simplifies test setup and configuration.
1719

1820
### Deprecations
1921

Diff for: ‎tests/pom.xml

+14-39
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<?xml version="1.0" encoding="UTF-8"?>
22
<!--
3-
Copyright (c) 2016, 2022 Contributors to the Eclipse Foundation
3+
Copyright (c) 2016, 2023 Contributors to the Eclipse Foundation
44
55
See the NOTICE file(s) distributed with this work for additional
66
information regarding copyright ownership.
@@ -1211,34 +1211,6 @@
12111211
</env>
12121212
</run>
12131213
</image>
1214-
<!-- ##### Zookeeper ##### -->
1215-
<image>
1216-
<name>${zookeeper.image.name}</name>
1217-
<alias>hono-zookeeper-test</alias>
1218-
<run>
1219-
<skip>${hono.kafka-messaging.disabled}</skip>
1220-
<network>
1221-
<mode>custom</mode>
1222-
<name>${custom.network.name}</name>
1223-
<alias>zookeeper</alias>
1224-
</network>
1225-
<memorySwap>400000000</memorySwap>
1226-
<memory>400000000</memory>
1227-
<log>
1228-
<prefix>Zookeeper</prefix>
1229-
<color>${log.color.kafka}</color>
1230-
</log>
1231-
<wait>
1232-
<time>${service.startup.timeout}</time>
1233-
<log>.*(binding to port).*</log>
1234-
</wait>
1235-
<env>
1236-
<KAFKA_HEAP_OPTS>-Xms240M -Xmx240M</KAFKA_HEAP_OPTS>
1237-
<ZOOKEEPER_CLIENT_PORT>2181</ZOOKEEPER_CLIENT_PORT>
1238-
<ZOOKEEPER_TICK_TIME>2000</ZOOKEEPER_TICK_TIME>
1239-
</env>
1240-
</run>
1241-
</image>
12421214
<!-- ##### Kafka ##### -->
12431215
<image>
12441216
<name>${kafka.image.name}</name>
@@ -1261,25 +1233,28 @@
12611233
</log>
12621234
<wait>
12631235
<time>${service.startup.timeout}</time>
1264-
<log>.*(\[KafkaServer id=1\] started).*</log>
1236+
<log>.*(Kafka startTimeMs:).*</log>
12651237
</wait>
12661238
<env>
1239+
<CLUSTER_ID>q1Sh-9_ISia_zwGINzRvyQ</CLUSTER_ID>
12671240
<KAFKA_ADVERTISED_LISTENERS>DOCKER_INTERNAL://kafka:9092,DOCKER_EXTERNAL://${docker.host.address}:${kafka.port}</KAFKA_ADVERTISED_LISTENERS>
1268-
<KAFKA_BROKER_ID>1</KAFKA_BROKER_ID>
1241+
<KAFKA_CONTROLLER_LISTENER_NAMES>CONTROLLER</KAFKA_CONTROLLER_LISTENER_NAMES>
1242+
<KAFKA_CONTROLLER_QUORUM_VOTERS>[email protected]:9093</KAFKA_CONTROLLER_QUORUM_VOTERS>
1243+
<KAFKA_NODE_ID>1</KAFKA_NODE_ID>
12691244
<KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS>0</KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS>
1270-
<KAFKA_HEAP_OPTS>-Xms1024m -Xmx1024m</KAFKA_HEAP_OPTS>
12711245
<KAFKA_INTER_BROKER_LISTENER_NAME>DOCKER_INTERNAL</KAFKA_INTER_BROKER_LISTENER_NAME>
1272-
<KAFKA_JVM_PERFORMANCE_OPTS>-server -XX:+UseG1GC -XX:MaxGCPauseMillis=40 -XX:InitiatingHeapOccupancyPercent=35 -XX:+DisableExplicitGC -Djava.awt.headless=true</KAFKA_JVM_PERFORMANCE_OPTS>
1273-
<KAFKA_LISTENERS>DOCKER_INTERNAL://0.0.0.0:9092,DOCKER_EXTERNAL://0.0.0.0:9094</KAFKA_LISTENERS>
1274-
<KAFKA_LISTENER_SECURITY_PROTOCOL_MAP>DOCKER_INTERNAL:PLAINTEXT,DOCKER_EXTERNAL:PLAINTEXT</KAFKA_LISTENER_SECURITY_PROTOCOL_MAP>
1246+
<KAFKA_LISTENERS>DOCKER_INTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093,DOCKER_EXTERNAL://0.0.0.0:9094</KAFKA_LISTENERS>
1247+
<KAFKA_LISTENER_SECURITY_PROTOCOL_MAP>CONTROLLER:PLAINTEXT,DOCKER_INTERNAL:PLAINTEXT,DOCKER_EXTERNAL:PLAINTEXT</KAFKA_LISTENER_SECURITY_PROTOCOL_MAP>
12751248
<!-- increase log cleaner check frequency (default is 5min) for test where records shall be removed -->
1276-
<KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS>1100</KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS>
1277-
<!-- log level for kafka.server.KafkaServer needs to stay on INFO so that docker-maven-plugin can wait for container startup -->
1278-
<KAFKA_LOG4J_LOGGERS>kafka.cluster=${hono.kafka.log-level},kafka.controller=${hono.kafka.log-level},kafka.coordinator=${hono.kafka.log-level},kafka.log=${hono.kafka.log-level},kafka.authorizer=${hono.kafka.log-level},kafka.zk=${hono.kafka.log-level},state.change.logger=${hono.kafka.log-level},kafka.server=${hono.kafka.log-level},kafka.server.KafkaServer=INFO</KAFKA_LOG4J_LOGGERS>
1249+
<KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS>1000</KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS>
1250+
<KAFKA_LOG_SEGMENT_DELETE_DELAY_MS>0</KAFKA_LOG_SEGMENT_DELETE_DELAY_MS>
1251+
<!-- log level for AppInfoParser needs to be pinned to INFO so that docker-maven-plugin can wait for container startup -->
1252+
<KAFKA_LOG4J_LOGGERS>kafka=${hono.kafka.log-level},org.apache.kafka=${hono.kafka.log-level},state.change.logger=${hono.kafka.log-level},org.apache.kafka.common.utils.AppInfoParser=INFO</KAFKA_LOG4J_LOGGERS>
12791253
<KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR>1</KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR>
1254+
<KAFKA_PROCESS_ROLES>broker,controller</KAFKA_PROCESS_ROLES>
1255+
<KAFKA_REST_BOOTSTRAP_SERVERS>127.0.0.1:9092</KAFKA_REST_BOOTSTRAP_SERVERS>
12801256
<KAFKA_TRANSACTION_STATE_LOG_MIN_ISR>1</KAFKA_TRANSACTION_STATE_LOG_MIN_ISR>
12811257
<KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR>1</KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR>
1282-
<KAFKA_ZOOKEEPER_CONNECT>zookeeper:2181</KAFKA_ZOOKEEPER_CONNECT>
12831258
</env>
12841259
</run>
12851260
</image>

Diff for: ‎tests/readme.md

-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ This starts the following Docker containers and runs the test cases against them
2727
* MongoDB server
2828
* Hono Authentication service
2929
* Hono Device Registration service
30-
* Apache Zookeeper
3130
* Apache Kafka
3231
* Hono Command Router service
3332
* Hono HTTP adapter

Diff for: ‎tests/src/test/java/org/eclipse/hono/tests/amqp/CommandAndControlAmqpIT.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ private Future<ProtonReceiver> subscribeToCommands(
138138

139139
final Promise<ProtonReceiver> result = Promise.promise();
140140
context.runOnContext(go -> {
141+
log.debug("creating command consumer for tenant [{}]", tenantId);
141142
final ProtonReceiver recv = connection.createReceiver(endpointConfig.getSubscriptionAddress(
142143
tenantId, commandTargetDeviceId));
143144
recv.setAutoAccept(false);
@@ -321,7 +322,10 @@ public void testSendAsyncCommandsSucceeds(
321322
: deviceId;
322323

323324
final int totalNoOfCommandsToSend = 60;
324-
connectAndSubscribe(ctx, commandTargetDeviceId, endpointConfig,
325+
connectAndSubscribe(
326+
ctx,
327+
commandTargetDeviceId,
328+
endpointConfig,
325329
(cmdReceiver, cmdResponseSender) -> createCommandConsumer(ctx, cmdReceiver, cmdResponseSender),
326330
totalNoOfCommandsToSend);
327331
if (ctx.failed()) {

0 commit comments

Comments
 (0)