From 8b36e87e1207394ab872565415522574c1681db2 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Sun, 10 May 2026 10:48:29 +0900 Subject: [PATCH 01/16] KAFKA-20169: Support static membership for Kafka Streams with the streams rebalance protocol - server side --- .../group/GroupCoordinatorService.java | 1 - .../group/GroupMetadataManager.java | 337 +++++++++++++++++- .../group/streams/StreamsGroup.java | 31 +- .../group/streams/StreamsGroupMember.java | 45 ++- .../group/GroupCoordinatorServiceTest.java | 25 +- .../group/GroupMetadataManagerTest.java | 8 +- 6 files changed, 399 insertions(+), 48 deletions(-) diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorService.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorService.java index 93fd360be24a5..2911736a01001 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorService.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorService.java @@ -584,7 +584,6 @@ private static void throwIfStreamsGroupHeartbeatRequestIsInvalid( private static void throwIfStreamsGroupHeartbeatRequestIsUsingUnsupportedFeatures( StreamsGroupHeartbeatRequestData request ) throws InvalidRequestException { - throwIfNotNull(request.instanceId(), "Static membership is not yet supported."); throwIfNotNull(request.taskOffsets(), "TaskOffsets are not supported yet."); throwIfNotNull(request.taskEndOffsets(), "TaskEndOffsets are not supported yet."); throwIfNotNullOrEmpty(request.warmupTasks(), "WarmupTasks are not supported yet."); diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java index faa36fcb18e22..5b725641451de 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java @@ -76,6 +76,7 @@ import org.apache.kafka.common.requests.JoinGroupRequest; import org.apache.kafka.common.requests.ShareGroupHeartbeatRequest; import org.apache.kafka.common.requests.ShareGroupHeartbeatResponse; +import org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest; import org.apache.kafka.common.requests.StreamsGroupHeartbeatResponse; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.internals.LogContext; @@ -150,6 +151,7 @@ import org.apache.kafka.coordinator.group.modern.share.ShareGroup.ShareGroupStatePartitionMetadataInfo; import org.apache.kafka.coordinator.group.modern.share.ShareGroupAssignmentBuilder; import org.apache.kafka.coordinator.group.modern.share.ShareGroupMember; +import org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers; import org.apache.kafka.coordinator.group.streams.StreamsGroup; import org.apache.kafka.coordinator.group.streams.StreamsGroupDescribeResult; import org.apache.kafka.coordinator.group.streams.StreamsGroupHeartbeatResult; @@ -312,7 +314,7 @@ private static UpdateTargetAssignmentResult fromLastTargetAssignment if (member.isPresent()) { return new UpdateTargetAssignmentResult<>( group.assignmentEpoch(), - group.targetAssignment(member.get().memberId()) + group.targetAssignment(member.get().memberId(), member.get().instanceId()) ); } else { return new UpdateTargetAssignmentResult<>( @@ -1690,6 +1692,26 @@ private void throwIfInstanceIdIsUnreleased(ConsumerGroupMember member, String gr } } + /** + * Validates if the received instanceId has been released from the group + * + * @param member The streams group member. + * @param groupId The streams group id. + * @param receivedMemberId The member id received in the request. + * @param receivedInstanceId The instance id received in the request. + * + * @throws UnreleasedInstanceIdException if the instance id received in the request is still in use by an existing static member. + */ + private void throwIfInstanceIdIsUnreleased(StreamsGroupMember member, String groupId, String receivedMemberId, String receivedInstanceId) { + if (member.memberEpoch() != StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) { + // The new member can't join. + log.info("[GroupId {}] Static member {} with instance id {} cannot join the group because the instance id is" + + " owned by member {}.", groupId, receivedMemberId, receivedInstanceId, member.memberId()); + throw Errors.UNRELEASED_INSTANCE_ID.exception("Static member " + receivedMemberId + " with instance id " + + receivedInstanceId + " cannot join the group because the instance id is owned by " + member.memberId() + " member."); + } + } + /** * Validates if the received instanceId has been released from the group * @@ -1709,6 +1731,26 @@ private void throwIfInstanceIdIsFenced(ConsumerGroupMember member, String groupI } } + /** + * Validates if the received instanceId has been fenced from the group + * + * @param member The streams group member. + * @param groupId The streams group id. + * @param receivedMemberId The member id received in the request. + * @param receivedInstanceId The instance id received in the request. + * + * @throws FencedInstanceIdException if the instance id provided is fenced because of another static member. + */ + private void throwIfInstanceIdIsFenced(StreamsGroupMember member, String groupId, String receivedMemberId, String receivedInstanceId) { + if (!member.memberId().equals(receivedMemberId)) { + log.info("[GroupId {}] Static member {} with instance id {} is fenced by existing member {}.", + groupId, receivedMemberId, receivedInstanceId, member.memberId()); + throw Errors.FENCED_INSTANCE_ID.exception("Static member " + receivedMemberId + " with instance id " + + receivedInstanceId + " was fenced by member " + member.memberId() + "."); + } + } + + /** * Validates if the received instanceId has been released from the group * @@ -1723,17 +1765,41 @@ private void throwIfStaticMemberIsUnknown(ConsumerGroupMember staticMember, Stri } } + /** + * Validates whether a static member exists for the given instanceId. + * + * @param staticMember The static member in the group. + * @param receivedInstanceId The instance id received in the request. + * + * @throws UnknownMemberIdException if no static member exists in the group against the provided instance id. + */ + private void throwIfStaticMemberIsUnknown(StreamsGroupMember staticMember, String receivedInstanceId) { + if (staticMember == null) { + throw Errors.UNKNOWN_MEMBER_ID.exception("Instance id " + receivedInstanceId + " is unknown."); + } + } + /** * Checks whether the streams group can accept a new member or not based on the * max group size defined. * * @param group The streams group. + * @param instanceId The instance id. * * @throws GroupMaxSizeReachedException if the maximum capacity has been reached. */ private void throwIfStreamsGroupIsFull( - StreamsGroup group + StreamsGroup group, + String instanceId ) throws GroupMaxSizeReachedException { + // If a static member already exists, we do not enforce the maximum group size check. + // An existing static member will fall into one of the following two cases, + // and neither affects the group size: + // 1. The member is replaced due to the static member rejoining. + // 2. 'UnreleasedInstanceIdException' is raised due to an epoch mismatch. + if (instanceId != null && group.hasStaticMember(instanceId)) + return; + // If the streams group has reached its maximum capacity, the member is rejected if it is not // already a member of the streams group. if (group.numMembers() >= config.streamsGroupMaxSize()) { @@ -1984,18 +2050,20 @@ private CoordinatorResult stream final List returnedStatus = new ArrayList<>(); // Get or create the streams group. - boolean isJoining = memberEpoch == 0; + boolean isJoining = memberEpoch == StreamsGroupHeartbeatRequest.JOIN_GROUP_MEMBER_EPOCH; StreamsGroup group; if (isJoining) { group = getOrCreateStreamsGroup(groupId, records); - throwIfStreamsGroupIsFull(group); + throwIfStreamsGroupIsFull(group, instanceId); } else { group = getStreamsGroupOrThrow(groupId); } // Get or create the member. StreamsGroupMember member; + StreamsGroupMember maybeReplacedStaticMember; if (instanceId == null) { + maybeReplacedStaticMember = null; member = getOrMaybeCreateDynamicStreamsGroupMember( group, memberId, @@ -2006,25 +2074,44 @@ private CoordinatorResult stream isJoining ); } else { - throw new UnsupportedOperationException("Static members are not supported yet."); + maybeReplacedStaticMember = group.staticMember(instanceId); + member = getOrMaybeCreateStaticStreamsGroupMember( + group, + memberId, + memberEpoch, + instanceId, + ownedActiveTasks, + ownedStandbyTasks, + ownedWarmupTasks, + isJoining, + records + ); } // 1. Create or update the member. - StreamsGroupMember updatedMember = new StreamsGroupMember.Builder(member) - .maybeUpdateInstanceId(Optional.empty()) + StreamsGroupMember.Builder updatedMemberBuilder = new StreamsGroupMember.Builder(member) + .maybeUpdateInstanceId(Optional.ofNullable(instanceId)) .maybeUpdateRackId(Optional.ofNullable(rackId)) .maybeUpdateRebalanceTimeoutMs(ofSentinel(rebalanceTimeoutMs)) .maybeUpdateTopologyEpoch(topology != null ? OptionalInt.of(topology.epoch()) : OptionalInt.empty()) .setClientId(clientId) .setClientHost(clientHost) .maybeUpdateProcessId(Optional.ofNullable(processId)) - .maybeUpdateClientTags(Optional.ofNullable(clientTags).map(x -> x.stream().collect(Collectors.toMap(KeyValue::key, KeyValue::value)))) - .maybeUpdateUserEndpoint(Optional.ofNullable(userEndpoint).map(x -> new StreamsGroupMemberMetadataValue.Endpoint().setHost(x.host()).setPort(x.port()))) - .build(); + .maybeUpdateClientTags(Optional.ofNullable(clientTags).map(x -> x.stream().collect(Collectors.toMap(KeyValue::key, KeyValue::value)))); + if (isJoining) { + StreamsGroupMemberMetadataValue.Endpoint userEndpointMetadata = userEndpoint == null ? null : + new StreamsGroupMemberMetadataValue.Endpoint().setHost(userEndpoint.host()).setPort(userEndpoint.port()); + updatedMemberBuilder.setUserEndpoint(userEndpointMetadata); + } else { + updatedMemberBuilder + .maybeUpdateUserEndpoint(Optional.ofNullable(userEndpoint).map(x -> new StreamsGroupMemberMetadataValue.Endpoint().setHost(x.host()).setPort(x.port()))); + } + + StreamsGroupMember updatedMember = updatedMemberBuilder.build(); // If the member is new or has changed, a StreamsGroupMemberMetadataValue record is written to the __consumer_offsets partition // to persist the change, and bump the group epoch later. - boolean bumpGroupEpoch = hasStreamsMemberMetadataChanged(groupId, member, updatedMember, records); + boolean bumpGroupEpoch = hasStreamsMemberMetadataChanged(groupId, instanceId, member, updatedMember, records); // 2. Initialize/Update the group topology. // If the topology is new or has changed, a StreamsGroupTopologyValue record is written to the __consumer_offsets partition to persist @@ -2170,7 +2257,10 @@ private CoordinatorResult stream response.setStandbyTasks(createStreamsGroupHeartbeatResponseTaskIds(updatedMember.assignedTasks().standbyTasks())); response.setWarmupTasks(createStreamsGroupHeartbeatResponseTaskIds(updatedMember.assignedTasks().warmupTasks())); group.invalidateCachedEndpointToPartitions(updatedMember.memberId()); - if (updatedMember.userEndpoint().isPresent()) { + if (maybeReplacedStaticMember != null && !maybeReplacedStaticMember.memberId().equals(updatedMember.memberId())) { + group.invalidateCachedEndpointToPartitions(maybeReplacedStaticMember.memberId()); + } + if (hasUserEndpointChanged(maybeReplacedStaticMember, updatedMember)) { // If no user endpoint is defined, there is no change in the endpoint information. // Otherwise, bump the endpoint information epoch group.setEndpointInformationEpoch(group.endpointInformationEpoch() + 1); @@ -2178,7 +2268,7 @@ private CoordinatorResult stream } if (group.endpointInformationEpoch() != memberEndpointEpoch) { - response.setPartitionsByUserEndpoint(group.buildEndpointToPartitions(updatedMember, metadataImage)); + response.setPartitionsByUserEndpoint(group.buildEndpointToPartitions(updatedMember, metadataImage, maybeReplacedStaticMember)); } if (groups.containsKey(group.groupId())) { // If we just created the group, the endpoint information epoch will not be persisted, so return epoch 0. @@ -3206,6 +3296,82 @@ private ConsumerGroupMember getOrMaybeSubscribeStaticConsumerGroupMember( } } + /** + * Gets an existing static Streams group member or creates/replaces one for static membership. + * + * If the member is joining: + * 1. Creates a new static member when no member exists for the instance ID. + * 2. Replaces the previous static member when the instance ID is released. + * + * If the member is not joining, validates static member identity and member epoch + * and returns the existing static member. + * + * @param group The streams group. + * @param memberId The member id from the request. + * @param memberEpoch The member epoch from the request. + * @param instanceId The static instance id from the request. + * @param ownedActiveTasks The owned active tasks from the request. + * @param ownedStandbyTasks The owned standby tasks from the request. + * @param ownedWarmupTasks The owned warmup tasks from the request. + * @param memberIsJoining Whether the member is joining (epoch 0). + * @param records The records accumulator used for member replacement. + * + * @return The resolved streams group member. + */ + private StreamsGroupMember getOrMaybeCreateStaticStreamsGroupMember( + StreamsGroup group, + String memberId, + int memberEpoch, + String instanceId, + List ownedActiveTasks, + List ownedStandbyTasks, + List ownedWarmupTasks, + boolean memberIsJoining, + List records + ) { + StreamsGroupMember existingStaticMemberOrNull = group.staticMember(instanceId); + if (memberIsJoining) { + // A new static member joins or the existing static member rejoins. + if (existingStaticMemberOrNull == null) { + // New static member. + StreamsGroupMember newMember = group.getOrCreateDefaultMember(memberId); + log.info("[GroupId {}][MemberId {}] Static member {} with instance id {} joins the streams group.", + group.groupId(), memberId, memberId, instanceId); + return newMember; + } else { + throwIfInstanceIdIsUnreleased(existingStaticMemberOrNull, group.groupId(), memberId, instanceId); + + // Copy the member but with its new member id. + StreamsGroupMember newMember = new StreamsGroupMember.Builder(existingStaticMemberOrNull, memberId) + .setMemberEpoch(0) + .setPreviousMemberEpoch(0) + .build(); + + // Generate the records to replace the member. We don't care about the regular expression + // here because it is taken care of later after the static membership replacement. + replaceStreamsMember(records, group, existingStaticMemberOrNull, newMember); + + log.info("[GroupId {}][MemberId {}] Static member with instance id {} re-joins the stream group " + + "using the streams protocol. Created a new member {} to replace the existing member {}.", + group.groupId(), memberId, instanceId, memberId, existingStaticMemberOrNull.memberId()); + + return newMember; + } + } else { + throwIfStaticMemberIsUnknown(existingStaticMemberOrNull, instanceId); + throwIfInstanceIdIsFenced(existingStaticMemberOrNull, group.groupId(), memberId, instanceId); + throwIfStreamsGroupMemberEpochIsInvalid( + existingStaticMemberOrNull, + memberEpoch, + ownedActiveTasks, + ownedStandbyTasks, + ownedWarmupTasks + ); + return existingStaticMemberOrNull; + } + } + + /** * Gets or subscribes a new dynamic share group member. * @@ -3547,6 +3713,7 @@ private boolean hasMemberSubscriptionChanged( * when a member is first created. * * @param groupId The group id. + * @param instanceId The instance id. * @param member The old member. * @param updatedMember The updated member. * @param records The list to accumulate any new records. @@ -3555,6 +3722,7 @@ private boolean hasMemberSubscriptionChanged( */ private boolean hasStreamsMemberMetadataChanged( String groupId, + String instanceId, StreamsGroupMember member, StreamsGroupMember updatedMember, List records @@ -3564,8 +3732,17 @@ private boolean hasStreamsMemberMetadataChanged( records.add(newStreamsGroupMemberRecord(groupId, updatedMember)); log.info("[GroupId {}][MemberId {}] Member updated its member metadata to {}.", groupId, memberId, updatedMember); - - return true; + if (instanceId == null) { + return true; + } else { + // When a static member rejoins, its member ID may change. + // According to KIP-1071, the group epoch should be bumped only + // when the topology metadata, rack ID, client tags, or process ID changes. + // Therefore, if we compare the old and new members using equals(), + // the group epoch can be bumped unintentionally, + // which may trigger unnecessary task rebalancing. + return hasEpochRelevantMemberConfigChanged(member, updatedMember); + } } return false; } @@ -4108,9 +4285,17 @@ private UpdateTargetAssignmentResult maybeUpdateStreamsTargetAssignm .withMetadataImage(metadataImage) .withTargetAssignment(group.targetAssignment()); - updatedMember.ifPresent(member -> - assignmentResultBuilder.addOrUpdateMember(member.memberId(), member) - ); + updatedMember.ifPresent(member -> { + assignmentResultBuilder.addOrUpdateMember(member.memberId(), member); + // If the instance id was associated to a different member, it means that the + // static member is replaced by the current member hence we remove the previous one. + member.instanceId().ifPresent(instanceId -> { + StreamsGroupMember previousMember = group.staticMember(instanceId); + if (previousMember != null && !member.memberId().equals(previousMember.memberId())) { + assignmentResultBuilder.removeMember(previousMember.memberId()); + } + }); + }); long startTimeMs = time.milliseconds(); org.apache.kafka.coordinator.group.streams.TargetAssignmentBuilder.TargetAssignmentResult assignmentResult = @@ -4256,7 +4441,18 @@ private CoordinatorResult stream log.info("[GroupId {}][MemberId {}] Member {} left the streams group.", groupId, memberId, memberId); return streamsGroupFenceMember(group, member, new StreamsGroupHeartbeatResult(response, Map.of(), group.currentTopologyEpoch())); } else { - throw new UnsupportedOperationException("Static members are not supported in streams groups."); + StreamsGroupMember member = group.staticMember(instanceId); + throwIfStaticMemberIsUnknown(member, instanceId); + throwIfInstanceIdIsFenced(member, groupId, memberId, instanceId); + if (memberEpoch == StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) { + log.info("[GroupId {}][MemberId {}] Static member {} with instance id {} temporarily left the streams group.", + group.groupId(), memberId, memberId, instanceId); + return streamsGroupStaticMemberGroupLeave(group, member); + } else { + log.info("[GroupId {}][MemberId {}] Static member {} with instance id {} left the streams group.", + group.groupId(), memberId, memberId, instanceId); + return streamsGroupFenceMember(group, member, new StreamsGroupHeartbeatResult(response, Map.of())); + } } } @@ -4293,6 +4489,45 @@ private CoordinatorResult ); } + /** + * Handles the case when a static member decides to leave the group. + * The member is not actually fenced from the group, and instead it's + * member epoch is updated to -2 to reflect that a member using the given + * instance id decided to leave the group and would be back within session + * timeout. + * + * @param group The group. + * @param member The static member in the group for the instance id. + * + * @return A CoordinatorResult with a single record signifying that the static member is leaving. + */ + private CoordinatorResult streamsGroupStaticMemberGroupLeave( + StreamsGroup group, + StreamsGroupMember member + ) { + // We will write a member epoch of -2 for this departing static member. + org.apache.kafka.coordinator.group.streams.MemberState nextState = member.isUnrevokedState() ? + org.apache.kafka.coordinator.group.streams.MemberState.STABLE : + member.state(); + StreamsGroupMember leavingStaticMember = new StreamsGroupMember.Builder(member) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setState(nextState) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .resetAssignedTasksEpochsToZero() + .build(); + + CoordinatorRecord record = newStreamsGroupCurrentAssignmentRecord(group.groupId(), leavingStaticMember); + StreamsGroupHeartbeatResponseData response = new StreamsGroupHeartbeatResponseData() + .setMemberId(member.memberId()) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setStatus(List.of()); + + return new CoordinatorResult<>( + List.of(record), + new StreamsGroupHeartbeatResult(response, Map.of()) + ); + } + /** * Handles leave request from a share group member. * @param groupId The group id from the request. @@ -4502,6 +4737,42 @@ private void replaceMember( )); } + /** + * Write records to replace the old member by the new member. + * + * @param records The list of records to append to. + * @param group The streams group. + * @param oldMember The old member. + * @param newMember The new member. + */ + private void replaceStreamsMember( + List records, + StreamsGroup group, + StreamsGroupMember oldMember, + StreamsGroupMember newMember + ) { + String groupId = group.groupId(); + + // Remove the member without canceling its timers in case the change is reverted. If the + // change is not reverted, the group validation will fail and the timer will do nothing. + records.addAll(removeStreamsMember(groupId, oldMember.memberId())); + + // Generate records. + records.add(StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord( + groupId, + newMember + )); + records.add(StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord( + groupId, + newMember.memberId(), + group.targetAssignment(oldMember.memberId(), oldMember.instanceId()) + )); + records.add(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord( + groupId, + newMember + )); + } + /** * Fences a member from a streams group. * @@ -9026,6 +9297,34 @@ private Map streamsGroupAssignmentConfigs(String groupId) { )); } + private boolean hasUserEndpointChanged(StreamsGroupMember maybeReplacedStaticMember, StreamsGroupMember updatedMember) { + + boolean hasPreviousUserEndpoint = maybeReplacedStaticMember != null && maybeReplacedStaticMember.userEndpoint().isPresent(); + boolean hasCurrentUserEndpoint = updatedMember.userEndpoint().isPresent(); + + if (hasPreviousUserEndpoint && hasCurrentUserEndpoint) { + return !maybeReplacedStaticMember.userEndpoint().get().equals(updatedMember.userEndpoint().get()); + } + + if (!hasPreviousUserEndpoint && !hasCurrentUserEndpoint) { + return false; + } + + return true; + } + + private static boolean hasEpochRelevantMemberConfigChanged( + StreamsGroupMember oldMember, + StreamsGroupMember newMember + ) { + // The group epoch is bumped: (KIP-1071) + // - When a member updates its topology metadata, rack ID, client tags or process ID. + return !Objects.equals(oldMember.topologyEpoch(), newMember.topologyEpoch()) + || !Objects.equals(oldMember.rackId(), newMember.rackId()) + || !Objects.equals(oldMember.clientTags(), newMember.clientTags()) + || !Objects.equals(oldMember.processId(), newMember.processId()); + } + /** * Generate a classic group heartbeat key for the timer. * diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java index 5f2878bba4355..5f553c1ae8af6 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java @@ -566,13 +566,33 @@ public Map staticMembers() { return Collections.unmodifiableMap(staticMembers); } + /** + * Returns true if the static member exists. + * + * @param instanceId The instance id. + * + * @return A boolean indicating whether the member exists or not. + */ + public boolean hasStaticMember(String instanceId) { + if (instanceId == null) return false; + return staticMembers.containsKey(instanceId); + } + /** * Returns the target assignment of the member. * * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist. */ - public TasksTuple targetAssignment(String memberId) { - return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY); + public TasksTuple targetAssignment(String memberId, Optional instanceId) { + if (instanceId.isEmpty()) { + return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY); + } else { + StreamsGroupMember previousMember = staticMember(instanceId.get()); + if (previousMember != null) { + return targetAssignment.getOrDefault(previousMember.memberId(), TasksTuple.EMPTY); + } + } + return TasksTuple.EMPTY; } /** @@ -1274,11 +1294,13 @@ public void invalidateCachedEndpointToPartitions(String memberId) { * * @param updatedMember The member that was just updated (may have a stale entry in the members map). * @param metadataImage The current metadata image for resolving topic partitions. + * @param maybeReplacedStaticMember The replaced static member. it can be null. * @return The list of endpoint-to-partitions mappings for all members with endpoints. */ public List buildEndpointToPartitions( StreamsGroupMember updatedMember, - CoordinatorMetadataImage metadataImage + CoordinatorMetadataImage metadataImage, + StreamsGroupMember maybeReplacedStaticMember ) { List endpointToPartitionsList = new ArrayList<>(); if (updatedMember == null) { @@ -1290,6 +1312,9 @@ public List buildEndpoin if (entry.getKey().equals(updatedMember.memberId())) { continue; } + if (maybeReplacedStaticMember != null && entry.getKey().equals(maybeReplacedStaticMember.memberId())) { + continue; + } getOrComputeEndpointToPartitions(entry.getValue(), metadataImage) .ifPresent(endpointToPartitionsList::add); } diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroupMember.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroupMember.java index a43431ce47ef5..05b133511eeae 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroupMember.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroupMember.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -104,9 +105,13 @@ public Builder(String memberId) { } public Builder(StreamsGroupMember member) { + this(Objects.requireNonNull(member, "member cannot be null"), member.memberId); + } + + public Builder(StreamsGroupMember member, String memberId) { Objects.requireNonNull(member, "member cannot be null"); - this.memberId = member.memberId; + this.memberId = memberId; this.memberEpoch = member.memberEpoch; this.previousMemberEpoch = member.previousMemberEpoch; this.instanceId = member.instanceId; @@ -315,6 +320,37 @@ public StreamsGroupMember build() { tasksPendingRevocation ); } + + /** + * Resets the assignment epochs to 0 for all assigned active tasks. + * Used when a static member leaves, so that the rejoining member's + * active tasks will be assigned from epoch 0 to the new member ID. + * All commits using the old member ID will be fenced. + */ + public Builder resetAssignedTasksEpochsToZero() { + if (this.assignedTasks.isEmpty()) { + return this; + } + + if (this.assignedTasks.activeTasksWithEpochs().isEmpty()) { + return this; + } + + Map> resetActiveTasks = new HashMap<>(); + for (Map.Entry> entry : this.assignedTasks.activeTasksWithEpochs().entrySet()) { + Map resetActiveTaskEpochs = new HashMap<>(); + for (Integer partitionId : entry.getValue().keySet()) { + resetActiveTaskEpochs.put(partitionId, 0); + } + resetActiveTasks.put(entry.getKey(), resetActiveTaskEpochs); + } + this.assignedTasks = new TasksTupleWithEpochs( + resetActiveTasks, + this.assignedTasks.standbyTasks(), + this.assignedTasks.warmupTasks() + ); + return this; + } } /** @@ -324,6 +360,13 @@ public boolean isReconciledTo(int targetAssignmentEpoch) { return state == MemberState.STABLE && memberEpoch == targetAssignmentEpoch; } + /** + * @return True if the member is in the Unrevoked state. + */ + public boolean isUnrevokedState() { + return state == MemberState.UNREVOKED_TASKS; + } + /** * Creates a member description for the streams group describe response from this member. * diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorServiceTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorServiceTest.java index a50610237886c..9ffbf69a16a63 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorServiceTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorServiceTest.java @@ -591,21 +591,6 @@ public void testStreamsGroupHeartbeatFailsForUnsupportedFeatures() throws Except AuthorizableRequestContext context = mock(AuthorizableRequestContext.class); when(context.requestVersion()).thenReturn((int) ApiKeys.STREAMS_GROUP_HEARTBEAT.latestVersion()); - assertEquals( - new StreamsGroupHeartbeatResult( - new StreamsGroupHeartbeatResponseData() - .setErrorCode(Errors.INVALID_REQUEST.code()) - .setErrorMessage("Static membership is not yet supported."), - Map.of(), - -1 - ), - service.streamsGroupHeartbeat( - context, - new StreamsGroupHeartbeatRequestData() - .setInstanceId(Uuid.randomUuid().toString()) - ).get(5, TimeUnit.SECONDS) - ); - assertEquals( new StreamsGroupHeartbeatResult( new StreamsGroupHeartbeatResponseData() @@ -2125,7 +2110,7 @@ public void testStreamsGroupDescribe() throws InterruptedException, ExecutionExc int partitionCount = 2; service.startup(() -> partitionCount); @SuppressWarnings("unchecked") - ArgumentCaptor> readOperationCaptor = + ArgumentCaptor>> readOperationCaptor = ArgumentCaptor.forClass(CoordinatorRuntime.CoordinatorReadOperation.class); StreamsGroupDescribeResponseData.DescribedGroup describedGroup1 = new StreamsGroupDescribeResponseData.DescribedGroup() @@ -2141,9 +2126,9 @@ public void testStreamsGroupDescribe() throws InterruptedException, ExecutionExc ArgumentMatchers.eq("streams-group-describe"), ArgumentMatchers.eq(new TopicPartition("__consumer_offsets", 0)), readOperationCaptor.capture() - )).thenReturn(CompletableFuture.completedFuture(new StreamsGroupDescribeResult(List.of(describedGroup1), Map.of()))); + )).thenReturn(CompletableFuture.completedFuture(List.of(describedGroup1))); - CompletableFuture describedGroupFuture = new CompletableFuture<>(); + CompletableFuture> describedGroupFuture = new CompletableFuture<>(); when(runtime.scheduleReadOperation( ArgumentMatchers.eq("streams-group-describe"), ArgumentMatchers.eq(new TopicPartition("__consumer_offsets", 1)), @@ -2154,7 +2139,7 @@ public void testStreamsGroupDescribe() throws InterruptedException, ExecutionExc service.streamsGroupDescribe(requestContext(ApiKeys.STREAMS_GROUP_DESCRIBE), Arrays.asList("group-id-1", "group-id-2")); assertFalse(future.isDone()); - describedGroupFuture.complete(new StreamsGroupDescribeResult(List.of(describedGroup2), Map.of())); + describedGroupFuture.complete(List.of(describedGroup2)); assertEquals(expectedDescribedGroups, future.get()); // Validate that the captured read operations, on the first and the second partition @@ -2188,7 +2173,7 @@ public void testStreamsGroupDescribeInvalidGroupId() throws ExecutionException, ArgumentMatchers.eq("streams-group-describe"), ArgumentMatchers.eq(new TopicPartition("__consumer_offsets", 0)), ArgumentMatchers.any() - )).thenReturn(CompletableFuture.completedFuture(new StreamsGroupDescribeResult(List.of(describedGroup), Map.of()))); + )).thenReturn(CompletableFuture.completedFuture(List.of(describedGroup))); CompletableFuture> future = service.streamsGroupDescribe(requestContext(ApiKeys.STREAMS_GROUP_DESCRIBE), Arrays.asList("", null)); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java index 7500ad7e711a9..9c1d3a2b2bca8 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java @@ -22710,9 +22710,9 @@ public void testReplayStreamsGroupTargetAssignmentMember() { TaskAssignmentTestUtil.mkTasks("subtopology-1", 6, 7, 8)) ); context.replay(StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord("foo", "m1", tasks)); - assertEquals(tasks.activeTasks(), context.groupMetadataManager.streamsGroup("foo").targetAssignment("m1").activeTasks()); - assertEquals(tasks.standbyTasks(), context.groupMetadataManager.streamsGroup("foo").targetAssignment("m1").standbyTasks()); - assertEquals(tasks.warmupTasks(), context.groupMetadataManager.streamsGroup("foo").targetAssignment("m1").warmupTasks()); + assertEquals(tasks.activeTasks(), context.groupMetadataManager.streamsGroup("foo").targetAssignment("m1", Optional.empty()).activeTasks()); + assertEquals(tasks.standbyTasks(), context.groupMetadataManager.streamsGroup("foo").targetAssignment("m1", Optional.empty()).standbyTasks()); + assertEquals(tasks.warmupTasks(), context.groupMetadataManager.streamsGroup("foo").targetAssignment("m1", Optional.empty()).warmupTasks()); } @Test @@ -22743,7 +22743,7 @@ public void testReplayStreamsGroupTargetAssignmentMemberTombstoneExisting() { context.replay(StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord("foo", "m1")); - assertTrue(context.groupMetadataManager.streamsGroup("foo").targetAssignment("m1").isEmpty()); + assertTrue(context.groupMetadataManager.streamsGroup("foo").targetAssignment("m1", Optional.empty()).isEmpty()); } @Test From 676622fb3eaf2af367e0ac312bfd6cf09fe62728 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Sun, 10 May 2026 10:58:36 +0900 Subject: [PATCH 02/16] Fix new lines. --- .../apache/kafka/coordinator/group/GroupMetadataManager.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java index 5b725641451de..07ac5259fecac 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java @@ -1750,7 +1750,6 @@ private void throwIfInstanceIdIsFenced(StreamsGroupMember member, String groupId } } - /** * Validates if the received instanceId has been released from the group * @@ -2107,8 +2106,8 @@ private CoordinatorResult stream updatedMemberBuilder .maybeUpdateUserEndpoint(Optional.ofNullable(userEndpoint).map(x -> new StreamsGroupMemberMetadataValue.Endpoint().setHost(x.host()).setPort(x.port()))); } - StreamsGroupMember updatedMember = updatedMemberBuilder.build(); + // If the member is new or has changed, a StreamsGroupMemberMetadataValue record is written to the __consumer_offsets partition // to persist the change, and bump the group epoch later. boolean bumpGroupEpoch = hasStreamsMemberMetadataChanged(groupId, instanceId, member, updatedMember, records); @@ -3371,7 +3370,6 @@ private StreamsGroupMember getOrMaybeCreateStaticStreamsGroupMember( } } - /** * Gets or subscribes a new dynamic share group member. * From 32398a9f9c905934a507860223a40e396beeebcc Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Sun, 10 May 2026 11:25:02 +0900 Subject: [PATCH 03/16] Add server-side test codes. --- ...amsGroupMixedGroupMetadataManagerTest.java | 781 +++++++++ ...pStaticMemberGroupMetadataManagerTest.java | 1481 +++++++++++++++++ .../group/StreamsGroupTestUtil.java | 353 ++++ 3 files changed, 2615 insertions(+) create mode 100644 group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java create mode 100644 group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java create mode 100644 group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java new file mode 100644 index 0000000000000..b8d0503588c9c --- /dev/null +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java @@ -0,0 +1,781 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.FencedInstanceIdException; +import org.apache.kafka.common.errors.GroupMaxSizeReachedException; +import org.apache.kafka.common.message.StreamsGroupHeartbeatRequestData; +import org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest; +import org.apache.kafka.coordinator.common.runtime.MetadataImageBuilder; +import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord; +import org.apache.kafka.coordinator.common.runtime.CoordinatorResult; +import org.apache.kafka.coordinator.common.runtime.MockCoordinatorTimer; +import org.apache.kafka.coordinator.group.streams.StreamsGroupBuilder; +import org.apache.kafka.coordinator.group.streams.StreamsTopology; +import org.apache.kafka.coordinator.group.streams.TasksTuple; +import org.apache.kafka.coordinator.group.streams.TasksTupleWithEpochs; +import org.apache.kafka.coordinator.group.streams.MockTaskAssignor; +import org.apache.kafka.coordinator.group.streams.StreamsGroupHeartbeatResult; +import org.apache.kafka.coordinator.group.streams.StreamsGroup; +import org.apache.kafka.coordinator.group.streams.StreamsGroupMember; +import org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers; +import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil; + +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.List; +import java.util.Optional; + +import static org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH; +import static org.apache.kafka.coordinator.group.Assertions.assertResponseEquals; +import static org.apache.kafka.coordinator.group.Assertions.assertRecordsEquals; +import static org.apache.kafka.coordinator.group.Assertions.assertUnorderedRecordsEquals; +import static org.apache.kafka.coordinator.group.GroupMetadataManager.groupSessionTimeoutKey; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticJoinHeartbeat; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.StreamsTopicFixture; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticHeartbeat; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.heartbeatResponseWithActiveTasks; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticLeaveResponseWithNullTasks; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.heartbeatResponseWithNullTasks; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.contextWithStreamsGroup; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.getDefaultAssignmentConfigs; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.streamsTopicFixture; +import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTupleWithEpochs; +import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole; +import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksWithEpochs; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; + +class StreamsGroupMixedGroupMetadataManagerTest { + + private static final int DEFAULT_GROUP_EPOCH = 10; + + @Test + public void testDynamicJoinFailsAtMaxSizeWhileStaticMemberIsTemporarilyLeftAndDynamicMemberStillExists() { + // STREAMS_GROUP_MAX_SIZE_CONFIG is 2. + // There are 2 members. (1 dynamic member, 1 static member) + // - one static member leaves temporarily with -2 epoch. + // - one dynamic member alive in group. + // Another dynamic member try to join. + + String groupId = "fooup"; + String staticMemberId = Uuid.randomUuid().toString(); + String staticInstanceId = Uuid.randomUuid().toString(); + String dynamicMemberId = Uuid.randomUuid().toString(); + String newDynamicMemberId = Uuid.randomUuid().toString(); + StreamsGroupHeartbeatRequestData.Topology topology = new StreamsGroupHeartbeatRequestData.Topology().setSubtopologies(List.of()); + + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withMetadataImage(new MetadataImageBuilder().buildCoordinatorMetadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, 2) + .withStreamsGroup(new StreamsGroupBuilder(groupId, 10) + .withMember(streamsGroupMemberBuilderWithDefaults(staticMemberId, staticInstanceId) + .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(10) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setMemberEpoch(10) + .setPreviousMemberEpoch(9) + .build()) + .withTargetAssignmentEpoch(10) + .withTopology(StreamsTopology.fromHeartbeatRequest(topology))) + .build(); + + assertThrows(GroupMaxSizeReachedException.class, () -> + context.streamsGroupHeartbeat( + staticJoinHeartbeat(groupId, newDynamicMemberId, null, "new-process-id") + .setRebalanceTimeoutMs(1500) + .setTopology(topology) + ) + ); + } + + @Test + public void testStaticRejoinSucceedsAtMaxSizeWhileDynamicMemberStillExists() { + // Scenario + // STREAMS_GROUP_MAX_SIZE_CONFIG is 2. + // There are 2 members. (1 dynamic member, 1 static member) + // - static member leaves temporarily with -2 epoch. + // - dynamic member alive in group + // static member try to rejoin. + + int groupEpoch = 10; + String groupId = "fooup"; + + String subtopologyId = "subtopology-1"; + StreamsTopicFixture topic = streamsTopicFixture(subtopologyId, "foo", 4); + + + String oldStaticMemberId = Uuid.randomUuid().toString(); + String newStaticMemberId = Uuid.randomUuid().toString(); + String staticInstanceId = Uuid.randomUuid().toString(); + String dynamicMemberId = Uuid.randomUuid().toString(); + + // GIVEN Task for static member + TasksTupleWithEpochs staticAssignedTasks = topic.assignedTasks(groupEpoch, 0, 1); + TasksTuple staticTargetAssignment = topic.targetAssignment(0, 1); + + // GIVEN Task for dynamic member + TasksTupleWithEpochs dynamicAssignedTasks = topic.assignedTasks(groupEpoch, 2, 3); + TasksTuple dynamicTargetAssignment = topic.targetAssignment(2, 3); + + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, 2) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) + .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(10) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setMemberEpoch(10) + .setPreviousMemberEpoch(9) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(oldStaticMemberId, staticTargetAssignment) + .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .build(); + + + // WHEN - static member rejoin. + CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, newStaticMemberId, staticInstanceId, StreamsGroupHeartbeatRequest.JOIN_GROUP_MEMBER_EPOCH) + ); + + // THEN - At the maxsize, static member still can rejoin if static member leaves with epoch -2. + assertResponseEquals(heartbeatResponseWithActiveTasks(newStaticMemberId, groupEpoch, subtopologyId, 0, 1), rejoinResult.response().data()); + + StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); + assertFalse(group.hasMember(oldStaticMemberId)); + assertTrue(group.hasMember(newStaticMemberId)); + assertEquals(newStaticMemberId, group.staticMember(staticInstanceId).memberId()); + + assertTrue(group.hasMember(dynamicMemberId)); + assertEquals(dynamicAssignedTasks, group.getMemberOrThrow(dynamicMemberId).assignedTasks()); + assertEquals(dynamicTargetAssignment, group.targetAssignment(dynamicMemberId, Optional.empty())); + assertEquals(groupEpoch, group.groupEpoch()); + } + + @Test + public void testDynamicJoinSucceedsAfterTemporarilyLeftStaticMemberSessionTimeoutInMixedGroup() { + // Scenario: + // STREAMS_GROUP_MAX_SIZE_CONFIG is 2. + // There are 2 members. (1 dynamic member, 1 static member) + // - static member leaves temporarily with -2 epoch. + // - dynamic member alive in group. + // After the static member session timeout expires, a new dynamic member can join. + + int groupEpoch = 10; + int timeoutGroupEpoch = groupEpoch + 1; + int joinGroupEpoch = timeoutGroupEpoch + 1; + + String groupId = "fooup"; + + String subtopologyId = "subtopology-1"; + StreamsTopicFixture topic = streamsTopicFixture(subtopologyId, "foo", 4); + StreamsGroupHeartbeatRequestData.Topology topology = topic.topology(); + + String staticMemberId = Uuid.randomUuid().toString(); + String staticInstanceId = Uuid.randomUuid().toString(); + String dynamicMemberId = Uuid.randomUuid().toString(); + String newDynamicMemberId = Uuid.randomUuid().toString(); + + String staticProcessId = "static-process-id"; + String dynamicProcessId = "dynamic-process-id"; + String newDynamicProcessId = "new-dynamic-process-id"; + + // GIVEN assignment + TasksTupleWithEpochs staticAssignedTasks = topic.assignedTasks(groupEpoch, 0, 1); + TasksTuple staticTargetAssignment = topic.targetAssignment(0, 1); + + TasksTupleWithEpochs dynamicAssignedTasks = topic.assignedTasks(groupEpoch, 2, 3); + TasksTuple dynamicTargetAssignment = topic.targetAssignment(2, 3); + + long groupMetadataHash = topic.metadataHash(); + + MockTaskAssignor assignor = new MockTaskAssignor("sticky"); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, 2) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(staticMemberId, staticInstanceId) + .setProcessId(staticProcessId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setProcessId(dynamicProcessId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(staticMemberId, staticTargetAssignment) + .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topology)) + .withValidatedTopologyEpoch(0) + .withMetadataHash(groupMetadataHash) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .build(); + context.onLoaded(); + + // WHEN1 - static member leaves with epoch -2. + CoordinatorResult leaveResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, staticMemberId, staticInstanceId, StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) + ); + + // THEN1 + assertResponseEquals(staticLeaveResponseWithNullTasks(staticMemberId, StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH), leaveResult.response().data()); + + // To prevent session timeout from dynamic member. + // Sleep 1, and dynamic member send a heartbeat. + GroupMetadataManagerTestContext.assertNoOrEmptyResult(context.sleep(1)); + CoordinatorResult dynamicHeartbeatResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, dynamicMemberId, null, groupEpoch) + ); + + assertResponseEquals(heartbeatResponseWithNullTasks(dynamicMemberId, groupEpoch), dynamicHeartbeatResult.response().data()); + assertTrue(dynamicHeartbeatResult.records().isEmpty()); + + // WHEN2: static member session timeout. + context.assertSessionTimeout(groupId, staticMemberId, 45000 - 1); + List> timeouts = context.sleep(45000 - 1); + + // THEN2 + List expectedTimeoutRecords = List.of( + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, staticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, staticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, staticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord( + groupId, timeoutGroupEpoch, groupMetadataHash, 0, getDefaultAssignmentConfigs() + ) + ); + + assertEquals( + List.of(new MockCoordinatorTimer.ExpiredTimeout<>( + groupSessionTimeoutKey(groupId, staticMemberId), + new CoordinatorResult<>(expectedTimeoutRecords) + )), + timeouts + ); + + StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); + assertFalse(group.hasMember(staticMemberId)); + assertTrue(group.hasMember(dynamicMemberId)); + assertEquals(timeoutGroupEpoch, group.groupEpoch()); + + assignor.prepareGroupAssignment(Map.of( + dynamicMemberId, dynamicTargetAssignment, + newDynamicMemberId, staticTargetAssignment + )); + + // WHEN3 - new dynamic member try to join. + CoordinatorResult joinResult = context.streamsGroupHeartbeat( + staticJoinHeartbeat(groupId, newDynamicMemberId, null, newDynamicProcessId) + .setRebalanceTimeoutMs(1500) + .setTopology(topology) + ); + + // THEN3 : accept join. + assertResponseEquals(heartbeatResponseWithActiveTasks(newDynamicMemberId, joinGroupEpoch, subtopologyId, 0, 1), joinResult.response().data()); + + StreamsGroupMember expectedJoiningDynamicMember = streamsGroupMemberBuilderWithDefaults(newDynamicMemberId) + .setProcessId(newDynamicProcessId) + .setMemberEpoch(0) + .setPreviousMemberEpoch(0) + .build(); + + StreamsGroupMember expectedReconciledDynamicMember = streamsGroupMemberBuilderWithDefaults(newDynamicMemberId) + .setProcessId(newDynamicProcessId) + .setMemberEpoch(joinGroupEpoch) + .setPreviousMemberEpoch(0) + .setAssignedTasks(mkTasksTupleWithEpochs( + TaskRole.ACTIVE, + mkTasksWithEpochs(subtopologyId, Map.of( + 0, joinGroupEpoch, + 1, joinGroupEpoch + )) + )) + .build(); + + List expectedJoinRecords = List.of( + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, expectedJoiningDynamicMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord( + groupId, + joinGroupEpoch, + groupMetadataHash, + 0, + getDefaultAssignmentConfigs() + ), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, newDynamicMemberId, staticTargetAssignment), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, joinGroupEpoch, context.time.milliseconds()), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedReconciledDynamicMember) + ); + + assertRecordsEquals(expectedJoinRecords, joinResult.records()); + + assertTrue(group.hasMember(dynamicMemberId)); + assertTrue(group.hasMember(newDynamicMemberId)); + assertEquals(joinGroupEpoch, group.groupEpoch()); + assertEquals(staticTargetAssignment, group.targetAssignment(newDynamicMemberId, Optional.empty())); + assertEquals(dynamicTargetAssignment, group.targetAssignment(dynamicMemberId, Optional.empty())); + } + + @Test + public void testStaticTemporaryLeaveDoesNotTransferTasksToExistingDynamicMember() { + // Scenario + // STREAMS_GROUP_MAX_SIZE_CONFIG is 2. + // There are 2 members. (1 dynamic member, 1 static member) + // - static member leaves temporarily with -2 epoch. + // - dynamic member alive in group + // If dynamic send a heartbeat, there is no new assignment. + int groupEpoch = DEFAULT_GROUP_EPOCH; + + String groupId = "fooup"; + String staticMemberId = Uuid.randomUuid().toString(); + String staticInstanceId = Uuid.randomUuid().toString(); + String dynamicMemberId = Uuid.randomUuid().toString(); + + + StreamsTopicFixture topic = streamsTopicFixture("subtopology-1", "foo", 3); + + // GIVEN Task for static member + TasksTupleWithEpochs staticAssignedTasks = topic.assignedTasks(groupEpoch, 0, 1); + TasksTuple staticTargetAssignment = topic.targetAssignment(0, 1); + + // GIVEN Task for dynamic member + TasksTupleWithEpochs dynamicAssignedTasks = topic.assignedTasks(groupEpoch, 2); + TasksTuple dynamicTargetAssignment = topic.targetAssignment(2); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group + .withMember(streamsGroupMemberBuilderWithDefaults(staticMemberId, staticInstanceId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(staticMemberId, staticTargetAssignment) + .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment)); + + // WHEN - static member leaves with epoch -2. + CoordinatorResult leaveResult = + context.streamsGroupHeartbeat(staticHeartbeat( + groupId, + staticMemberId, + staticInstanceId, + LEAVE_GROUP_STATIC_MEMBER_EPOCH + )); + + // THEN + assertResponseEquals( + staticLeaveResponseWithNullTasks(staticMemberId, LEAVE_GROUP_STATIC_MEMBER_EPOCH), + leaveResult.response().data() + ); + + // WHEN2 - dynamic member send a heartbeat. + CoordinatorResult dynamicHeartbeatResult = + context.streamsGroupHeartbeat( + new StreamsGroupHeartbeatRequestData() + .setGroupId(groupId) + .setMemberId(dynamicMemberId) + .setMemberEpoch(groupEpoch) + ); + + // THEN2 : There is no new assignment. + assertResponseEquals(heartbeatResponseWithNullTasks(dynamicMemberId, groupEpoch), dynamicHeartbeatResult.response().data()); + assertTrue(dynamicHeartbeatResult.records().isEmpty()); + + StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); + assertEquals(dynamicAssignedTasks, group.getMemberOrThrow(dynamicMemberId).assignedTasks()); + assertEquals(dynamicTargetAssignment, group.targetAssignment(dynamicMemberId, Optional.empty())); + assertEquals(groupEpoch, group.groupEpoch()); + } + + @Test + public void testStaticRejoinWithUpdatedProcessIdRecomputesTargetAssignmentAndDynamicMemberReconcilesInMixedGroup() { + // Scenario: + // There are 2 members. + // - static member left temporarily with -2 epoch. + // - dynamic member alive in group. + // When the same static instance rejoins with a different processId, the coordinator + // bumps the group epoch and recomputes the target assignment. + // The dynamic member keeps its current assignment until its next heartbeat. + // When the dynamic member sends the next heartbeat, it reconciles to the recomputed target assignment. + + String groupId = "fooup"; + int groupEpoch = 10; + int bumpedGroupEpoch = groupEpoch + 1; + + String subtopologyId = "subtopology-1"; + StreamsTopicFixture topic = streamsTopicFixture(subtopologyId, "foo", 4); + + String oldStaticMemberId = Uuid.randomUuid().toString(); + String rejoinStaticMemberId = Uuid.randomUuid().toString(); + String staticInstanceId = Uuid.randomUuid().toString(); + String dynamicMemberId = Uuid.randomUuid().toString(); + + String oldProcessId = "old-process-id"; + String newProcessId = "new-process-id"; + + // Initial assignment + TasksTupleWithEpochs staticAssignedTasks = topic.assignedTasks(groupEpoch, 0, 1); + TasksTuple oldStaticTargetAssignment = topic.targetAssignment(0, 1); + + TasksTupleWithEpochs dynamicAssignedTasks = topic.assignedTasks(groupEpoch, 2, 3); + TasksTuple oldDynamicTargetAssignment = topic.targetAssignment(2, 3); + + // Recomputed target assignment after static member rejoins with updated processId + TasksTuple newStaticTargetAssignment = topic.targetAssignment(0); + TasksTuple newDynamicTargetAssignment = topic.targetAssignment(1, 2, 3); + + MockTaskAssignor assignor = new MockTaskAssignor("sticky"); + assignor.prepareGroupAssignment(Map.of( + rejoinStaticMemberId, newStaticTargetAssignment, + dynamicMemberId, newDynamicTargetAssignment + )); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, group -> group + .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) + .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(groupEpoch) + .setProcessId(oldProcessId) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(oldStaticMemberId, oldStaticTargetAssignment) + .withTargetAssignment(dynamicMemberId, oldDynamicTargetAssignment)); + + // WHEN 1: static member try to rejoin with new process id. + CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( + staticJoinHeartbeat(groupId, rejoinStaticMemberId, staticInstanceId, newProcessId) + ); + + // THEN 1: + assertResponseEquals( + heartbeatResponseWithActiveTasks(rejoinStaticMemberId, bumpedGroupEpoch, topic, 0), + rejoinResult.response().data() + ); + + StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); + + // group epoch should be bumped up. + assertEquals(bumpedGroupEpoch, group.groupEpoch()); + + // static member should be replaced. + assertFalse(group.hasMember(oldStaticMemberId)); + assertTrue(group.hasMember(rejoinStaticMemberId)); + assertEquals(rejoinStaticMemberId, group.staticMember(staticInstanceId).memberId()); + + // Because group epoch is bumped up, target assignment should be recomupted. + assertEquals(newStaticTargetAssignment, group.targetAssignment(rejoinStaticMemberId, Optional.of(staticInstanceId))); + assertEquals(newDynamicTargetAssignment, group.targetAssignment(dynamicMemberId, Optional.empty())); + assertEquals(dynamicAssignedTasks, group.getMemberOrThrow(dynamicMemberId).assignedTasks()); + assertEquals(groupEpoch, group.getMemberOrThrow(dynamicMemberId).memberEpoch()); + + StreamsGroupMember expectedCopiedStaticMember = streamsGroupMemberBuilderWithDefaults(rejoinStaticMemberId, staticInstanceId) + .setProcessId(oldProcessId) + .setMemberEpoch(0) + .setPreviousMemberEpoch(0) + .setAssignedTasks(staticAssignedTasks) + .build(); + + StreamsGroupMember expectedUpdatedStaticMember = streamsGroupMemberBuilderWithDefaults(rejoinStaticMemberId, staticInstanceId) + .setProcessId(newProcessId) + .setMemberEpoch(0) + .setPreviousMemberEpoch(0) + .setAssignedTasks(staticAssignedTasks) + .build(); + + StreamsGroupMember expectedReconciledStaticMember = streamsGroupMemberBuilderWithDefaults(rejoinStaticMemberId, + staticInstanceId) + .setProcessId(newProcessId) + .setMemberEpoch(bumpedGroupEpoch) + .setPreviousMemberEpoch(0) + .setAssignedTasks(mkTasksTupleWithEpochs( + TaskAssignmentTestUtil.TaskRole.ACTIVE, + mkTasksWithEpochs(subtopologyId, Map.of(0, groupEpoch)) + )) + .build(); + + List expectedRecordsBeforeRecomputedAssignments = List.of( + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, oldStaticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, oldStaticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, oldStaticMemberId), + + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, expectedCopiedStaticMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, rejoinStaticMemberId, + oldStaticTargetAssignment), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedCopiedStaticMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, expectedUpdatedStaticMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord( + groupId, + bumpedGroupEpoch, + topic.metadataHash(), + 0, + getDefaultAssignmentConfigs() + ) + ); + + List expectedRecomputedTargetAssignmentRecords = List.of( + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, rejoinStaticMemberId, + newStaticTargetAssignment), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, dynamicMemberId, + newDynamicTargetAssignment) + ); + + List expectedRecordsAfterRecomputedAssignments = List.of( + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, bumpedGroupEpoch, + context.time.milliseconds()), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedReconciledStaticMember) + ); + + assertRecordsEquals( + expectedRecordsBeforeRecomputedAssignments, + rejoinResult.records().subList(0, 8) + ); + assertUnorderedRecordsEquals( + List.of(expectedRecomputedTargetAssignmentRecords), + rejoinResult.records().subList(8, 10) + ); + assertRecordsEquals( + expectedRecordsAfterRecomputedAssignments, + rejoinResult.records().subList(10, 12) + ); + + // WHEN 2: dynamic member send a heartbeat and reconciles to the new target assignment. + CoordinatorResult dynamicHeartbeatResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, dynamicMemberId, null, groupEpoch) + ); + + assertResponseEquals( + heartbeatResponseWithActiveTasks(dynamicMemberId, bumpedGroupEpoch, topic, 1, 2, 3), + dynamicHeartbeatResult.response().data() + ); + + StreamsGroupMember expectedUpdatedDynamicMember = streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setMemberEpoch(bumpedGroupEpoch) + .setPreviousMemberEpoch(groupEpoch) + .setAssignedTasks(mkTasksTupleWithEpochs( + TaskAssignmentTestUtil.TaskRole.ACTIVE, + mkTasksWithEpochs(subtopologyId, Map.of( + 1, bumpedGroupEpoch, + 2, groupEpoch, + 3, groupEpoch + )) + )) + .build(); + + List expectedRecordsAfterDynamicHeartbeat = List.of( + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedUpdatedDynamicMember) + ); + assertRecordsEquals(expectedRecordsAfterDynamicHeartbeat, dynamicHeartbeatResult.records()); + + assertEquals(expectedUpdatedDynamicMember.assignedTasks(), group.getMemberOrThrow(dynamicMemberId).assignedTasks()); + assertEquals(bumpedGroupEpoch, group.getMemberOrThrow(dynamicMemberId).memberEpoch()); + assertEquals(newDynamicTargetAssignment, group.targetAssignment(dynamicMemberId, Optional.empty())); + } + + @Test + public void testStaticRejoinWithSameProcessIdDoesNotBumpEpochAndDynamicHeartbeatRemainsNoOpInMixedGroup() { + // Scenario: + // There are 2 members. + // - static member left temporarily with -2 epoch. + // - dynamic member alive in group. + // When the same static instance rejoins with the same processId, the coordinator + // does not bump the group epoch or recompute the target assignment. + // When the dynamic member sends a heartbeat, there is still no new assignment. + + int groupEpoch = DEFAULT_GROUP_EPOCH; + String groupId = "fooup"; + + String oldStaticMemberId = Uuid.randomUuid().toString(); + String newStaticMemberId = Uuid.randomUuid().toString(); + String staticInstanceId = Uuid.randomUuid().toString(); + String dynamicMemberId = Uuid.randomUuid().toString(); + + String staticProcessId = "static-process-id"; + String dynamicProcessId = "dynamic-process-id"; + + StreamsTopicFixture topic = streamsTopicFixture("subtopology-1", "foo", 4); + + // GIVEN Tasks + TasksTupleWithEpochs staticAssignedTasks = topic.assignedTasks(groupEpoch, 0, 1); + TasksTuple staticTargetAssignment = topic.targetAssignment(0, 1); + + TasksTupleWithEpochs dynamicAssignedTasks = topic.assignedTasks(groupEpoch, 2, 3); + TasksTuple dynamicTargetAssignment = topic.targetAssignment(2, 3); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group + .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) + .setProcessId(staticProcessId) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(groupEpoch) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setProcessId(dynamicProcessId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(oldStaticMemberId, staticTargetAssignment) + .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment)); + + // WHEN1 : static member try to rejoin with same process id. + CoordinatorResult rejoinResult = + context.streamsGroupHeartbeat( + staticJoinHeartbeat(groupId, newStaticMemberId, staticInstanceId, staticProcessId) + ); + + + // THEN1 + assertResponseEquals(heartbeatResponseWithActiveTasks(newStaticMemberId, groupEpoch, topic, 0, 1), rejoinResult.response().data()); + + StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); + + // no group epoch bump up. + assertEquals(groupEpoch, group.groupEpoch()); + assertFalse(group.hasMember(oldStaticMemberId)); + assertTrue(group.hasMember(newStaticMemberId)); + assertTrue(group.hasMember(dynamicMemberId)); + assertEquals(newStaticMemberId, group.staticMember(staticInstanceId).memberId()); + assertEquals(staticTargetAssignment, group.targetAssignment(newStaticMemberId, Optional.of(staticInstanceId))); + assertEquals(dynamicTargetAssignment, group.targetAssignment(dynamicMemberId, Optional.empty())); + + StreamsGroupMember expectedCopiedStaticMember = streamsGroupMemberBuilderWithDefaults(newStaticMemberId, staticInstanceId) + .setProcessId(staticProcessId) + .setMemberEpoch(0) + .setPreviousMemberEpoch(0) + .setAssignedTasks(staticAssignedTasks) + .build(); + + StreamsGroupMember expectedRejoinedStaticMember = streamsGroupMemberBuilderWithDefaults(newStaticMemberId, staticInstanceId) + .setProcessId(staticProcessId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(0) + .setAssignedTasks(staticAssignedTasks) + .build(); + + // no new target assignment. + List expectedRejoinRecords = List.of( + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, oldStaticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, oldStaticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, oldStaticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, expectedCopiedStaticMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, newStaticMemberId, staticTargetAssignment), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedCopiedStaticMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedRejoinedStaticMember) + ); + + assertRecordsEquals(expectedRejoinRecords, rejoinResult.records()); + + // WHEN2 - dynamic member send a heartbeat request + CoordinatorResult dynamicHeartbeatResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, dynamicMemberId, null, groupEpoch) + ); + + // THEN2 - no new target assignment. + assertResponseEquals(heartbeatResponseWithNullTasks(dynamicMemberId, groupEpoch), dynamicHeartbeatResult.response().data()); + assertTrue(dynamicHeartbeatResult.records().isEmpty()); + + assertEquals(dynamicAssignedTasks, group.getMemberOrThrow(dynamicMemberId).assignedTasks()); + assertEquals(groupEpoch, group.getMemberOrThrow(dynamicMemberId).memberEpoch()); + assertEquals(dynamicTargetAssignment, group.targetAssignment(dynamicMemberId, Optional.empty())); + } + + @Test + public void testOldStaticMemberIdIsFencedAfterReplacementInMixedGroup() { + // Scenario: + // There are 2 members. + // - static member left temporarily with -2 epoch. + // - dynamic member alive in group. + // When the static member rejoins with a new memberId, the old memberId is fenced. + int groupEpoch = DEFAULT_GROUP_EPOCH; + String groupId = "fooup"; + + String oldStaticMemberId = Uuid.randomUuid().toString(); + String newStaticMemberId = Uuid.randomUuid().toString(); + String staticInstanceId = Uuid.randomUuid().toString(); + String dynamicMemberId = Uuid.randomUuid().toString(); + + String staticProcessId = "static-process-id"; + String dynamicProcessId = "dynamic-process-id"; + + StreamsTopicFixture topic = streamsTopicFixture("subtopology-1", "foo", 4); + + TasksTupleWithEpochs staticAssignedTasks = topic.assignedTasks(groupEpoch, 0, 1); + TasksTuple staticTargetAssignment = topic.targetAssignment(0, 1); + + TasksTupleWithEpochs dynamicAssignedTasks = topic.assignedTasks(groupEpoch, 2, 3); + TasksTuple dynamicTargetAssignment = topic.targetAssignment(2, 3); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group + .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) + .setProcessId(staticProcessId) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(groupEpoch) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setProcessId(dynamicProcessId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(oldStaticMemberId, staticTargetAssignment) + .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment)); + + // WHEN1 - static member try to rejoin with new member id. + context.streamsGroupHeartbeat( + staticJoinHeartbeat(groupId, newStaticMemberId, staticInstanceId, staticProcessId) + ); + + // WHEN2 + THEN2 - stale static member send a heartbeat with stale member id. + assertThrows(FencedInstanceIdException.class, () -> + context.streamsGroupHeartbeat(staticHeartbeat(groupId, oldStaticMemberId, staticInstanceId, groupEpoch)) + ); + + StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); + assertFalse(group.hasMember(oldStaticMemberId)); + assertTrue(group.hasMember(newStaticMemberId)); + assertTrue(group.hasMember(dynamicMemberId)); + assertEquals(newStaticMemberId, group.staticMember(staticInstanceId).memberId()); + } +} \ No newline at end of file diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java new file mode 100644 index 0000000000000..5c14e1cb72bd3 --- /dev/null +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java @@ -0,0 +1,1481 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.FencedInstanceIdException; +import org.apache.kafka.common.errors.FencedMemberEpochException; +import org.apache.kafka.common.errors.GroupIdNotFoundException; +import org.apache.kafka.common.errors.GroupMaxSizeReachedException; +import org.apache.kafka.common.errors.UnknownMemberIdException; +import org.apache.kafka.common.errors.UnreleasedInstanceIdException; +import org.apache.kafka.common.message.StreamsGroupHeartbeatRequestData; +import org.apache.kafka.common.message.StreamsGroupHeartbeatResponseData; +import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord; +import org.apache.kafka.coordinator.common.runtime.CoordinatorResult; +import org.apache.kafka.coordinator.common.runtime.MetadataImageBuilder; +import org.apache.kafka.coordinator.common.runtime.MockCoordinatorTimer; +import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataKey; +import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataValue; +import org.apache.kafka.coordinator.group.streams.MockTaskAssignor; +import org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers; +import org.apache.kafka.coordinator.group.streams.StreamsGroup; +import org.apache.kafka.coordinator.group.streams.StreamsGroupBuilder; +import org.apache.kafka.coordinator.group.streams.StreamsGroupHeartbeatResult; +import org.apache.kafka.coordinator.group.streams.StreamsGroupMember; +import org.apache.kafka.coordinator.group.streams.StreamsTopology; +import org.apache.kafka.coordinator.group.streams.TasksTuple; +import org.apache.kafka.coordinator.group.streams.TasksTupleWithEpochs; +import org.apache.kafka.coordinator.group.streams.MemberState; +import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue; + +import org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +import static org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH; +import static org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest.LEAVE_GROUP_MEMBER_EPOCH; +import static org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest.JOIN_GROUP_MEMBER_EPOCH; +import static org.apache.kafka.coordinator.group.Assertions.assertResponseEquals; +import static org.apache.kafka.coordinator.group.Assertions.assertRecordsEquals; +import static org.apache.kafka.coordinator.group.GroupMetadataManager.groupSessionTimeoutKey; +import static org.apache.kafka.coordinator.group.GroupMetadataManagerTestContext.DEFAULT_PROCESS_ID; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.kafka.coordinator.group.StreamsGroupTestUtil.StreamsTopicFixture; + +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.contextWithStreamsGroup; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.getDefaultAssignmentConfigs; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.heartbeatResponseWithActiveTasks; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.heartbeatResponseWithNullTasks; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.resetAssignedTasksEpochsToZero; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticHeartbeat; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticJoinHeartbeat; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticLeaveResponse; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticLeaveResponseWithNullTasks; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.streamsTopicFixture; + +class StreamsGroupStaticMemberGroupMetadataManagerTest { + + private static final int DEFAULT_MEMBER_EPOCH = 10; + private static final int DEFAULT_GROUP_EPOCH = 10; + + @Test + public void testUnknownStaticMemberLeaveStreamsGroup() { + StaticMemberFixtureWith2Members fixture = new StaticMemberFixtureWith2Members(2); + + fixture.assertLeaveFails( + fixture.staticLeaveRequest("unknown-member-id", "unknown-instance-id"), + UnknownMemberIdException.class + ); + } + + @Test + public void testStreamsStaticJoinWithNewInstanceAtMaxSizeThrowsGroupMaxSizeReached() { + // With max.size=2 already reached, + // joining with a new static instanceId must throw GroupMaxSizeReachedException. + int streamsGroupMaxSize = 2; + StaticMemberFixtureWith2Members fixture = new StaticMemberFixtureWith2Members(streamsGroupMaxSize); + + fixture.assertJoinFails( + fixture.newMemberJoinsWithNewInstanceId(), + GroupMaxSizeReachedException.class + ); + } + + @Test + public void testStreamsStaticRejoinWithLeaveGroupStaticEpochAtMaxSizeSucceeds() { + // If a static member is in leave epoch (-2), + // rejoining with the same memberId/instanceId is allowed even at max size. + int streamsGroupMaxSize = 2; + StaticMemberFixtureWith2Members fixture = new StaticMemberFixtureWith2Members(streamsGroupMaxSize); + + fixture.assertJoinSucceeds( + fixture.leftMemberRejoinsWithSameInstanceId() + ); + } + + @Test + public void testStreamsStaticJoinWithUnreleasedInstanceThrowsUnreleasedInstanceIdAtMaxSize() { + // If an active static member (epoch=10) still owns the instanceId, + // a different memberId joining with that instanceId must fail even at max size. + int streamsGroupMaxSize = 2; + StaticMemberFixtureWith2Members fixture = new StaticMemberFixtureWith2Members(streamsGroupMaxSize); + + fixture.assertJoinFails( + fixture.newMemberJoinsWithActiveInstanceId(), + UnreleasedInstanceIdException.class + ); + } + + @ParameterizedTest + @MethodSource("staticMemberReusedInstanceErrorCases") + public void testStaticMemberSendHeartbeatWithVariousEpochThenThrowError(int whenMemberEpoch, Class expectedException) { + StaticMemberFixtureWith2Members fixture = new StaticMemberFixtureWith2Members(3); + // WHEN: same instance id is reused with mismatched member identity/epoch. THEN: expected exception is thrown. + fixture.assertHeartbeatFails( + fixture.newMemberHeartbeatsWithActiveInstanceId(whenMemberEpoch), + expectedException + ); + } + + private static Stream staticMemberReusedInstanceErrorCases() { + return Stream.of( + Arguments.of(0, UnreleasedInstanceIdException.class), // static member try to join when static member already existed, then throw UnreleasedInstanceIdException. + Arguments.of(1000, FencedInstanceIdException.class) // static member try to send bigger epoch when static member already existed, then throw FencedInstanceIdException. + ); + } + + @Test + public void testStaticMemberJoinThenRevokeAndReceiveTasks() { + int enoughMaxSize = 100; + testStaticMemberJoinThenRevokeAndReceiveTasksWith2Members(enoughMaxSize); + } + + @Test + public void testStaticMemberJoinThenRevokeAndReceiveTasksInMaxSizeBoundary() { + int boundarySize = 2; + testStaticMemberJoinThenRevokeAndReceiveTasksWith2Members(boundarySize); + } + + private void testStaticMemberJoinThenRevokeAndReceiveTasksWith2Members(int maxSize) { + String groupId = "fooup"; + + String memberId1 = Uuid.randomUuid().toString(); + String memberId2 = Uuid.randomUuid().toString(); + String otherMemberId2 = Uuid.randomUuid().toString(); + + String instanceId1 = Uuid.randomUuid().toString(); + String instanceId2 = Uuid.randomUuid().toString(); + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 4); + + MockTaskAssignor assignor = new MockTaskAssignor("sticky"); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, maxSize) + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, 10) + .withMember(StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults(memberId1) + .setInstanceId(instanceId1) + .setMemberEpoch(10) + .setPreviousMemberEpoch(9) + .setAssignedTasks(topic.assignedTasks(10, 0, 1, 2, 3)) + .build()) + .withTargetAssignment(memberId1, topic.targetAssignment(0, 1, 2, 3)) + .withTargetAssignmentEpoch(10) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withMetadataHash(topic.metadataHash()) + .withValidatedTopologyEpoch(0) + ) + .build(); + + // Next target assignment after member2 joins. + assignor.prepareGroupAssignment(Map.of( + memberId1, topic.targetAssignment(0, 1), + memberId2, topic.targetAssignment(2, 3) + )); + + // 1) Static member2 joins. It gets no active tasks yet because member1 still owns them. + CoordinatorResult joinResult = context.streamsGroupHeartbeat( + staticJoinHeartbeat(groupId, memberId2, instanceId2, topic) + ); + + assertResponseEquals( + heartbeatResponseWithActiveTasks(memberId2, 11, List.of()), + joinResult.response().data() + ); + + // 2) member1 receives revocation instruction: keep only [0,1]. + CoordinatorResult revokeInstructionResult = context.streamsGroupHeartbeat( + new StreamsGroupHeartbeatRequestData() + .setGroupId(groupId) + .setInstanceId(instanceId1) + .setMemberId(memberId1) + .setMemberEpoch(10) + ); + + assertResponseEquals(heartbeatResponseWithActiveTasks(memberId1, 10, topic, 0, 1), revokeInstructionResult.response().data()); + + // 3) member1 acknowledges revocation by reporting owned active tasks [0,1]. + CoordinatorResult revokeAckResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, memberId1, instanceId1, 10) + .setActiveTasks(List.of( + new StreamsGroupHeartbeatRequestData.TaskIds() + .setSubtopologyId("subtopology1") + .setPartitions(List.of(0, 1)) + )) + .setStandbyTasks(List.of()) + .setWarmupTasks(List.of()) + ); + + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId1) + .setMemberEpoch(11) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setStatus(List.of()), + revokeAckResult.response().data() + ); + + // 4) member2 heartbeats again and now receives [2,3]. + CoordinatorResult member2ReceiveResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, memberId2, instanceId2, 11) + ); + + assertResponseEquals(heartbeatResponseWithActiveTasks(memberId2, 11, topic, 2, 3), member2ReceiveResult.response().data()); + + // 5) member2 leave. + CoordinatorResult member2LeaveResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, memberId2, instanceId2, LEAVE_GROUP_STATIC_MEMBER_EPOCH) + ); + + assertResponseEquals( + staticLeaveResponseWithNullTasks(memberId2, LEAVE_GROUP_STATIC_MEMBER_EPOCH).setHeartbeatIntervalMs(0), + member2LeaveResult.response().data() + ); + + // 6) member2 re-join with other memberId. + CoordinatorResult member2rejoinResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, otherMemberId2, instanceId2, JOIN_GROUP_MEMBER_EPOCH) + ); + + assertResponseEquals( + heartbeatResponseWithActiveTasks(otherMemberId2, 11, topic, 2, 3), + member2rejoinResult.response().data() + ); + } + + @Test + public void testStaticMemberRejoinWithUpdatedProcessIdBumpsStreamsGroupEpoch() { + String groupId = "fooup"; + int groupEpoch = DEFAULT_GROUP_EPOCH; + int bumpedGroupEpoch = groupEpoch + 1; + + String oldMemberId = Uuid.randomUuid().toString(); + String rejoinMemberId = Uuid.randomUuid().toString(); + String instanceId = Uuid.randomUuid().toString(); + + String oldProcessId = "old-process-id"; + String newProcessId = "new-process-id"; + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 4); + TasksTupleWithEpochs assignedTasks = topic.assignedTasks(groupEpoch, 0, 1, 2, 3); + TasksTuple targetAssignment = topic.targetAssignment(0, 1, 2, 3); + + MockTaskAssignor assignor = new MockTaskAssignor("sticky"); + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, group -> group + .withMember(streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(groupEpoch) + .setProcessId(oldProcessId) + .setAssignedTasks(assignedTasks) + .build()) + .withTargetAssignment(oldMemberId, targetAssignment)); + + assignor.prepareGroupAssignment(Map.of(rejoinMemberId, targetAssignment)); + + CoordinatorResult result = context.streamsGroupHeartbeat( + staticJoinHeartbeat(groupId, rejoinMemberId, instanceId, newProcessId) + ); + + assertEquals(rejoinMemberId, result.response().data().memberId()); + assertEquals(bumpedGroupEpoch, result.response().data().memberEpoch()); + + CoordinatorRecord metadataRecord = result.records().stream() + .filter(record -> record.key() instanceof StreamsGroupMetadataKey) + .findFirst() + .orElse(null); + + assertNotNull(metadataRecord, "Expected a StreamsGroupMetadata record when static member config changes."); + StreamsGroupMetadataValue metadataValue = (StreamsGroupMetadataValue) metadataRecord.value().message(); + assertEquals(bumpedGroupEpoch, metadataValue.epoch()); + + assertTrue(result.records().contains( + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord( + groupId, + bumpedGroupEpoch, + topic.metadataHash(), + 0, + getDefaultAssignmentConfigs() + ) + )); + } + + @Test + public void testStaticMemberLeaveWithMinusOneFencesMemberAndBumpsStreamsGroupEpoch() { + String groupId = "fooup"; + int groupEpoch = DEFAULT_GROUP_EPOCH; + int bumpedGroupEpoch = groupEpoch + 1; + + String memberId = Uuid.randomUuid().toString(); + String instanceId = Uuid.randomUuid().toString(); + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 4); + TasksTupleWithEpochs assignedTasks = topic.assignedTasks(groupEpoch, 0, 1, 2, 3); + TasksTuple targetAssignment = topic.targetAssignment(0, 1, 2, 3); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(assignedTasks) + .build()) + .withTargetAssignment(memberId, targetAssignment)); + + CoordinatorResult result = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, memberId, instanceId, LEAVE_GROUP_MEMBER_EPOCH) + ); + + assertResponseEquals(staticLeaveResponse(memberId, LEAVE_GROUP_MEMBER_EPOCH), result.response().data()); + + assertRecordsEquals( + List.of( + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, bumpedGroupEpoch, topic.metadataHash(), 0, getDefaultAssignmentConfigs()), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, bumpedGroupEpoch, 0L) + ), + result.records() + ); + } + + @Test + public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpoch() { + // GIVEN + int leaveEpoch = LEAVE_GROUP_STATIC_MEMBER_EPOCH; + int memberEpoch = DEFAULT_MEMBER_EPOCH; + int groupEpoch = DEFAULT_GROUP_EPOCH; + + String groupId = "fooup"; + String memberId = Uuid.randomUuid().toString(); + String instanceId = Uuid.randomUuid().toString(); + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 4); + TasksTupleWithEpochs assignedTasks = topic.assignedTasks(groupEpoch, 0, 1, 2, 3); + TasksTupleWithEpochs pendingRevocationTasks = topic.assignedTasks(groupEpoch, 2, 3); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(assignedTasks) + .setTasksPendingRevocation(pendingRevocationTasks) + .build()) + .withTargetAssignment(memberId, topic.targetAssignment(0, 1, 2, 3))); + + // WHEN + CoordinatorResult result = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) + ); + + // THEN + assertResponseEquals(staticLeaveResponse(memberId, leaveEpoch), result.response().data()); + + // No group epoch bump. + // Member epoch should be -2. + // task still remain. + // pendingRevocationTasks should be EMPTY. + StreamsGroupMember expectedMemberInResponse = streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(leaveEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) + .build(); + assertRecordsEquals( + List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedMemberInResponse)), + result.records() + ); + assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); + } + + @Test + public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochThenShouldBeIdempotence() { + // GIVEN + int leaveEpoch = LEAVE_GROUP_STATIC_MEMBER_EPOCH; + int memberEpoch = DEFAULT_MEMBER_EPOCH; + int groupEpoch = DEFAULT_GROUP_EPOCH; + + String groupId = "fooup"; + String memberId = Uuid.randomUuid().toString(); + String instanceId = Uuid.randomUuid().toString(); + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 4); + TasksTupleWithEpochs assignedTasks = topic.assignedTasks(groupEpoch, 0, 1, 2, 3); + TasksTuple targetAssignment = topic.targetAssignment(0, 1, 2, 3); + + StreamsGroupMember alreadyLeftStaticMember = streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(leaveEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .build(); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group + .withMember(alreadyLeftStaticMember) + .withTargetAssignment(memberId, targetAssignment)); + + CoordinatorResult result = + context.streamsGroupHeartbeat(staticHeartbeat(groupId, memberId, instanceId, leaveEpoch)); + + + // THEN + assertResponseEquals(staticLeaveResponse(memberId, leaveEpoch), result.response().data()); + + assertRecordsEquals( + List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, alreadyLeftStaticMember)), + result.records() + ); + assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); + } + + @Test + public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochAndRejoinWithNewMemberId() { + String instanceId = Uuid.randomUuid().toString(); + String oldMemberId = Uuid.randomUuid().toString(); + String newMemberId = Uuid.randomUuid().toString(); + + verifyStaticMemberLeaveAndRejoinNoGroupBump(instanceId, oldMemberId, newMemberId); + } + + @Test + public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochAndRejoinWithSameMemberId() { + String instanceId = Uuid.randomUuid().toString(); + String memberId = Uuid.randomUuid().toString(); + + verifyStaticMemberLeaveAndRejoinNoGroupBump(instanceId, memberId, memberId); + } + + private void verifyStaticMemberLeaveAndRejoinNoGroupBump(String instanceId, String oldMemberId, String newMemberId) { + /* + * Verifies: + * 1. leave(-2) does not bump group epoch. + * 2. rejoin restores member epoch and assignment. + * 3. replacement/tombstone records are written as expected. + */ + + // GIVEN + int leaveEpoch = LEAVE_GROUP_STATIC_MEMBER_EPOCH; + int memberEpoch = DEFAULT_MEMBER_EPOCH; + int groupEpoch = DEFAULT_GROUP_EPOCH; + + String groupId = "fooup"; + + String subtopology1 = "subtopology1"; + StreamsTopicFixture topic = streamsTopicFixture(subtopology1, "foo", 4); + + // GIVEN Task + TasksTupleWithEpochs givenAssignedTasks = topic.assignedTasks(memberEpoch, 0, 1, 2, 3); + TasksTuple givenTargetAssignment = topic.targetAssignment(0, 1, 2, 3); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group + .withMember(streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(givenAssignedTasks) + .build()) + .withTargetAssignment(oldMemberId, givenTargetAssignment)); + + // WHEN1 : normal heart beat. + CoordinatorResult normalHeartbeatResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, oldMemberId, instanceId, memberEpoch) + ); + + // THEN1 : + // - all tasks should be null because assigned tasks unchanged. + // - Keep the group epoch. + assertResponseEquals(heartbeatResponseWithNullTasks(oldMemberId, memberEpoch), normalHeartbeatResult.response().data()); + assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); + + + // WHEN2 : Stream Member leave with -2 + CoordinatorResult leaveResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, oldMemberId, instanceId, leaveEpoch) + ); + + // THEN2 + // - Keep the group epoch. + assertResponseEquals(staticLeaveResponseWithNullTasks(oldMemberId, leaveEpoch), leaveResult.response().data()); + assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); + + // WHEN3 : Streams Member rejoin with other memberId + CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) + ); + + // THEN3 : + // - Inherit previous member's member epoch, and assigned tasks. + // - Keep the member epoch bump. + // - Keep the group epoch bump. + assertResponseEquals(heartbeatResponseWithActiveTasks(newMemberId, memberEpoch, topic, 0, 1, 2, 3), rejoinResult.response().data()); + assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); + + StreamsGroupMember newJoinStaticMember = streamsGroupMemberBuilderWithDefaults(newMemberId, instanceId) + .setMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) + .setPreviousMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) + .setAssignedTasks(resetAssignedTasksEpochsToZero(givenAssignedTasks)) + .build(); + + StreamsGroupMember withPrevMemberId = streamsGroupMemberBuilderWithDefaults(newMemberId, instanceId) + .setMemberEpoch(memberEpoch) // 0 -> 10 + .setPreviousMemberEpoch(0) // 0 -> 0 + .setAssignedTasks(resetAssignedTasksEpochsToZero(givenAssignedTasks)) + .build(); + + assertRecordsEquals( + List.of( + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, oldMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, oldMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, oldMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, newJoinStaticMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, newJoinStaticMember.memberId(), givenTargetAssignment), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, newJoinStaticMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, withPrevMemberId) + ), + rejoinResult.records() + ); + } + + @Test + public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnrevokedState() { + int leaveEpoch = LEAVE_GROUP_STATIC_MEMBER_EPOCH; + int memberEpoch = DEFAULT_MEMBER_EPOCH; + int groupEpoch = DEFAULT_GROUP_EPOCH; + + String groupId = "fooup"; + String instanceId = Uuid.randomUuid().toString(); + String memberId = Uuid.randomUuid().toString(); + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 4); + TasksTupleWithEpochs assignedTasks = topic.assignedTasks(memberEpoch, 0, 1); + TasksTupleWithEpochs tasksPendingRevocation = topic.assignedTasks(memberEpoch, 2, 3); + TasksTuple targetAssignment = topic.targetAssignment(0, 1); + + StreamsGroupMember unrevokedMember = streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setState(MemberState.UNREVOKED_TASKS) + .setAssignedTasks(assignedTasks) + .setTasksPendingRevocation(tasksPendingRevocation) + .build(); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group + .withMember(unrevokedMember) + .withTargetAssignment(memberId, targetAssignment)); + + // WHEN + CoordinatorResult result = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) + ); + + // THEN + StreamsGroupMember expectedMember = streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(leaveEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) + .build(); + + assertResponseEquals(staticLeaveResponseWithNullTasks(memberId, leaveEpoch), result.response().data()); + assertRecordsEquals( + List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedMember)), + result.records() + ); + assertEquals(MemberState.STABLE, context.streamsGroupMemberState(groupId, memberId)); + assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); + } + + @Test + public void testStaticMemberRejoinsAfterTemporaryLeave() { + int leaveEpoch = LEAVE_GROUP_STATIC_MEMBER_EPOCH; + int memberEpoch = DEFAULT_MEMBER_EPOCH; + int groupEpoch = DEFAULT_GROUP_EPOCH; + + String groupId = "fooup"; + String instanceId = Uuid.randomUuid().toString(); + String oldMemberId = Uuid.randomUuid().toString(); + String newMemberId = Uuid.randomUuid().toString(); + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 4); + TasksTupleWithEpochs assignedTasks = topic.assignedTasks(memberEpoch, 0, 1); + TasksTuple targetAssignment = topic.targetAssignment(0, 1); + + StreamsGroupMember temporarilyLeftMember = streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) + .setMemberEpoch(leaveEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) + .build(); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group + .withMember(temporarilyLeftMember) + .withTargetAssignment(oldMemberId, targetAssignment)); + + CoordinatorResult result = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) + ); + + assertResponseEquals(heartbeatResponseWithActiveTasks(newMemberId, memberEpoch, topic, 0, 1), result.response().data()); + assertEquals(MemberState.STABLE, context.streamsGroupMemberState(groupId, newMemberId)); + assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); + } + + @Test + public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnreleasedState() { + int leaveEpoch = LEAVE_GROUP_STATIC_MEMBER_EPOCH; + int memberEpoch = DEFAULT_MEMBER_EPOCH; + int groupEpoch = DEFAULT_GROUP_EPOCH; + + String groupId = "fooup"; + String instanceId = Uuid.randomUuid().toString(); + String memberId = Uuid.randomUuid().toString(); + String processId = Uuid.randomUuid().toString(); + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 3); + TasksTupleWithEpochs assignedTasks = topic.assignedTasks(memberEpoch, 0, 1); + TasksTuple targetAssignment = topic.targetAssignment(0, 1, 2); + + StreamsGroupMember unreleasedMember = streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setProcessId(processId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setState(MemberState.UNRELEASED_TASKS) + .setAssignedTasks(assignedTasks) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .build(); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group + .withMember(unreleasedMember) + .withTargetAssignment(memberId, targetAssignment)); + + CoordinatorResult result = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) + ); + + StreamsGroupMember expectedMember = streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setProcessId(processId) + .setMemberEpoch(leaveEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setState(MemberState.UNRELEASED_TASKS) + .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .build(); + + assertResponseEquals(staticLeaveResponseWithNullTasks(memberId, leaveEpoch), result.response().data()); + assertRecordsEquals( + List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedMember)), + result.records() + ); + assertEquals(MemberState.UNRELEASED_TASKS, context.streamsGroupMemberState(groupId, memberId)); + assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); + } + + @Test + public void testStaticMemberRejoinsAfterTemporaryLeaveFromUnreleasedState() { + int leaveEpoch = LEAVE_GROUP_STATIC_MEMBER_EPOCH; + int memberEpoch = DEFAULT_MEMBER_EPOCH; + int groupEpoch = DEFAULT_GROUP_EPOCH; + + String groupId = "fooup"; + String instanceId = Uuid.randomUuid().toString(); + String oldMemberId = "old-member-id"; + String newMemberId = "new-member-id"; + String otherMemberId = "other-member-id"; + String oldProcessId = "old-process-id"; + String otherProcessId = "other-process-id"; + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 3); + TasksTupleWithEpochs assignedTasks = topic.assignedTasks(memberEpoch, 0, 1); + TasksTupleWithEpochs otherTasksPendingRevocation = topic.assignedTasks(memberEpoch, 2); + TasksTuple targetAssignment = topic.targetAssignment(0, 1, 2); + + StreamsGroupMember temporarilyLeftMember = StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) + .setProcessId(oldProcessId) + .setMemberEpoch(leaveEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setState(MemberState.UNRELEASED_TASKS) + .setAssignedTasks(assignedTasks) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .build(); + + StreamsGroupMember otherMember = StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults(otherMemberId) + .setProcessId(otherProcessId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setState(MemberState.UNREVOKED_TASKS) + .setTasksPendingRevocation(otherTasksPendingRevocation) + .build(); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group + .withMember(temporarilyLeftMember) + .withMember(otherMember) + .withTargetAssignment(oldMemberId, targetAssignment) + .withTargetAssignment(otherMemberId, TasksTuple.EMPTY)); + + CoordinatorResult result = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) + ); + + assertResponseEquals(heartbeatResponseWithActiveTasks(newMemberId, memberEpoch, topic, 0, 1), result.response().data()); + assertEquals(MemberState.UNRELEASED_TASKS, context.streamsGroupMemberState(groupId, newMemberId)); + assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); + } + + + @Test + public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnrevokedStateAllowsUnreleasedMemberToProgress() { + int leaveEpoch = LEAVE_GROUP_STATIC_MEMBER_EPOCH; + int memberEpoch = DEFAULT_MEMBER_EPOCH; + int groupEpoch = DEFAULT_GROUP_EPOCH; + + String groupId = "fooup"; + String instanceId = Uuid.randomUuid().toString(); + String leavingMemberId = Uuid.randomUuid().toString(); + String waitingMemberId = Uuid.randomUuid().toString(); + + String subtopology1 = "subtopology1"; + StreamsTopicFixture topic = streamsTopicFixture(subtopology1, "foo", 3); + + // GIVEN Tasks + TasksTupleWithEpochs assignedTasks = topic.assignedTasks(memberEpoch, 0, 1); + TasksTupleWithEpochs tasksPendingRevocation = topic.assignedTasks(memberEpoch, 2); + TasksTuple leavingTargetAssignment = topic.targetAssignment(0, 1); + TasksTuple waitingTargetAssignment = topic.targetAssignment(2); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group + .withMember(streamsGroupMemberBuilderWithDefaults(leavingMemberId, instanceId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setState(MemberState.UNREVOKED_TASKS) + .setAssignedTasks(assignedTasks) + .setTasksPendingRevocation(tasksPendingRevocation) + .build()) + .withMember(StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults(waitingMemberId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch) + .setState(MemberState.UNRELEASED_TASKS) + .build()) + .withTargetAssignment(leavingMemberId, leavingTargetAssignment) + .withTargetAssignment(waitingMemberId, waitingTargetAssignment)); + + // WHEN1 - leave + CoordinatorResult leaveResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, leavingMemberId, instanceId, leaveEpoch) + ); + + // THEN1 + StreamsGroupHeartbeatResponseData expectedLeavingResponse = staticLeaveResponseWithNullTasks(leavingMemberId, leaveEpoch); + assertResponseEquals(expectedLeavingResponse, leaveResult.response().data()); + List expectedRecordsTriggeredByLeave = List.of( + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, + streamsGroupMemberBuilderWithDefaults(leavingMemberId, instanceId) + .setMemberEpoch(leaveEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) + .build())); + assertRecordsEquals(expectedRecordsTriggeredByLeave, leaveResult.records()); + assertEquals(MemberState.STABLE, context.streamsGroupMemberState(groupId, leavingMemberId)); + + // When2 - Waiting member send a heartbeat expecting get unreleased tasks. + CoordinatorResult waitingMemberResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, waitingMemberId, null, memberEpoch) + ); + + // THEN2 + StreamsGroupHeartbeatResponseData expectedWaitngMemberResponse = heartbeatResponseWithActiveTasks(waitingMemberId, memberEpoch, topic, 2); + assertResponseEquals(expectedWaitngMemberResponse, waitingMemberResult.response().data()); + + List expectedRecordsTriggeredByWaitngMember = List.of( + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, + StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults(waitingMemberId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch) + .setAssignedTasks(topic.assignedTasks(memberEpoch, 2)) + .build()) + ); + assertRecordsEquals(expectedRecordsTriggeredByWaitngMember, waitingMemberResult.records()); + + assertEquals(MemberState.STABLE, context.streamsGroupMemberState(groupId, waitingMemberId)); + assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); + } + + + @Test + public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochAndRejoinAndOtherRackIdThenGroupBumpOccur() { + // GIVEN + int leaveEpoch = LEAVE_GROUP_STATIC_MEMBER_EPOCH; + int memberEpoch = DEFAULT_MEMBER_EPOCH; + int groupEpoch = DEFAULT_GROUP_EPOCH; + + String groupId = "fooup"; + String rackId = Uuid.randomUuid().toString(); + String memberId = Uuid.randomUuid().toString(); + String instanceId = Uuid.randomUuid().toString(); + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 4); + + // GIVEN Task + TasksTupleWithEpochs givenAssignedTasks = topic.assignedTasks(memberEpoch, 0, 1, 2, 3); + TasksTuple givenTargetAssignment = topic.targetAssignment(0, 1, 2, 3); + + // GIVEN Assignor + MockTaskAssignor assignor = new MockTaskAssignor("sticky"); + + long groupMetadataHash = topic.metadataHash(); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, group -> group + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(memberEpoch) + .setRackId(rackId) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(givenAssignedTasks) + .build()) + .withTargetAssignment(memberId, givenTargetAssignment)); + + // WHEN1 : normal heart beat. + CoordinatorResult normalHeartbeatResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, memberId, instanceId, memberEpoch) + .setRackId(rackId) + ); + + // THEN1 : + // - all tasks should be null because assigned tasks unchanged. + // - Keep the group epoch. + assertResponseEquals(heartbeatResponseWithNullTasks(memberId, memberEpoch), normalHeartbeatResult.response().data()); + assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); + + + // WHEN2 : Stream Member leave with -2 + CoordinatorResult leaveResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) + .setRackId(rackId) + ); + + // THEN2 + // - Keep the group epoch. + assertResponseEquals(staticLeaveResponseWithNullTasks(memberId, leaveEpoch), leaveResult.response().data()); + assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); + + // GIVEN3 + String newMemberId = Uuid.randomUuid().toString(); + String newRackId = Uuid.randomUuid().toString(); + assignor.prepareGroupAssignment(Map.of(newMemberId, givenTargetAssignment)); + + int bumpedGroupEpoch = groupEpoch + 1; + int bumpedMemberEpoch = memberEpoch + 1; + + // WHEN3 : Streams Member rejoin with other memberId and rackId + CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) + .setRackId(newRackId) + ); + + // THEN3 : + // - Inherit previous member's member epoch, and assigned tasks. + // - member epoch should be bumped. + // - group epoch should be bumped. + assertResponseEquals(heartbeatResponseWithActiveTasks(newMemberId, bumpedMemberEpoch, topic, 0, 1, 2, 3), rejoinResult.response().data()); + assertEquals(bumpedGroupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); + + StreamsGroupMember transationStaticInitMember = streamsGroupMemberBuilderWithDefaults(newMemberId, instanceId) + .setMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) + .setPreviousMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) + .setRackId(rackId) + .setAssignedTasks(resetAssignedTasksEpochsToZero(givenAssignedTasks)) + .build(); + + StreamsGroupMember newJoinStaticMember = streamsGroupMemberBuilderWithDefaults(newMemberId, instanceId) + .setMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) + .setPreviousMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) + .setRackId(newRackId) + .setAssignedTasks(resetAssignedTasksEpochsToZero(givenAssignedTasks)) + .build(); + + StreamsGroupMember reconciledMember = streamsGroupMemberBuilderWithDefaults(newMemberId, instanceId) + .setMemberEpoch(bumpedMemberEpoch) + .setPreviousMemberEpoch(0) + .setRackId(newRackId) + .setAssignedTasks(resetAssignedTasksEpochsToZero(givenAssignedTasks)) + .build(); + + assertRecordsEquals( + List.of( + // From eplaceStreamsMembers + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, transationStaticInitMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, transationStaticInitMember.memberId(), givenTargetAssignment), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, transationStaticInitMember), + + // From hasStreamsMemberMetadataChanged + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, newJoinStaticMember), + + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, bumpedGroupEpoch, groupMetadataHash, 0, getDefaultAssignmentConfigs()), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, newJoinStaticMember.memberId(), givenTargetAssignment), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, bumpedGroupEpoch, context.time.milliseconds()), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, reconciledMember) + ), + rejoinResult.records() + ); + } + + + @Test + public void testStaticMemberRejoinWritesReplacementRecordsInStreamsGroup() { + int groupEpoch = DEFAULT_GROUP_EPOCH; + + + String groupId = "fooup"; + String oldMemberId = Uuid.randomUuid().toString(); + String rejoinMemberId = Uuid.randomUuid().toString(); + String instanceId = Uuid.randomUuid().toString(); + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 4); + TasksTuple oldTargetAssignment = topic.targetAssignment(0, 1, 2, 3); + TasksTupleWithEpochs assignedTasks = topic.assignedTasks(groupEpoch, 0, 1, 2, 3); + + StreamsGroupMember oldMember = streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(groupEpoch) + .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) + .build(); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group + .withMember(oldMember) + .withTargetAssignment(oldMemberId, oldTargetAssignment)); + + CoordinatorResult result = context.streamsGroupHeartbeat( + staticJoinHeartbeat(groupId, rejoinMemberId, instanceId, DEFAULT_PROCESS_ID) + ); + + assertEquals(rejoinMemberId, result.response().data().memberId()); + assertEquals(groupEpoch, result.response().data().memberEpoch()); + + StreamsGroupMember expectedCopiedMember = new StreamsGroupMember.Builder(oldMember, rejoinMemberId) + .setMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) + .setPreviousMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) + .build(); + + assertTrue(result.records().contains( + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, oldMemberId) + )); + assertTrue(result.records().contains( + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, oldMemberId) + )); + assertTrue(result.records().contains( + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, oldMemberId) + )); + assertTrue(result.records().contains( + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, expectedCopiedMember) + )); + assertTrue(result.records().contains( + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, rejoinMemberId, oldTargetAssignment) + )); + assertTrue(result.records().contains( + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedCopiedMember) + )); + } + + @Test + public void testStaticMemberLeaveWithMismatchedMemberIdThrowsFencedInstanceIdInStreamsGroup() { + String groupId = "fooup"; + String memberId = Uuid.randomUuid().toString(); + String differentMemberId = Uuid.randomUuid().toString(); + String instanceId = Uuid.randomUuid().toString(); + + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroup(new StreamsGroupBuilder(groupId, 10) + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(10) + .setPreviousMemberEpoch(9) + .build()) + .withTargetAssignmentEpoch(10) + ) + .build(); + + assertThrows(FencedInstanceIdException.class, () -> + context.streamsGroupHeartbeat( + staticHeartbeat(groupId, differentMemberId, instanceId, LEAVE_GROUP_STATIC_MEMBER_EPOCH) + )); + } + + @Test + public void testUnknownStaticMemberHeartbeatWithPositiveEpochThrowsUnknownMemberIdInStreamsGroup() { + String groupId = "fooup"; + String memberId = Uuid.randomUuid().toString(); + String instanceId = Uuid.randomUuid().toString(); + String unknownInstanceId = Uuid.randomUuid().toString(); + String unknownMemberId = Uuid.randomUuid().toString(); + + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroup(new StreamsGroupBuilder(groupId, 10) + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(10) + .setPreviousMemberEpoch(9) + .build()) + .withTargetAssignmentEpoch(10) + ) + .build(); + + UnknownMemberIdException e = assertThrows(UnknownMemberIdException.class, () -> + context.streamsGroupHeartbeat(staticHeartbeat(groupId, unknownMemberId, unknownInstanceId, 1)) + ); + assertEquals(String.format("Instance id %s is unknown.", unknownInstanceId), e.getMessage()); + } + + @ParameterizedTest + @MethodSource("ownedActiveTasksAtPreviousEpochCases") + public void testStreamsStaticMemberHeartbeatWithPreviousEpochAndOwnedActiveTasks( + List requestAssignedTaskIds, Class expectedException + ) { + Integer[] givenAssignedTasksIds = new Integer[]{0, 1, 2, 3}; + + verifyStreamsStaticMemberHeartbeatWithOwnedActiveTasksAtPreviousEpoch( + givenAssignedTasksIds, + requestAssignedTaskIds, + expectedException + ); + } + + private static Stream ownedActiveTasksAtPreviousEpochCases() { + return Stream.of( + Arguments.of(List.of(0, 1, 2), null), // Subset Owned Active Tasks + Arguments.of(List.of(0, 1, 2, 3), null), // Exact Owned Active Tasks + Arguments.of(List.of(0, 1, 2, 3, 4), FencedMemberEpochException.class) // Non Subset active tasks + ); + } + + private void verifyStreamsStaticMemberHeartbeatWithOwnedActiveTasksAtPreviousEpoch( + Integer[] givenTaskIds, + List requestAssignedTaskIds, + Class expectedException) { + int groupEpoch = 10; + int partitionSize = 5; + int currentMemberEpoch = 10; + int previousMemberEpoch = 9; + int requestMemberEpoch = 9; + + String groupId = "fooup"; + String memberId = Uuid.randomUuid().toString(); + String instanceId = Uuid.randomUuid().toString(); + + // GIVEN TASK and Topology + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", partitionSize); + TasksTupleWithEpochs givenAssignedTask = topic.assignedTasks(groupEpoch, givenTaskIds); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(currentMemberEpoch) + .setPreviousMemberEpoch(previousMemberEpoch) + .setAssignedTasks(givenAssignedTask) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .build()) + .withTargetAssignment(memberId, topic.targetAssignment(givenTaskIds))); + + // WHEN + StreamsGroupHeartbeatRequestData requestData = staticHeartbeat(groupId, memberId, instanceId, requestMemberEpoch) + .setProcessId("process-id") + .setRebalanceTimeoutMs(1500) + .setTopology(topic.topology()) + .setActiveTasks(topic.requestTasks(requestAssignedTaskIds)) + .setStandbyTasks(List.of()) + .setWarmupTasks(List.of()); + + // THEN + if (expectedException != null) { + assertThrows(expectedException, () -> context.streamsGroupHeartbeat(requestData)); + } else { + assertDoesNotThrow(() -> context.streamsGroupHeartbeat(requestData)); + } + } + + @Test + public void testStreamsStaticMemberTemporaryLeaveSessionTimeoutExpiration() { + String groupId = "fooup"; + String memberId = Uuid.randomUuid().toString(); + String instanceId = Uuid.randomUuid().toString(); + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 4); + MockTaskAssignor assignor = new MockTaskAssignor("sticky"); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, 0) + .build(); + + assignor.prepareGroupAssignment(Map.of(memberId, topic.targetAssignment(0, 1, 2, 3))); + + // WHEN1 : static member joins (session timeout should be scheduled) + CoordinatorResult firstJoinResult = context.streamsGroupHeartbeat( + staticJoinHeartbeat(groupId, memberId, instanceId, topic).setRebalanceTimeoutMs(90000) + ); + + // THEN1 + // - member epoch should be bumped up. + // - session timeout should be 45000ms. + assertEquals(2, firstJoinResult.response().data().memberEpoch()); + context.assertSessionTimeout(groupId, memberId, 45000); + + // WHEN2: static member leaves temporarily. + CoordinatorResult temporaryLeaveResult = context.streamsGroupHeartbeat( + staticHeartbeat(groupId, memberId, instanceId, LEAVE_GROUP_STATIC_MEMBER_EPOCH) + ); + + // THEN2: + // member epoch should be -2. + // session timeout still 45000ms. + assertResponseEquals(staticLeaveResponse(memberId, LEAVE_GROUP_STATIC_MEMBER_EPOCH), temporaryLeaveResult.response().data()); + context.assertSessionTimeout(groupId, memberId, 45000); + + // WHEN3: no rejoin, session timeout expires. + List> timeouts = context.sleep(45000 + 1); + + // THEN3 + List expectedRecords = List.of( + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, 3, topic.metadataHash(), 0, getDefaultAssignmentConfigs()), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, 3, 0L) + ); + assertEquals( + List.of(new MockCoordinatorTimer.ExpiredTimeout<>( + groupSessionTimeoutKey(groupId, memberId), + new CoordinatorResult<>(expectedRecords) + )), + timeouts + ); + context.assertNoSessionTimeout(groupId, memberId); + context.assertNoRebalanceTimeout(groupId, memberId); + } + + @Test + public void testStaticMemberJoinEmptyStreamsGroupRegistersStaticMember1() { + String groupId = "fooup"; + String memberId = Uuid.randomUuid().toString(); + String instanceId = Uuid.randomUuid().toString(); + + StreamsGroupHeartbeatRequestData.Topology topology = new StreamsGroupHeartbeatRequestData.Topology().setSubtopologies(List.of()); + MockTaskAssignor assignor = new MockTaskAssignor("sticky"); + assignor.prepareGroupAssignment(Map.of(memberId, TasksTuple.EMPTY)); + + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withMetadataImage(new MetadataImageBuilder().buildCoordinatorMetadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, 0) + .withStreamsGroupTaskAssignors(List.of(assignor)) + .build(); + + // There is no group at all. + assertThrows(GroupIdNotFoundException.class, () -> + context.groupMetadataManager.streamsGroup(groupId)); + + // WHEN + context.streamsGroupHeartbeat( + staticHeartbeat(groupId, memberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) + .setProcessId(DEFAULT_PROCESS_ID) + .setRebalanceTimeoutMs(1500) + .setTopology(topology) + .setActiveTasks(List.of()) + .setStandbyTasks(List.of()) + .setWarmupTasks(List.of()) + ); + + // THEN + StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); + assertEquals(memberId, group.staticMember(instanceId).memberId()); + assertEquals(Optional.of(instanceId), group.getMemberOrThrow(memberId).instanceId()); + } + + @ParameterizedTest + @MethodSource("userEndpointTestCases") + public void testStaticMemberRejoinUpdatesUserEndpointInformationEpoch( + StreamsGroupHeartbeatRequestData.Endpoint firstUserEndpoint, + int firstExpectedUserEndpointEpoch, + List firstExpectedPartitionsByUserEndpoint, + StreamsGroupMemberMetadataValue.Endpoint firstExpectedUserEndpointMetadata, + + StreamsGroupHeartbeatRequestData.Endpoint secondUserEndpoint, + int secondExpectedUserEndpointEpoch, + List secondExpectedPartitionsByUserEndpoint, + StreamsGroupMemberMetadataValue.Endpoint secondExpectedUserEndpointMetadata + ) { + int memberEpoch = DEFAULT_MEMBER_EPOCH; + int groupEpoch = DEFAULT_GROUP_EPOCH; + int bumpedEpoch = memberEpoch + 1; + + String groupId = "fooup"; + String instanceId = Uuid.randomUuid().toString(); + String oldMemberId = "old-member-id"; + String rejoinMemberId = "new-member-id"; + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 3); + TasksTuple targetAssignment = topic.targetAssignment(0, 1, 2); + + + MockTaskAssignor assignor = new MockTaskAssignor("sticky"); + assignor.prepareGroupAssignment(Map.of(oldMemberId, topic.targetAssignment(0, 1, 2))); + + GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, 0, group -> group + .withTargetAssignment(oldMemberId, targetAssignment)); + + assertEquals(0, context.groupMetadataManager.streamsGroup(groupId).endpointInformationEpoch()); + + // First Join -> First Input + CoordinatorResult result = context.streamsGroupHeartbeat( + staticJoinHeartbeat(groupId, oldMemberId, instanceId, topic) + .setUserEndpoint(firstUserEndpoint) // first input + ); + + // First Check + assertResponseEquals( + heartbeatResponseWithActiveTasks(oldMemberId, bumpedEpoch, topic, 0, 1, 2) + .setEndpointInformationEpoch(firstExpectedUserEndpointEpoch) // first endpoint epoch + .setPartitionsByUserEndpoint(firstExpectedPartitionsByUserEndpoint), // first partitions by user endpoint + result.response().data() + ); + + if (firstExpectedUserEndpointMetadata != null) { + assertEquals(firstExpectedUserEndpointMetadata, context.groupMetadataManager.streamsGroup(groupId).getMemberOrThrow(oldMemberId).userEndpoint().get()); + } else { + assertTrue(context.groupMetadataManager.streamsGroup(groupId).getMemberOrThrow(oldMemberId).userEndpoint().isEmpty()); + } + assertEquals(firstExpectedUserEndpointEpoch, context.groupMetadataManager.streamsGroup(groupId).endpointInformationEpoch()); + + // static leave + context.streamsGroupHeartbeat( + staticHeartbeat(groupId, oldMemberId, instanceId, StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) + ); + + // second - static member rejoins + CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( + staticJoinHeartbeat(groupId, rejoinMemberId, instanceId, topic) + .setUserEndpoint(secondUserEndpoint) + ); + + // second check. + assertResponseEquals( + heartbeatResponseWithActiveTasks(rejoinMemberId, bumpedEpoch, topic, 0, 1, 2) + .setEndpointInformationEpoch(secondExpectedUserEndpointEpoch) + .setPartitionsByUserEndpoint(secondExpectedPartitionsByUserEndpoint), + rejoinResult.response().data() + ); + + if (secondExpectedUserEndpointMetadata != null) { + assertEquals(secondExpectedUserEndpointMetadata, context.groupMetadataManager.streamsGroup(groupId).getMemberOrThrow(rejoinMemberId).userEndpoint().get()); + } else { + assertTrue(context.groupMetadataManager.streamsGroup(groupId).getMemberOrThrow(rejoinMemberId).userEndpoint().isEmpty()); + } + assertEquals(secondExpectedUserEndpointEpoch, context.groupMetadataManager.streamsGroup(groupId).endpointInformationEpoch()); + } + + private static Stream userEndpointTestCases() { + return Stream.of( + Arguments.of( + null, // firstInput + 0, // first endpoint Epoch + null, // first partitionsByUserEndpoint + null, // first group metadata userEndpoint + userEndpoint("bar.com", 8080), // second input + 1, // second endpoint epoch + buildEndpoints("bar.com", 8080, "foo", List.of(0, 1, 2)), // second partitionsByUserEndpoint + userEndpointForMetadata("bar.com", 8080) + ), + Arguments.of( + null, // firstInput + 0, // first endpoint Epoch + null, // first partitionsByUserEndpoint + null, // first group metadata userEndpoint + null, // second input + 0, // second endpoint epoch + null, // second partitionsByUserEndpoint + null + ), + Arguments.of( + userEndpoint("foo.com", 8080), // firstInput + 1, // first endpoint Epoch + buildEndpoints("foo.com", 8080, "foo", List.of(0, 1, 2)), // first partitionsByUserEndpoint + userEndpointForMetadata("foo.com", 8080), // first group metadata userEndpoint + null, // second input + 2, // second endpoint epoch + List.of(), // second partitionsByUserEndpoint + null + ), + Arguments.of( + userEndpoint("foo.com", 8080), // firstInput + 1, // first endpoint Epoch + buildEndpoints("foo.com", 8080, "foo", List.of(0, 1, 2)), // first partitionsByUserEndpoint + userEndpointForMetadata("foo.com", 8080), // first group metadata userEndpoint + userEndpoint("foo.com", 8080), // second input + 1, // second endpoint epoch + buildEndpoints("foo.com", 8080, "foo", List.of(0, 1, 2)), // second partitionsByUserEndpoint + userEndpointForMetadata("foo.com", 8080) + ), + Arguments.of( + userEndpoint("foo.com", 8080), // firstInput + 1, // first endpoint Epoch + buildEndpoints("foo.com", 8080, "foo", List.of(0, 1, 2)), // first partitionsByUserEndpoint + userEndpointForMetadata("foo.com", 8080), // first group metadata userEndpoint + userEndpoint("bar.com", 8080), // second input + 2, // second endpoint epoch + buildEndpoints("bar.com", 8080, "foo", List.of(0, 1, 2)), // second partitionsByUserEndpoint + userEndpointForMetadata("bar.com", 8080) + ) + ); + } + + private static class StaticMemberFixtureWith2Members { + // a single static member is active, the other static member leaves with -2. + private static final String GROUP_ID = "streams-group"; + private static final int GROUP_EPOCH = 10; + private static final int MEMBER_EPOCH = 10; + private static final int PREVIOUS_MEMBER_EPOCH = 9; + private static final int REBALANCE_TIMEOUT_MS = 1500; + + private final String activeMemberId = "active-member"; + private final String activeInstanceId = "active-instance"; + + private final String leftMemberId = "left-member"; + private final String leftInstanceId = "left-instance"; + + private final String newMemberId = "new-member"; + private final String newInstanceId = "new-instance"; + + private final StreamsGroupHeartbeatRequestData.Topology topology = + new StreamsGroupHeartbeatRequestData.Topology().setSubtopologies(List.of()); + + private final GroupMetadataManagerTestContext context; + + private StaticMemberFixtureWith2Members(int streamsGroupMaxSize) { + MockTaskAssignor assignor = new MockTaskAssignor("sticky"); + assignor.prepareGroupAssignment(Map.of(activeMemberId, TasksTuple.EMPTY)); + + this.context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(new MetadataImageBuilder().buildCoordinatorMetadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, streamsGroupMaxSize) + .withStreamsGroup(new StreamsGroupBuilder(GROUP_ID, GROUP_EPOCH) + .withMember(streamsGroupMemberBuilderWithDefaults(activeMemberId, activeInstanceId) + .setMemberEpoch(MEMBER_EPOCH) + .setPreviousMemberEpoch(PREVIOUS_MEMBER_EPOCH) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(leftMemberId, leftInstanceId) + .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(PREVIOUS_MEMBER_EPOCH) + .build()) + .withTargetAssignmentEpoch(GROUP_EPOCH) + .withTopology(StreamsTopology.fromHeartbeatRequest(topology)) + ) + .build(); + } + + private StreamsGroupHeartbeatRequestData newMemberJoinsWithNewInstanceId() { + return joinRequest(newMemberId, newInstanceId); + } + + private StreamsGroupHeartbeatRequestData leftMemberRejoinsWithSameInstanceId() { + return joinRequest(leftMemberId, leftInstanceId); + } + + private StreamsGroupHeartbeatRequestData newMemberJoinsWithActiveInstanceId() { + return joinRequest(newMemberId, activeInstanceId); + } + + private StreamsGroupHeartbeatRequestData newMemberHeartbeatsWithActiveInstanceId(int memberEpoch) { + return request(newMemberId, activeInstanceId, memberEpoch); + } + + private StreamsGroupHeartbeatRequestData joinRequest(String memberId, String instanceId) { + return request(memberId, instanceId, StreamsGroupHeartbeatRequest.JOIN_GROUP_MEMBER_EPOCH); + } + + private StreamsGroupHeartbeatRequestData staticLeaveRequest(String memberId, String instanceId) { + return request(memberId, instanceId, StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH); + } + + private StreamsGroupHeartbeatRequestData request(String memberId, String instanceId, int memberEpoch) { + return new StreamsGroupHeartbeatRequestData() + .setGroupId(GROUP_ID) + .setInstanceId(instanceId) + .setMemberId(memberId) + .setMemberEpoch(memberEpoch) + .setProcessId(DEFAULT_PROCESS_ID) + .setRebalanceTimeoutMs(REBALANCE_TIMEOUT_MS) + .setTopology(topology) + .setActiveTasks(List.of()) + .setStandbyTasks(List.of()) + .setWarmupTasks(List.of()); + } + + private void assertJoinSucceeds(StreamsGroupHeartbeatRequestData request) { + assertDoesNotThrow(() -> context.streamsGroupHeartbeat(request)); + } + + private void assertJoinFails(StreamsGroupHeartbeatRequestData request, Class expectedException) { + assertHeartbeatFails(request, expectedException); + } + + private void assertHeartbeatFails(StreamsGroupHeartbeatRequestData request, Class expectedException + ) { + assertThrows(expectedException, () -> context.streamsGroupHeartbeat(request)); + } + + private void assertLeaveFails(StreamsGroupHeartbeatRequestData request, Class expectedException + ) { + assertHeartbeatFails(request, expectedException); + } + } + + private static StreamsGroupHeartbeatRequestData.Endpoint userEndpoint(String host, int port) { + return new StreamsGroupHeartbeatRequestData.Endpoint() + .setHost(host) + .setPort(port); + } + + private static List buildEndpoints(String host, int port, String topic, List partitions) { + List endpoints = new ArrayList<>(); + endpoints.add(new StreamsGroupHeartbeatResponseData.EndpointToPartitions() + .setUserEndpoint(new StreamsGroupHeartbeatResponseData.Endpoint() + .setHost(host) + .setPort(port)) + .setActivePartitions(List.of(topicPartition(topic, partitions)))); + return endpoints; + } + + private static StreamsGroupMemberMetadataValue.Endpoint userEndpointForMetadata(String host, int port) { + return new StreamsGroupMemberMetadataValue.Endpoint().setHost(host).setPort(port); + } + + private static StreamsGroupHeartbeatResponseData.TopicPartition topicPartition(String topic, List partitions) { + return new StreamsGroupHeartbeatResponseData.TopicPartition().setTopic(topic).setPartitions(partitions); + } + +} \ No newline at end of file diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java new file mode 100644 index 0000000000000..4f7ffdd038f11 --- /dev/null +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java @@ -0,0 +1,353 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.StreamsGroupHeartbeatRequestData; +import org.apache.kafka.common.message.StreamsGroupHeartbeatResponseData; +import org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest; +import org.apache.kafka.coordinator.common.runtime.CoordinatorMetadataImage; +import org.apache.kafka.coordinator.common.runtime.MetadataImageBuilder; +import org.apache.kafka.coordinator.group.streams.MemberState; +import org.apache.kafka.coordinator.group.streams.MockTaskAssignor; +import org.apache.kafka.coordinator.group.streams.StreamsGroupBuilder; +import org.apache.kafka.coordinator.group.streams.StreamsGroupMember; +import org.apache.kafka.coordinator.group.streams.StreamsTopology; +import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil; +import org.apache.kafka.coordinator.group.streams.TasksTuple; +import org.apache.kafka.coordinator.group.streams.TasksTupleWithEpochs; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import static org.apache.kafka.coordinator.group.GroupMetadataManagerTestContext.DEFAULT_CLIENT_ADDRESS; +import static org.apache.kafka.coordinator.group.GroupMetadataManagerTestContext.DEFAULT_CLIENT_ID; +import static org.apache.kafka.coordinator.group.GroupMetadataManagerTestContext.DEFAULT_PROCESS_ID; +import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTupleWithCommonEpoch; +import static org.apache.kafka.coordinator.group.Utils.computeGroupHash; +import static org.apache.kafka.coordinator.group.Utils.computeTopicHash; + +class StreamsGroupTestUtil { + + static StreamsGroupMember.Builder streamsGroupMemberBuilderWithDefaults(String memberId) { + return streamsGroupMemberBuilderWithDefaults(memberId, null); + } + + static StreamsGroupMember.Builder streamsGroupMemberBuilderWithDefaults(String memberId, String instanceId) { + return new StreamsGroupMember.Builder(memberId) + .setMemberEpoch(1) + .setPreviousMemberEpoch(0) + .setState(MemberState.STABLE) + .setRackId(null) + .setInstanceId(instanceId) + .setRebalanceTimeoutMs(1500) + .setAssignedTasks(TasksTupleWithEpochs.EMPTY) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .setTopologyEpoch(0) + .setClientTags(Map.of()) + .setClientId(DEFAULT_CLIENT_ID) + .setClientHost(DEFAULT_CLIENT_ADDRESS.toString()) + .setProcessId(DEFAULT_PROCESS_ID) + .setUserEndpoint(null); + } + + /** + * Returns the default assignment configurations that would be used by the system. + * This matches what streamsGroupAssignmentConfigs() would return. + */ + static Map getDefaultAssignmentConfigs() { + // Use the same default value as GroupCoordinatorConfig.STREAMS_GROUP_NUM_STANDBY_REPLICAS_DEFAULT + return new TreeMap<>(Map.of( + "num.standby.replicas", String.valueOf(GroupCoordinatorConfig.STREAMS_GROUP_NUM_STANDBY_REPLICAS_DEFAULT) + )); + } + + static List mkResponseTasks( + String subtopologyId, + Integer... partitions + ) { + return List.of( + new StreamsGroupHeartbeatResponseData.TaskIds() + .setSubtopologyId(subtopologyId) + .setPartitions(Arrays.asList(partitions)) + ); + } + + static final int DEFAULT_REBALANCE_TIMEOUT_MS = 1500; + + static StreamsTopicFixture streamsTopicFixture(String subtopologyId, String topicName, int partitions) { + return new StreamsTopicFixture(subtopologyId, topicName, partitions); + } + + static StreamsGroupHeartbeatRequestData staticHeartbeat(String groupId, String memberId, String instanceId, int memberEpoch) { + return new StreamsGroupHeartbeatRequestData() + .setGroupId(groupId) + .setInstanceId(instanceId) + .setMemberId(memberId) + .setMemberEpoch(memberEpoch); + } + + static StreamsGroupHeartbeatRequestData staticJoinHeartbeat(String groupId, String memberId, String instanceId, StreamsTopicFixture topic) { + return staticHeartbeat(groupId, memberId, instanceId, StreamsGroupHeartbeatRequest.JOIN_GROUP_MEMBER_EPOCH) + .setProcessId(DEFAULT_PROCESS_ID) + .setRebalanceTimeoutMs(DEFAULT_REBALANCE_TIMEOUT_MS) + .setTopology(topic.topology) + .setActiveTasks(List.of()) + .setStandbyTasks(List.of()) + .setWarmupTasks(List.of()); + } + + static StreamsGroupHeartbeatResponseData staticLeaveResponse(String memberId, int leaveEpoch) { + return new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId) + .setMemberEpoch(leaveEpoch) + .setStatus(List.of()); + } + + + static StreamsGroupHeartbeatResponseData staticLeaveResponseWithNullTasks(String memberId, int leaveEpoch) { + return staticLeaveResponse(memberId, leaveEpoch) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null); + } + + + static class StreamsTopicFixture { + private final String subtopologyId; + private final String topicName; + private final Uuid topicId; + private final StreamsGroupHeartbeatRequestData.Topology topology; + private final CoordinatorMetadataImage metadataImage; + private final long metadataHash; + + private StreamsTopicFixture( + String subtopologyId, + String topicName, + int partitions + ) { + this.subtopologyId = subtopologyId; + this.topicName = topicName; + this.topicId = Uuid.randomUuid(); + this.topology = new StreamsGroupHeartbeatRequestData.Topology() + .setSubtopologies(List.of( + new StreamsGroupHeartbeatRequestData.Subtopology() + .setSubtopologyId(subtopologyId) + .setSourceTopics(List.of(topicName)) + )); + this.metadataImage = new MetadataImageBuilder() + .addTopic(topicId, topicName, partitions) + .buildCoordinatorMetadataImage(); + this.metadataHash = computeGroupHash(Map.of( + topicName, + computeTopicHash(topicName, metadataImage) + )); + } + + public Map.Entry> tasks(Integer... partitions) { + return TaskAssignmentTestUtil.mkTasks(subtopologyId, partitions); + } + + public TasksTuple targetAssignment(Integer... partitions) { + return TaskAssignmentTestUtil.mkTasksTuple( + TaskAssignmentTestUtil.TaskRole.ACTIVE, + tasks(partitions) + ); + } + + public TasksTupleWithEpochs assignedTasks( + int epoch, + Integer... partitions + ) { + return mkTasksTupleWithCommonEpoch( + TaskAssignmentTestUtil.TaskRole.ACTIVE, + epoch, + tasks(partitions) + ); + } + + public CoordinatorMetadataImage metadataImage() { + return metadataImage; + } + + public long metadataHash() { + return metadataHash; + } + + public StreamsGroupHeartbeatRequestData.Topology topology() { + return topology; + } + + public List responseTasks(Integer... partitions) { + return mkResponseTasks(subtopologyId, partitions); + } + + public List requestTasks(List partitions) { + return List.of( + new StreamsGroupHeartbeatRequestData.TaskIds() + .setSubtopologyId(subtopologyId) + .setPartitions(partitions) + ); + } + } + + static GroupMetadataManagerTestContext contextWithStreamsGroup( + String groupId, + int groupEpoch, + StreamsTopicFixture topic, + java.util.function.UnaryOperator configureGroup + ) { + return contextWithStreamsGroup( + groupId, + groupEpoch, + topic, + new MockTaskAssignor("sticky"), + configureGroup + ); + } + + static GroupMetadataManagerTestContext contextWithStreamsGroup( + String groupId, + int groupEpoch, + StreamsTopicFixture topic, + MockTaskAssignor assignor, + java.util.function.UnaryOperator configureGroup + ) { + return contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT, configureGroup); + } + + static GroupMetadataManagerTestContext contextWithStreamsGroup( + String groupId, + int groupEpoch, + StreamsTopicFixture topic, + MockTaskAssignor assignor, + int initialRebalanceDelayMs, + java.util.function.UnaryOperator configureGroup + ) { + StreamsGroupBuilder group = new StreamsGroupBuilder(groupId, groupEpoch) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs()); + + return new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(configureGroup.apply(group)) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, initialRebalanceDelayMs) + .build(); + } + + static StreamsGroupHeartbeatResponseData heartbeatResponseWithActiveTasks( + String memberId, + int memberEpoch, + StreamsTopicFixture topic, + Integer... activeTasks + ) { + return heartbeatResponseWithActiveTasks( + memberId, + memberEpoch, + topic.responseTasks(activeTasks) + ); + } + + static StreamsGroupHeartbeatResponseData heartbeatResponseWithActiveTasks( + String memberId, + int memberEpoch, + List activeTasks + ) { + return new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId) + .setMemberEpoch(memberEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(activeTasks) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()); + } + + static StreamsGroupHeartbeatResponseData heartbeatResponseWithActiveTasks( + String memberId, + int memberEpoch, + String subtopologyId, + Integer... activeTasks + ) { + return heartbeatResponseWithActiveTasks( + memberId, + memberEpoch, + mkResponseTasks(subtopologyId, activeTasks) + ); + } + + + static StreamsGroupHeartbeatResponseData heartbeatResponseWithNullTasks( + String memberId, + int memberEpoch + ) { + return new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId) + .setMemberEpoch(memberEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null); + } + + static StreamsGroupHeartbeatRequestData staticJoinHeartbeat( + String groupId, + String memberId, + String instanceId, + String processId + ) { + return staticHeartbeat(groupId, memberId, instanceId, StreamsGroupHeartbeatRequest.JOIN_GROUP_MEMBER_EPOCH) + .setProcessId(processId) + .setActiveTasks(List.of()) + .setStandbyTasks(List.of()) + .setWarmupTasks(List.of()); + } + + static TasksTupleWithEpochs resetAssignedTasksEpochsToZero(TasksTupleWithEpochs assignedTasks) { + if (assignedTasks.isEmpty()) { + return assignedTasks; + } + + if (assignedTasks.activeTasksWithEpochs().isEmpty()) { + return assignedTasks; + } + + Map> resetActiveTasks = new HashMap<>(); + for (Map.Entry> entry : assignedTasks.activeTasksWithEpochs().entrySet()) { + Map resetActiveTaskEpochs = new HashMap<>(); + for (Integer partitionId : entry.getValue().keySet()) { + resetActiveTaskEpochs.put(partitionId, 0); + } + resetActiveTasks.put(entry.getKey(), resetActiveTaskEpochs); + } + return new TasksTupleWithEpochs( + resetActiveTasks, + assignedTasks.standbyTasks(), + assignedTasks.warmupTasks() + ); + } + +} From d5908c5830d169141a1f41b868d92980b10ca353 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Mon, 11 May 2026 09:09:06 +0900 Subject: [PATCH 04/16] Update userEndpoint epoch bump logic. --- .../group/GroupMetadataManager.java | 38 +++++++++++-------- .../group/streams/StreamsGroup.java | 10 +++++ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java index 07ac5259fecac..34097ec9c4e79 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java @@ -2060,9 +2060,9 @@ private CoordinatorResult stream // Get or create the member. StreamsGroupMember member; - StreamsGroupMember maybeReplacedStaticMember; + StreamsGroupMember maybeOldMember; if (instanceId == null) { - maybeReplacedStaticMember = null; + maybeOldMember = group.dynamicMember(memberId); member = getOrMaybeCreateDynamicStreamsGroupMember( group, memberId, @@ -2073,7 +2073,7 @@ private CoordinatorResult stream isJoining ); } else { - maybeReplacedStaticMember = group.staticMember(instanceId); + maybeOldMember = group.staticMember(instanceId); member = getOrMaybeCreateStaticStreamsGroupMember( group, memberId, @@ -2251,23 +2251,28 @@ private CoordinatorResult stream // The assignment is only provided in the following cases: // 1. The member is joining. // 2. The member's assignment has been updated. - if (memberEpoch == 0 || hasAssignedTasksChanged(member, updatedMember)) { + boolean newlyJoinOrAssignmentChanged = memberEpoch == 0 || hasAssignedTasksChanged(member, updatedMember); + boolean hasReplacedStaticMember = maybeOldMember != null && !maybeOldMember.memberId().equals(updatedMember.memberId()); + boolean userEndpointChanged = hasUserEndpointChanged(maybeOldMember, updatedMember); + if (newlyJoinOrAssignmentChanged) { response.setActiveTasks(createStreamsGroupHeartbeatResponseTaskIdsFromEpochs(updatedMember.assignedTasks().activeTasksWithEpochs())); response.setStandbyTasks(createStreamsGroupHeartbeatResponseTaskIds(updatedMember.assignedTasks().standbyTasks())); response.setWarmupTasks(createStreamsGroupHeartbeatResponseTaskIds(updatedMember.assignedTasks().warmupTasks())); + } + + if (newlyJoinOrAssignmentChanged || hasReplacedStaticMember || userEndpointChanged) { group.invalidateCachedEndpointToPartitions(updatedMember.memberId()); - if (maybeReplacedStaticMember != null && !maybeReplacedStaticMember.memberId().equals(updatedMember.memberId())) { - group.invalidateCachedEndpointToPartitions(maybeReplacedStaticMember.memberId()); - } - if (hasUserEndpointChanged(maybeReplacedStaticMember, updatedMember)) { - // If no user endpoint is defined, there is no change in the endpoint information. - // Otherwise, bump the endpoint information epoch - group.setEndpointInformationEpoch(group.endpointInformationEpoch() + 1); + if (hasReplacedStaticMember) { + group.invalidateCachedEndpointToPartitions(maybeOldMember.memberId()); } } + if (userEndpointChanged || (hasAssignedTasksChanged(member, updatedMember) && updatedMember.userEndpoint().isPresent())) { + group.setEndpointInformationEpoch(group.endpointInformationEpoch() + 1); + } + if (group.endpointInformationEpoch() != memberEndpointEpoch) { - response.setPartitionsByUserEndpoint(group.buildEndpointToPartitions(updatedMember, metadataImage, maybeReplacedStaticMember)); + response.setPartitionsByUserEndpoint(group.buildEndpointToPartitions(updatedMember, metadataImage, maybeOldMember)); } if (groups.containsKey(group.groupId())) { // If we just created the group, the endpoint information epoch will not be persisted, so return epoch 0. @@ -9295,13 +9300,14 @@ private Map streamsGroupAssignmentConfigs(String groupId) { )); } - private boolean hasUserEndpointChanged(StreamsGroupMember maybeReplacedStaticMember, StreamsGroupMember updatedMember) { - - boolean hasPreviousUserEndpoint = maybeReplacedStaticMember != null && maybeReplacedStaticMember.userEndpoint().isPresent(); + private boolean hasUserEndpointChanged(StreamsGroupMember maybeOldMember, StreamsGroupMember updatedMember) { + boolean hasPreviousUserEndpoint = maybeOldMember != null && maybeOldMember.userEndpoint().isPresent(); boolean hasCurrentUserEndpoint = updatedMember.userEndpoint().isPresent(); + // Both the previous and updated member have a user endpoint. This can happen for + // regular heartbeats after join, or for static member rejoins with an endpoint. if (hasPreviousUserEndpoint && hasCurrentUserEndpoint) { - return !maybeReplacedStaticMember.userEndpoint().get().equals(updatedMember.userEndpoint().get()); + return !maybeOldMember.userEndpoint().get().equals(updatedMember.userEndpoint().get()); } if (!hasPreviousUserEndpoint && !hasCurrentUserEndpoint) { diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java index 5f553c1ae8af6..8e60b5cbb745a 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java @@ -484,6 +484,16 @@ public StreamsGroupMember staticMember(String instanceId) { return existingMemberId == null ? null : getMemberOrThrow(existingMemberId); } + /** + * Gets a dynamic member. + * + * @param memberId The Member ID. + * @return The member corresponding to the given member ID or null if it does not exist + */ + public StreamsGroupMember dynamicMember(String memberId) { + return members.get(memberId); + } + /** * Adds or updates the member. * From ad340317c72cf6ccadde963cf68df59017d882e8 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Thu, 21 May 2026 22:43:25 +0900 Subject: [PATCH 05/16] Addressing review. --- .../group/GroupMetadataManager.java | 86 +++++++++++-------- .../group/streams/StreamsGroup.java | 20 ++--- .../group/streams/StreamsGroupMember.java | 7 -- 3 files changed, 58 insertions(+), 55 deletions(-) diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java index 34097ec9c4e79..47ea9135ce118 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java @@ -1703,7 +1703,7 @@ private void throwIfInstanceIdIsUnreleased(ConsumerGroupMember member, String gr * @throws UnreleasedInstanceIdException if the instance id received in the request is still in use by an existing static member. */ private void throwIfInstanceIdIsUnreleased(StreamsGroupMember member, String groupId, String receivedMemberId, String receivedInstanceId) { - if (member.memberEpoch() != StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) { + if (member.memberEpoch() != LEAVE_GROUP_STATIC_MEMBER_EPOCH) { // The new member can't join. log.info("[GroupId {}] Static member {} with instance id {} cannot join the group because the instance id is" + " owned by member {}.", groupId, receivedMemberId, receivedInstanceId, member.memberId()); @@ -1783,13 +1783,14 @@ private void throwIfStaticMemberIsUnknown(StreamsGroupMember staticMember, Strin * max group size defined. * * @param group The streams group. - * @param instanceId The instance id. + * @param memberId The member id. * * @throws GroupMaxSizeReachedException if the maximum capacity has been reached. */ private void throwIfStreamsGroupIsFull( StreamsGroup group, - String instanceId + String instanceId, + String memberId ) throws GroupMaxSizeReachedException { // If a static member already exists, we do not enforce the maximum group size check. // An existing static member will fall into one of the following two cases, @@ -1801,7 +1802,7 @@ private void throwIfStreamsGroupIsFull( // If the streams group has reached its maximum capacity, the member is rejected if it is not // already a member of the streams group. - if (group.numMembers() >= config.streamsGroupMaxSize()) { + if (group.numMembers() >= config.streamsGroupMaxSize() && (memberId.isEmpty() || !group.hasMember(memberId))) { throw new GroupMaxSizeReachedException("The streams group has reached its maximum capacity of " + config.streamsGroupMaxSize() + " members."); } @@ -2053,16 +2054,15 @@ private CoordinatorResult stream StreamsGroup group; if (isJoining) { group = getOrCreateStreamsGroup(groupId, records); - throwIfStreamsGroupIsFull(group, instanceId); + throwIfStreamsGroupIsFull(group, instanceId, memberId); } else { group = getStreamsGroupOrThrow(groupId); } // Get or create the member. StreamsGroupMember member; - StreamsGroupMember maybeOldMember; + StreamsGroupMember replaceStaticOldMember = null; if (instanceId == null) { - maybeOldMember = group.dynamicMember(memberId); member = getOrMaybeCreateDynamicStreamsGroupMember( group, memberId, @@ -2073,17 +2073,20 @@ private CoordinatorResult stream isJoining ); } else { - maybeOldMember = group.staticMember(instanceId); + StreamsGroupMember maybeOldStaticMember = group.staticMember(instanceId); + if (maybeOldStaticMember != null && !maybeOldStaticMember.memberId().equals(memberId)) { + replaceStaticOldMember = maybeOldStaticMember; + } member = getOrMaybeCreateStaticStreamsGroupMember( - group, - memberId, - memberEpoch, - instanceId, - ownedActiveTasks, - ownedStandbyTasks, - ownedWarmupTasks, - isJoining, - records + group, + memberId, + memberEpoch, + instanceId, + ownedActiveTasks, + ownedStandbyTasks, + ownedWarmupTasks, + isJoining, + records ); } @@ -2251,28 +2254,31 @@ private CoordinatorResult stream // The assignment is only provided in the following cases: // 1. The member is joining. // 2. The member's assignment has been updated. - boolean newlyJoinOrAssignmentChanged = memberEpoch == 0 || hasAssignedTasksChanged(member, updatedMember); - boolean hasReplacedStaticMember = maybeOldMember != null && !maybeOldMember.memberId().equals(updatedMember.memberId()); - boolean userEndpointChanged = hasUserEndpointChanged(maybeOldMember, updatedMember); - if (newlyJoinOrAssignmentChanged) { + boolean assignedTaskChanged = hasAssignedTasksChanged(member, updatedMember); + boolean endpointChanged = hasUserEndpointChanged(member, updatedMember); + + // Echo the assignment back when joining or the assignment changed. + if (isJoining || assignedTaskChanged) { response.setActiveTasks(createStreamsGroupHeartbeatResponseTaskIdsFromEpochs(updatedMember.assignedTasks().activeTasksWithEpochs())); response.setStandbyTasks(createStreamsGroupHeartbeatResponseTaskIds(updatedMember.assignedTasks().standbyTasks())); response.setWarmupTasks(createStreamsGroupHeartbeatResponseTaskIds(updatedMember.assignedTasks().warmupTasks())); } - if (newlyJoinOrAssignmentChanged || hasReplacedStaticMember || userEndpointChanged) { + // Drop stale per-member endpoint mappings. + if (isJoining || assignedTaskChanged || endpointChanged || replaceStaticOldMember != null) { group.invalidateCachedEndpointToPartitions(updatedMember.memberId()); - if (hasReplacedStaticMember) { - group.invalidateCachedEndpointToPartitions(maybeOldMember.memberId()); + if (replaceStaticOldMember != null) { + group.invalidateCachedEndpointToPartitions(replaceStaticOldMember.memberId()); } } - if (userEndpointChanged || (hasAssignedTasksChanged(member, updatedMember) && updatedMember.userEndpoint().isPresent())) { + // Bump the group's endpoint epoch so peers refetch endpoint-to-partition mappings. + if (endpointChanged || (assignedTaskChanged && updatedMember.userEndpoint().isPresent())) { group.setEndpointInformationEpoch(group.endpointInformationEpoch() + 1); } if (group.endpointInformationEpoch() != memberEndpointEpoch) { - response.setPartitionsByUserEndpoint(group.buildEndpointToPartitions(updatedMember, metadataImage, maybeOldMember)); + response.setPartitionsByUserEndpoint(group.buildEndpointToPartitions(updatedMember, metadataImage, replaceStaticOldMember)); } if (groups.containsKey(group.groupId())) { // If we just created the group, the endpoint information epoch will not be persisted, so return epoch 0. @@ -3351,11 +3357,9 @@ private StreamsGroupMember getOrMaybeCreateStaticStreamsGroupMember( .setPreviousMemberEpoch(0) .build(); - // Generate the records to replace the member. We don't care about the regular expression - // here because it is taken care of later after the static membership replacement. replaceStreamsMember(records, group, existingStaticMemberOrNull, newMember); - log.info("[GroupId {}][MemberId {}] Static member with instance id {} re-joins the stream group " + + log.info("[GroupId {}][MemberId {}] Static member with instance id {} re-joins the streams group " + "using the streams protocol. Created a new member {} to replace the existing member {}.", group.groupId(), memberId, instanceId, memberId, existingStaticMemberOrNull.memberId()); @@ -3711,9 +3715,10 @@ private boolean hasMemberSubscriptionChanged( } /** - * Creates the member metadata record record if the updatedMember is different from - * the old member. Returns true if the metadata has changed, which is always the case - * when a member is first created. + * Creates the member metadata record if the updatedMember is different from the + * old member. Returns whether the group epoch should be bumped. Dynamic members + * bump the group epoch on any member metadata change, while static members bump + * it only when an epoch-relevant member configuration changes. * * @param groupId The group id. * @param instanceId The instance id. @@ -4447,7 +4452,7 @@ private CoordinatorResult stream StreamsGroupMember member = group.staticMember(instanceId); throwIfStaticMemberIsUnknown(member, instanceId); throwIfInstanceIdIsFenced(member, groupId, memberId, instanceId); - if (memberEpoch == StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) { + if (memberEpoch == LEAVE_GROUP_STATIC_MEMBER_EPOCH) { log.info("[GroupId {}][MemberId {}] Static member {} with instance id {} temporarily left the streams group.", group.groupId(), memberId, memberId, instanceId); return streamsGroupStaticMemberGroupLeave(group, member); @@ -4508,8 +4513,15 @@ private CoordinatorResult stream StreamsGroup group, StreamsGroupMember member ) { - // We will write a member epoch of -2 for this departing static member. - org.apache.kafka.coordinator.group.streams.MemberState nextState = member.isUnrevokedState() ? + // A static member leaving with epoch -2 may later be replaced with a new + // member id for the same instance id. Since we clear the pending revocations + // and reset the assigned task epochs for that replacement, keeping the member + // in UNREVOKED_TASKS would leave an inconsistent state: there are no pending + // revocations left to acknowledge, but reconciliation would still treat the + // member as waiting for revocation acknowledgement. Move it back to STABLE so + // the rejoining static member can be reconciled from the reset assignment. + org.apache.kafka.coordinator.group.streams.MemberState nextState = + member.state() == org.apache.kafka.coordinator.group.streams.MemberState.UNREVOKED_TASKS ? org.apache.kafka.coordinator.group.streams.MemberState.STABLE : member.state(); StreamsGroupMember leavingStaticMember = new StreamsGroupMember.Builder(member) @@ -4768,7 +4780,7 @@ private void replaceStreamsMember( records.add(StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord( groupId, newMember.memberId(), - group.targetAssignment(oldMember.memberId(), oldMember.instanceId()) + group.targetAssignment(oldMember.memberId(), Optional.empty()) )); records.add(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord( groupId, @@ -9300,7 +9312,7 @@ private Map streamsGroupAssignmentConfigs(String groupId) { )); } - private boolean hasUserEndpointChanged(StreamsGroupMember maybeOldMember, StreamsGroupMember updatedMember) { + private static boolean hasUserEndpointChanged(StreamsGroupMember maybeOldMember, StreamsGroupMember updatedMember) { boolean hasPreviousUserEndpoint = maybeOldMember != null && maybeOldMember.userEndpoint().isPresent(); boolean hasCurrentUserEndpoint = updatedMember.userEndpoint().isPresent(); diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java index 8e60b5cbb745a..39caa8ddb9d46 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java @@ -484,16 +484,6 @@ public StreamsGroupMember staticMember(String instanceId) { return existingMemberId == null ? null : getMemberOrThrow(existingMemberId); } - /** - * Gets a dynamic member. - * - * @param memberId The Member ID. - * @return The member corresponding to the given member ID or null if it does not exist - */ - public StreamsGroupMember dynamicMember(String memberId) { - return members.get(memberId); - } - /** * Adds or updates the member. * @@ -590,8 +580,16 @@ public boolean hasStaticMember(String instanceId) { /** * Returns the target assignment of the member. + *

+ * If {@code instanceId} is empty, the assignment is looked up by {@code memberId}. + * If {@code instanceId} is present, the assignment is looked up by the member ID + * associated with that static member instance ID. * - * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist. + * @param memberId The member id. + * @param instanceId The instance id. + * + * @return The StreamsGroupMemberAssignment for the resolved member ID, or {@link TasksTuple#EMPTY} + * if no assignment exists or no static member exists for {@code instanceId}. */ public TasksTuple targetAssignment(String memberId, Optional instanceId) { if (instanceId.isEmpty()) { diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroupMember.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroupMember.java index 05b133511eeae..8dafecac4961a 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroupMember.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroupMember.java @@ -360,13 +360,6 @@ public boolean isReconciledTo(int targetAssignmentEpoch) { return state == MemberState.STABLE && memberEpoch == targetAssignmentEpoch; } - /** - * @return True if the member is in the Unrevoked state. - */ - public boolean isUnrevokedState() { - return state == MemberState.UNREVOKED_TASKS; - } - /** * Creates a member description for the streams group describe response from this member. * From b9c4a71acc1719ea68ad69ccb2e93c7fac46cf35 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Thu, 21 May 2026 22:47:43 +0900 Subject: [PATCH 06/16] Fix lint error. --- ...amsGroupMixedGroupMetadataManagerTest.java | 37 +++++++++---------- ...pStaticMemberGroupMetadataManagerTest.java | 27 ++++++-------- .../group/StreamsGroupTestUtil.java | 2 +- 3 files changed, 31 insertions(+), 35 deletions(-) diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java index b8d0503588c9c..59a646d1082bd 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java @@ -21,50 +21,49 @@ import org.apache.kafka.common.errors.GroupMaxSizeReachedException; import org.apache.kafka.common.message.StreamsGroupHeartbeatRequestData; import org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest; -import org.apache.kafka.coordinator.common.runtime.MetadataImageBuilder; import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord; import org.apache.kafka.coordinator.common.runtime.CoordinatorResult; +import org.apache.kafka.coordinator.common.runtime.MetadataImageBuilder; import org.apache.kafka.coordinator.common.runtime.MockCoordinatorTimer; -import org.apache.kafka.coordinator.group.streams.StreamsGroupBuilder; -import org.apache.kafka.coordinator.group.streams.StreamsTopology; -import org.apache.kafka.coordinator.group.streams.TasksTuple; -import org.apache.kafka.coordinator.group.streams.TasksTupleWithEpochs; import org.apache.kafka.coordinator.group.streams.MockTaskAssignor; -import org.apache.kafka.coordinator.group.streams.StreamsGroupHeartbeatResult; +import org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers; import org.apache.kafka.coordinator.group.streams.StreamsGroup; +import org.apache.kafka.coordinator.group.streams.StreamsGroupBuilder; +import org.apache.kafka.coordinator.group.streams.StreamsGroupHeartbeatResult; import org.apache.kafka.coordinator.group.streams.StreamsGroupMember; -import org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers; +import org.apache.kafka.coordinator.group.streams.StreamsTopology; import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil; +import org.apache.kafka.coordinator.group.streams.TasksTuple; +import org.apache.kafka.coordinator.group.streams.TasksTupleWithEpochs; import org.junit.jupiter.api.Test; -import java.util.Map; import java.util.List; +import java.util.Map; import java.util.Optional; import static org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH; -import static org.apache.kafka.coordinator.group.Assertions.assertResponseEquals; import static org.apache.kafka.coordinator.group.Assertions.assertRecordsEquals; +import static org.apache.kafka.coordinator.group.Assertions.assertResponseEquals; import static org.apache.kafka.coordinator.group.Assertions.assertUnorderedRecordsEquals; import static org.apache.kafka.coordinator.group.GroupMetadataManager.groupSessionTimeoutKey; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticJoinHeartbeat; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.StreamsTopicFixture; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticHeartbeat; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.heartbeatResponseWithActiveTasks; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticLeaveResponseWithNullTasks; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.heartbeatResponseWithNullTasks; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.contextWithStreamsGroup; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.getDefaultAssignmentConfigs; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.heartbeatResponseWithActiveTasks; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.heartbeatResponseWithNullTasks; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticHeartbeat; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticJoinHeartbeat; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticLeaveResponseWithNullTasks; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.streamsTopicFixture; -import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTupleWithEpochs; import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole; +import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTupleWithEpochs; import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksWithEpochs; - -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertEquals; class StreamsGroupMixedGroupMetadataManagerTest { diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java index 5c14e1cb72bd3..15e6b9dfccb01 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java @@ -25,12 +25,16 @@ import org.apache.kafka.common.errors.UnreleasedInstanceIdException; import org.apache.kafka.common.message.StreamsGroupHeartbeatRequestData; import org.apache.kafka.common.message.StreamsGroupHeartbeatResponseData; +import org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest; import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord; import org.apache.kafka.coordinator.common.runtime.CoordinatorResult; import org.apache.kafka.coordinator.common.runtime.MetadataImageBuilder; import org.apache.kafka.coordinator.common.runtime.MockCoordinatorTimer; +import org.apache.kafka.coordinator.group.StreamsGroupTestUtil.StreamsTopicFixture; +import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue; import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataKey; import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataValue; +import org.apache.kafka.coordinator.group.streams.MemberState; import org.apache.kafka.coordinator.group.streams.MockTaskAssignor; import org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers; import org.apache.kafka.coordinator.group.streams.StreamsGroup; @@ -40,10 +44,6 @@ import org.apache.kafka.coordinator.group.streams.StreamsTopology; import org.apache.kafka.coordinator.group.streams.TasksTuple; import org.apache.kafka.coordinator.group.streams.TasksTupleWithEpochs; -import org.apache.kafka.coordinator.group.streams.MemberState; -import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue; - -import org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -56,21 +56,13 @@ import java.util.Optional; import java.util.stream.Stream; -import static org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH; -import static org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest.LEAVE_GROUP_MEMBER_EPOCH; import static org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest.JOIN_GROUP_MEMBER_EPOCH; -import static org.apache.kafka.coordinator.group.Assertions.assertResponseEquals; +import static org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest.LEAVE_GROUP_MEMBER_EPOCH; +import static org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH; import static org.apache.kafka.coordinator.group.Assertions.assertRecordsEquals; +import static org.apache.kafka.coordinator.group.Assertions.assertResponseEquals; import static org.apache.kafka.coordinator.group.GroupMetadataManager.groupSessionTimeoutKey; import static org.apache.kafka.coordinator.group.GroupMetadataManagerTestContext.DEFAULT_PROCESS_ID; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import org.apache.kafka.coordinator.group.StreamsGroupTestUtil.StreamsTopicFixture; - import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.contextWithStreamsGroup; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.getDefaultAssignmentConfigs; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.heartbeatResponseWithActiveTasks; @@ -82,6 +74,11 @@ import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticLeaveResponseWithNullTasks; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.streamsTopicFixture; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; class StreamsGroupStaticMemberGroupMetadataManagerTest { diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java index 4f7ffdd038f11..1279a845b5a34 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java @@ -41,9 +41,9 @@ import static org.apache.kafka.coordinator.group.GroupMetadataManagerTestContext.DEFAULT_CLIENT_ADDRESS; import static org.apache.kafka.coordinator.group.GroupMetadataManagerTestContext.DEFAULT_CLIENT_ID; import static org.apache.kafka.coordinator.group.GroupMetadataManagerTestContext.DEFAULT_PROCESS_ID; -import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTupleWithCommonEpoch; import static org.apache.kafka.coordinator.group.Utils.computeGroupHash; import static org.apache.kafka.coordinator.group.Utils.computeTopicHash; +import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTupleWithCommonEpoch; class StreamsGroupTestUtil { From d7673c578f482de68b821bc271f01d5d98726e62 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Thu, 28 May 2026 21:44:44 +0900 Subject: [PATCH 07/16] Fix white space. --- .../group/GroupMetadataManager.java | 114 +++++++++--------- .../group/streams/StreamsGroupMember.java | 6 +- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java index 47ea9135ce118..c61d4a70ee138 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java @@ -1706,9 +1706,9 @@ private void throwIfInstanceIdIsUnreleased(StreamsGroupMember member, String gro if (member.memberEpoch() != LEAVE_GROUP_STATIC_MEMBER_EPOCH) { // The new member can't join. log.info("[GroupId {}] Static member {} with instance id {} cannot join the group because the instance id is" + - " owned by member {}.", groupId, receivedMemberId, receivedInstanceId, member.memberId()); + " owned by member {}.", groupId, receivedMemberId, receivedInstanceId, member.memberId()); throw Errors.UNRELEASED_INSTANCE_ID.exception("Static member " + receivedMemberId + " with instance id " - + receivedInstanceId + " cannot join the group because the instance id is owned by " + member.memberId() + " member."); + + receivedInstanceId + " cannot join the group because the instance id is owned by " + member.memberId() + " member."); } } @@ -1744,9 +1744,9 @@ private void throwIfInstanceIdIsFenced(ConsumerGroupMember member, String groupI private void throwIfInstanceIdIsFenced(StreamsGroupMember member, String groupId, String receivedMemberId, String receivedInstanceId) { if (!member.memberId().equals(receivedMemberId)) { log.info("[GroupId {}] Static member {} with instance id {} is fenced by existing member {}.", - groupId, receivedMemberId, receivedInstanceId, member.memberId()); + groupId, receivedMemberId, receivedInstanceId, member.memberId()); throw Errors.FENCED_INSTANCE_ID.exception("Static member " + receivedMemberId + " with instance id " - + receivedInstanceId + " was fenced by member " + member.memberId() + "."); + + receivedInstanceId + " was fenced by member " + member.memberId() + "."); } } @@ -2103,11 +2103,11 @@ private CoordinatorResult stream if (isJoining) { StreamsGroupMemberMetadataValue.Endpoint userEndpointMetadata = userEndpoint == null ? null : - new StreamsGroupMemberMetadataValue.Endpoint().setHost(userEndpoint.host()).setPort(userEndpoint.port()); + new StreamsGroupMemberMetadataValue.Endpoint().setHost(userEndpoint.host()).setPort(userEndpoint.port()); updatedMemberBuilder.setUserEndpoint(userEndpointMetadata); } else { updatedMemberBuilder - .maybeUpdateUserEndpoint(Optional.ofNullable(userEndpoint).map(x -> new StreamsGroupMemberMetadataValue.Endpoint().setHost(x.host()).setPort(x.port()))); + .maybeUpdateUserEndpoint(Optional.ofNullable(userEndpoint).map(x -> new StreamsGroupMemberMetadataValue.Endpoint().setHost(x.host()).setPort(x.port()))); } StreamsGroupMember updatedMember = updatedMemberBuilder.build(); @@ -3329,15 +3329,15 @@ private ConsumerGroupMember getOrMaybeSubscribeStaticConsumerGroupMember( * @return The resolved streams group member. */ private StreamsGroupMember getOrMaybeCreateStaticStreamsGroupMember( - StreamsGroup group, - String memberId, - int memberEpoch, - String instanceId, - List ownedActiveTasks, - List ownedStandbyTasks, - List ownedWarmupTasks, - boolean memberIsJoining, - List records + StreamsGroup group, + String memberId, + int memberEpoch, + String instanceId, + List ownedActiveTasks, + List ownedStandbyTasks, + List ownedWarmupTasks, + boolean memberIsJoining, + List records ) { StreamsGroupMember existingStaticMemberOrNull = group.staticMember(instanceId); if (memberIsJoining) { @@ -3346,22 +3346,22 @@ private StreamsGroupMember getOrMaybeCreateStaticStreamsGroupMember( // New static member. StreamsGroupMember newMember = group.getOrCreateDefaultMember(memberId); log.info("[GroupId {}][MemberId {}] Static member {} with instance id {} joins the streams group.", - group.groupId(), memberId, memberId, instanceId); + group.groupId(), memberId, memberId, instanceId); return newMember; } else { throwIfInstanceIdIsUnreleased(existingStaticMemberOrNull, group.groupId(), memberId, instanceId); // Copy the member but with its new member id. StreamsGroupMember newMember = new StreamsGroupMember.Builder(existingStaticMemberOrNull, memberId) - .setMemberEpoch(0) - .setPreviousMemberEpoch(0) - .build(); + .setMemberEpoch(0) + .setPreviousMemberEpoch(0) + .build(); replaceStreamsMember(records, group, existingStaticMemberOrNull, newMember); log.info("[GroupId {}][MemberId {}] Static member with instance id {} re-joins the streams group " + - "using the streams protocol. Created a new member {} to replace the existing member {}.", - group.groupId(), memberId, instanceId, memberId, existingStaticMemberOrNull.memberId()); + "using the streams protocol. Created a new member {} to replace the existing member {}.", + group.groupId(), memberId, instanceId, memberId, existingStaticMemberOrNull.memberId()); return newMember; } @@ -3369,11 +3369,11 @@ private StreamsGroupMember getOrMaybeCreateStaticStreamsGroupMember( throwIfStaticMemberIsUnknown(existingStaticMemberOrNull, instanceId); throwIfInstanceIdIsFenced(existingStaticMemberOrNull, group.groupId(), memberId, instanceId); throwIfStreamsGroupMemberEpochIsInvalid( - existingStaticMemberOrNull, - memberEpoch, - ownedActiveTasks, - ownedStandbyTasks, - ownedWarmupTasks + existingStaticMemberOrNull, + memberEpoch, + ownedActiveTasks, + ownedStandbyTasks, + ownedWarmupTasks ); return existingStaticMemberOrNull; } @@ -4454,11 +4454,11 @@ private CoordinatorResult stream throwIfInstanceIdIsFenced(member, groupId, memberId, instanceId); if (memberEpoch == LEAVE_GROUP_STATIC_MEMBER_EPOCH) { log.info("[GroupId {}][MemberId {}] Static member {} with instance id {} temporarily left the streams group.", - group.groupId(), memberId, memberId, instanceId); + group.groupId(), memberId, memberId, instanceId); return streamsGroupStaticMemberGroupLeave(group, member); } else { log.info("[GroupId {}][MemberId {}] Static member {} with instance id {} left the streams group.", - group.groupId(), memberId, memberId, instanceId); + group.groupId(), memberId, memberId, instanceId); return streamsGroupFenceMember(group, member, new StreamsGroupHeartbeatResult(response, Map.of())); } } @@ -4510,8 +4510,8 @@ private CoordinatorResult * @return A CoordinatorResult with a single record signifying that the static member is leaving. */ private CoordinatorResult streamsGroupStaticMemberGroupLeave( - StreamsGroup group, - StreamsGroupMember member + StreamsGroup group, + StreamsGroupMember member ) { // A static member leaving with epoch -2 may later be replaced with a new // member id for the same instance id. Since we clear the pending revocations @@ -4521,25 +4521,25 @@ private CoordinatorResult stream // member as waiting for revocation acknowledgement. Move it back to STABLE so // the rejoining static member can be reconciled from the reset assignment. org.apache.kafka.coordinator.group.streams.MemberState nextState = - member.state() == org.apache.kafka.coordinator.group.streams.MemberState.UNREVOKED_TASKS ? + member.state() == org.apache.kafka.coordinator.group.streams.MemberState.UNREVOKED_TASKS ? org.apache.kafka.coordinator.group.streams.MemberState.STABLE : member.state(); StreamsGroupMember leavingStaticMember = new StreamsGroupMember.Builder(member) - .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setState(nextState) - .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) - .resetAssignedTasksEpochsToZero() - .build(); + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setState(nextState) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .resetAssignedTasksEpochsToZero() + .build(); CoordinatorRecord record = newStreamsGroupCurrentAssignmentRecord(group.groupId(), leavingStaticMember); StreamsGroupHeartbeatResponseData response = new StreamsGroupHeartbeatResponseData() - .setMemberId(member.memberId()) - .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setStatus(List.of()); + .setMemberId(member.memberId()) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setStatus(List.of()); return new CoordinatorResult<>( - List.of(record), - new StreamsGroupHeartbeatResult(response, Map.of()) + List.of(record), + new StreamsGroupHeartbeatResult(response, Map.of()) ); } @@ -4761,10 +4761,10 @@ private void replaceMember( * @param newMember The new member. */ private void replaceStreamsMember( - List records, - StreamsGroup group, - StreamsGroupMember oldMember, - StreamsGroupMember newMember + List records, + StreamsGroup group, + StreamsGroupMember oldMember, + StreamsGroupMember newMember ) { String groupId = group.groupId(); @@ -4774,17 +4774,17 @@ private void replaceStreamsMember( // Generate records. records.add(StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord( - groupId, - newMember + groupId, + newMember )); records.add(StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord( - groupId, - newMember.memberId(), - group.targetAssignment(oldMember.memberId(), Optional.empty()) + groupId, + newMember.memberId(), + group.targetAssignment(oldMember.memberId(), Optional.empty()) )); records.add(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord( - groupId, - newMember + groupId, + newMember )); } @@ -9330,15 +9330,15 @@ private static boolean hasUserEndpointChanged(StreamsGroupMember maybeOldMember, } private static boolean hasEpochRelevantMemberConfigChanged( - StreamsGroupMember oldMember, - StreamsGroupMember newMember + StreamsGroupMember oldMember, + StreamsGroupMember newMember ) { // The group epoch is bumped: (KIP-1071) // - When a member updates its topology metadata, rack ID, client tags or process ID. return !Objects.equals(oldMember.topologyEpoch(), newMember.topologyEpoch()) - || !Objects.equals(oldMember.rackId(), newMember.rackId()) - || !Objects.equals(oldMember.clientTags(), newMember.clientTags()) - || !Objects.equals(oldMember.processId(), newMember.processId()); + || !Objects.equals(oldMember.rackId(), newMember.rackId()) + || !Objects.equals(oldMember.clientTags(), newMember.clientTags()) + || !Objects.equals(oldMember.processId(), newMember.processId()); } /** diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroupMember.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroupMember.java index 8dafecac4961a..1c042778bbe89 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroupMember.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroupMember.java @@ -345,9 +345,9 @@ public Builder resetAssignedTasksEpochsToZero() { resetActiveTasks.put(entry.getKey(), resetActiveTaskEpochs); } this.assignedTasks = new TasksTupleWithEpochs( - resetActiveTasks, - this.assignedTasks.standbyTasks(), - this.assignedTasks.warmupTasks() + resetActiveTasks, + this.assignedTasks.standbyTasks(), + this.assignedTasks.warmupTasks() ); return this; } From e0c596c696916f1e0179ec4a2bb5c62d66cf8df9 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Thu, 28 May 2026 21:56:49 +0900 Subject: [PATCH 08/16] Fix white space. --- ...amsGroupMixedGroupMetadataManagerTest.java | 538 +++++----- ...pStaticMemberGroupMetadataManagerTest.java | 949 +++++++++--------- .../group/StreamsGroupTestUtil.java | 266 ++--- 3 files changed, 875 insertions(+), 878 deletions(-) diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java index 59a646d1082bd..b8bdfba431414 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java @@ -85,27 +85,27 @@ public void testDynamicJoinFailsAtMaxSizeWhileStaticMemberIsTemporarilyLeftAndDy StreamsGroupHeartbeatRequestData.Topology topology = new StreamsGroupHeartbeatRequestData.Topology().setSubtopologies(List.of()); GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() - .withMetadataImage(new MetadataImageBuilder().buildCoordinatorMetadataImage()) - .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, 2) - .withStreamsGroup(new StreamsGroupBuilder(groupId, 10) - .withMember(streamsGroupMemberBuilderWithDefaults(staticMemberId, staticInstanceId) - .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setPreviousMemberEpoch(10) - .build()) - .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) - .setMemberEpoch(10) - .setPreviousMemberEpoch(9) - .build()) - .withTargetAssignmentEpoch(10) - .withTopology(StreamsTopology.fromHeartbeatRequest(topology))) - .build(); + .withMetadataImage(new MetadataImageBuilder().buildCoordinatorMetadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, 2) + .withStreamsGroup(new StreamsGroupBuilder(groupId, 10) + .withMember(streamsGroupMemberBuilderWithDefaults(staticMemberId, staticInstanceId) + .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(10) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setMemberEpoch(10) + .setPreviousMemberEpoch(9) + .build()) + .withTargetAssignmentEpoch(10) + .withTopology(StreamsTopology.fromHeartbeatRequest(topology))) + .build(); assertThrows(GroupMaxSizeReachedException.class, () -> - context.streamsGroupHeartbeat( - staticJoinHeartbeat(groupId, newDynamicMemberId, null, "new-process-id") - .setRebalanceTimeoutMs(1500) - .setTopology(topology) - ) + context.streamsGroupHeartbeat( + staticJoinHeartbeat(groupId, newDynamicMemberId, null, "new-process-id") + .setRebalanceTimeoutMs(1500) + .setTopology(topology) + ) ); } @@ -139,33 +139,33 @@ public void testStaticRejoinSucceedsAtMaxSizeWhileDynamicMemberStillExists() { TasksTuple dynamicTargetAssignment = topic.targetAssignment(2, 3); GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() - .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) - .withMetadataImage(topic.metadataImage()) - .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, 2) - .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) - .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) - .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setPreviousMemberEpoch(10) - .setAssignedTasks(staticAssignedTasks) - .build()) - .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) - .setMemberEpoch(10) - .setPreviousMemberEpoch(9) - .setAssignedTasks(dynamicAssignedTasks) - .build()) - .withTargetAssignment(oldStaticMemberId, staticTargetAssignment) - .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment) - .withTargetAssignmentEpoch(groupEpoch) - .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) - .withValidatedTopologyEpoch(0) - .withMetadataHash(topic.metadataHash()) - .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) - .build(); + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, 2) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) + .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(10) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setMemberEpoch(10) + .setPreviousMemberEpoch(9) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(oldStaticMemberId, staticTargetAssignment) + .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .build(); // WHEN - static member rejoin. CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, newStaticMemberId, staticInstanceId, StreamsGroupHeartbeatRequest.JOIN_GROUP_MEMBER_EPOCH) + staticHeartbeat(groupId, newStaticMemberId, staticInstanceId, StreamsGroupHeartbeatRequest.JOIN_GROUP_MEMBER_EPOCH) ); // THEN - At the maxsize, static member still can rejoin if static member leaves with epoch -2. @@ -221,35 +221,35 @@ public void testDynamicJoinSucceedsAfterTemporarilyLeftStaticMemberSessionTimeou MockTaskAssignor assignor = new MockTaskAssignor("sticky"); GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() - .withStreamsGroupTaskAssignors(List.of(assignor)) - .withMetadataImage(topic.metadataImage()) - .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, 2) - .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) - .withMember(streamsGroupMemberBuilderWithDefaults(staticMemberId, staticInstanceId) - .setProcessId(staticProcessId) - .setMemberEpoch(groupEpoch) - .setPreviousMemberEpoch(groupEpoch - 1) - .setAssignedTasks(staticAssignedTasks) - .build()) - .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) - .setProcessId(dynamicProcessId) - .setMemberEpoch(groupEpoch) - .setPreviousMemberEpoch(groupEpoch - 1) - .setAssignedTasks(dynamicAssignedTasks) - .build()) - .withTargetAssignment(staticMemberId, staticTargetAssignment) - .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment) - .withTargetAssignmentEpoch(groupEpoch) - .withTopology(StreamsTopology.fromHeartbeatRequest(topology)) - .withValidatedTopologyEpoch(0) - .withMetadataHash(groupMetadataHash) - .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) - .build(); + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, 2) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(staticMemberId, staticInstanceId) + .setProcessId(staticProcessId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setProcessId(dynamicProcessId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(staticMemberId, staticTargetAssignment) + .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topology)) + .withValidatedTopologyEpoch(0) + .withMetadataHash(groupMetadataHash) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .build(); context.onLoaded(); // WHEN1 - static member leaves with epoch -2. CoordinatorResult leaveResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, staticMemberId, staticInstanceId, StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) + staticHeartbeat(groupId, staticMemberId, staticInstanceId, StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) ); // THEN1 @@ -259,7 +259,7 @@ public void testDynamicJoinSucceedsAfterTemporarilyLeftStaticMemberSessionTimeou // Sleep 1, and dynamic member send a heartbeat. GroupMetadataManagerTestContext.assertNoOrEmptyResult(context.sleep(1)); CoordinatorResult dynamicHeartbeatResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, dynamicMemberId, null, groupEpoch) + staticHeartbeat(groupId, dynamicMemberId, null, groupEpoch) ); assertResponseEquals(heartbeatResponseWithNullTasks(dynamicMemberId, groupEpoch), dynamicHeartbeatResult.response().data()); @@ -271,20 +271,20 @@ public void testDynamicJoinSucceedsAfterTemporarilyLeftStaticMemberSessionTimeou // THEN2 List expectedTimeoutRecords = List.of( - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, staticMemberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, staticMemberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, staticMemberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord( - groupId, timeoutGroupEpoch, groupMetadataHash, 0, getDefaultAssignmentConfigs() - ) + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, staticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, staticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, staticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord( + groupId, timeoutGroupEpoch, groupMetadataHash, 0, getDefaultAssignmentConfigs() + ) ); assertEquals( - List.of(new MockCoordinatorTimer.ExpiredTimeout<>( - groupSessionTimeoutKey(groupId, staticMemberId), - new CoordinatorResult<>(expectedTimeoutRecords) - )), - timeouts + List.of(new MockCoordinatorTimer.ExpiredTimeout<>( + groupSessionTimeoutKey(groupId, staticMemberId), + new CoordinatorResult<>(expectedTimeoutRecords) + )), + timeouts ); StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); @@ -293,51 +293,51 @@ public void testDynamicJoinSucceedsAfterTemporarilyLeftStaticMemberSessionTimeou assertEquals(timeoutGroupEpoch, group.groupEpoch()); assignor.prepareGroupAssignment(Map.of( - dynamicMemberId, dynamicTargetAssignment, - newDynamicMemberId, staticTargetAssignment + dynamicMemberId, dynamicTargetAssignment, + newDynamicMemberId, staticTargetAssignment )); // WHEN3 - new dynamic member try to join. CoordinatorResult joinResult = context.streamsGroupHeartbeat( - staticJoinHeartbeat(groupId, newDynamicMemberId, null, newDynamicProcessId) - .setRebalanceTimeoutMs(1500) - .setTopology(topology) + staticJoinHeartbeat(groupId, newDynamicMemberId, null, newDynamicProcessId) + .setRebalanceTimeoutMs(1500) + .setTopology(topology) ); // THEN3 : accept join. assertResponseEquals(heartbeatResponseWithActiveTasks(newDynamicMemberId, joinGroupEpoch, subtopologyId, 0, 1), joinResult.response().data()); StreamsGroupMember expectedJoiningDynamicMember = streamsGroupMemberBuilderWithDefaults(newDynamicMemberId) - .setProcessId(newDynamicProcessId) - .setMemberEpoch(0) - .setPreviousMemberEpoch(0) - .build(); + .setProcessId(newDynamicProcessId) + .setMemberEpoch(0) + .setPreviousMemberEpoch(0) + .build(); StreamsGroupMember expectedReconciledDynamicMember = streamsGroupMemberBuilderWithDefaults(newDynamicMemberId) - .setProcessId(newDynamicProcessId) - .setMemberEpoch(joinGroupEpoch) - .setPreviousMemberEpoch(0) - .setAssignedTasks(mkTasksTupleWithEpochs( - TaskRole.ACTIVE, - mkTasksWithEpochs(subtopologyId, Map.of( - 0, joinGroupEpoch, - 1, joinGroupEpoch - )) + .setProcessId(newDynamicProcessId) + .setMemberEpoch(joinGroupEpoch) + .setPreviousMemberEpoch(0) + .setAssignedTasks(mkTasksTupleWithEpochs( + TaskRole.ACTIVE, + mkTasksWithEpochs(subtopologyId, Map.of( + 0, joinGroupEpoch, + 1, joinGroupEpoch )) - .build(); + )) + .build(); List expectedJoinRecords = List.of( - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, expectedJoiningDynamicMember), - StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord( - groupId, - joinGroupEpoch, - groupMetadataHash, - 0, - getDefaultAssignmentConfigs() - ), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, newDynamicMemberId, staticTargetAssignment), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, joinGroupEpoch, context.time.milliseconds()), - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedReconciledDynamicMember) + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, expectedJoiningDynamicMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord( + groupId, + joinGroupEpoch, + groupMetadataHash, + 0, + getDefaultAssignmentConfigs() + ), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, newDynamicMemberId, staticTargetAssignment), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, joinGroupEpoch, context.time.milliseconds()), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedReconciledDynamicMember) ); assertRecordsEquals(expectedJoinRecords, joinResult.records()); @@ -376,42 +376,42 @@ public void testStaticTemporaryLeaveDoesNotTransferTasksToExistingDynamicMember( TasksTuple dynamicTargetAssignment = topic.targetAssignment(2); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(staticMemberId, staticInstanceId) - .setMemberEpoch(groupEpoch) - .setPreviousMemberEpoch(groupEpoch - 1) - .setAssignedTasks(staticAssignedTasks) - .build()) - .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) - .setMemberEpoch(groupEpoch) - .setPreviousMemberEpoch(groupEpoch - 1) - .setAssignedTasks(dynamicAssignedTasks) - .build()) - .withTargetAssignment(staticMemberId, staticTargetAssignment) - .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment)); + .withMember(streamsGroupMemberBuilderWithDefaults(staticMemberId, staticInstanceId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(staticMemberId, staticTargetAssignment) + .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment)); // WHEN - static member leaves with epoch -2. CoordinatorResult leaveResult = - context.streamsGroupHeartbeat(staticHeartbeat( - groupId, - staticMemberId, - staticInstanceId, - LEAVE_GROUP_STATIC_MEMBER_EPOCH - )); + context.streamsGroupHeartbeat(staticHeartbeat( + groupId, + staticMemberId, + staticInstanceId, + LEAVE_GROUP_STATIC_MEMBER_EPOCH + )); // THEN assertResponseEquals( - staticLeaveResponseWithNullTasks(staticMemberId, LEAVE_GROUP_STATIC_MEMBER_EPOCH), - leaveResult.response().data() + staticLeaveResponseWithNullTasks(staticMemberId, LEAVE_GROUP_STATIC_MEMBER_EPOCH), + leaveResult.response().data() ); // WHEN2 - dynamic member send a heartbeat. CoordinatorResult dynamicHeartbeatResult = - context.streamsGroupHeartbeat( - new StreamsGroupHeartbeatRequestData() - .setGroupId(groupId) - .setMemberId(dynamicMemberId) - .setMemberEpoch(groupEpoch) - ); + context.streamsGroupHeartbeat( + new StreamsGroupHeartbeatRequestData() + .setGroupId(groupId) + .setMemberId(dynamicMemberId) + .setMemberEpoch(groupEpoch) + ); // THEN2 : There is no new assignment. assertResponseEquals(heartbeatResponseWithNullTasks(dynamicMemberId, groupEpoch), dynamicHeartbeatResult.response().data()); @@ -462,34 +462,34 @@ public void testStaticRejoinWithUpdatedProcessIdRecomputesTargetAssignmentAndDyn MockTaskAssignor assignor = new MockTaskAssignor("sticky"); assignor.prepareGroupAssignment(Map.of( - rejoinStaticMemberId, newStaticTargetAssignment, - dynamicMemberId, newDynamicTargetAssignment + rejoinStaticMemberId, newStaticTargetAssignment, + dynamicMemberId, newDynamicTargetAssignment )); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) - .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setPreviousMemberEpoch(groupEpoch) - .setProcessId(oldProcessId) - .setAssignedTasks(staticAssignedTasks) - .build()) - .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) - .setMemberEpoch(groupEpoch) - .setPreviousMemberEpoch(groupEpoch - 1) - .setAssignedTasks(dynamicAssignedTasks) - .build()) - .withTargetAssignment(oldStaticMemberId, oldStaticTargetAssignment) - .withTargetAssignment(dynamicMemberId, oldDynamicTargetAssignment)); + .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) + .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(groupEpoch) + .setProcessId(oldProcessId) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(oldStaticMemberId, oldStaticTargetAssignment) + .withTargetAssignment(dynamicMemberId, oldDynamicTargetAssignment)); // WHEN 1: static member try to rejoin with new process id. CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( - staticJoinHeartbeat(groupId, rejoinStaticMemberId, staticInstanceId, newProcessId) + staticJoinHeartbeat(groupId, rejoinStaticMemberId, staticInstanceId, newProcessId) ); // THEN 1: assertResponseEquals( - heartbeatResponseWithActiveTasks(rejoinStaticMemberId, bumpedGroupEpoch, topic, 0), - rejoinResult.response().data() + heartbeatResponseWithActiveTasks(rejoinStaticMemberId, bumpedGroupEpoch, topic, 0), + rejoinResult.response().data() ); StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); @@ -509,100 +509,99 @@ public void testStaticRejoinWithUpdatedProcessIdRecomputesTargetAssignmentAndDyn assertEquals(groupEpoch, group.getMemberOrThrow(dynamicMemberId).memberEpoch()); StreamsGroupMember expectedCopiedStaticMember = streamsGroupMemberBuilderWithDefaults(rejoinStaticMemberId, staticInstanceId) - .setProcessId(oldProcessId) - .setMemberEpoch(0) - .setPreviousMemberEpoch(0) - .setAssignedTasks(staticAssignedTasks) - .build(); + .setProcessId(oldProcessId) + .setMemberEpoch(0) + .setPreviousMemberEpoch(0) + .setAssignedTasks(staticAssignedTasks) + .build(); StreamsGroupMember expectedUpdatedStaticMember = streamsGroupMemberBuilderWithDefaults(rejoinStaticMemberId, staticInstanceId) - .setProcessId(newProcessId) - .setMemberEpoch(0) - .setPreviousMemberEpoch(0) - .setAssignedTasks(staticAssignedTasks) - .build(); - - StreamsGroupMember expectedReconciledStaticMember = streamsGroupMemberBuilderWithDefaults(rejoinStaticMemberId, - staticInstanceId) - .setProcessId(newProcessId) - .setMemberEpoch(bumpedGroupEpoch) - .setPreviousMemberEpoch(0) - .setAssignedTasks(mkTasksTupleWithEpochs( - TaskAssignmentTestUtil.TaskRole.ACTIVE, - mkTasksWithEpochs(subtopologyId, Map.of(0, groupEpoch)) - )) - .build(); - + .setProcessId(newProcessId) + .setMemberEpoch(0) + .setPreviousMemberEpoch(0) + .setAssignedTasks(staticAssignedTasks) + .build(); + + StreamsGroupMember expectedReconciledStaticMember = streamsGroupMemberBuilderWithDefaults(rejoinStaticMemberId, staticInstanceId) + .setProcessId(newProcessId) + .setMemberEpoch(bumpedGroupEpoch) + .setPreviousMemberEpoch(0) + .setAssignedTasks(mkTasksTupleWithEpochs( + TaskAssignmentTestUtil.TaskRole.ACTIVE, + mkTasksWithEpochs(subtopologyId, Map.of(0, groupEpoch)) + )) + .build(); + List expectedRecordsBeforeRecomputedAssignments = List.of( - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, oldStaticMemberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, oldStaticMemberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, oldStaticMemberId), - - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, expectedCopiedStaticMember), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, rejoinStaticMemberId, - oldStaticTargetAssignment), - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedCopiedStaticMember), - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, expectedUpdatedStaticMember), - StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord( - groupId, - bumpedGroupEpoch, - topic.metadataHash(), - 0, - getDefaultAssignmentConfigs() - ) + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, oldStaticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, oldStaticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, oldStaticMemberId), + + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, expectedCopiedStaticMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, rejoinStaticMemberId, + oldStaticTargetAssignment), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedCopiedStaticMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, expectedUpdatedStaticMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord( + groupId, + bumpedGroupEpoch, + topic.metadataHash(), + 0, + getDefaultAssignmentConfigs() + ) ); List expectedRecomputedTargetAssignmentRecords = List.of( - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, rejoinStaticMemberId, - newStaticTargetAssignment), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, dynamicMemberId, - newDynamicTargetAssignment) + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, rejoinStaticMemberId, + newStaticTargetAssignment), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, dynamicMemberId, + newDynamicTargetAssignment) ); List expectedRecordsAfterRecomputedAssignments = List.of( - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, bumpedGroupEpoch, - context.time.milliseconds()), - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedReconciledStaticMember) + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, bumpedGroupEpoch, + context.time.milliseconds()), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedReconciledStaticMember) ); assertRecordsEquals( - expectedRecordsBeforeRecomputedAssignments, - rejoinResult.records().subList(0, 8) + expectedRecordsBeforeRecomputedAssignments, + rejoinResult.records().subList(0, 8) ); assertUnorderedRecordsEquals( - List.of(expectedRecomputedTargetAssignmentRecords), - rejoinResult.records().subList(8, 10) + List.of(expectedRecomputedTargetAssignmentRecords), + rejoinResult.records().subList(8, 10) ); assertRecordsEquals( - expectedRecordsAfterRecomputedAssignments, - rejoinResult.records().subList(10, 12) + expectedRecordsAfterRecomputedAssignments, + rejoinResult.records().subList(10, 12) ); // WHEN 2: dynamic member send a heartbeat and reconciles to the new target assignment. CoordinatorResult dynamicHeartbeatResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, dynamicMemberId, null, groupEpoch) + staticHeartbeat(groupId, dynamicMemberId, null, groupEpoch) ); assertResponseEquals( - heartbeatResponseWithActiveTasks(dynamicMemberId, bumpedGroupEpoch, topic, 1, 2, 3), - dynamicHeartbeatResult.response().data() + heartbeatResponseWithActiveTasks(dynamicMemberId, bumpedGroupEpoch, topic, 1, 2, 3), + dynamicHeartbeatResult.response().data() ); StreamsGroupMember expectedUpdatedDynamicMember = streamsGroupMemberBuilderWithDefaults(dynamicMemberId) - .setMemberEpoch(bumpedGroupEpoch) - .setPreviousMemberEpoch(groupEpoch) - .setAssignedTasks(mkTasksTupleWithEpochs( - TaskAssignmentTestUtil.TaskRole.ACTIVE, - mkTasksWithEpochs(subtopologyId, Map.of( - 1, bumpedGroupEpoch, - 2, groupEpoch, - 3, groupEpoch - )) + .setMemberEpoch(bumpedGroupEpoch) + .setPreviousMemberEpoch(groupEpoch) + .setAssignedTasks(mkTasksTupleWithEpochs( + TaskAssignmentTestUtil.TaskRole.ACTIVE, + mkTasksWithEpochs(subtopologyId, Map.of( + 1, bumpedGroupEpoch, + 2, groupEpoch, + 3, groupEpoch )) - .build(); + )) + .build(); List expectedRecordsAfterDynamicHeartbeat = List.of( - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedUpdatedDynamicMember) + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedUpdatedDynamicMember) ); assertRecordsEquals(expectedRecordsAfterDynamicHeartbeat, dynamicHeartbeatResult.records()); @@ -642,26 +641,25 @@ public void testStaticRejoinWithSameProcessIdDoesNotBumpEpochAndDynamicHeartbeat TasksTuple dynamicTargetAssignment = topic.targetAssignment(2, 3); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) - .setProcessId(staticProcessId) - .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setPreviousMemberEpoch(groupEpoch) - .setAssignedTasks(staticAssignedTasks) - .build()) - .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) - .setProcessId(dynamicProcessId) - .setMemberEpoch(groupEpoch) - .setPreviousMemberEpoch(groupEpoch - 1) - .setAssignedTasks(dynamicAssignedTasks) - .build()) - .withTargetAssignment(oldStaticMemberId, staticTargetAssignment) - .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment)); + .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) + .setProcessId(staticProcessId) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(groupEpoch) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setProcessId(dynamicProcessId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(oldStaticMemberId, staticTargetAssignment) + .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment)); // WHEN1 : static member try to rejoin with same process id. - CoordinatorResult rejoinResult = - context.streamsGroupHeartbeat( - staticJoinHeartbeat(groupId, newStaticMemberId, staticInstanceId, staticProcessId) - ); + CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( + staticJoinHeartbeat(groupId, newStaticMemberId, staticInstanceId, staticProcessId) + ); // THEN1 @@ -679,35 +677,35 @@ public void testStaticRejoinWithSameProcessIdDoesNotBumpEpochAndDynamicHeartbeat assertEquals(dynamicTargetAssignment, group.targetAssignment(dynamicMemberId, Optional.empty())); StreamsGroupMember expectedCopiedStaticMember = streamsGroupMemberBuilderWithDefaults(newStaticMemberId, staticInstanceId) - .setProcessId(staticProcessId) - .setMemberEpoch(0) - .setPreviousMemberEpoch(0) - .setAssignedTasks(staticAssignedTasks) - .build(); + .setProcessId(staticProcessId) + .setMemberEpoch(0) + .setPreviousMemberEpoch(0) + .setAssignedTasks(staticAssignedTasks) + .build(); StreamsGroupMember expectedRejoinedStaticMember = streamsGroupMemberBuilderWithDefaults(newStaticMemberId, staticInstanceId) - .setProcessId(staticProcessId) - .setMemberEpoch(groupEpoch) - .setPreviousMemberEpoch(0) - .setAssignedTasks(staticAssignedTasks) - .build(); + .setProcessId(staticProcessId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(0) + .setAssignedTasks(staticAssignedTasks) + .build(); // no new target assignment. List expectedRejoinRecords = List.of( - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, oldStaticMemberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, oldStaticMemberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, oldStaticMemberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, expectedCopiedStaticMember), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, newStaticMemberId, staticTargetAssignment), - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedCopiedStaticMember), - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedRejoinedStaticMember) + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, oldStaticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, oldStaticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, oldStaticMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, expectedCopiedStaticMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, newStaticMemberId, staticTargetAssignment), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedCopiedStaticMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedRejoinedStaticMember) ); assertRecordsEquals(expectedRejoinRecords, rejoinResult.records()); // WHEN2 - dynamic member send a heartbeat request CoordinatorResult dynamicHeartbeatResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, dynamicMemberId, null, groupEpoch) + staticHeartbeat(groupId, dynamicMemberId, null, groupEpoch) ); // THEN2 - no new target assignment. @@ -746,29 +744,29 @@ public void testOldStaticMemberIdIsFencedAfterReplacementInMixedGroup() { TasksTuple dynamicTargetAssignment = topic.targetAssignment(2, 3); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) - .setProcessId(staticProcessId) - .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setPreviousMemberEpoch(groupEpoch) - .setAssignedTasks(staticAssignedTasks) - .build()) - .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) - .setProcessId(dynamicProcessId) - .setMemberEpoch(groupEpoch) - .setPreviousMemberEpoch(groupEpoch - 1) - .setAssignedTasks(dynamicAssignedTasks) - .build()) - .withTargetAssignment(oldStaticMemberId, staticTargetAssignment) - .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment)); + .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) + .setProcessId(staticProcessId) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(groupEpoch) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setProcessId(dynamicProcessId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(oldStaticMemberId, staticTargetAssignment) + .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment)); // WHEN1 - static member try to rejoin with new member id. context.streamsGroupHeartbeat( - staticJoinHeartbeat(groupId, newStaticMemberId, staticInstanceId, staticProcessId) + staticJoinHeartbeat(groupId, newStaticMemberId, staticInstanceId, staticProcessId) ); // WHEN2 + THEN2 - stale static member send a heartbeat with stale member id. assertThrows(FencedInstanceIdException.class, () -> - context.streamsGroupHeartbeat(staticHeartbeat(groupId, oldStaticMemberId, staticInstanceId, groupEpoch)) + context.streamsGroupHeartbeat(staticHeartbeat(groupId, oldStaticMemberId, staticInstanceId, groupEpoch)) ); StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java index 15e6b9dfccb01..e87a06aca7aba 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java @@ -90,8 +90,8 @@ public void testUnknownStaticMemberLeaveStreamsGroup() { StaticMemberFixtureWith2Members fixture = new StaticMemberFixtureWith2Members(2); fixture.assertLeaveFails( - fixture.staticLeaveRequest("unknown-member-id", "unknown-instance-id"), - UnknownMemberIdException.class + fixture.staticLeaveRequest("unknown-member-id", "unknown-instance-id"), + UnknownMemberIdException.class ); } @@ -103,8 +103,8 @@ public void testStreamsStaticJoinWithNewInstanceAtMaxSizeThrowsGroupMaxSizeReach StaticMemberFixtureWith2Members fixture = new StaticMemberFixtureWith2Members(streamsGroupMaxSize); fixture.assertJoinFails( - fixture.newMemberJoinsWithNewInstanceId(), - GroupMaxSizeReachedException.class + fixture.newMemberJoinsWithNewInstanceId(), + GroupMaxSizeReachedException.class ); } @@ -116,7 +116,7 @@ public void testStreamsStaticRejoinWithLeaveGroupStaticEpochAtMaxSizeSucceeds() StaticMemberFixtureWith2Members fixture = new StaticMemberFixtureWith2Members(streamsGroupMaxSize); fixture.assertJoinSucceeds( - fixture.leftMemberRejoinsWithSameInstanceId() + fixture.leftMemberRejoinsWithSameInstanceId() ); } @@ -128,8 +128,8 @@ public void testStreamsStaticJoinWithUnreleasedInstanceThrowsUnreleasedInstanceI StaticMemberFixtureWith2Members fixture = new StaticMemberFixtureWith2Members(streamsGroupMaxSize); fixture.assertJoinFails( - fixture.newMemberJoinsWithActiveInstanceId(), - UnreleasedInstanceIdException.class + fixture.newMemberJoinsWithActiveInstanceId(), + UnreleasedInstanceIdException.class ); } @@ -139,15 +139,15 @@ public void testStaticMemberSendHeartbeatWithVariousEpochThenThrowError(int when StaticMemberFixtureWith2Members fixture = new StaticMemberFixtureWith2Members(3); // WHEN: same instance id is reused with mismatched member identity/epoch. THEN: expected exception is thrown. fixture.assertHeartbeatFails( - fixture.newMemberHeartbeatsWithActiveInstanceId(whenMemberEpoch), - expectedException + fixture.newMemberHeartbeatsWithActiveInstanceId(whenMemberEpoch), + expectedException ); } private static Stream staticMemberReusedInstanceErrorCases() { return Stream.of( - Arguments.of(0, UnreleasedInstanceIdException.class), // static member try to join when static member already existed, then throw UnreleasedInstanceIdException. - Arguments.of(1000, FencedInstanceIdException.class) // static member try to send bigger epoch when static member already existed, then throw FencedInstanceIdException. + Arguments.of(0, UnreleasedInstanceIdException.class), // static member try to join when static member already existed, then throw UnreleasedInstanceIdException. + Arguments.of(1000, FencedInstanceIdException.class) // static member try to send bigger epoch when static member already existed, then throw FencedInstanceIdException. ); } @@ -177,98 +177,98 @@ private void testStaticMemberJoinThenRevokeAndReceiveTasksWith2Members(int maxSi MockTaskAssignor assignor = new MockTaskAssignor("sticky"); GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() - .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, maxSize) - .withStreamsGroupTaskAssignors(List.of(assignor)) - .withMetadataImage(topic.metadataImage()) - .withStreamsGroup(new StreamsGroupBuilder(groupId, 10) - .withMember(StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults(memberId1) - .setInstanceId(instanceId1) - .setMemberEpoch(10) - .setPreviousMemberEpoch(9) - .setAssignedTasks(topic.assignedTasks(10, 0, 1, 2, 3)) - .build()) - .withTargetAssignment(memberId1, topic.targetAssignment(0, 1, 2, 3)) - .withTargetAssignmentEpoch(10) - .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) - .withMetadataHash(topic.metadataHash()) - .withValidatedTopologyEpoch(0) - ) - .build(); + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, maxSize) + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, 10) + .withMember(StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults(memberId1) + .setInstanceId(instanceId1) + .setMemberEpoch(10) + .setPreviousMemberEpoch(9) + .setAssignedTasks(topic.assignedTasks(10, 0, 1, 2, 3)) + .build()) + .withTargetAssignment(memberId1, topic.targetAssignment(0, 1, 2, 3)) + .withTargetAssignmentEpoch(10) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withMetadataHash(topic.metadataHash()) + .withValidatedTopologyEpoch(0) + ) + .build(); // Next target assignment after member2 joins. assignor.prepareGroupAssignment(Map.of( - memberId1, topic.targetAssignment(0, 1), - memberId2, topic.targetAssignment(2, 3) + memberId1, topic.targetAssignment(0, 1), + memberId2, topic.targetAssignment(2, 3) )); // 1) Static member2 joins. It gets no active tasks yet because member1 still owns them. CoordinatorResult joinResult = context.streamsGroupHeartbeat( - staticJoinHeartbeat(groupId, memberId2, instanceId2, topic) + staticJoinHeartbeat(groupId, memberId2, instanceId2, topic) ); assertResponseEquals( - heartbeatResponseWithActiveTasks(memberId2, 11, List.of()), - joinResult.response().data() + heartbeatResponseWithActiveTasks(memberId2, 11, List.of()), + joinResult.response().data() ); // 2) member1 receives revocation instruction: keep only [0,1]. CoordinatorResult revokeInstructionResult = context.streamsGroupHeartbeat( - new StreamsGroupHeartbeatRequestData() - .setGroupId(groupId) - .setInstanceId(instanceId1) - .setMemberId(memberId1) - .setMemberEpoch(10) + new StreamsGroupHeartbeatRequestData() + .setGroupId(groupId) + .setInstanceId(instanceId1) + .setMemberId(memberId1) + .setMemberEpoch(10) ); assertResponseEquals(heartbeatResponseWithActiveTasks(memberId1, 10, topic, 0, 1), revokeInstructionResult.response().data()); // 3) member1 acknowledges revocation by reporting owned active tasks [0,1]. CoordinatorResult revokeAckResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, memberId1, instanceId1, 10) - .setActiveTasks(List.of( - new StreamsGroupHeartbeatRequestData.TaskIds() - .setSubtopologyId("subtopology1") - .setPartitions(List.of(0, 1)) - )) - .setStandbyTasks(List.of()) - .setWarmupTasks(List.of()) + staticHeartbeat(groupId, memberId1, instanceId1, 10) + .setActiveTasks(List.of( + new StreamsGroupHeartbeatRequestData.TaskIds() + .setSubtopologyId("subtopology1") + .setPartitions(List.of(0, 1)) + )) + .setStandbyTasks(List.of()) + .setWarmupTasks(List.of()) ); assertResponseEquals( - new StreamsGroupHeartbeatResponseData() - .setMemberId(memberId1) - .setMemberEpoch(11) - .setHeartbeatIntervalMs(5000) - .setTaskOffsetIntervalMs(60000) - .setStatus(List.of()), - revokeAckResult.response().data() + new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId1) + .setMemberEpoch(11) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setStatus(List.of()), + revokeAckResult.response().data() ); // 4) member2 heartbeats again and now receives [2,3]. CoordinatorResult member2ReceiveResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, memberId2, instanceId2, 11) + staticHeartbeat(groupId, memberId2, instanceId2, 11) ); assertResponseEquals(heartbeatResponseWithActiveTasks(memberId2, 11, topic, 2, 3), member2ReceiveResult.response().data()); // 5) member2 leave. CoordinatorResult member2LeaveResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, memberId2, instanceId2, LEAVE_GROUP_STATIC_MEMBER_EPOCH) + staticHeartbeat(groupId, memberId2, instanceId2, LEAVE_GROUP_STATIC_MEMBER_EPOCH) ); assertResponseEquals( - staticLeaveResponseWithNullTasks(memberId2, LEAVE_GROUP_STATIC_MEMBER_EPOCH).setHeartbeatIntervalMs(0), - member2LeaveResult.response().data() + staticLeaveResponseWithNullTasks(memberId2, LEAVE_GROUP_STATIC_MEMBER_EPOCH).setHeartbeatIntervalMs(0), + member2LeaveResult.response().data() ); // 6) member2 re-join with other memberId. CoordinatorResult member2rejoinResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, otherMemberId2, instanceId2, JOIN_GROUP_MEMBER_EPOCH) + staticHeartbeat(groupId, otherMemberId2, instanceId2, JOIN_GROUP_MEMBER_EPOCH) ); assertResponseEquals( - heartbeatResponseWithActiveTasks(otherMemberId2, 11, topic, 2, 3), - member2rejoinResult.response().data() + heartbeatResponseWithActiveTasks(otherMemberId2, 11, topic, 2, 3), + member2rejoinResult.response().data() ); } @@ -291,40 +291,40 @@ public void testStaticMemberRejoinWithUpdatedProcessIdBumpsStreamsGroupEpoch() { MockTaskAssignor assignor = new MockTaskAssignor("sticky"); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) - .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setPreviousMemberEpoch(groupEpoch) - .setProcessId(oldProcessId) - .setAssignedTasks(assignedTasks) - .build()) - .withTargetAssignment(oldMemberId, targetAssignment)); + .withMember(streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(groupEpoch) + .setProcessId(oldProcessId) + .setAssignedTasks(assignedTasks) + .build()) + .withTargetAssignment(oldMemberId, targetAssignment)); assignor.prepareGroupAssignment(Map.of(rejoinMemberId, targetAssignment)); CoordinatorResult result = context.streamsGroupHeartbeat( - staticJoinHeartbeat(groupId, rejoinMemberId, instanceId, newProcessId) + staticJoinHeartbeat(groupId, rejoinMemberId, instanceId, newProcessId) ); assertEquals(rejoinMemberId, result.response().data().memberId()); assertEquals(bumpedGroupEpoch, result.response().data().memberEpoch()); CoordinatorRecord metadataRecord = result.records().stream() - .filter(record -> record.key() instanceof StreamsGroupMetadataKey) - .findFirst() - .orElse(null); + .filter(record -> record.key() instanceof StreamsGroupMetadataKey) + .findFirst() + .orElse(null); assertNotNull(metadataRecord, "Expected a StreamsGroupMetadata record when static member config changes."); StreamsGroupMetadataValue metadataValue = (StreamsGroupMetadataValue) metadataRecord.value().message(); assertEquals(bumpedGroupEpoch, metadataValue.epoch()); assertTrue(result.records().contains( - StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord( - groupId, - bumpedGroupEpoch, - topic.metadataHash(), - 0, - getDefaultAssignmentConfigs() - ) + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord( + groupId, + bumpedGroupEpoch, + topic.metadataHash(), + 0, + getDefaultAssignmentConfigs() + ) )); } @@ -342,28 +342,28 @@ public void testStaticMemberLeaveWithMinusOneFencesMemberAndBumpsStreamsGroupEpo TasksTuple targetAssignment = topic.targetAssignment(0, 1, 2, 3); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setMemberEpoch(groupEpoch) - .setPreviousMemberEpoch(groupEpoch - 1) - .setAssignedTasks(assignedTasks) - .build()) - .withTargetAssignment(memberId, targetAssignment)); + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(assignedTasks) + .build()) + .withTargetAssignment(memberId, targetAssignment)); CoordinatorResult result = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, memberId, instanceId, LEAVE_GROUP_MEMBER_EPOCH) + staticHeartbeat(groupId, memberId, instanceId, LEAVE_GROUP_MEMBER_EPOCH) ); assertResponseEquals(staticLeaveResponse(memberId, LEAVE_GROUP_MEMBER_EPOCH), result.response().data()); assertRecordsEquals( - List.of( - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, memberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, memberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, memberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, bumpedGroupEpoch, topic.metadataHash(), 0, getDefaultAssignmentConfigs()), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, bumpedGroupEpoch, 0L) - ), - result.records() + List.of( + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, bumpedGroupEpoch, topic.metadataHash(), 0, getDefaultAssignmentConfigs()), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, bumpedGroupEpoch, 0L) + ), + result.records() ); } @@ -383,17 +383,17 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpoch() { TasksTupleWithEpochs pendingRevocationTasks = topic.assignedTasks(groupEpoch, 2, 3); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setMemberEpoch(memberEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setAssignedTasks(assignedTasks) - .setTasksPendingRevocation(pendingRevocationTasks) - .build()) - .withTargetAssignment(memberId, topic.targetAssignment(0, 1, 2, 3))); + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(assignedTasks) + .setTasksPendingRevocation(pendingRevocationTasks) + .build()) + .withTargetAssignment(memberId, topic.targetAssignment(0, 1, 2, 3))); // WHEN CoordinatorResult result = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) + staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) ); // THEN @@ -404,14 +404,14 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpoch() { // task still remain. // pendingRevocationTasks should be EMPTY. StreamsGroupMember expectedMemberInResponse = streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setMemberEpoch(leaveEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) - .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) - .build(); + .setMemberEpoch(leaveEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) + .build(); assertRecordsEquals( - List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedMemberInResponse)), - result.records() + List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedMemberInResponse)), + result.records() ); assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); } @@ -432,26 +432,24 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochThenShouldBeIdem TasksTuple targetAssignment = topic.targetAssignment(0, 1, 2, 3); StreamsGroupMember alreadyLeftStaticMember = streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setMemberEpoch(leaveEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) - .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) - .build(); + .setMemberEpoch(leaveEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .build(); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(alreadyLeftStaticMember) - .withTargetAssignment(memberId, targetAssignment)); - - CoordinatorResult result = - context.streamsGroupHeartbeat(staticHeartbeat(groupId, memberId, instanceId, leaveEpoch)); + .withMember(alreadyLeftStaticMember) + .withTargetAssignment(memberId, targetAssignment)); + CoordinatorResult result = context.streamsGroupHeartbeat(staticHeartbeat(groupId, memberId, instanceId, leaveEpoch)); // THEN assertResponseEquals(staticLeaveResponse(memberId, leaveEpoch), result.response().data()); assertRecordsEquals( - List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, alreadyLeftStaticMember)), - result.records() + List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, alreadyLeftStaticMember)), + result.records() ); assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); } @@ -496,16 +494,16 @@ private void verifyStaticMemberLeaveAndRejoinNoGroupBump(String instanceId, Stri TasksTuple givenTargetAssignment = topic.targetAssignment(0, 1, 2, 3); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) - .setMemberEpoch(memberEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setAssignedTasks(givenAssignedTasks) - .build()) - .withTargetAssignment(oldMemberId, givenTargetAssignment)); + .withMember(streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(givenAssignedTasks) + .build()) + .withTargetAssignment(oldMemberId, givenTargetAssignment)); // WHEN1 : normal heart beat. CoordinatorResult normalHeartbeatResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, oldMemberId, instanceId, memberEpoch) + staticHeartbeat(groupId, oldMemberId, instanceId, memberEpoch) ); // THEN1 : @@ -517,7 +515,7 @@ private void verifyStaticMemberLeaveAndRejoinNoGroupBump(String instanceId, Stri // WHEN2 : Stream Member leave with -2 CoordinatorResult leaveResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, oldMemberId, instanceId, leaveEpoch) + staticHeartbeat(groupId, oldMemberId, instanceId, leaveEpoch) ); // THEN2 @@ -527,7 +525,7 @@ private void verifyStaticMemberLeaveAndRejoinNoGroupBump(String instanceId, Stri // WHEN3 : Streams Member rejoin with other memberId CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) + staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) ); // THEN3 : @@ -538,28 +536,28 @@ private void verifyStaticMemberLeaveAndRejoinNoGroupBump(String instanceId, Stri assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); StreamsGroupMember newJoinStaticMember = streamsGroupMemberBuilderWithDefaults(newMemberId, instanceId) - .setMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) - .setPreviousMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) - .setAssignedTasks(resetAssignedTasksEpochsToZero(givenAssignedTasks)) - .build(); + .setMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) + .setPreviousMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) + .setAssignedTasks(resetAssignedTasksEpochsToZero(givenAssignedTasks)) + .build(); StreamsGroupMember withPrevMemberId = streamsGroupMemberBuilderWithDefaults(newMemberId, instanceId) - .setMemberEpoch(memberEpoch) // 0 -> 10 - .setPreviousMemberEpoch(0) // 0 -> 0 - .setAssignedTasks(resetAssignedTasksEpochsToZero(givenAssignedTasks)) - .build(); + .setMemberEpoch(memberEpoch) // 0 -> 10 + .setPreviousMemberEpoch(0) // 0 -> 0 + .setAssignedTasks(resetAssignedTasksEpochsToZero(givenAssignedTasks)) + .build(); assertRecordsEquals( - List.of( - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, oldMemberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, oldMemberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, oldMemberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, newJoinStaticMember), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, newJoinStaticMember.memberId(), givenTargetAssignment), - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, newJoinStaticMember), - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, withPrevMemberId) - ), - rejoinResult.records() + List.of( + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, oldMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, oldMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, oldMemberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, newJoinStaticMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, newJoinStaticMember.memberId(), givenTargetAssignment), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, newJoinStaticMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, withPrevMemberId) + ), + rejoinResult.records() ); } @@ -579,33 +577,33 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnrevokedSta TasksTuple targetAssignment = topic.targetAssignment(0, 1); StreamsGroupMember unrevokedMember = streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setMemberEpoch(memberEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setState(MemberState.UNREVOKED_TASKS) - .setAssignedTasks(assignedTasks) - .setTasksPendingRevocation(tasksPendingRevocation) - .build(); + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setState(MemberState.UNREVOKED_TASKS) + .setAssignedTasks(assignedTasks) + .setTasksPendingRevocation(tasksPendingRevocation) + .build(); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(unrevokedMember) - .withTargetAssignment(memberId, targetAssignment)); + .withMember(unrevokedMember) + .withTargetAssignment(memberId, targetAssignment)); // WHEN CoordinatorResult result = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) + staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) ); // THEN StreamsGroupMember expectedMember = streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setMemberEpoch(leaveEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) - .build(); + .setMemberEpoch(leaveEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) + .build(); assertResponseEquals(staticLeaveResponseWithNullTasks(memberId, leaveEpoch), result.response().data()); assertRecordsEquals( - List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedMember)), - result.records() + List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedMember)), + result.records() ); assertEquals(MemberState.STABLE, context.streamsGroupMemberState(groupId, memberId)); assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); @@ -627,17 +625,17 @@ public void testStaticMemberRejoinsAfterTemporaryLeave() { TasksTuple targetAssignment = topic.targetAssignment(0, 1); StreamsGroupMember temporarilyLeftMember = streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) - .setMemberEpoch(leaveEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) - .build(); + .setMemberEpoch(leaveEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) + .build(); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(temporarilyLeftMember) - .withTargetAssignment(oldMemberId, targetAssignment)); + .withMember(temporarilyLeftMember) + .withTargetAssignment(oldMemberId, targetAssignment)); CoordinatorResult result = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) + staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) ); assertResponseEquals(heartbeatResponseWithActiveTasks(newMemberId, memberEpoch, topic, 0, 1), result.response().data()); @@ -661,35 +659,35 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnreleasedSt TasksTuple targetAssignment = topic.targetAssignment(0, 1, 2); StreamsGroupMember unreleasedMember = streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setProcessId(processId) - .setMemberEpoch(memberEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setState(MemberState.UNRELEASED_TASKS) - .setAssignedTasks(assignedTasks) - .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) - .build(); + .setProcessId(processId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setState(MemberState.UNRELEASED_TASKS) + .setAssignedTasks(assignedTasks) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .build(); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(unreleasedMember) - .withTargetAssignment(memberId, targetAssignment)); + .withMember(unreleasedMember) + .withTargetAssignment(memberId, targetAssignment)); CoordinatorResult result = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) + staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) ); StreamsGroupMember expectedMember = streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setProcessId(processId) - .setMemberEpoch(leaveEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setState(MemberState.UNRELEASED_TASKS) - .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) - .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) - .build(); + .setProcessId(processId) + .setMemberEpoch(leaveEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setState(MemberState.UNRELEASED_TASKS) + .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .build(); assertResponseEquals(staticLeaveResponseWithNullTasks(memberId, leaveEpoch), result.response().data()); assertRecordsEquals( - List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedMember)), - result.records() + List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedMember)), + result.records() ); assertEquals(MemberState.UNRELEASED_TASKS, context.streamsGroupMemberState(groupId, memberId)); assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); @@ -715,30 +713,30 @@ public void testStaticMemberRejoinsAfterTemporaryLeaveFromUnreleasedState() { TasksTuple targetAssignment = topic.targetAssignment(0, 1, 2); StreamsGroupMember temporarilyLeftMember = StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) - .setProcessId(oldProcessId) - .setMemberEpoch(leaveEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setState(MemberState.UNRELEASED_TASKS) - .setAssignedTasks(assignedTasks) - .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) - .build(); + .setProcessId(oldProcessId) + .setMemberEpoch(leaveEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setState(MemberState.UNRELEASED_TASKS) + .setAssignedTasks(assignedTasks) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .build(); StreamsGroupMember otherMember = StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults(otherMemberId) - .setProcessId(otherProcessId) - .setMemberEpoch(memberEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setState(MemberState.UNREVOKED_TASKS) - .setTasksPendingRevocation(otherTasksPendingRevocation) - .build(); + .setProcessId(otherProcessId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setState(MemberState.UNREVOKED_TASKS) + .setTasksPendingRevocation(otherTasksPendingRevocation) + .build(); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(temporarilyLeftMember) - .withMember(otherMember) - .withTargetAssignment(oldMemberId, targetAssignment) - .withTargetAssignment(otherMemberId, TasksTuple.EMPTY)); + .withMember(temporarilyLeftMember) + .withMember(otherMember) + .withTargetAssignment(oldMemberId, targetAssignment) + .withTargetAssignment(otherMemberId, TasksTuple.EMPTY)); CoordinatorResult result = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) + staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) ); assertResponseEquals(heartbeatResponseWithActiveTasks(newMemberId, memberEpoch, topic, 0, 1), result.response().data()); @@ -768,42 +766,42 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnrevokedSta TasksTuple waitingTargetAssignment = topic.targetAssignment(2); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(leavingMemberId, instanceId) - .setMemberEpoch(memberEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setState(MemberState.UNREVOKED_TASKS) - .setAssignedTasks(assignedTasks) - .setTasksPendingRevocation(tasksPendingRevocation) - .build()) - .withMember(StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults(waitingMemberId) - .setMemberEpoch(memberEpoch) - .setPreviousMemberEpoch(memberEpoch) - .setState(MemberState.UNRELEASED_TASKS) - .build()) - .withTargetAssignment(leavingMemberId, leavingTargetAssignment) - .withTargetAssignment(waitingMemberId, waitingTargetAssignment)); + .withMember(streamsGroupMemberBuilderWithDefaults(leavingMemberId, instanceId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setState(MemberState.UNREVOKED_TASKS) + .setAssignedTasks(assignedTasks) + .setTasksPendingRevocation(tasksPendingRevocation) + .build()) + .withMember(StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults(waitingMemberId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch) + .setState(MemberState.UNRELEASED_TASKS) + .build()) + .withTargetAssignment(leavingMemberId, leavingTargetAssignment) + .withTargetAssignment(waitingMemberId, waitingTargetAssignment)); // WHEN1 - leave CoordinatorResult leaveResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, leavingMemberId, instanceId, leaveEpoch) + staticHeartbeat(groupId, leavingMemberId, instanceId, leaveEpoch) ); // THEN1 StreamsGroupHeartbeatResponseData expectedLeavingResponse = staticLeaveResponseWithNullTasks(leavingMemberId, leaveEpoch); assertResponseEquals(expectedLeavingResponse, leaveResult.response().data()); List expectedRecordsTriggeredByLeave = List.of( - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, - streamsGroupMemberBuilderWithDefaults(leavingMemberId, instanceId) - .setMemberEpoch(leaveEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) - .build())); + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, + streamsGroupMemberBuilderWithDefaults(leavingMemberId, instanceId) + .setMemberEpoch(leaveEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) + .build())); assertRecordsEquals(expectedRecordsTriggeredByLeave, leaveResult.records()); assertEquals(MemberState.STABLE, context.streamsGroupMemberState(groupId, leavingMemberId)); // When2 - Waiting member send a heartbeat expecting get unreleased tasks. CoordinatorResult waitingMemberResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, waitingMemberId, null, memberEpoch) + staticHeartbeat(groupId, waitingMemberId, null, memberEpoch) ); // THEN2 @@ -811,12 +809,12 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnrevokedSta assertResponseEquals(expectedWaitngMemberResponse, waitingMemberResult.response().data()); List expectedRecordsTriggeredByWaitngMember = List.of( - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, - StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults(waitingMemberId) - .setMemberEpoch(memberEpoch) - .setPreviousMemberEpoch(memberEpoch) - .setAssignedTasks(topic.assignedTasks(memberEpoch, 2)) - .build()) + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, + StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults(waitingMemberId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch) + .setAssignedTasks(topic.assignedTasks(memberEpoch, 2)) + .build()) ); assertRecordsEquals(expectedRecordsTriggeredByWaitngMember, waitingMemberResult.records()); @@ -849,18 +847,18 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochAndRejoinAndOthe long groupMetadataHash = topic.metadataHash(); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setMemberEpoch(memberEpoch) - .setRackId(rackId) - .setPreviousMemberEpoch(memberEpoch - 1) - .setAssignedTasks(givenAssignedTasks) - .build()) - .withTargetAssignment(memberId, givenTargetAssignment)); + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(memberEpoch) + .setRackId(rackId) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(givenAssignedTasks) + .build()) + .withTargetAssignment(memberId, givenTargetAssignment)); // WHEN1 : normal heart beat. CoordinatorResult normalHeartbeatResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, memberId, instanceId, memberEpoch) - .setRackId(rackId) + staticHeartbeat(groupId, memberId, instanceId, memberEpoch) + .setRackId(rackId) ); // THEN1 : @@ -872,8 +870,8 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochAndRejoinAndOthe // WHEN2 : Stream Member leave with -2 CoordinatorResult leaveResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) - .setRackId(rackId) + staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) + .setRackId(rackId) ); // THEN2 @@ -891,8 +889,8 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochAndRejoinAndOthe // WHEN3 : Streams Member rejoin with other memberId and rackId CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) - .setRackId(newRackId) + staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) + .setRackId(newRackId) ); // THEN3 : @@ -903,45 +901,45 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochAndRejoinAndOthe assertEquals(bumpedGroupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); StreamsGroupMember transationStaticInitMember = streamsGroupMemberBuilderWithDefaults(newMemberId, instanceId) - .setMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) - .setPreviousMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) - .setRackId(rackId) - .setAssignedTasks(resetAssignedTasksEpochsToZero(givenAssignedTasks)) - .build(); + .setMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) + .setPreviousMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) + .setRackId(rackId) + .setAssignedTasks(resetAssignedTasksEpochsToZero(givenAssignedTasks)) + .build(); StreamsGroupMember newJoinStaticMember = streamsGroupMemberBuilderWithDefaults(newMemberId, instanceId) - .setMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) - .setPreviousMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) - .setRackId(newRackId) - .setAssignedTasks(resetAssignedTasksEpochsToZero(givenAssignedTasks)) - .build(); + .setMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) + .setPreviousMemberEpoch(JOIN_GROUP_MEMBER_EPOCH) + .setRackId(newRackId) + .setAssignedTasks(resetAssignedTasksEpochsToZero(givenAssignedTasks)) + .build(); StreamsGroupMember reconciledMember = streamsGroupMemberBuilderWithDefaults(newMemberId, instanceId) - .setMemberEpoch(bumpedMemberEpoch) - .setPreviousMemberEpoch(0) - .setRackId(newRackId) - .setAssignedTasks(resetAssignedTasksEpochsToZero(givenAssignedTasks)) - .build(); + .setMemberEpoch(bumpedMemberEpoch) + .setPreviousMemberEpoch(0) + .setRackId(newRackId) + .setAssignedTasks(resetAssignedTasksEpochsToZero(givenAssignedTasks)) + .build(); assertRecordsEquals( - List.of( - // From eplaceStreamsMembers - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, memberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, memberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, memberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, transationStaticInitMember), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, transationStaticInitMember.memberId(), givenTargetAssignment), - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, transationStaticInitMember), - - // From hasStreamsMemberMetadataChanged - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, newJoinStaticMember), - - StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, bumpedGroupEpoch, groupMetadataHash, 0, getDefaultAssignmentConfigs()), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, newJoinStaticMember.memberId(), givenTargetAssignment), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, bumpedGroupEpoch, context.time.milliseconds()), - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, reconciledMember) - ), - rejoinResult.records() + List.of( + // From eplaceStreamsMembers + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, transationStaticInitMember), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, transationStaticInitMember.memberId(), givenTargetAssignment), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, transationStaticInitMember), + + // From hasStreamsMemberMetadataChanged + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, newJoinStaticMember), + + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, bumpedGroupEpoch, groupMetadataHash, 0, getDefaultAssignmentConfigs()), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, newJoinStaticMember.memberId(), givenTargetAssignment), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, bumpedGroupEpoch, context.time.milliseconds()), + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, reconciledMember) + ), + rejoinResult.records() ); } @@ -961,17 +959,17 @@ public void testStaticMemberRejoinWritesReplacementRecordsInStreamsGroup() { TasksTupleWithEpochs assignedTasks = topic.assignedTasks(groupEpoch, 0, 1, 2, 3); StreamsGroupMember oldMember = streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) - .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setPreviousMemberEpoch(groupEpoch) - .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) - .build(); + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(groupEpoch) + .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) + .build(); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(oldMember) - .withTargetAssignment(oldMemberId, oldTargetAssignment)); + .withMember(oldMember) + .withTargetAssignment(oldMemberId, oldTargetAssignment)); CoordinatorResult result = context.streamsGroupHeartbeat( - staticJoinHeartbeat(groupId, rejoinMemberId, instanceId, DEFAULT_PROCESS_ID) + staticJoinHeartbeat(groupId, rejoinMemberId, instanceId, DEFAULT_PROCESS_ID) ); assertEquals(rejoinMemberId, result.response().data().memberId()); @@ -983,22 +981,22 @@ public void testStaticMemberRejoinWritesReplacementRecordsInStreamsGroup() { .build(); assertTrue(result.records().contains( - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, oldMemberId) + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, oldMemberId) )); assertTrue(result.records().contains( - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, oldMemberId) + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, oldMemberId) )); assertTrue(result.records().contains( - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, oldMemberId) + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, oldMemberId) )); assertTrue(result.records().contains( - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, expectedCopiedMember) + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, expectedCopiedMember) )); assertTrue(result.records().contains( - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, rejoinMemberId, oldTargetAssignment) + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, rejoinMemberId, oldTargetAssignment) )); assertTrue(result.records().contains( - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedCopiedMember) + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedCopiedMember) )); } @@ -1010,19 +1008,20 @@ public void testStaticMemberLeaveWithMismatchedMemberIdThrowsFencedInstanceIdInS String instanceId = Uuid.randomUuid().toString(); GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() - .withStreamsGroup(new StreamsGroupBuilder(groupId, 10) - .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setMemberEpoch(10) - .setPreviousMemberEpoch(9) - .build()) - .withTargetAssignmentEpoch(10) - ) - .build(); + .withStreamsGroup(new StreamsGroupBuilder(groupId, 10) + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(10) + .setPreviousMemberEpoch(9) + .build()) + .withTargetAssignmentEpoch(10) + ) + .build(); assertThrows(FencedInstanceIdException.class, () -> - context.streamsGroupHeartbeat( - staticHeartbeat(groupId, differentMemberId, instanceId, LEAVE_GROUP_STATIC_MEMBER_EPOCH) - )); + context.streamsGroupHeartbeat( + staticHeartbeat(groupId, differentMemberId, instanceId, LEAVE_GROUP_STATIC_MEMBER_EPOCH) + ) + ); } @Test @@ -1034,17 +1033,17 @@ public void testUnknownStaticMemberHeartbeatWithPositiveEpochThrowsUnknownMember String unknownMemberId = Uuid.randomUuid().toString(); GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() - .withStreamsGroup(new StreamsGroupBuilder(groupId, 10) - .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setMemberEpoch(10) - .setPreviousMemberEpoch(9) - .build()) - .withTargetAssignmentEpoch(10) - ) - .build(); + .withStreamsGroup(new StreamsGroupBuilder(groupId, 10) + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(10) + .setPreviousMemberEpoch(9) + .build()) + .withTargetAssignmentEpoch(10) + ) + .build(); UnknownMemberIdException e = assertThrows(UnknownMemberIdException.class, () -> - context.streamsGroupHeartbeat(staticHeartbeat(groupId, unknownMemberId, unknownInstanceId, 1)) + context.streamsGroupHeartbeat(staticHeartbeat(groupId, unknownMemberId, unknownInstanceId, 1)) ); assertEquals(String.format("Instance id %s is unknown.", unknownInstanceId), e.getMessage()); } @@ -1052,29 +1051,29 @@ public void testUnknownStaticMemberHeartbeatWithPositiveEpochThrowsUnknownMember @ParameterizedTest @MethodSource("ownedActiveTasksAtPreviousEpochCases") public void testStreamsStaticMemberHeartbeatWithPreviousEpochAndOwnedActiveTasks( - List requestAssignedTaskIds, Class expectedException + List requestAssignedTaskIds, Class expectedException ) { Integer[] givenAssignedTasksIds = new Integer[]{0, 1, 2, 3}; verifyStreamsStaticMemberHeartbeatWithOwnedActiveTasksAtPreviousEpoch( - givenAssignedTasksIds, - requestAssignedTaskIds, - expectedException + givenAssignedTasksIds, + requestAssignedTaskIds, + expectedException ); } private static Stream ownedActiveTasksAtPreviousEpochCases() { return Stream.of( - Arguments.of(List.of(0, 1, 2), null), // Subset Owned Active Tasks - Arguments.of(List.of(0, 1, 2, 3), null), // Exact Owned Active Tasks - Arguments.of(List.of(0, 1, 2, 3, 4), FencedMemberEpochException.class) // Non Subset active tasks + Arguments.of(List.of(0, 1, 2), null), // Subset Owned Active Tasks + Arguments.of(List.of(0, 1, 2, 3), null), // Exact Owned Active Tasks + Arguments.of(List.of(0, 1, 2, 3, 4), FencedMemberEpochException.class) // Non Subset active tasks ); } private void verifyStreamsStaticMemberHeartbeatWithOwnedActiveTasksAtPreviousEpoch( - Integer[] givenTaskIds, - List requestAssignedTaskIds, - Class expectedException) { + Integer[] givenTaskIds, + List requestAssignedTaskIds, + Class expectedException) { int groupEpoch = 10; int partitionSize = 5; int currentMemberEpoch = 10; @@ -1090,22 +1089,22 @@ private void verifyStreamsStaticMemberHeartbeatWithOwnedActiveTasksAtPreviousEpo TasksTupleWithEpochs givenAssignedTask = topic.assignedTasks(groupEpoch, givenTaskIds); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setMemberEpoch(currentMemberEpoch) - .setPreviousMemberEpoch(previousMemberEpoch) - .setAssignedTasks(givenAssignedTask) - .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) - .build()) - .withTargetAssignment(memberId, topic.targetAssignment(givenTaskIds))); - + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(currentMemberEpoch) + .setPreviousMemberEpoch(previousMemberEpoch) + .setAssignedTasks(givenAssignedTask) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .build()) + .withTargetAssignment(memberId, topic.targetAssignment(givenTaskIds))); + // WHEN StreamsGroupHeartbeatRequestData requestData = staticHeartbeat(groupId, memberId, instanceId, requestMemberEpoch) - .setProcessId("process-id") - .setRebalanceTimeoutMs(1500) - .setTopology(topic.topology()) - .setActiveTasks(topic.requestTasks(requestAssignedTaskIds)) - .setStandbyTasks(List.of()) - .setWarmupTasks(List.of()); + .setProcessId("process-id") + .setRebalanceTimeoutMs(1500) + .setTopology(topic.topology()) + .setActiveTasks(topic.requestTasks(requestAssignedTaskIds)) + .setStandbyTasks(List.of()) + .setWarmupTasks(List.of()); // THEN if (expectedException != null) { @@ -1124,16 +1123,16 @@ public void testStreamsStaticMemberTemporaryLeaveSessionTimeoutExpiration() { StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 4); MockTaskAssignor assignor = new MockTaskAssignor("sticky"); GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() - .withStreamsGroupTaskAssignors(List.of(assignor)) - .withMetadataImage(topic.metadataImage()) - .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, 0) - .build(); + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, 0) + .build(); assignor.prepareGroupAssignment(Map.of(memberId, topic.targetAssignment(0, 1, 2, 3))); // WHEN1 : static member joins (session timeout should be scheduled) CoordinatorResult firstJoinResult = context.streamsGroupHeartbeat( - staticJoinHeartbeat(groupId, memberId, instanceId, topic).setRebalanceTimeoutMs(90000) + staticJoinHeartbeat(groupId, memberId, instanceId, topic).setRebalanceTimeoutMs(90000) ); // THEN1 @@ -1144,7 +1143,7 @@ public void testStreamsStaticMemberTemporaryLeaveSessionTimeoutExpiration() { // WHEN2: static member leaves temporarily. CoordinatorResult temporaryLeaveResult = context.streamsGroupHeartbeat( - staticHeartbeat(groupId, memberId, instanceId, LEAVE_GROUP_STATIC_MEMBER_EPOCH) + staticHeartbeat(groupId, memberId, instanceId, LEAVE_GROUP_STATIC_MEMBER_EPOCH) ); // THEN2: @@ -1158,18 +1157,18 @@ public void testStreamsStaticMemberTemporaryLeaveSessionTimeoutExpiration() { // THEN3 List expectedRecords = List.of( - StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, memberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, memberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, memberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, 3, topic.metadataHash(), 0, getDefaultAssignmentConfigs()), - StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, 3, 0L) + StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, memberId), + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, 3, topic.metadataHash(), 0, getDefaultAssignmentConfigs()), + StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, 3, 0L) ); assertEquals( - List.of(new MockCoordinatorTimer.ExpiredTimeout<>( - groupSessionTimeoutKey(groupId, memberId), - new CoordinatorResult<>(expectedRecords) - )), - timeouts + List.of(new MockCoordinatorTimer.ExpiredTimeout<>( + groupSessionTimeoutKey(groupId, memberId), + new CoordinatorResult<>(expectedRecords) + )), + timeouts ); context.assertNoSessionTimeout(groupId, memberId); context.assertNoRebalanceTimeout(groupId, memberId); @@ -1186,24 +1185,25 @@ public void testStaticMemberJoinEmptyStreamsGroupRegistersStaticMember1() { assignor.prepareGroupAssignment(Map.of(memberId, TasksTuple.EMPTY)); GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() - .withMetadataImage(new MetadataImageBuilder().buildCoordinatorMetadataImage()) - .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, 0) - .withStreamsGroupTaskAssignors(List.of(assignor)) - .build(); + .withMetadataImage(new MetadataImageBuilder().buildCoordinatorMetadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, 0) + .withStreamsGroupTaskAssignors(List.of(assignor)) + .build(); // There is no group at all. assertThrows(GroupIdNotFoundException.class, () -> - context.groupMetadataManager.streamsGroup(groupId)); + context.groupMetadataManager.streamsGroup(groupId) + ); // WHEN context.streamsGroupHeartbeat( - staticHeartbeat(groupId, memberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) - .setProcessId(DEFAULT_PROCESS_ID) - .setRebalanceTimeoutMs(1500) - .setTopology(topology) - .setActiveTasks(List.of()) - .setStandbyTasks(List.of()) - .setWarmupTasks(List.of()) + staticHeartbeat(groupId, memberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) + .setProcessId(DEFAULT_PROCESS_ID) + .setRebalanceTimeoutMs(1500) + .setTopology(topology) + .setActiveTasks(List.of()) + .setStandbyTasks(List.of()) + .setWarmupTasks(List.of()) ); // THEN @@ -1215,15 +1215,15 @@ public void testStaticMemberJoinEmptyStreamsGroupRegistersStaticMember1() { @ParameterizedTest @MethodSource("userEndpointTestCases") public void testStaticMemberRejoinUpdatesUserEndpointInformationEpoch( - StreamsGroupHeartbeatRequestData.Endpoint firstUserEndpoint, - int firstExpectedUserEndpointEpoch, - List firstExpectedPartitionsByUserEndpoint, - StreamsGroupMemberMetadataValue.Endpoint firstExpectedUserEndpointMetadata, - - StreamsGroupHeartbeatRequestData.Endpoint secondUserEndpoint, - int secondExpectedUserEndpointEpoch, - List secondExpectedPartitionsByUserEndpoint, - StreamsGroupMemberMetadataValue.Endpoint secondExpectedUserEndpointMetadata + StreamsGroupHeartbeatRequestData.Endpoint firstUserEndpoint, + int firstExpectedUserEndpointEpoch, + List firstExpectedPartitionsByUserEndpoint, + StreamsGroupMemberMetadataValue.Endpoint firstExpectedUserEndpointMetadata, + + StreamsGroupHeartbeatRequestData.Endpoint secondUserEndpoint, + int secondExpectedUserEndpointEpoch, + List secondExpectedPartitionsByUserEndpoint, + StreamsGroupMemberMetadataValue.Endpoint secondExpectedUserEndpointMetadata ) { int memberEpoch = DEFAULT_MEMBER_EPOCH; int groupEpoch = DEFAULT_GROUP_EPOCH; @@ -1242,22 +1242,23 @@ public void testStaticMemberRejoinUpdatesUserEndpointInformationEpoch( assignor.prepareGroupAssignment(Map.of(oldMemberId, topic.targetAssignment(0, 1, 2))); GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, 0, group -> group - .withTargetAssignment(oldMemberId, targetAssignment)); + .withTargetAssignment(oldMemberId, targetAssignment) + ); assertEquals(0, context.groupMetadataManager.streamsGroup(groupId).endpointInformationEpoch()); // First Join -> First Input CoordinatorResult result = context.streamsGroupHeartbeat( - staticJoinHeartbeat(groupId, oldMemberId, instanceId, topic) - .setUserEndpoint(firstUserEndpoint) // first input + staticJoinHeartbeat(groupId, oldMemberId, instanceId, topic) + .setUserEndpoint(firstUserEndpoint) // first input ); // First Check assertResponseEquals( - heartbeatResponseWithActiveTasks(oldMemberId, bumpedEpoch, topic, 0, 1, 2) - .setEndpointInformationEpoch(firstExpectedUserEndpointEpoch) // first endpoint epoch - .setPartitionsByUserEndpoint(firstExpectedPartitionsByUserEndpoint), // first partitions by user endpoint - result.response().data() + heartbeatResponseWithActiveTasks(oldMemberId, bumpedEpoch, topic, 0, 1, 2) + .setEndpointInformationEpoch(firstExpectedUserEndpointEpoch) // first endpoint epoch + .setPartitionsByUserEndpoint(firstExpectedPartitionsByUserEndpoint), // first partitions by user endpoint + result.response().data() ); if (firstExpectedUserEndpointMetadata != null) { @@ -1269,21 +1270,21 @@ public void testStaticMemberRejoinUpdatesUserEndpointInformationEpoch( // static leave context.streamsGroupHeartbeat( - staticHeartbeat(groupId, oldMemberId, instanceId, StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) + staticHeartbeat(groupId, oldMemberId, instanceId, StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) ); // second - static member rejoins CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( - staticJoinHeartbeat(groupId, rejoinMemberId, instanceId, topic) - .setUserEndpoint(secondUserEndpoint) + staticJoinHeartbeat(groupId, rejoinMemberId, instanceId, topic) + .setUserEndpoint(secondUserEndpoint) ); // second check. assertResponseEquals( - heartbeatResponseWithActiveTasks(rejoinMemberId, bumpedEpoch, topic, 0, 1, 2) - .setEndpointInformationEpoch(secondExpectedUserEndpointEpoch) - .setPartitionsByUserEndpoint(secondExpectedPartitionsByUserEndpoint), - rejoinResult.response().data() + heartbeatResponseWithActiveTasks(rejoinMemberId, bumpedEpoch, topic, 0, 1, 2) + .setEndpointInformationEpoch(secondExpectedUserEndpointEpoch) + .setPartitionsByUserEndpoint(secondExpectedPartitionsByUserEndpoint), + rejoinResult.response().data() ); if (secondExpectedUserEndpointMetadata != null) { @@ -1296,56 +1297,56 @@ public void testStaticMemberRejoinUpdatesUserEndpointInformationEpoch( private static Stream userEndpointTestCases() { return Stream.of( - Arguments.of( - null, // firstInput - 0, // first endpoint Epoch - null, // first partitionsByUserEndpoint - null, // first group metadata userEndpoint - userEndpoint("bar.com", 8080), // second input - 1, // second endpoint epoch - buildEndpoints("bar.com", 8080, "foo", List.of(0, 1, 2)), // second partitionsByUserEndpoint - userEndpointForMetadata("bar.com", 8080) - ), - Arguments.of( - null, // firstInput - 0, // first endpoint Epoch - null, // first partitionsByUserEndpoint - null, // first group metadata userEndpoint - null, // second input - 0, // second endpoint epoch - null, // second partitionsByUserEndpoint - null - ), - Arguments.of( - userEndpoint("foo.com", 8080), // firstInput - 1, // first endpoint Epoch - buildEndpoints("foo.com", 8080, "foo", List.of(0, 1, 2)), // first partitionsByUserEndpoint - userEndpointForMetadata("foo.com", 8080), // first group metadata userEndpoint - null, // second input - 2, // second endpoint epoch - List.of(), // second partitionsByUserEndpoint - null - ), - Arguments.of( - userEndpoint("foo.com", 8080), // firstInput - 1, // first endpoint Epoch - buildEndpoints("foo.com", 8080, "foo", List.of(0, 1, 2)), // first partitionsByUserEndpoint - userEndpointForMetadata("foo.com", 8080), // first group metadata userEndpoint - userEndpoint("foo.com", 8080), // second input - 1, // second endpoint epoch - buildEndpoints("foo.com", 8080, "foo", List.of(0, 1, 2)), // second partitionsByUserEndpoint - userEndpointForMetadata("foo.com", 8080) - ), - Arguments.of( - userEndpoint("foo.com", 8080), // firstInput - 1, // first endpoint Epoch - buildEndpoints("foo.com", 8080, "foo", List.of(0, 1, 2)), // first partitionsByUserEndpoint - userEndpointForMetadata("foo.com", 8080), // first group metadata userEndpoint - userEndpoint("bar.com", 8080), // second input - 2, // second endpoint epoch - buildEndpoints("bar.com", 8080, "foo", List.of(0, 1, 2)), // second partitionsByUserEndpoint - userEndpointForMetadata("bar.com", 8080) - ) + Arguments.of( + null, // firstInput + 0, // first endpoint Epoch + null, // first partitionsByUserEndpoint + null, // first group metadata userEndpoint + userEndpoint("bar.com", 8080), // second input + 1, // second endpoint epoch + buildEndpoints("bar.com", 8080, "foo", List.of(0, 1, 2)), // second partitionsByUserEndpoint + userEndpointForMetadata("bar.com", 8080) + ), + Arguments.of( + null, // firstInput + 0, // first endpoint Epoch + null, // first partitionsByUserEndpoint + null, // first group metadata userEndpoint + null, // second input + 0, // second endpoint epoch + null, // second partitionsByUserEndpoint + null + ), + Arguments.of( + userEndpoint("foo.com", 8080), // firstInput + 1, // first endpoint Epoch + buildEndpoints("foo.com", 8080, "foo", List.of(0, 1, 2)), // first partitionsByUserEndpoint + userEndpointForMetadata("foo.com", 8080), // first group metadata userEndpoint + null, // second input + 2, // second endpoint epoch + List.of(), // second partitionsByUserEndpoint + null + ), + Arguments.of( + userEndpoint("foo.com", 8080), // firstInput + 1, // first endpoint Epoch + buildEndpoints("foo.com", 8080, "foo", List.of(0, 1, 2)), // first partitionsByUserEndpoint + userEndpointForMetadata("foo.com", 8080), // first group metadata userEndpoint + userEndpoint("foo.com", 8080), // second input + 1, // second endpoint epoch + buildEndpoints("foo.com", 8080, "foo", List.of(0, 1, 2)), // second partitionsByUserEndpoint + userEndpointForMetadata("foo.com", 8080) + ), + Arguments.of( + userEndpoint("foo.com", 8080), // firstInput + 1, // first endpoint Epoch + buildEndpoints("foo.com", 8080, "foo", List.of(0, 1, 2)), // first partitionsByUserEndpoint + userEndpointForMetadata("foo.com", 8080), // first group metadata userEndpoint + userEndpoint("bar.com", 8080), // second input + 2, // second endpoint epoch + buildEndpoints("bar.com", 8080, "foo", List.of(0, 1, 2)), // second partitionsByUserEndpoint + userEndpointForMetadata("bar.com", 8080) + ) ); } @@ -1367,7 +1368,7 @@ private static class StaticMemberFixtureWith2Members { private final String newInstanceId = "new-instance"; private final StreamsGroupHeartbeatRequestData.Topology topology = - new StreamsGroupHeartbeatRequestData.Topology().setSubtopologies(List.of()); + new StreamsGroupHeartbeatRequestData.Topology().setSubtopologies(List.of()); private final GroupMetadataManagerTestContext context; @@ -1376,22 +1377,22 @@ private StaticMemberFixtureWith2Members(int streamsGroupMaxSize) { assignor.prepareGroupAssignment(Map.of(activeMemberId, TasksTuple.EMPTY)); this.context = new GroupMetadataManagerTestContext.Builder() - .withStreamsGroupTaskAssignors(List.of(assignor)) - .withMetadataImage(new MetadataImageBuilder().buildCoordinatorMetadataImage()) - .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, streamsGroupMaxSize) - .withStreamsGroup(new StreamsGroupBuilder(GROUP_ID, GROUP_EPOCH) - .withMember(streamsGroupMemberBuilderWithDefaults(activeMemberId, activeInstanceId) - .setMemberEpoch(MEMBER_EPOCH) - .setPreviousMemberEpoch(PREVIOUS_MEMBER_EPOCH) - .build()) - .withMember(streamsGroupMemberBuilderWithDefaults(leftMemberId, leftInstanceId) - .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setPreviousMemberEpoch(PREVIOUS_MEMBER_EPOCH) - .build()) - .withTargetAssignmentEpoch(GROUP_EPOCH) - .withTopology(StreamsTopology.fromHeartbeatRequest(topology)) - ) - .build(); + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(new MetadataImageBuilder().buildCoordinatorMetadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, streamsGroupMaxSize) + .withStreamsGroup(new StreamsGroupBuilder(GROUP_ID, GROUP_EPOCH) + .withMember(streamsGroupMemberBuilderWithDefaults(activeMemberId, activeInstanceId) + .setMemberEpoch(MEMBER_EPOCH) + .setPreviousMemberEpoch(PREVIOUS_MEMBER_EPOCH) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(leftMemberId, leftInstanceId) + .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(PREVIOUS_MEMBER_EPOCH) + .build()) + .withTargetAssignmentEpoch(GROUP_EPOCH) + .withTopology(StreamsTopology.fromHeartbeatRequest(topology)) + ) + .build(); } private StreamsGroupHeartbeatRequestData newMemberJoinsWithNewInstanceId() { @@ -1420,16 +1421,16 @@ private StreamsGroupHeartbeatRequestData staticLeaveRequest(String memberId, Str private StreamsGroupHeartbeatRequestData request(String memberId, String instanceId, int memberEpoch) { return new StreamsGroupHeartbeatRequestData() - .setGroupId(GROUP_ID) - .setInstanceId(instanceId) - .setMemberId(memberId) - .setMemberEpoch(memberEpoch) - .setProcessId(DEFAULT_PROCESS_ID) - .setRebalanceTimeoutMs(REBALANCE_TIMEOUT_MS) - .setTopology(topology) - .setActiveTasks(List.of()) - .setStandbyTasks(List.of()) - .setWarmupTasks(List.of()); + .setGroupId(GROUP_ID) + .setInstanceId(instanceId) + .setMemberId(memberId) + .setMemberEpoch(memberEpoch) + .setProcessId(DEFAULT_PROCESS_ID) + .setRebalanceTimeoutMs(REBALANCE_TIMEOUT_MS) + .setTopology(topology) + .setActiveTasks(List.of()) + .setStandbyTasks(List.of()) + .setWarmupTasks(List.of()); } private void assertJoinSucceeds(StreamsGroupHeartbeatRequestData request) { @@ -1440,30 +1441,28 @@ private void assertJoinFails(StreamsGroupHeartbeatRequestData request, Class expectedException - ) { + private void assertHeartbeatFails(StreamsGroupHeartbeatRequestData request, Class expectedException) { assertThrows(expectedException, () -> context.streamsGroupHeartbeat(request)); } - private void assertLeaveFails(StreamsGroupHeartbeatRequestData request, Class expectedException - ) { + private void assertLeaveFails(StreamsGroupHeartbeatRequestData request, Class expectedException) { assertHeartbeatFails(request, expectedException); } } private static StreamsGroupHeartbeatRequestData.Endpoint userEndpoint(String host, int port) { return new StreamsGroupHeartbeatRequestData.Endpoint() - .setHost(host) - .setPort(port); + .setHost(host) + .setPort(port); } private static List buildEndpoints(String host, int port, String topic, List partitions) { List endpoints = new ArrayList<>(); endpoints.add(new StreamsGroupHeartbeatResponseData.EndpointToPartitions() - .setUserEndpoint(new StreamsGroupHeartbeatResponseData.Endpoint() - .setHost(host) - .setPort(port)) - .setActivePartitions(List.of(topicPartition(topic, partitions)))); + .setUserEndpoint(new StreamsGroupHeartbeatResponseData.Endpoint() + .setHost(host) + .setPort(port)) + .setActivePartitions(List.of(topicPartition(topic, partitions)))); return endpoints; } diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java index 1279a845b5a34..69a1a3694bbb9 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java @@ -53,20 +53,20 @@ static StreamsGroupMember.Builder streamsGroupMemberBuilderWithDefaults(String m static StreamsGroupMember.Builder streamsGroupMemberBuilderWithDefaults(String memberId, String instanceId) { return new StreamsGroupMember.Builder(memberId) - .setMemberEpoch(1) - .setPreviousMemberEpoch(0) - .setState(MemberState.STABLE) - .setRackId(null) - .setInstanceId(instanceId) - .setRebalanceTimeoutMs(1500) - .setAssignedTasks(TasksTupleWithEpochs.EMPTY) - .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) - .setTopologyEpoch(0) - .setClientTags(Map.of()) - .setClientId(DEFAULT_CLIENT_ID) - .setClientHost(DEFAULT_CLIENT_ADDRESS.toString()) - .setProcessId(DEFAULT_PROCESS_ID) - .setUserEndpoint(null); + .setMemberEpoch(1) + .setPreviousMemberEpoch(0) + .setState(MemberState.STABLE) + .setRackId(null) + .setInstanceId(instanceId) + .setRebalanceTimeoutMs(1500) + .setAssignedTasks(TasksTupleWithEpochs.EMPTY) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .setTopologyEpoch(0) + .setClientTags(Map.of()) + .setClientId(DEFAULT_CLIENT_ID) + .setClientHost(DEFAULT_CLIENT_ADDRESS.toString()) + .setProcessId(DEFAULT_PROCESS_ID) + .setUserEndpoint(null); } /** @@ -76,18 +76,18 @@ static StreamsGroupMember.Builder streamsGroupMemberBuilderWithDefaults(String m static Map getDefaultAssignmentConfigs() { // Use the same default value as GroupCoordinatorConfig.STREAMS_GROUP_NUM_STANDBY_REPLICAS_DEFAULT return new TreeMap<>(Map.of( - "num.standby.replicas", String.valueOf(GroupCoordinatorConfig.STREAMS_GROUP_NUM_STANDBY_REPLICAS_DEFAULT) + "num.standby.replicas", String.valueOf(GroupCoordinatorConfig.STREAMS_GROUP_NUM_STANDBY_REPLICAS_DEFAULT) )); } static List mkResponseTasks( - String subtopologyId, - Integer... partitions + String subtopologyId, + Integer... partitions ) { return List.of( - new StreamsGroupHeartbeatResponseData.TaskIds() - .setSubtopologyId(subtopologyId) - .setPartitions(Arrays.asList(partitions)) + new StreamsGroupHeartbeatResponseData.TaskIds() + .setSubtopologyId(subtopologyId) + .setPartitions(Arrays.asList(partitions)) ); } @@ -99,35 +99,35 @@ static StreamsTopicFixture streamsTopicFixture(String subtopologyId, String topi static StreamsGroupHeartbeatRequestData staticHeartbeat(String groupId, String memberId, String instanceId, int memberEpoch) { return new StreamsGroupHeartbeatRequestData() - .setGroupId(groupId) - .setInstanceId(instanceId) - .setMemberId(memberId) - .setMemberEpoch(memberEpoch); + .setGroupId(groupId) + .setInstanceId(instanceId) + .setMemberId(memberId) + .setMemberEpoch(memberEpoch); } static StreamsGroupHeartbeatRequestData staticJoinHeartbeat(String groupId, String memberId, String instanceId, StreamsTopicFixture topic) { return staticHeartbeat(groupId, memberId, instanceId, StreamsGroupHeartbeatRequest.JOIN_GROUP_MEMBER_EPOCH) - .setProcessId(DEFAULT_PROCESS_ID) - .setRebalanceTimeoutMs(DEFAULT_REBALANCE_TIMEOUT_MS) - .setTopology(topic.topology) - .setActiveTasks(List.of()) - .setStandbyTasks(List.of()) - .setWarmupTasks(List.of()); + .setProcessId(DEFAULT_PROCESS_ID) + .setRebalanceTimeoutMs(DEFAULT_REBALANCE_TIMEOUT_MS) + .setTopology(topic.topology) + .setActiveTasks(List.of()) + .setStandbyTasks(List.of()) + .setWarmupTasks(List.of()); } static StreamsGroupHeartbeatResponseData staticLeaveResponse(String memberId, int leaveEpoch) { return new StreamsGroupHeartbeatResponseData() - .setMemberId(memberId) - .setMemberEpoch(leaveEpoch) - .setStatus(List.of()); + .setMemberId(memberId) + .setMemberEpoch(leaveEpoch) + .setStatus(List.of()); } static StreamsGroupHeartbeatResponseData staticLeaveResponseWithNullTasks(String memberId, int leaveEpoch) { return staticLeaveResponse(memberId, leaveEpoch) - .setActiveTasks(null) - .setWarmupTasks(null) - .setStandbyTasks(null); + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null); } @@ -140,25 +140,25 @@ static class StreamsTopicFixture { private final long metadataHash; private StreamsTopicFixture( - String subtopologyId, - String topicName, - int partitions + String subtopologyId, + String topicName, + int partitions ) { this.subtopologyId = subtopologyId; this.topicName = topicName; this.topicId = Uuid.randomUuid(); this.topology = new StreamsGroupHeartbeatRequestData.Topology() - .setSubtopologies(List.of( - new StreamsGroupHeartbeatRequestData.Subtopology() - .setSubtopologyId(subtopologyId) - .setSourceTopics(List.of(topicName)) - )); + .setSubtopologies(List.of( + new StreamsGroupHeartbeatRequestData.Subtopology() + .setSubtopologyId(subtopologyId) + .setSourceTopics(List.of(topicName)) + )); this.metadataImage = new MetadataImageBuilder() - .addTopic(topicId, topicName, partitions) - .buildCoordinatorMetadataImage(); + .addTopic(topicId, topicName, partitions) + .buildCoordinatorMetadataImage(); this.metadataHash = computeGroupHash(Map.of( - topicName, - computeTopicHash(topicName, metadataImage) + topicName, + computeTopicHash(topicName, metadataImage) )); } @@ -168,19 +168,19 @@ public Map.Entry> tasks(Integer... partitions) { public TasksTuple targetAssignment(Integer... partitions) { return TaskAssignmentTestUtil.mkTasksTuple( - TaskAssignmentTestUtil.TaskRole.ACTIVE, - tasks(partitions) + TaskAssignmentTestUtil.TaskRole.ACTIVE, + tasks(partitions) ); } public TasksTupleWithEpochs assignedTasks( - int epoch, - Integer... partitions + int epoch, + Integer... partitions ) { return mkTasksTupleWithCommonEpoch( - TaskAssignmentTestUtil.TaskRole.ACTIVE, - epoch, - tasks(partitions) + TaskAssignmentTestUtil.TaskRole.ACTIVE, + epoch, + tasks(partitions) ); } @@ -202,128 +202,128 @@ public List responseTasks(Integer... public List requestTasks(List partitions) { return List.of( - new StreamsGroupHeartbeatRequestData.TaskIds() - .setSubtopologyId(subtopologyId) - .setPartitions(partitions) + new StreamsGroupHeartbeatRequestData.TaskIds() + .setSubtopologyId(subtopologyId) + .setPartitions(partitions) ); } } static GroupMetadataManagerTestContext contextWithStreamsGroup( - String groupId, - int groupEpoch, - StreamsTopicFixture topic, - java.util.function.UnaryOperator configureGroup + String groupId, + int groupEpoch, + StreamsTopicFixture topic, + java.util.function.UnaryOperator configureGroup ) { return contextWithStreamsGroup( - groupId, - groupEpoch, - topic, - new MockTaskAssignor("sticky"), - configureGroup + groupId, + groupEpoch, + topic, + new MockTaskAssignor("sticky"), + configureGroup ); } static GroupMetadataManagerTestContext contextWithStreamsGroup( - String groupId, - int groupEpoch, - StreamsTopicFixture topic, - MockTaskAssignor assignor, - java.util.function.UnaryOperator configureGroup + String groupId, + int groupEpoch, + StreamsTopicFixture topic, + MockTaskAssignor assignor, + java.util.function.UnaryOperator configureGroup ) { - return contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT, configureGroup); + return contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT, configureGroup); } static GroupMetadataManagerTestContext contextWithStreamsGroup( - String groupId, - int groupEpoch, - StreamsTopicFixture topic, - MockTaskAssignor assignor, - int initialRebalanceDelayMs, - java.util.function.UnaryOperator configureGroup + String groupId, + int groupEpoch, + StreamsTopicFixture topic, + MockTaskAssignor assignor, + int initialRebalanceDelayMs, + java.util.function.UnaryOperator configureGroup ) { StreamsGroupBuilder group = new StreamsGroupBuilder(groupId, groupEpoch) - .withTargetAssignmentEpoch(groupEpoch) - .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) - .withValidatedTopologyEpoch(0) - .withMetadataHash(topic.metadataHash()) - .withLastAssignmentConfigs(getDefaultAssignmentConfigs()); + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs()); return new GroupMetadataManagerTestContext.Builder() - .withStreamsGroupTaskAssignors(List.of(assignor)) - .withMetadataImage(topic.metadataImage()) - .withStreamsGroup(configureGroup.apply(group)) - .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, initialRebalanceDelayMs) - .build(); + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(configureGroup.apply(group)) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, initialRebalanceDelayMs) + .build(); } static StreamsGroupHeartbeatResponseData heartbeatResponseWithActiveTasks( - String memberId, - int memberEpoch, - StreamsTopicFixture topic, - Integer... activeTasks + String memberId, + int memberEpoch, + StreamsTopicFixture topic, + Integer... activeTasks ) { return heartbeatResponseWithActiveTasks( - memberId, - memberEpoch, - topic.responseTasks(activeTasks) + memberId, + memberEpoch, + topic.responseTasks(activeTasks) ); } static StreamsGroupHeartbeatResponseData heartbeatResponseWithActiveTasks( - String memberId, - int memberEpoch, - List activeTasks + String memberId, + int memberEpoch, + List activeTasks ) { return new StreamsGroupHeartbeatResponseData() - .setMemberId(memberId) - .setMemberEpoch(memberEpoch) - .setHeartbeatIntervalMs(5000) - .setTaskOffsetIntervalMs(60000) - .setActiveTasks(activeTasks) - .setWarmupTasks(List.of()) - .setStandbyTasks(List.of()); + .setMemberId(memberId) + .setMemberEpoch(memberEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(activeTasks) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()); } static StreamsGroupHeartbeatResponseData heartbeatResponseWithActiveTasks( - String memberId, - int memberEpoch, - String subtopologyId, - Integer... activeTasks + String memberId, + int memberEpoch, + String subtopologyId, + Integer... activeTasks ) { return heartbeatResponseWithActiveTasks( - memberId, - memberEpoch, - mkResponseTasks(subtopologyId, activeTasks) + memberId, + memberEpoch, + mkResponseTasks(subtopologyId, activeTasks) ); } static StreamsGroupHeartbeatResponseData heartbeatResponseWithNullTasks( - String memberId, - int memberEpoch + String memberId, + int memberEpoch ) { return new StreamsGroupHeartbeatResponseData() - .setMemberId(memberId) - .setMemberEpoch(memberEpoch) - .setHeartbeatIntervalMs(5000) - .setTaskOffsetIntervalMs(60000) - .setActiveTasks(null) - .setWarmupTasks(null) - .setStandbyTasks(null); + .setMemberId(memberId) + .setMemberEpoch(memberEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null); } static StreamsGroupHeartbeatRequestData staticJoinHeartbeat( - String groupId, - String memberId, - String instanceId, - String processId + String groupId, + String memberId, + String instanceId, + String processId ) { return staticHeartbeat(groupId, memberId, instanceId, StreamsGroupHeartbeatRequest.JOIN_GROUP_MEMBER_EPOCH) - .setProcessId(processId) - .setActiveTasks(List.of()) - .setStandbyTasks(List.of()) - .setWarmupTasks(List.of()); + .setProcessId(processId) + .setActiveTasks(List.of()) + .setStandbyTasks(List.of()) + .setWarmupTasks(List.of()); } static TasksTupleWithEpochs resetAssignedTasksEpochsToZero(TasksTupleWithEpochs assignedTasks) { @@ -344,9 +344,9 @@ static TasksTupleWithEpochs resetAssignedTasksEpochsToZero(TasksTupleWithEpochs resetActiveTasks.put(entry.getKey(), resetActiveTaskEpochs); } return new TasksTupleWithEpochs( - resetActiveTasks, - assignedTasks.standbyTasks(), - assignedTasks.warmupTasks() + resetActiveTasks, + assignedTasks.standbyTasks(), + assignedTasks.warmupTasks() ); } From 6bd6d0d000cbbe76681e0374e2a81cd4b76b1ff1 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Thu, 28 May 2026 21:57:40 +0900 Subject: [PATCH 09/16] Remove redundant code. --- .../apache/kafka/coordinator/group/GroupMetadataManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java index c61d4a70ee138..4749a85a2a5d3 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java @@ -1797,7 +1797,7 @@ private void throwIfStreamsGroupIsFull( // and neither affects the group size: // 1. The member is replaced due to the static member rejoining. // 2. 'UnreleasedInstanceIdException' is raised due to an epoch mismatch. - if (instanceId != null && group.hasStaticMember(instanceId)) + if (group.hasStaticMember(instanceId)) return; // If the streams group has reached its maximum capacity, the member is rejected if it is not From ace0788634c5c6581b1383f8d431560b0b69f071 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Fri, 29 May 2026 23:32:38 +0900 Subject: [PATCH 10/16] Addressing review. --- .../GroupMetadataManagerTestContext.java | 18 +- ...amsGroupMixedGroupMetadataManagerTest.java | 327 +++-- ...pStaticMemberGroupMetadataManagerTest.java | 1051 ++++++++++++----- .../group/StreamsGroupTestUtil.java | 123 -- 4 files changed, 974 insertions(+), 545 deletions(-) diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTestContext.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTestContext.java index a9f4c2998b06b..1e3010f0a1cff 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTestContext.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTestContext.java @@ -734,24 +734,28 @@ public CoordinatorResult streamsGroupHeartbeat( - StreamsGroupHeartbeatRequestData request + StreamsGroupHeartbeatRequestData request, + String clientId, + InetAddress clientAddress ) { - return streamsGroupHeartbeat(request, ApiKeys.STREAMS_GROUP_HEARTBEAT.latestVersion()); + return streamsGroupHeartbeat(request, clientId, clientAddress, ApiKeys.STREAMS_GROUP_HEARTBEAT.latestVersion()); } public CoordinatorResult streamsGroupHeartbeat( StreamsGroupHeartbeatRequestData request, + String clientId, + InetAddress clientAddress, short version ) { RequestContext context = new RequestContext( new RequestHeader( ApiKeys.STREAMS_GROUP_HEARTBEAT, version, - "client", + clientId, 0 ), "1", - InetAddress.getLoopbackAddress(), + clientAddress, KafkaPrincipal.ANONYMOUS, ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT), SecurityProtocol.PLAINTEXT, @@ -770,6 +774,12 @@ public CoordinatorResult streams return result; } + public CoordinatorResult streamsGroupHeartbeat( + StreamsGroupHeartbeatRequestData request + ) { + return streamsGroupHeartbeat(request, "client", InetAddress.getLoopbackAddress()); + } + public List> sleep(long ms) { time.sleep(ms); List> timeouts = timer.poll(); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java index b8bdfba431414..23585a9a1186e 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.errors.FencedInstanceIdException; import org.apache.kafka.common.errors.GroupMaxSizeReachedException; import org.apache.kafka.common.message.StreamsGroupHeartbeatRequestData; +import org.apache.kafka.common.message.StreamsGroupHeartbeatResponseData; import org.apache.kafka.common.requests.StreamsGroupHeartbeatRequest; import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord; import org.apache.kafka.coordinator.common.runtime.CoordinatorResult; @@ -47,14 +48,10 @@ import static org.apache.kafka.coordinator.group.Assertions.assertResponseEquals; import static org.apache.kafka.coordinator.group.Assertions.assertUnorderedRecordsEquals; import static org.apache.kafka.coordinator.group.GroupMetadataManager.groupSessionTimeoutKey; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.StreamsTopicFixture; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.contextWithStreamsGroup; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.getDefaultAssignmentConfigs; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.heartbeatResponseWithActiveTasks; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.heartbeatResponseWithNullTasks; +import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.mkResponseTasks; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticHeartbeat; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticJoinHeartbeat; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticLeaveResponseWithNullTasks; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.streamsTopicFixture; import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole; @@ -122,19 +119,16 @@ public void testStaticRejoinSucceedsAtMaxSizeWhileDynamicMemberStillExists() { String groupId = "fooup"; String subtopologyId = "subtopology-1"; - StreamsTopicFixture topic = streamsTopicFixture(subtopologyId, "foo", 4); - + StreamsGroupTestUtil.StreamsTopicFixture topic = streamsTopicFixture(subtopologyId, "foo", 4); String oldStaticMemberId = Uuid.randomUuid().toString(); String newStaticMemberId = Uuid.randomUuid().toString(); String staticInstanceId = Uuid.randomUuid().toString(); String dynamicMemberId = Uuid.randomUuid().toString(); - // GIVEN Task for static member TasksTupleWithEpochs staticAssignedTasks = topic.assignedTasks(groupEpoch, 0, 1); TasksTuple staticTargetAssignment = topic.targetAssignment(0, 1); - // GIVEN Task for dynamic member TasksTupleWithEpochs dynamicAssignedTasks = topic.assignedTasks(groupEpoch, 2, 3); TasksTuple dynamicTargetAssignment = topic.targetAssignment(2, 3); @@ -163,13 +157,22 @@ public void testStaticRejoinSucceedsAtMaxSizeWhileDynamicMemberStillExists() { .build(); - // WHEN - static member rejoin. + // static member rejoins. CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( staticHeartbeat(groupId, newStaticMemberId, staticInstanceId, StreamsGroupHeartbeatRequest.JOIN_GROUP_MEMBER_EPOCH) ); - // THEN - At the maxsize, static member still can rejoin if static member leaves with epoch -2. - assertResponseEquals(heartbeatResponseWithActiveTasks(newStaticMemberId, groupEpoch, subtopologyId, 0, 1), rejoinResult.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(newStaticMemberId) + .setMemberEpoch(groupEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(mkResponseTasks(subtopologyId, 0, 1)) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()), + rejoinResult.response().data() + ); StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); assertFalse(group.hasMember(oldStaticMemberId)); @@ -198,7 +201,7 @@ public void testDynamicJoinSucceedsAfterTemporarilyLeftStaticMemberSessionTimeou String groupId = "fooup"; String subtopologyId = "subtopology-1"; - StreamsTopicFixture topic = streamsTopicFixture(subtopologyId, "foo", 4); + StreamsGroupTestUtil.StreamsTopicFixture topic = streamsTopicFixture(subtopologyId, "foo", 4); StreamsGroupHeartbeatRequestData.Topology topology = topic.topology(); String staticMemberId = Uuid.randomUuid().toString(); @@ -210,7 +213,6 @@ public void testDynamicJoinSucceedsAfterTemporarilyLeftStaticMemberSessionTimeou String dynamicProcessId = "dynamic-process-id"; String newDynamicProcessId = "new-dynamic-process-id"; - // GIVEN assignment TasksTupleWithEpochs staticAssignedTasks = topic.assignedTasks(groupEpoch, 0, 1); TasksTuple staticTargetAssignment = topic.targetAssignment(0, 1); @@ -247,13 +249,21 @@ public void testDynamicJoinSucceedsAfterTemporarilyLeftStaticMemberSessionTimeou .build(); context.onLoaded(); - // WHEN1 - static member leaves with epoch -2. + // static member leaves with epoch -2. CoordinatorResult leaveResult = context.streamsGroupHeartbeat( staticHeartbeat(groupId, staticMemberId, staticInstanceId, StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) ); - // THEN1 - assertResponseEquals(staticLeaveResponseWithNullTasks(staticMemberId, StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH), leaveResult.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(staticMemberId) + .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setStatus(List.of()) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null), + leaveResult.response().data() + ); // To prevent session timeout from dynamic member. // Sleep 1, and dynamic member send a heartbeat. @@ -262,14 +272,22 @@ public void testDynamicJoinSucceedsAfterTemporarilyLeftStaticMemberSessionTimeou staticHeartbeat(groupId, dynamicMemberId, null, groupEpoch) ); - assertResponseEquals(heartbeatResponseWithNullTasks(dynamicMemberId, groupEpoch), dynamicHeartbeatResult.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(dynamicMemberId) + .setMemberEpoch(groupEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null), + dynamicHeartbeatResult.response().data()); assertTrue(dynamicHeartbeatResult.records().isEmpty()); - // WHEN2: static member session timeout. + // static member session timeout. context.assertSessionTimeout(groupId, staticMemberId, 45000 - 1); List> timeouts = context.sleep(45000 - 1); - // THEN2 List expectedTimeoutRecords = List.of( StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, staticMemberId), StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, staticMemberId), @@ -297,15 +315,24 @@ public void testDynamicJoinSucceedsAfterTemporarilyLeftStaticMemberSessionTimeou newDynamicMemberId, staticTargetAssignment )); - // WHEN3 - new dynamic member try to join. + // new dynamic member try to join. CoordinatorResult joinResult = context.streamsGroupHeartbeat( staticJoinHeartbeat(groupId, newDynamicMemberId, null, newDynamicProcessId) .setRebalanceTimeoutMs(1500) .setTopology(topology) ); - // THEN3 : accept join. - assertResponseEquals(heartbeatResponseWithActiveTasks(newDynamicMemberId, joinGroupEpoch, subtopologyId, 0, 1), joinResult.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(newDynamicMemberId) + .setMemberEpoch(joinGroupEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(mkResponseTasks(subtopologyId, 0, 1)) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()), + joinResult.response().data() + ); StreamsGroupMember expectedJoiningDynamicMember = streamsGroupMemberBuilderWithDefaults(newDynamicMemberId) .setProcessId(newDynamicProcessId) @@ -365,31 +392,40 @@ public void testStaticTemporaryLeaveDoesNotTransferTasksToExistingDynamicMember( String dynamicMemberId = Uuid.randomUuid().toString(); - StreamsTopicFixture topic = streamsTopicFixture("subtopology-1", "foo", 3); + StreamsGroupTestUtil.StreamsTopicFixture topic = streamsTopicFixture("subtopology-1", "foo", 3); - // GIVEN Task for static member TasksTupleWithEpochs staticAssignedTasks = topic.assignedTasks(groupEpoch, 0, 1); TasksTuple staticTargetAssignment = topic.targetAssignment(0, 1); - // GIVEN Task for dynamic member TasksTupleWithEpochs dynamicAssignedTasks = topic.assignedTasks(groupEpoch, 2); TasksTuple dynamicTargetAssignment = topic.targetAssignment(2); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(staticMemberId, staticInstanceId) - .setMemberEpoch(groupEpoch) - .setPreviousMemberEpoch(groupEpoch - 1) - .setAssignedTasks(staticAssignedTasks) - .build()) - .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) - .setMemberEpoch(groupEpoch) - .setPreviousMemberEpoch(groupEpoch - 1) - .setAssignedTasks(dynamicAssignedTasks) - .build()) - .withTargetAssignment(staticMemberId, staticTargetAssignment) - .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment)); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(staticMemberId, staticInstanceId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(staticMemberId, staticTargetAssignment) + .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs()) + ) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); - // WHEN - static member leaves with epoch -2. + // static member leaves with epoch -2. CoordinatorResult leaveResult = context.streamsGroupHeartbeat(staticHeartbeat( groupId, @@ -398,13 +434,18 @@ public void testStaticTemporaryLeaveDoesNotTransferTasksToExistingDynamicMember( LEAVE_GROUP_STATIC_MEMBER_EPOCH )); - // THEN assertResponseEquals( - staticLeaveResponseWithNullTasks(staticMemberId, LEAVE_GROUP_STATIC_MEMBER_EPOCH), + new StreamsGroupHeartbeatResponseData() + .setMemberId(staticMemberId) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setStatus(List.of()) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null), leaveResult.response().data() ); - // WHEN2 - dynamic member send a heartbeat. + // dynamic member send a heartbeat. CoordinatorResult dynamicHeartbeatResult = context.streamsGroupHeartbeat( new StreamsGroupHeartbeatRequestData() @@ -413,8 +454,17 @@ public void testStaticTemporaryLeaveDoesNotTransferTasksToExistingDynamicMember( .setMemberEpoch(groupEpoch) ); - // THEN2 : There is no new assignment. - assertResponseEquals(heartbeatResponseWithNullTasks(dynamicMemberId, groupEpoch), dynamicHeartbeatResult.response().data()); + // There is no new assignment. + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(dynamicMemberId) + .setMemberEpoch(groupEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null), + dynamicHeartbeatResult.response().data()); assertTrue(dynamicHeartbeatResult.records().isEmpty()); StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); @@ -439,7 +489,7 @@ public void testStaticRejoinWithUpdatedProcessIdRecomputesTargetAssignmentAndDyn int bumpedGroupEpoch = groupEpoch + 1; String subtopologyId = "subtopology-1"; - StreamsTopicFixture topic = streamsTopicFixture(subtopologyId, "foo", 4); + StreamsGroupTestUtil.StreamsTopicFixture topic = streamsTopicFixture(subtopologyId, "foo", 4); String oldStaticMemberId = Uuid.randomUuid().toString(); String rejoinStaticMemberId = Uuid.randomUuid().toString(); @@ -466,29 +516,45 @@ public void testStaticRejoinWithUpdatedProcessIdRecomputesTargetAssignmentAndDyn dynamicMemberId, newDynamicTargetAssignment )); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) - .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setPreviousMemberEpoch(groupEpoch) - .setProcessId(oldProcessId) - .setAssignedTasks(staticAssignedTasks) - .build()) - .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) - .setMemberEpoch(groupEpoch) - .setPreviousMemberEpoch(groupEpoch - 1) - .setAssignedTasks(dynamicAssignedTasks) - .build()) - .withTargetAssignment(oldStaticMemberId, oldStaticTargetAssignment) - .withTargetAssignment(dynamicMemberId, oldDynamicTargetAssignment)); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) + .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(groupEpoch) + .setProcessId(oldProcessId) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(oldStaticMemberId, oldStaticTargetAssignment) + .withTargetAssignment(dynamicMemberId, oldDynamicTargetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); - // WHEN 1: static member try to rejoin with new process id. + // static member try to rejoin with new process id. CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( staticJoinHeartbeat(groupId, rejoinStaticMemberId, staticInstanceId, newProcessId) ); - // THEN 1: assertResponseEquals( - heartbeatResponseWithActiveTasks(rejoinStaticMemberId, bumpedGroupEpoch, topic, 0), + new StreamsGroupHeartbeatResponseData() + .setMemberId(rejoinStaticMemberId) + .setMemberEpoch(bumpedGroupEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(topic.responseTasks(0)) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()), rejoinResult.response().data() ); @@ -502,7 +568,7 @@ public void testStaticRejoinWithUpdatedProcessIdRecomputesTargetAssignmentAndDyn assertTrue(group.hasMember(rejoinStaticMemberId)); assertEquals(rejoinStaticMemberId, group.staticMember(staticInstanceId).memberId()); - // Because group epoch is bumped up, target assignment should be recomupted. + // Because group epoch is bumped up, target assignment should be recomputed. assertEquals(newStaticTargetAssignment, group.targetAssignment(rejoinStaticMemberId, Optional.of(staticInstanceId))); assertEquals(newDynamicTargetAssignment, group.targetAssignment(dynamicMemberId, Optional.empty())); assertEquals(dynamicAssignedTasks, group.getMemberOrThrow(dynamicMemberId).assignedTasks()); @@ -577,13 +643,20 @@ public void testStaticRejoinWithUpdatedProcessIdRecomputesTargetAssignmentAndDyn rejoinResult.records().subList(10, 12) ); - // WHEN 2: dynamic member send a heartbeat and reconciles to the new target assignment. + // dynamic member send a heartbeat and reconciles to the new target assignment. CoordinatorResult dynamicHeartbeatResult = context.streamsGroupHeartbeat( staticHeartbeat(groupId, dynamicMemberId, null, groupEpoch) ); assertResponseEquals( - heartbeatResponseWithActiveTasks(dynamicMemberId, bumpedGroupEpoch, topic, 1, 2, 3), + new StreamsGroupHeartbeatResponseData() + .setMemberId(dynamicMemberId) + .setMemberEpoch(bumpedGroupEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(topic.responseTasks(1, 2, 3)) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()), dynamicHeartbeatResult.response().data() ); @@ -631,39 +704,56 @@ public void testStaticRejoinWithSameProcessIdDoesNotBumpEpochAndDynamicHeartbeat String staticProcessId = "static-process-id"; String dynamicProcessId = "dynamic-process-id"; - StreamsTopicFixture topic = streamsTopicFixture("subtopology-1", "foo", 4); + StreamsGroupTestUtil.StreamsTopicFixture topic = streamsTopicFixture("subtopology-1", "foo", 4); - // GIVEN Tasks TasksTupleWithEpochs staticAssignedTasks = topic.assignedTasks(groupEpoch, 0, 1); TasksTuple staticTargetAssignment = topic.targetAssignment(0, 1); TasksTupleWithEpochs dynamicAssignedTasks = topic.assignedTasks(groupEpoch, 2, 3); TasksTuple dynamicTargetAssignment = topic.targetAssignment(2, 3); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) - .setProcessId(staticProcessId) - .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setPreviousMemberEpoch(groupEpoch) - .setAssignedTasks(staticAssignedTasks) - .build()) - .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) - .setProcessId(dynamicProcessId) - .setMemberEpoch(groupEpoch) - .setPreviousMemberEpoch(groupEpoch - 1) - .setAssignedTasks(dynamicAssignedTasks) - .build()) - .withTargetAssignment(oldStaticMemberId, staticTargetAssignment) - .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment)); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) + .setProcessId(staticProcessId) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(groupEpoch) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setProcessId(dynamicProcessId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(oldStaticMemberId, staticTargetAssignment) + .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); - // WHEN1 : static member try to rejoin with same process id. + // static member try to rejoin with same process id. CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( staticJoinHeartbeat(groupId, newStaticMemberId, staticInstanceId, staticProcessId) ); - - // THEN1 - assertResponseEquals(heartbeatResponseWithActiveTasks(newStaticMemberId, groupEpoch, topic, 0, 1), rejoinResult.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(newStaticMemberId) + .setMemberEpoch(groupEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(topic.responseTasks(0, 1)) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()), + rejoinResult.response().data() + ); StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); @@ -703,13 +793,22 @@ public void testStaticRejoinWithSameProcessIdDoesNotBumpEpochAndDynamicHeartbeat assertRecordsEquals(expectedRejoinRecords, rejoinResult.records()); - // WHEN2 - dynamic member send a heartbeat request + // dynamic member send a heartbeat request CoordinatorResult dynamicHeartbeatResult = context.streamsGroupHeartbeat( staticHeartbeat(groupId, dynamicMemberId, null, groupEpoch) ); - // THEN2 - no new target assignment. - assertResponseEquals(heartbeatResponseWithNullTasks(dynamicMemberId, groupEpoch), dynamicHeartbeatResult.response().data()); + // no new target assignment. + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(dynamicMemberId) + .setMemberEpoch(groupEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null), + dynamicHeartbeatResult.response().data()); assertTrue(dynamicHeartbeatResult.records().isEmpty()); assertEquals(dynamicAssignedTasks, group.getMemberOrThrow(dynamicMemberId).assignedTasks()); @@ -735,7 +834,7 @@ public void testOldStaticMemberIdIsFencedAfterReplacementInMixedGroup() { String staticProcessId = "static-process-id"; String dynamicProcessId = "dynamic-process-id"; - StreamsTopicFixture topic = streamsTopicFixture("subtopology-1", "foo", 4); + StreamsGroupTestUtil.StreamsTopicFixture topic = streamsTopicFixture("subtopology-1", "foo", 4); TasksTupleWithEpochs staticAssignedTasks = topic.assignedTasks(groupEpoch, 0, 1); TasksTuple staticTargetAssignment = topic.targetAssignment(0, 1); @@ -743,28 +842,38 @@ public void testOldStaticMemberIdIsFencedAfterReplacementInMixedGroup() { TasksTupleWithEpochs dynamicAssignedTasks = topic.assignedTasks(groupEpoch, 2, 3); TasksTuple dynamicTargetAssignment = topic.targetAssignment(2, 3); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) - .setProcessId(staticProcessId) - .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setPreviousMemberEpoch(groupEpoch) - .setAssignedTasks(staticAssignedTasks) - .build()) - .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) - .setProcessId(dynamicProcessId) - .setMemberEpoch(groupEpoch) - .setPreviousMemberEpoch(groupEpoch - 1) - .setAssignedTasks(dynamicAssignedTasks) - .build()) - .withTargetAssignment(oldStaticMemberId, staticTargetAssignment) - .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment)); - - // WHEN1 - static member try to rejoin with new member id. + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(oldStaticMemberId, staticInstanceId) + .setProcessId(staticProcessId) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(groupEpoch) + .setAssignedTasks(staticAssignedTasks) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(dynamicMemberId) + .setProcessId(dynamicProcessId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(dynamicAssignedTasks) + .build()) + .withTargetAssignment(oldStaticMemberId, staticTargetAssignment) + .withTargetAssignment(dynamicMemberId, dynamicTargetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); + + // static member try to rejoin with new member id. context.streamsGroupHeartbeat( staticJoinHeartbeat(groupId, newStaticMemberId, staticInstanceId, staticProcessId) ); - // WHEN2 + THEN2 - stale static member send a heartbeat with stale member id. + // stale static member send a heartbeat with stale member id. assertThrows(FencedInstanceIdException.class, () -> context.streamsGroupHeartbeat(staticHeartbeat(groupId, oldStaticMemberId, staticInstanceId, groupEpoch)) ); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java index e87a06aca7aba..bf895c090e20e 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java @@ -31,6 +31,7 @@ import org.apache.kafka.coordinator.common.runtime.MetadataImageBuilder; import org.apache.kafka.coordinator.common.runtime.MockCoordinatorTimer; import org.apache.kafka.coordinator.group.StreamsGroupTestUtil.StreamsTopicFixture; +import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataKey; import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue; import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataKey; import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataValue; @@ -50,6 +51,8 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -62,16 +65,13 @@ import static org.apache.kafka.coordinator.group.Assertions.assertRecordsEquals; import static org.apache.kafka.coordinator.group.Assertions.assertResponseEquals; import static org.apache.kafka.coordinator.group.GroupMetadataManager.groupSessionTimeoutKey; +import static org.apache.kafka.coordinator.group.GroupMetadataManagerTestContext.DEFAULT_CLIENT_ADDRESS; +import static org.apache.kafka.coordinator.group.GroupMetadataManagerTestContext.DEFAULT_CLIENT_ID; import static org.apache.kafka.coordinator.group.GroupMetadataManagerTestContext.DEFAULT_PROCESS_ID; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.contextWithStreamsGroup; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.getDefaultAssignmentConfigs; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.heartbeatResponseWithActiveTasks; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.heartbeatResponseWithNullTasks; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.resetAssignedTasksEpochsToZero; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticHeartbeat; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticJoinHeartbeat; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticLeaveResponse; -import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.staticLeaveResponseWithNullTasks; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults; import static org.apache.kafka.coordinator.group.StreamsGroupTestUtil.streamsTopicFixture; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -85,65 +85,241 @@ class StreamsGroupStaticMemberGroupMetadataManagerTest { private static final int DEFAULT_MEMBER_EPOCH = 10; private static final int DEFAULT_GROUP_EPOCH = 10; + @Test public void testUnknownStaticMemberLeaveStreamsGroup() { - StaticMemberFixtureWith2Members fixture = new StaticMemberFixtureWith2Members(2); + String groupId = "streams-group"; + + String activeMemberId = "active-member"; + String activeInstanceId = "active-instance"; - fixture.assertLeaveFails( - fixture.staticLeaveRequest("unknown-member-id", "unknown-instance-id"), - UnknownMemberIdException.class + String unknownMemberId = "unknown-member-id"; + String unknownInstanceId = "unknown-instance-id"; + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 1); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, DEFAULT_GROUP_EPOCH) + .withMember(streamsGroupMemberBuilderWithDefaults(activeMemberId, activeInstanceId) + .setMemberEpoch(DEFAULT_MEMBER_EPOCH) + .setPreviousMemberEpoch(DEFAULT_MEMBER_EPOCH - 1) + .build()) + .withTargetAssignmentEpoch(DEFAULT_GROUP_EPOCH) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); + + assertThrows( + UnknownMemberIdException.class, + () -> context.streamsGroupHeartbeat(staticHeartbeat( + groupId, + unknownMemberId, + unknownInstanceId, + LEAVE_GROUP_STATIC_MEMBER_EPOCH + )) ); } @Test public void testStreamsStaticJoinWithNewInstanceAtMaxSizeThrowsGroupMaxSizeReached() { - // With max.size=2 already reached, + // With max.size=2 already reached, // joining with a new static instanceId must throw GroupMaxSizeReachedException. int streamsGroupMaxSize = 2; - StaticMemberFixtureWith2Members fixture = new StaticMemberFixtureWith2Members(streamsGroupMaxSize); + String groupId = "streams-group"; + + String activeMemberId = "active-member"; + String activeInstanceId = "active-instance"; - fixture.assertJoinFails( - fixture.newMemberJoinsWithNewInstanceId(), - GroupMaxSizeReachedException.class + String leftMemberId = "left-member"; + String leftInstanceId = "left-instance"; + + String newMemberId = "new-member"; + String newInstanceId = "new-instance"; + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 1); + + MockTaskAssignor assignor = new MockTaskAssignor("sticky"); + assignor.prepareGroupAssignment(Map.of(activeMemberId, TasksTuple.EMPTY)); + + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, streamsGroupMaxSize) + .withStreamsGroup(new StreamsGroupBuilder(groupId, DEFAULT_GROUP_EPOCH) + .withMember(streamsGroupMemberBuilderWithDefaults(activeMemberId, activeInstanceId) + .setMemberEpoch(DEFAULT_MEMBER_EPOCH) + .setPreviousMemberEpoch(DEFAULT_MEMBER_EPOCH - 1) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(leftMemberId, leftInstanceId) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(DEFAULT_MEMBER_EPOCH - 1) + .build()) + .withTargetAssignmentEpoch(DEFAULT_GROUP_EPOCH) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withMetadataHash(topic.metadataHash()) + .withValidatedTopologyEpoch(0) + ) + .build(); + + assertThrows( + GroupMaxSizeReachedException.class, + () -> context.streamsGroupHeartbeat(staticJoinHeartbeat( + groupId, + newMemberId, + newInstanceId, + topic + )) ); + } @Test public void testStreamsStaticRejoinWithLeaveGroupStaticEpochAtMaxSizeSucceeds() { - // If a static member is in leave epoch (-2), + // If a static member is in leave epoch (-2), // rejoining with the same memberId/instanceId is allowed even at max size. int streamsGroupMaxSize = 2; - StaticMemberFixtureWith2Members fixture = new StaticMemberFixtureWith2Members(streamsGroupMaxSize); + String groupId = "streams-group"; + + String activeMemberId = "active-member"; + String activeInstanceId = "active-instance"; + + String leftMemberId = "left-member"; + String leftInstanceId = "left-instance"; + String newJoinMemberId = "new-join-member"; + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 1); + + MockTaskAssignor assignor = new MockTaskAssignor("sticky"); + assignor.prepareGroupAssignment(Map.of( + activeMemberId, TasksTuple.EMPTY, + leftMemberId, TasksTuple.EMPTY + )); + + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, streamsGroupMaxSize) + .withStreamsGroup(new StreamsGroupBuilder(groupId, DEFAULT_GROUP_EPOCH) + .withMember(streamsGroupMemberBuilderWithDefaults(activeMemberId, activeInstanceId) + .setMemberEpoch(DEFAULT_MEMBER_EPOCH) + .setPreviousMemberEpoch(DEFAULT_MEMBER_EPOCH - 1) + .build()) + .withMember(streamsGroupMemberBuilderWithDefaults(leftMemberId, leftInstanceId) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(DEFAULT_MEMBER_EPOCH - 1) + .build()) + .withTargetAssignmentEpoch(DEFAULT_GROUP_EPOCH) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withMetadataHash(topic.metadataHash()) + .withValidatedTopologyEpoch(0) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs()) + ) + .build(); + + assertDoesNotThrow(() -> context.streamsGroupHeartbeat(staticJoinHeartbeat( + groupId, + newJoinMemberId, + leftInstanceId, + topic + ))); - fixture.assertJoinSucceeds( - fixture.leftMemberRejoinsWithSameInstanceId() - ); } @Test public void testStreamsStaticJoinWithUnreleasedInstanceThrowsUnreleasedInstanceIdAtMaxSize() { - // If an active static member (epoch=10) still owns the instanceId, + // If an active static member (epoch=10) still owns the instanceId, // a different memberId joining with that instanceId must fail even at max size. - int streamsGroupMaxSize = 2; - StaticMemberFixtureWith2Members fixture = new StaticMemberFixtureWith2Members(streamsGroupMaxSize); + int streamsGroupMaxSize = 1; + String groupId = "streams-group"; + + String activeMemberId = "active-member"; + String activeInstanceId = "active-instance"; + + String newMemberId = "new-member"; + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 1); + + MockTaskAssignor assignor = new MockTaskAssignor("sticky"); + assignor.prepareGroupAssignment(Map.of(activeMemberId, TasksTuple.EMPTY)); + + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, streamsGroupMaxSize) + .withStreamsGroup(new StreamsGroupBuilder(groupId, DEFAULT_GROUP_EPOCH) + .withMember(streamsGroupMemberBuilderWithDefaults(activeMemberId, activeInstanceId) + .setMemberEpoch(DEFAULT_MEMBER_EPOCH) + .setPreviousMemberEpoch(DEFAULT_MEMBER_EPOCH - 1) + .build()) + .withTargetAssignmentEpoch(DEFAULT_GROUP_EPOCH) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withMetadataHash(topic.metadataHash()) + .withValidatedTopologyEpoch(0) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs()) + ) + .build(); - fixture.assertJoinFails( - fixture.newMemberJoinsWithActiveInstanceId(), - UnreleasedInstanceIdException.class + assertThrows( + UnreleasedInstanceIdException.class, + () -> context.streamsGroupHeartbeat(staticJoinHeartbeat( + groupId, + newMemberId, + activeInstanceId, + topic + )) ); } - + @ParameterizedTest @MethodSource("staticMemberReusedInstanceErrorCases") - public void testStaticMemberSendHeartbeatWithVariousEpochThenThrowError(int whenMemberEpoch, Class expectedException) { - StaticMemberFixtureWith2Members fixture = new StaticMemberFixtureWith2Members(3); - // WHEN: same instance id is reused with mismatched member identity/epoch. THEN: expected exception is thrown. - fixture.assertHeartbeatFails( - fixture.newMemberHeartbeatsWithActiveInstanceId(whenMemberEpoch), - expectedException + public void testStaticMemberSendHeartbeatWithVariousEpochThenThrowError( + int heartbeatMemberEpoch, + Class expectedException + ) { + // same instance id is reused with mismatched member identity/epoch. + String groupId = "streams-group"; + + String activeMemberId = "active-member"; + String activeInstanceId = "active-instance"; + String newMemberId = "new-member"; + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 1); + + MockTaskAssignor assignor = new MockTaskAssignor("sticky"); + assignor.prepareGroupAssignment(Map.of(activeMemberId, TasksTuple.EMPTY)); + + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, 2) + .withStreamsGroup(new StreamsGroupBuilder(groupId, DEFAULT_GROUP_EPOCH) + .withMember(streamsGroupMemberBuilderWithDefaults(activeMemberId, activeInstanceId) + .setMemberEpoch(DEFAULT_MEMBER_EPOCH) + .setPreviousMemberEpoch(DEFAULT_MEMBER_EPOCH - 1) + .build()) + .withTargetAssignmentEpoch(DEFAULT_GROUP_EPOCH) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withMetadataHash(topic.metadataHash()) + .withValidatedTopologyEpoch(0) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs()) + ) + .build(); + + assertThrows( + expectedException, + () -> context.streamsGroupHeartbeat(staticHeartbeat( + groupId, + newMemberId, + activeInstanceId, + heartbeatMemberEpoch + )) ); } - + private static Stream staticMemberReusedInstanceErrorCases() { return Stream.of( Arguments.of(0, UnreleasedInstanceIdException.class), // static member try to join when static member already existed, then throw UnreleasedInstanceIdException. @@ -207,7 +383,14 @@ private void testStaticMemberJoinThenRevokeAndReceiveTasksWith2Members(int maxSi ); assertResponseEquals( - heartbeatResponseWithActiveTasks(memberId2, 11, List.of()), + new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId2) + .setMemberEpoch(11) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(List.of()) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()), joinResult.response().data() ); @@ -220,7 +403,17 @@ private void testStaticMemberJoinThenRevokeAndReceiveTasksWith2Members(int maxSi .setMemberEpoch(10) ); - assertResponseEquals(heartbeatResponseWithActiveTasks(memberId1, 10, topic, 0, 1), revokeInstructionResult.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId1) + .setMemberEpoch(10) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(topic.responseTasks(0, 1)) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()), + revokeInstructionResult.response().data() + ); // 3) member1 acknowledges revocation by reporting owned active tasks [0,1]. CoordinatorResult revokeAckResult = context.streamsGroupHeartbeat( @@ -249,7 +442,17 @@ private void testStaticMemberJoinThenRevokeAndReceiveTasksWith2Members(int maxSi staticHeartbeat(groupId, memberId2, instanceId2, 11) ); - assertResponseEquals(heartbeatResponseWithActiveTasks(memberId2, 11, topic, 2, 3), member2ReceiveResult.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId2) + .setMemberEpoch(11) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(topic.responseTasks(2, 3)) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()), + member2ReceiveResult.response().data() + ); // 5) member2 leave. CoordinatorResult member2LeaveResult = context.streamsGroupHeartbeat( @@ -257,7 +460,14 @@ private void testStaticMemberJoinThenRevokeAndReceiveTasksWith2Members(int maxSi ); assertResponseEquals( - staticLeaveResponseWithNullTasks(memberId2, LEAVE_GROUP_STATIC_MEMBER_EPOCH).setHeartbeatIntervalMs(0), + new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId2) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setStatus(List.of()) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null) + .setHeartbeatIntervalMs(0), member2LeaveResult.response().data() ); @@ -267,7 +477,14 @@ private void testStaticMemberJoinThenRevokeAndReceiveTasksWith2Members(int maxSi ); assertResponseEquals( - heartbeatResponseWithActiveTasks(otherMemberId2, 11, topic, 2, 3), + new StreamsGroupHeartbeatResponseData() + .setMemberId(otherMemberId2) + .setMemberEpoch(11) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(topic.responseTasks(2, 3)) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()), member2rejoinResult.response().data() ); } @@ -290,14 +507,24 @@ public void testStaticMemberRejoinWithUpdatedProcessIdBumpsStreamsGroupEpoch() { TasksTuple targetAssignment = topic.targetAssignment(0, 1, 2, 3); MockTaskAssignor assignor = new MockTaskAssignor("sticky"); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) - .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setPreviousMemberEpoch(groupEpoch) - .setProcessId(oldProcessId) - .setAssignedTasks(assignedTasks) - .build()) - .withTargetAssignment(oldMemberId, targetAssignment)); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(groupEpoch) + .setProcessId(oldProcessId) + .setAssignedTasks(assignedTasks) + .build()) + .withTargetAssignment(oldMemberId, targetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); assignor.prepareGroupAssignment(Map.of(rejoinMemberId, targetAssignment)); @@ -328,6 +555,77 @@ public void testStaticMemberRejoinWithUpdatedProcessIdBumpsStreamsGroupEpoch() { )); } + @Test + public void testStaticMemberRejoinWithUpdatedProcessIdBumpsStreamsGroupEpoch1() throws UnknownHostException { + String groupId = "fooup"; + int groupEpoch = DEFAULT_GROUP_EPOCH; + + String oldMemberId = Uuid.randomUuid().toString(); + String rejoinMemberId = Uuid.randomUuid().toString(); + String instanceId = Uuid.randomUuid().toString(); + + String processId = "process-id"; + String newClientId = "new-client-id"; + InetAddress newClientAddress = InetAddress.getByName("127.0.0.2"); + + StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 4); + TasksTupleWithEpochs assignedTasks = topic.assignedTasks(groupEpoch, 0, 1, 2, 3); + TasksTuple targetAssignment = topic.targetAssignment(0, 1, 2, 3); + + MockTaskAssignor assignor = new MockTaskAssignor("sticky"); + assignor.prepareGroupAssignment(Map.of(rejoinMemberId, targetAssignment)); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) + .setClientId(DEFAULT_CLIENT_ID) + .setClientHost(DEFAULT_CLIENT_ADDRESS.toString()) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setPreviousMemberEpoch(groupEpoch) + .setProcessId(processId) + .setAssignedTasks(assignedTasks) + .build()) + .withTargetAssignment(oldMemberId, targetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); + + CoordinatorResult result = context.streamsGroupHeartbeat( + staticJoinHeartbeat(groupId, rejoinMemberId, instanceId, processId), + newClientId, + newClientAddress + ); + + assertEquals(rejoinMemberId, result.response().data().memberId()); + assertEquals(groupEpoch, result.response().data().memberEpoch()); + + Optional updatedMemberMetadataValue = result.records().stream() + .filter(record -> record.key() instanceof StreamsGroupMemberMetadataKey) + .filter(record -> ((StreamsGroupMemberMetadataKey) record.key()).memberId().equals(rejoinMemberId)) + .filter(record -> record.value() != null) + .map(record -> (StreamsGroupMemberMetadataValue) record.value().message()) + .filter(value -> newClientId.equals(value.clientId())) + .filter(value -> newClientAddress.toString().equals(value.clientHost())) + .findFirst(); + + assertTrue( + updatedMemberMetadataValue.isPresent(), + "Expected a StreamsGroupMemberMetadata record with the updated client id/host." + ); + + assertTrue( + result.records().stream().noneMatch(record -> record.key() instanceof StreamsGroupMetadataKey), + "Expected no StreamsGroupMetadata record when only client id/host changes." + ); + + assertEquals(groupEpoch, result.response().data().memberEpoch()); + } + @Test public void testStaticMemberLeaveWithMinusOneFencesMemberAndBumpsStreamsGroupEpoch() { String groupId = "fooup"; @@ -341,19 +639,35 @@ public void testStaticMemberLeaveWithMinusOneFencesMemberAndBumpsStreamsGroupEpo TasksTupleWithEpochs assignedTasks = topic.assignedTasks(groupEpoch, 0, 1, 2, 3); TasksTuple targetAssignment = topic.targetAssignment(0, 1, 2, 3); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setMemberEpoch(groupEpoch) - .setPreviousMemberEpoch(groupEpoch - 1) - .setAssignedTasks(assignedTasks) - .build()) - .withTargetAssignment(memberId, targetAssignment)); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(groupEpoch) + .setPreviousMemberEpoch(groupEpoch - 1) + .setAssignedTasks(assignedTasks) + .build()) + .withTargetAssignment(memberId, targetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); CoordinatorResult result = context.streamsGroupHeartbeat( staticHeartbeat(groupId, memberId, instanceId, LEAVE_GROUP_MEMBER_EPOCH) ); - assertResponseEquals(staticLeaveResponse(memberId, LEAVE_GROUP_MEMBER_EPOCH), result.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId) + .setMemberEpoch(LEAVE_GROUP_MEMBER_EPOCH) + .setStatus(List.of()), + result.response().data() + ); assertRecordsEquals( List.of( @@ -369,7 +683,6 @@ public void testStaticMemberLeaveWithMinusOneFencesMemberAndBumpsStreamsGroupEpo @Test public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpoch() { - // GIVEN int leaveEpoch = LEAVE_GROUP_STATIC_MEMBER_EPOCH; int memberEpoch = DEFAULT_MEMBER_EPOCH; int groupEpoch = DEFAULT_GROUP_EPOCH; @@ -382,22 +695,36 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpoch() { TasksTupleWithEpochs assignedTasks = topic.assignedTasks(groupEpoch, 0, 1, 2, 3); TasksTupleWithEpochs pendingRevocationTasks = topic.assignedTasks(groupEpoch, 2, 3); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setMemberEpoch(memberEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setAssignedTasks(assignedTasks) - .setTasksPendingRevocation(pendingRevocationTasks) - .build()) - .withTargetAssignment(memberId, topic.targetAssignment(0, 1, 2, 3))); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(assignedTasks) + .setTasksPendingRevocation(pendingRevocationTasks) + .build()) + .withTargetAssignment(memberId, topic.targetAssignment(0, 1, 2, 3)) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); - // WHEN CoordinatorResult result = context.streamsGroupHeartbeat( staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) ); - // THEN - assertResponseEquals(staticLeaveResponse(memberId, leaveEpoch), result.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId) + .setMemberEpoch(leaveEpoch) + .setStatus(List.of()), + result.response().data() + ); // No group epoch bump. // Member epoch should be -2. @@ -418,7 +745,6 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpoch() { @Test public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochThenShouldBeIdempotence() { - // GIVEN int leaveEpoch = LEAVE_GROUP_STATIC_MEMBER_EPOCH; int memberEpoch = DEFAULT_MEMBER_EPOCH; int groupEpoch = DEFAULT_GROUP_EPOCH; @@ -438,14 +764,29 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochThenShouldBeIdem .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) .build(); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(alreadyLeftStaticMember) - .withTargetAssignment(memberId, targetAssignment)); - + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(alreadyLeftStaticMember) + .withTargetAssignment(memberId, targetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); + CoordinatorResult result = context.streamsGroupHeartbeat(staticHeartbeat(groupId, memberId, instanceId, leaveEpoch)); - // THEN - assertResponseEquals(staticLeaveResponse(memberId, leaveEpoch), result.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId) + .setMemberEpoch(leaveEpoch) + .setStatus(List.of()), + result.response().data() + ); assertRecordsEquals( List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, alreadyLeftStaticMember)), @@ -479,7 +820,6 @@ private void verifyStaticMemberLeaveAndRejoinNoGroupBump(String instanceId, Stri * 3. replacement/tombstone records are written as expected. */ - // GIVEN int leaveEpoch = LEAVE_GROUP_STATIC_MEMBER_EPOCH; int memberEpoch = DEFAULT_MEMBER_EPOCH; int groupEpoch = DEFAULT_GROUP_EPOCH; @@ -489,50 +829,84 @@ private void verifyStaticMemberLeaveAndRejoinNoGroupBump(String instanceId, Stri String subtopology1 = "subtopology1"; StreamsTopicFixture topic = streamsTopicFixture(subtopology1, "foo", 4); - // GIVEN Task TasksTupleWithEpochs givenAssignedTasks = topic.assignedTasks(memberEpoch, 0, 1, 2, 3); TasksTuple givenTargetAssignment = topic.targetAssignment(0, 1, 2, 3); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) - .setMemberEpoch(memberEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setAssignedTasks(givenAssignedTasks) - .build()) - .withTargetAssignment(oldMemberId, givenTargetAssignment)); - - // WHEN1 : normal heart beat. + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(oldMemberId, instanceId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(givenAssignedTasks) + .build()) + .withTargetAssignment(oldMemberId, givenTargetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); + CoordinatorResult normalHeartbeatResult = context.streamsGroupHeartbeat( staticHeartbeat(groupId, oldMemberId, instanceId, memberEpoch) ); - // THEN1 : - // - all tasks should be null because assigned tasks unchanged. - // - Keep the group epoch. - assertResponseEquals(heartbeatResponseWithNullTasks(oldMemberId, memberEpoch), normalHeartbeatResult.response().data()); + + // all tasks should be null because assigned tasks unchanged. + // Keep the group epoch. + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(oldMemberId) + .setMemberEpoch(memberEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null), + normalHeartbeatResult.response().data()); assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); - // WHEN2 : Stream Member leave with -2 + // Stream member leaves with epoch -2. CoordinatorResult leaveResult = context.streamsGroupHeartbeat( staticHeartbeat(groupId, oldMemberId, instanceId, leaveEpoch) ); - // THEN2 - // - Keep the group epoch. - assertResponseEquals(staticLeaveResponseWithNullTasks(oldMemberId, leaveEpoch), leaveResult.response().data()); + // Keep the group epoch. + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(oldMemberId) + .setMemberEpoch(leaveEpoch) + .setStatus(List.of()) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null), + leaveResult.response().data() + ); assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); - // WHEN3 : Streams Member rejoin with other memberId + // Streams Member rejoin with other memberId CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) ); - // THEN3 : - // - Inherit previous member's member epoch, and assigned tasks. - // - Keep the member epoch bump. - // - Keep the group epoch bump. - assertResponseEquals(heartbeatResponseWithActiveTasks(newMemberId, memberEpoch, topic, 0, 1, 2, 3), rejoinResult.response().data()); + // Inherit previous member's member epoch, and assigned tasks. + // Keep the member epoch bump. + // Keep the group epoch bump. + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(newMemberId) + .setMemberEpoch(memberEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(topic.responseTasks(0, 1, 2, 3)) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()), + rejoinResult.response().data() + ); assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); StreamsGroupMember newJoinStaticMember = streamsGroupMemberBuilderWithDefaults(newMemberId, instanceId) @@ -584,23 +958,40 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnrevokedSta .setTasksPendingRevocation(tasksPendingRevocation) .build(); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(unrevokedMember) - .withTargetAssignment(memberId, targetAssignment)); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(unrevokedMember) + .withTargetAssignment(memberId, targetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); - // WHEN CoordinatorResult result = context.streamsGroupHeartbeat( staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) ); - // THEN StreamsGroupMember expectedMember = streamsGroupMemberBuilderWithDefaults(memberId, instanceId) .setMemberEpoch(leaveEpoch) .setPreviousMemberEpoch(memberEpoch - 1) .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) .build(); - assertResponseEquals(staticLeaveResponseWithNullTasks(memberId, leaveEpoch), result.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId) + .setMemberEpoch(leaveEpoch) + .setStatus(List.of()) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null), + result.response().data() + ); assertRecordsEquals( List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedMember)), result.records() @@ -630,15 +1021,35 @@ public void testStaticMemberRejoinsAfterTemporaryLeave() { .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) .build(); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(temporarilyLeftMember) - .withTargetAssignment(oldMemberId, targetAssignment)); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(temporarilyLeftMember) + .withTargetAssignment(oldMemberId, targetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); CoordinatorResult result = context.streamsGroupHeartbeat( staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) ); - assertResponseEquals(heartbeatResponseWithActiveTasks(newMemberId, memberEpoch, topic, 0, 1), result.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(newMemberId) + .setMemberEpoch(memberEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(topic.responseTasks(0, 1)) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()), + result.response().data() + ); assertEquals(MemberState.STABLE, context.streamsGroupMemberState(groupId, newMemberId)); assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); } @@ -667,9 +1078,19 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnreleasedSt .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) .build(); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(unreleasedMember) - .withTargetAssignment(memberId, targetAssignment)); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(unreleasedMember) + .withTargetAssignment(memberId, targetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); CoordinatorResult result = context.streamsGroupHeartbeat( staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) @@ -684,7 +1105,16 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnreleasedSt .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) .build(); - assertResponseEquals(staticLeaveResponseWithNullTasks(memberId, leaveEpoch), result.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId) + .setMemberEpoch(leaveEpoch) + .setStatus(List.of()) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null), + result.response().data() + ); assertRecordsEquals( List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedMember)), result.records() @@ -729,17 +1159,37 @@ public void testStaticMemberRejoinsAfterTemporaryLeaveFromUnreleasedState() { .setTasksPendingRevocation(otherTasksPendingRevocation) .build(); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(temporarilyLeftMember) - .withMember(otherMember) - .withTargetAssignment(oldMemberId, targetAssignment) - .withTargetAssignment(otherMemberId, TasksTuple.EMPTY)); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(temporarilyLeftMember) + .withMember(otherMember) + .withTargetAssignment(oldMemberId, targetAssignment) + .withTargetAssignment(otherMemberId, TasksTuple.EMPTY) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); CoordinatorResult result = context.streamsGroupHeartbeat( staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) ); - assertResponseEquals(heartbeatResponseWithActiveTasks(newMemberId, memberEpoch, topic, 0, 1), result.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(newMemberId) + .setMemberEpoch(memberEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(topic.responseTasks(0, 1)) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()), + result.response().data() + ); assertEquals(MemberState.UNRELEASED_TASKS, context.streamsGroupMemberState(groupId, newMemberId)); assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); } @@ -759,36 +1209,51 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnrevokedSta String subtopology1 = "subtopology1"; StreamsTopicFixture topic = streamsTopicFixture(subtopology1, "foo", 3); - // GIVEN Tasks TasksTupleWithEpochs assignedTasks = topic.assignedTasks(memberEpoch, 0, 1); TasksTupleWithEpochs tasksPendingRevocation = topic.assignedTasks(memberEpoch, 2); TasksTuple leavingTargetAssignment = topic.targetAssignment(0, 1); TasksTuple waitingTargetAssignment = topic.targetAssignment(2); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(leavingMemberId, instanceId) - .setMemberEpoch(memberEpoch) - .setPreviousMemberEpoch(memberEpoch - 1) - .setState(MemberState.UNREVOKED_TASKS) - .setAssignedTasks(assignedTasks) - .setTasksPendingRevocation(tasksPendingRevocation) - .build()) - .withMember(StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults(waitingMemberId) - .setMemberEpoch(memberEpoch) - .setPreviousMemberEpoch(memberEpoch) - .setState(MemberState.UNRELEASED_TASKS) - .build()) - .withTargetAssignment(leavingMemberId, leavingTargetAssignment) - .withTargetAssignment(waitingMemberId, waitingTargetAssignment)); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(leavingMemberId, instanceId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch - 1) + .setState(MemberState.UNREVOKED_TASKS) + .setAssignedTasks(assignedTasks) + .setTasksPendingRevocation(tasksPendingRevocation) + .build()) + .withMember(StreamsGroupTestUtil.streamsGroupMemberBuilderWithDefaults(waitingMemberId) + .setMemberEpoch(memberEpoch) + .setPreviousMemberEpoch(memberEpoch) + .setState(MemberState.UNRELEASED_TASKS) + .build()) + .withTargetAssignment(leavingMemberId, leavingTargetAssignment) + .withTargetAssignment(waitingMemberId, waitingTargetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); - // WHEN1 - leave CoordinatorResult leaveResult = context.streamsGroupHeartbeat( staticHeartbeat(groupId, leavingMemberId, instanceId, leaveEpoch) ); - // THEN1 - StreamsGroupHeartbeatResponseData expectedLeavingResponse = staticLeaveResponseWithNullTasks(leavingMemberId, leaveEpoch); - assertResponseEquals(expectedLeavingResponse, leaveResult.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(leavingMemberId) + .setMemberEpoch(leaveEpoch) + .setStatus(List.of()) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null), + leaveResult.response().data() + ); List expectedRecordsTriggeredByLeave = List.of( StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, streamsGroupMemberBuilderWithDefaults(leavingMemberId, instanceId) @@ -799,14 +1264,20 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnrevokedSta assertRecordsEquals(expectedRecordsTriggeredByLeave, leaveResult.records()); assertEquals(MemberState.STABLE, context.streamsGroupMemberState(groupId, leavingMemberId)); - // When2 - Waiting member send a heartbeat expecting get unreleased tasks. + // Waiting member send a heartbeat expecting get unreleased tasks. CoordinatorResult waitingMemberResult = context.streamsGroupHeartbeat( staticHeartbeat(groupId, waitingMemberId, null, memberEpoch) ); - // THEN2 - StreamsGroupHeartbeatResponseData expectedWaitngMemberResponse = heartbeatResponseWithActiveTasks(waitingMemberId, memberEpoch, topic, 2); - assertResponseEquals(expectedWaitngMemberResponse, waitingMemberResult.response().data()); + StreamsGroupHeartbeatResponseData expectedWaitingMemberResponse = new StreamsGroupHeartbeatResponseData() + .setMemberId(waitingMemberId) + .setMemberEpoch(memberEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(topic.responseTasks(2)) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()); + assertResponseEquals(expectedWaitingMemberResponse, waitingMemberResult.response().data()); List expectedRecordsTriggeredByWaitngMember = List.of( StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, @@ -825,7 +1296,6 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnrevokedSta @Test public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochAndRejoinAndOtherRackIdThenGroupBumpOccur() { - // GIVEN int leaveEpoch = LEAVE_GROUP_STATIC_MEMBER_EPOCH; int memberEpoch = DEFAULT_MEMBER_EPOCH; int groupEpoch = DEFAULT_GROUP_EPOCH; @@ -837,49 +1307,63 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochAndRejoinAndOthe StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", 4); - // GIVEN Task TasksTupleWithEpochs givenAssignedTasks = topic.assignedTasks(memberEpoch, 0, 1, 2, 3); TasksTuple givenTargetAssignment = topic.targetAssignment(0, 1, 2, 3); - // GIVEN Assignor MockTaskAssignor assignor = new MockTaskAssignor("sticky"); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(memberEpoch) + .setRackId(rackId) + .setPreviousMemberEpoch(memberEpoch - 1) + .setAssignedTasks(givenAssignedTasks) + .build()) + .withTargetAssignment(memberId, givenTargetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); - long groupMetadataHash = topic.metadataHash(); - - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setMemberEpoch(memberEpoch) - .setRackId(rackId) - .setPreviousMemberEpoch(memberEpoch - 1) - .setAssignedTasks(givenAssignedTasks) - .build()) - .withTargetAssignment(memberId, givenTargetAssignment)); - - // WHEN1 : normal heart beat. CoordinatorResult normalHeartbeatResult = context.streamsGroupHeartbeat( staticHeartbeat(groupId, memberId, instanceId, memberEpoch) .setRackId(rackId) ); - // THEN1 : - // - all tasks should be null because assigned tasks unchanged. - // - Keep the group epoch. - assertResponseEquals(heartbeatResponseWithNullTasks(memberId, memberEpoch), normalHeartbeatResult.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId) + .setMemberEpoch(memberEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null), + normalHeartbeatResult.response().data()); assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); - - // WHEN2 : Stream Member leave with -2 CoordinatorResult leaveResult = context.streamsGroupHeartbeat( staticHeartbeat(groupId, memberId, instanceId, leaveEpoch) .setRackId(rackId) ); - // THEN2 - // - Keep the group epoch. - assertResponseEquals(staticLeaveResponseWithNullTasks(memberId, leaveEpoch), leaveResult.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId) + .setMemberEpoch(leaveEpoch) + .setStatus(List.of()) + .setActiveTasks(null) + .setWarmupTasks(null) + .setStandbyTasks(null), + leaveResult.response().data() + ); assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); - // GIVEN3 String newMemberId = Uuid.randomUuid().toString(); String newRackId = Uuid.randomUuid().toString(); assignor.prepareGroupAssignment(Map.of(newMemberId, givenTargetAssignment)); @@ -887,17 +1371,26 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochAndRejoinAndOthe int bumpedGroupEpoch = groupEpoch + 1; int bumpedMemberEpoch = memberEpoch + 1; - // WHEN3 : Streams Member rejoin with other memberId and rackId + // Streams Member rejoin with other memberId and rackId CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( staticHeartbeat(groupId, newMemberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) .setRackId(newRackId) ); - // THEN3 : - // - Inherit previous member's member epoch, and assigned tasks. - // - member epoch should be bumped. - // - group epoch should be bumped. - assertResponseEquals(heartbeatResponseWithActiveTasks(newMemberId, bumpedMemberEpoch, topic, 0, 1, 2, 3), rejoinResult.response().data()); + // Inherit previous member's member epoch, and assigned tasks. + // member epoch should be bumped. + // group epoch should be bumped. + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(newMemberId) + .setMemberEpoch(bumpedMemberEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(topic.responseTasks(0, 1, 2, 3)) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()), + rejoinResult.response().data() + ); assertEquals(bumpedGroupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); StreamsGroupMember transationStaticInitMember = streamsGroupMemberBuilderWithDefaults(newMemberId, instanceId) @@ -923,7 +1416,7 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochAndRejoinAndOthe assertRecordsEquals( List.of( - // From eplaceStreamsMembers + // From replaceStreamsMembers StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, memberId), StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, memberId), StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, memberId), @@ -934,7 +1427,7 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochAndRejoinAndOthe // From hasStreamsMemberMetadataChanged StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, newJoinStaticMember), - StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, bumpedGroupEpoch, groupMetadataHash, 0, getDefaultAssignmentConfigs()), + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, bumpedGroupEpoch, topic.metadataHash(), 0, getDefaultAssignmentConfigs()), StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, newJoinStaticMember.memberId(), givenTargetAssignment), StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, bumpedGroupEpoch, context.time.milliseconds()), StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, reconciledMember) @@ -964,9 +1457,19 @@ public void testStaticMemberRejoinWritesReplacementRecordsInStreamsGroup() { .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) .build(); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(oldMember) - .withTargetAssignment(oldMemberId, oldTargetAssignment)); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(oldMember) + .withTargetAssignment(oldMemberId, oldTargetAssignment) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); CoordinatorResult result = context.streamsGroupHeartbeat( staticJoinHeartbeat(groupId, rejoinMemberId, instanceId, DEFAULT_PROCESS_ID) @@ -1084,20 +1587,28 @@ private void verifyStreamsStaticMemberHeartbeatWithOwnedActiveTasksAtPreviousEpo String memberId = Uuid.randomUuid().toString(); String instanceId = Uuid.randomUuid().toString(); - // GIVEN TASK and Topology StreamsTopicFixture topic = streamsTopicFixture("subtopology1", "foo", partitionSize); TasksTupleWithEpochs givenAssignedTask = topic.assignedTasks(groupEpoch, givenTaskIds); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, group -> group - .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) - .setMemberEpoch(currentMemberEpoch) - .setPreviousMemberEpoch(previousMemberEpoch) - .setAssignedTasks(givenAssignedTask) - .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) - .build()) - .withTargetAssignment(memberId, topic.targetAssignment(givenTaskIds))); + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(new MockTaskAssignor("sticky"))) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withMember(streamsGroupMemberBuilderWithDefaults(memberId, instanceId) + .setMemberEpoch(currentMemberEpoch) + .setPreviousMemberEpoch(previousMemberEpoch) + .setAssignedTasks(givenAssignedTask) + .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) + .build()) + .withTargetAssignment(memberId, topic.targetAssignment(givenTaskIds)) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT) + .build(); - // WHEN StreamsGroupHeartbeatRequestData requestData = staticHeartbeat(groupId, memberId, instanceId, requestMemberEpoch) .setProcessId("process-id") .setRebalanceTimeoutMs(1500) @@ -1106,7 +1617,6 @@ private void verifyStreamsStaticMemberHeartbeatWithOwnedActiveTasksAtPreviousEpo .setStandbyTasks(List.of()) .setWarmupTasks(List.of()); - // THEN if (expectedException != null) { assertThrows(expectedException, () -> context.streamsGroupHeartbeat(requestData)); } else { @@ -1130,32 +1640,35 @@ public void testStreamsStaticMemberTemporaryLeaveSessionTimeoutExpiration() { assignor.prepareGroupAssignment(Map.of(memberId, topic.targetAssignment(0, 1, 2, 3))); - // WHEN1 : static member joins (session timeout should be scheduled) + // static member joins (session timeout should be scheduled) CoordinatorResult firstJoinResult = context.streamsGroupHeartbeat( staticJoinHeartbeat(groupId, memberId, instanceId, topic).setRebalanceTimeoutMs(90000) ); - // THEN1 - // - member epoch should be bumped up. - // - session timeout should be 45000ms. + // member epoch should be bumped up. + // session timeout should be 45000ms. assertEquals(2, firstJoinResult.response().data().memberEpoch()); context.assertSessionTimeout(groupId, memberId, 45000); - // WHEN2: static member leaves temporarily. + // static member leaves temporarily. CoordinatorResult temporaryLeaveResult = context.streamsGroupHeartbeat( staticHeartbeat(groupId, memberId, instanceId, LEAVE_GROUP_STATIC_MEMBER_EPOCH) ); - // THEN2: // member epoch should be -2. // session timeout still 45000ms. - assertResponseEquals(staticLeaveResponse(memberId, LEAVE_GROUP_STATIC_MEMBER_EPOCH), temporaryLeaveResult.response().data()); + assertResponseEquals( + new StreamsGroupHeartbeatResponseData() + .setMemberId(memberId) + .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) + .setStatus(List.of()), + temporaryLeaveResult.response().data() + ); context.assertSessionTimeout(groupId, memberId, 45000); - // WHEN3: no rejoin, session timeout expires. + // no rejoin, session timeout expires. List> timeouts = context.sleep(45000 + 1); - // THEN3 List expectedRecords = List.of( StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, memberId), StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, memberId), @@ -1195,7 +1708,6 @@ public void testStaticMemberJoinEmptyStreamsGroupRegistersStaticMember1() { context.groupMetadataManager.streamsGroup(groupId) ); - // WHEN context.streamsGroupHeartbeat( staticHeartbeat(groupId, memberId, instanceId, JOIN_GROUP_MEMBER_EPOCH) .setProcessId(DEFAULT_PROCESS_ID) @@ -1206,7 +1718,6 @@ public void testStaticMemberJoinEmptyStreamsGroupRegistersStaticMember1() { .setWarmupTasks(List.of()) ); - // THEN StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); assertEquals(memberId, group.staticMember(instanceId).memberId()); assertEquals(Optional.of(instanceId), group.getMemberOrThrow(memberId).instanceId()); @@ -1241,10 +1752,19 @@ public void testStaticMemberRejoinUpdatesUserEndpointInformationEpoch( MockTaskAssignor assignor = new MockTaskAssignor("sticky"); assignor.prepareGroupAssignment(Map.of(oldMemberId, topic.targetAssignment(0, 1, 2))); - GroupMetadataManagerTestContext context = contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, 0, group -> group - .withTargetAssignment(oldMemberId, targetAssignment) - ); - + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withStreamsGroupTaskAssignors(List.of(assignor)) + .withMetadataImage(topic.metadataImage()) + .withStreamsGroup(new StreamsGroupBuilder(groupId, groupEpoch) + .withTargetAssignmentEpoch(groupEpoch) + .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) + .withValidatedTopologyEpoch(0) + .withMetadataHash(topic.metadataHash()) + .withTargetAssignment(oldMemberId, targetAssignment) + .withLastAssignmentConfigs(getDefaultAssignmentConfigs())) + .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, 0) + .build(); + assertEquals(0, context.groupMetadataManager.streamsGroup(groupId).endpointInformationEpoch()); // First Join -> First Input @@ -1255,7 +1775,14 @@ public void testStaticMemberRejoinUpdatesUserEndpointInformationEpoch( // First Check assertResponseEquals( - heartbeatResponseWithActiveTasks(oldMemberId, bumpedEpoch, topic, 0, 1, 2) + new StreamsGroupHeartbeatResponseData() + .setMemberId(oldMemberId) + .setMemberEpoch(bumpedEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(topic.responseTasks(0, 1, 2)) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()) .setEndpointInformationEpoch(firstExpectedUserEndpointEpoch) // first endpoint epoch .setPartitionsByUserEndpoint(firstExpectedPartitionsByUserEndpoint), // first partitions by user endpoint result.response().data() @@ -1268,20 +1795,26 @@ public void testStaticMemberRejoinUpdatesUserEndpointInformationEpoch( } assertEquals(firstExpectedUserEndpointEpoch, context.groupMetadataManager.streamsGroup(groupId).endpointInformationEpoch()); - // static leave + // static member leaves with epoch -2. context.streamsGroupHeartbeat( staticHeartbeat(groupId, oldMemberId, instanceId, StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) ); - // second - static member rejoins + // static member rejoins. CoordinatorResult rejoinResult = context.streamsGroupHeartbeat( staticJoinHeartbeat(groupId, rejoinMemberId, instanceId, topic) .setUserEndpoint(secondUserEndpoint) ); - // second check. assertResponseEquals( - heartbeatResponseWithActiveTasks(rejoinMemberId, bumpedEpoch, topic, 0, 1, 2) + new StreamsGroupHeartbeatResponseData() + .setMemberId(rejoinMemberId) + .setMemberEpoch(bumpedEpoch) + .setHeartbeatIntervalMs(5000) + .setTaskOffsetIntervalMs(60000) + .setActiveTasks(topic.responseTasks(0, 1, 2)) + .setWarmupTasks(List.of()) + .setStandbyTasks(List.of()) .setEndpointInformationEpoch(secondExpectedUserEndpointEpoch) .setPartitionsByUserEndpoint(secondExpectedPartitionsByUserEndpoint), rejoinResult.response().data() @@ -1350,106 +1883,6 @@ private static Stream userEndpointTestCases() { ); } - private static class StaticMemberFixtureWith2Members { - // a single static member is active, the other static member leaves with -2. - private static final String GROUP_ID = "streams-group"; - private static final int GROUP_EPOCH = 10; - private static final int MEMBER_EPOCH = 10; - private static final int PREVIOUS_MEMBER_EPOCH = 9; - private static final int REBALANCE_TIMEOUT_MS = 1500; - - private final String activeMemberId = "active-member"; - private final String activeInstanceId = "active-instance"; - - private final String leftMemberId = "left-member"; - private final String leftInstanceId = "left-instance"; - - private final String newMemberId = "new-member"; - private final String newInstanceId = "new-instance"; - - private final StreamsGroupHeartbeatRequestData.Topology topology = - new StreamsGroupHeartbeatRequestData.Topology().setSubtopologies(List.of()); - - private final GroupMetadataManagerTestContext context; - - private StaticMemberFixtureWith2Members(int streamsGroupMaxSize) { - MockTaskAssignor assignor = new MockTaskAssignor("sticky"); - assignor.prepareGroupAssignment(Map.of(activeMemberId, TasksTuple.EMPTY)); - - this.context = new GroupMetadataManagerTestContext.Builder() - .withStreamsGroupTaskAssignors(List.of(assignor)) - .withMetadataImage(new MetadataImageBuilder().buildCoordinatorMetadataImage()) - .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_MAX_SIZE_CONFIG, streamsGroupMaxSize) - .withStreamsGroup(new StreamsGroupBuilder(GROUP_ID, GROUP_EPOCH) - .withMember(streamsGroupMemberBuilderWithDefaults(activeMemberId, activeInstanceId) - .setMemberEpoch(MEMBER_EPOCH) - .setPreviousMemberEpoch(PREVIOUS_MEMBER_EPOCH) - .build()) - .withMember(streamsGroupMemberBuilderWithDefaults(leftMemberId, leftInstanceId) - .setMemberEpoch(StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setPreviousMemberEpoch(PREVIOUS_MEMBER_EPOCH) - .build()) - .withTargetAssignmentEpoch(GROUP_EPOCH) - .withTopology(StreamsTopology.fromHeartbeatRequest(topology)) - ) - .build(); - } - - private StreamsGroupHeartbeatRequestData newMemberJoinsWithNewInstanceId() { - return joinRequest(newMemberId, newInstanceId); - } - - private StreamsGroupHeartbeatRequestData leftMemberRejoinsWithSameInstanceId() { - return joinRequest(leftMemberId, leftInstanceId); - } - - private StreamsGroupHeartbeatRequestData newMemberJoinsWithActiveInstanceId() { - return joinRequest(newMemberId, activeInstanceId); - } - - private StreamsGroupHeartbeatRequestData newMemberHeartbeatsWithActiveInstanceId(int memberEpoch) { - return request(newMemberId, activeInstanceId, memberEpoch); - } - - private StreamsGroupHeartbeatRequestData joinRequest(String memberId, String instanceId) { - return request(memberId, instanceId, StreamsGroupHeartbeatRequest.JOIN_GROUP_MEMBER_EPOCH); - } - - private StreamsGroupHeartbeatRequestData staticLeaveRequest(String memberId, String instanceId) { - return request(memberId, instanceId, StreamsGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH); - } - - private StreamsGroupHeartbeatRequestData request(String memberId, String instanceId, int memberEpoch) { - return new StreamsGroupHeartbeatRequestData() - .setGroupId(GROUP_ID) - .setInstanceId(instanceId) - .setMemberId(memberId) - .setMemberEpoch(memberEpoch) - .setProcessId(DEFAULT_PROCESS_ID) - .setRebalanceTimeoutMs(REBALANCE_TIMEOUT_MS) - .setTopology(topology) - .setActiveTasks(List.of()) - .setStandbyTasks(List.of()) - .setWarmupTasks(List.of()); - } - - private void assertJoinSucceeds(StreamsGroupHeartbeatRequestData request) { - assertDoesNotThrow(() -> context.streamsGroupHeartbeat(request)); - } - - private void assertJoinFails(StreamsGroupHeartbeatRequestData request, Class expectedException) { - assertHeartbeatFails(request, expectedException); - } - - private void assertHeartbeatFails(StreamsGroupHeartbeatRequestData request, Class expectedException) { - assertThrows(expectedException, () -> context.streamsGroupHeartbeat(request)); - } - - private void assertLeaveFails(StreamsGroupHeartbeatRequestData request, Class expectedException) { - assertHeartbeatFails(request, expectedException); - } - } - private static StreamsGroupHeartbeatRequestData.Endpoint userEndpoint(String host, int port) { return new StreamsGroupHeartbeatRequestData.Endpoint() .setHost(host) diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java index 69a1a3694bbb9..5276ab6b00a3c 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupTestUtil.java @@ -23,10 +23,7 @@ import org.apache.kafka.coordinator.common.runtime.CoordinatorMetadataImage; import org.apache.kafka.coordinator.common.runtime.MetadataImageBuilder; import org.apache.kafka.coordinator.group.streams.MemberState; -import org.apache.kafka.coordinator.group.streams.MockTaskAssignor; -import org.apache.kafka.coordinator.group.streams.StreamsGroupBuilder; import org.apache.kafka.coordinator.group.streams.StreamsGroupMember; -import org.apache.kafka.coordinator.group.streams.StreamsTopology; import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil; import org.apache.kafka.coordinator.group.streams.TasksTuple; import org.apache.kafka.coordinator.group.streams.TasksTupleWithEpochs; @@ -115,22 +112,6 @@ static StreamsGroupHeartbeatRequestData staticJoinHeartbeat(String groupId, Stri .setWarmupTasks(List.of()); } - static StreamsGroupHeartbeatResponseData staticLeaveResponse(String memberId, int leaveEpoch) { - return new StreamsGroupHeartbeatResponseData() - .setMemberId(memberId) - .setMemberEpoch(leaveEpoch) - .setStatus(List.of()); - } - - - static StreamsGroupHeartbeatResponseData staticLeaveResponseWithNullTasks(String memberId, int leaveEpoch) { - return staticLeaveResponse(memberId, leaveEpoch) - .setActiveTasks(null) - .setWarmupTasks(null) - .setStandbyTasks(null); - } - - static class StreamsTopicFixture { private final String subtopologyId; private final String topicName; @@ -209,110 +190,6 @@ public List requestTasks(List } } - static GroupMetadataManagerTestContext contextWithStreamsGroup( - String groupId, - int groupEpoch, - StreamsTopicFixture topic, - java.util.function.UnaryOperator configureGroup - ) { - return contextWithStreamsGroup( - groupId, - groupEpoch, - topic, - new MockTaskAssignor("sticky"), - configureGroup - ); - } - - static GroupMetadataManagerTestContext contextWithStreamsGroup( - String groupId, - int groupEpoch, - StreamsTopicFixture topic, - MockTaskAssignor assignor, - java.util.function.UnaryOperator configureGroup - ) { - return contextWithStreamsGroup(groupId, groupEpoch, topic, assignor, GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_DEFAULT, configureGroup); - } - - static GroupMetadataManagerTestContext contextWithStreamsGroup( - String groupId, - int groupEpoch, - StreamsTopicFixture topic, - MockTaskAssignor assignor, - int initialRebalanceDelayMs, - java.util.function.UnaryOperator configureGroup - ) { - StreamsGroupBuilder group = new StreamsGroupBuilder(groupId, groupEpoch) - .withTargetAssignmentEpoch(groupEpoch) - .withTopology(StreamsTopology.fromHeartbeatRequest(topic.topology())) - .withValidatedTopologyEpoch(0) - .withMetadataHash(topic.metadataHash()) - .withLastAssignmentConfigs(getDefaultAssignmentConfigs()); - - return new GroupMetadataManagerTestContext.Builder() - .withStreamsGroupTaskAssignors(List.of(assignor)) - .withMetadataImage(topic.metadataImage()) - .withStreamsGroup(configureGroup.apply(group)) - .withConfig(GroupCoordinatorConfig.STREAMS_GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, initialRebalanceDelayMs) - .build(); - } - - static StreamsGroupHeartbeatResponseData heartbeatResponseWithActiveTasks( - String memberId, - int memberEpoch, - StreamsTopicFixture topic, - Integer... activeTasks - ) { - return heartbeatResponseWithActiveTasks( - memberId, - memberEpoch, - topic.responseTasks(activeTasks) - ); - } - - static StreamsGroupHeartbeatResponseData heartbeatResponseWithActiveTasks( - String memberId, - int memberEpoch, - List activeTasks - ) { - return new StreamsGroupHeartbeatResponseData() - .setMemberId(memberId) - .setMemberEpoch(memberEpoch) - .setHeartbeatIntervalMs(5000) - .setTaskOffsetIntervalMs(60000) - .setActiveTasks(activeTasks) - .setWarmupTasks(List.of()) - .setStandbyTasks(List.of()); - } - - static StreamsGroupHeartbeatResponseData heartbeatResponseWithActiveTasks( - String memberId, - int memberEpoch, - String subtopologyId, - Integer... activeTasks - ) { - return heartbeatResponseWithActiveTasks( - memberId, - memberEpoch, - mkResponseTasks(subtopologyId, activeTasks) - ); - } - - - static StreamsGroupHeartbeatResponseData heartbeatResponseWithNullTasks( - String memberId, - int memberEpoch - ) { - return new StreamsGroupHeartbeatResponseData() - .setMemberId(memberId) - .setMemberEpoch(memberEpoch) - .setHeartbeatIntervalMs(5000) - .setTaskOffsetIntervalMs(60000) - .setActiveTasks(null) - .setWarmupTasks(null) - .setStandbyTasks(null); - } - static StreamsGroupHeartbeatRequestData staticJoinHeartbeat( String groupId, String memberId, From 8cd4b4820d90c87f686a468c8f63d102937ff6f4 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Sat, 30 May 2026 00:18:49 +0900 Subject: [PATCH 11/16] Addressing review. --- .../group/StreamsGroupStaticMemberGroupMetadataManagerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java index bf895c090e20e..66536ee261da8 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java @@ -556,7 +556,7 @@ public void testStaticMemberRejoinWithUpdatedProcessIdBumpsStreamsGroupEpoch() { } @Test - public void testStaticMemberRejoinWithUpdatedProcessIdBumpsStreamsGroupEpoch1() throws UnknownHostException { + public void testStaticMemberRejoinWithSameProcessIdDoesNotBumpStreamsGroupEpoch() throws UnknownHostException { String groupId = "fooup"; int groupEpoch = DEFAULT_GROUP_EPOCH; From 7cc07e98d56c9b6d82c8b9f1322bd9140814501b Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Tue, 9 Jun 2026 23:24:34 +0900 Subject: [PATCH 12/16] handle KIP-1331 changes after rebase. --- .../apache/kafka/coordinator/group/GroupMetadataManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java index 4749a85a2a5d3..1c4a277f27772 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java @@ -4459,7 +4459,7 @@ private CoordinatorResult stream } else { log.info("[GroupId {}][MemberId {}] Static member {} with instance id {} left the streams group.", group.groupId(), memberId, memberId, instanceId); - return streamsGroupFenceMember(group, member, new StreamsGroupHeartbeatResult(response, Map.of())); + return streamsGroupFenceMember(group, member, new StreamsGroupHeartbeatResult(response, Map.of(), group.currentTopologyEpoch())); } } } @@ -4539,7 +4539,7 @@ private CoordinatorResult stream return new CoordinatorResult<>( List.of(record), - new StreamsGroupHeartbeatResult(response, Map.of()) + new StreamsGroupHeartbeatResult(response, Map.of(), group.currentTopologyEpoch()) ); } From bf28af3f7804c073335b7a7783df2ead461e9240 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Tue, 9 Jun 2026 23:26:22 +0900 Subject: [PATCH 13/16] handle KIP-1331 test code changes after rebase. --- .../group/GroupMetadataManagerTestContext.java | 7 +++++++ .../StreamsGroupMixedGroupMetadataManagerTest.java | 10 +++++++--- ...reamsGroupStaticMemberGroupMetadataManagerTest.java | 10 ++++++---- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTestContext.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTestContext.java index 1e3010f0a1cff..6c81bcb03ec25 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTestContext.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTestContext.java @@ -779,6 +779,13 @@ public CoordinatorResult streams ) { return streamsGroupHeartbeat(request, "client", InetAddress.getLoopbackAddress()); } + + public CoordinatorResult streamsGroupHeartbeat( + StreamsGroupHeartbeatRequestData request, + short version + ) { + return streamsGroupHeartbeat(request, "client", InetAddress.getLoopbackAddress(), version); + } public List> sleep(long ms) { time.sleep(ms); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java index 23585a9a1186e..e788c45259fc1 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java @@ -293,7 +293,7 @@ public void testDynamicJoinSucceedsAfterTemporarilyLeftStaticMemberSessionTimeou StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, staticMemberId), StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, staticMemberId), StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord( - groupId, timeoutGroupEpoch, groupMetadataHash, 0, getDefaultAssignmentConfigs() + groupId, timeoutGroupEpoch, groupMetadataHash, 0, getDefaultAssignmentConfigs(), -1, -1 ) ); @@ -360,7 +360,9 @@ public void testDynamicJoinSucceedsAfterTemporarilyLeftStaticMemberSessionTimeou joinGroupEpoch, groupMetadataHash, 0, - getDefaultAssignmentConfigs() + getDefaultAssignmentConfigs(), + -1, + -1 ), StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, newDynamicMemberId, staticTargetAssignment), StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, joinGroupEpoch, context.time.milliseconds()), @@ -613,7 +615,9 @@ public void testStaticRejoinWithUpdatedProcessIdRecomputesTargetAssignmentAndDyn bumpedGroupEpoch, topic.metadataHash(), 0, - getDefaultAssignmentConfigs() + getDefaultAssignmentConfigs(), + -1, + -1 ) ); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java index 66536ee261da8..ad43cf403214f 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java @@ -550,7 +550,9 @@ public void testStaticMemberRejoinWithUpdatedProcessIdBumpsStreamsGroupEpoch() { bumpedGroupEpoch, topic.metadataHash(), 0, - getDefaultAssignmentConfigs() + getDefaultAssignmentConfigs(), + -1, + -1 ) )); } @@ -674,7 +676,7 @@ public void testStaticMemberLeaveWithMinusOneFencesMemberAndBumpsStreamsGroupEpo StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, memberId), StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, memberId), StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, memberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, bumpedGroupEpoch, topic.metadataHash(), 0, getDefaultAssignmentConfigs()), + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, bumpedGroupEpoch, topic.metadataHash(), 0, getDefaultAssignmentConfigs(), -1, -1), StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, bumpedGroupEpoch, 0L) ), result.records() @@ -1427,7 +1429,7 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochAndRejoinAndOthe // From hasStreamsMemberMetadataChanged StreamsCoordinatorRecordHelpers.newStreamsGroupMemberRecord(groupId, newJoinStaticMember), - StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, bumpedGroupEpoch, topic.metadataHash(), 0, getDefaultAssignmentConfigs()), + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, bumpedGroupEpoch, topic.metadataHash(), 0, getDefaultAssignmentConfigs(), -1, -1), StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(groupId, newJoinStaticMember.memberId(), givenTargetAssignment), StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, bumpedGroupEpoch, context.time.milliseconds()), StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, reconciledMember) @@ -1673,7 +1675,7 @@ public void testStreamsStaticMemberTemporaryLeaveSessionTimeoutExpiration() { StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentTombstoneRecord(groupId, memberId), StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentTombstoneRecord(groupId, memberId), StreamsCoordinatorRecordHelpers.newStreamsGroupMemberTombstoneRecord(groupId, memberId), - StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, 3, topic.metadataHash(), 0, getDefaultAssignmentConfigs()), + StreamsCoordinatorRecordHelpers.newStreamsGroupMetadataRecord(groupId, 3, topic.metadataHash(), 0, getDefaultAssignmentConfigs(), -1, -1), StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(groupId, 3, 0L) ); assertEquals( From 440ccbc6f55f489d620579da17ce23f52e184543 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Tue, 9 Jun 2026 23:29:22 +0900 Subject: [PATCH 14/16] fix broken test cases related with recovery lag. --- .../StreamsGroupMixedGroupMetadataManagerTest.java | 8 ++++++++ ...sGroupStaticMemberGroupMetadataManagerTest.java | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java index e788c45259fc1..b709426710bad 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupMixedGroupMetadataManagerTest.java @@ -168,6 +168,7 @@ public void testStaticRejoinSucceedsAtMaxSizeWhileDynamicMemberStillExists() { .setMemberEpoch(groupEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(mkResponseTasks(subtopologyId, 0, 1)) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()), @@ -278,6 +279,7 @@ public void testDynamicJoinSucceedsAfterTemporarilyLeftStaticMemberSessionTimeou .setMemberEpoch(groupEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(null) .setWarmupTasks(null) .setStandbyTasks(null), @@ -328,6 +330,7 @@ public void testDynamicJoinSucceedsAfterTemporarilyLeftStaticMemberSessionTimeou .setMemberEpoch(joinGroupEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(mkResponseTasks(subtopologyId, 0, 1)) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()), @@ -463,6 +466,7 @@ public void testStaticTemporaryLeaveDoesNotTransferTasksToExistingDynamicMember( .setMemberEpoch(groupEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(null) .setWarmupTasks(null) .setStandbyTasks(null), @@ -554,6 +558,7 @@ public void testStaticRejoinWithUpdatedProcessIdRecomputesTargetAssignmentAndDyn .setMemberEpoch(bumpedGroupEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(topic.responseTasks(0)) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()), @@ -658,6 +663,7 @@ public void testStaticRejoinWithUpdatedProcessIdRecomputesTargetAssignmentAndDyn .setMemberEpoch(bumpedGroupEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(topic.responseTasks(1, 2, 3)) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()), @@ -753,6 +759,7 @@ public void testStaticRejoinWithSameProcessIdDoesNotBumpEpochAndDynamicHeartbeat .setMemberEpoch(groupEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(topic.responseTasks(0, 1)) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()), @@ -809,6 +816,7 @@ public void testStaticRejoinWithSameProcessIdDoesNotBumpEpochAndDynamicHeartbeat .setMemberEpoch(groupEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(null) .setWarmupTasks(null) .setStandbyTasks(null), diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java index ad43cf403214f..a91db518653e1 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java @@ -388,6 +388,7 @@ private void testStaticMemberJoinThenRevokeAndReceiveTasksWith2Members(int maxSi .setMemberEpoch(11) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(List.of()) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()), @@ -409,6 +410,7 @@ private void testStaticMemberJoinThenRevokeAndReceiveTasksWith2Members(int maxSi .setMemberEpoch(10) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(topic.responseTasks(0, 1)) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()), @@ -433,6 +435,7 @@ private void testStaticMemberJoinThenRevokeAndReceiveTasksWith2Members(int maxSi .setMemberEpoch(11) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setStatus(List.of()), revokeAckResult.response().data() ); @@ -448,6 +451,7 @@ private void testStaticMemberJoinThenRevokeAndReceiveTasksWith2Members(int maxSi .setMemberEpoch(11) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(topic.responseTasks(2, 3)) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()), @@ -482,6 +486,7 @@ private void testStaticMemberJoinThenRevokeAndReceiveTasksWith2Members(int maxSi .setMemberEpoch(11) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(topic.responseTasks(2, 3)) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()), @@ -865,6 +870,7 @@ private void verifyStaticMemberLeaveAndRejoinNoGroupBump(String instanceId, Stri .setMemberEpoch(memberEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(null) .setWarmupTasks(null) .setStandbyTasks(null), @@ -904,6 +910,7 @@ private void verifyStaticMemberLeaveAndRejoinNoGroupBump(String instanceId, Stri .setMemberEpoch(memberEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(topic.responseTasks(0, 1, 2, 3)) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()), @@ -1047,6 +1054,7 @@ public void testStaticMemberRejoinsAfterTemporaryLeave() { .setMemberEpoch(memberEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(topic.responseTasks(0, 1)) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()), @@ -1187,6 +1195,7 @@ public void testStaticMemberRejoinsAfterTemporaryLeaveFromUnreleasedState() { .setMemberEpoch(memberEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(topic.responseTasks(0, 1)) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()), @@ -1276,6 +1285,7 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnrevokedSta .setMemberEpoch(memberEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(topic.responseTasks(2)) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()); @@ -1343,6 +1353,7 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochAndRejoinAndOthe .setMemberEpoch(memberEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(null) .setWarmupTasks(null) .setStandbyTasks(null), @@ -1388,6 +1399,7 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochAndRejoinAndOthe .setMemberEpoch(bumpedMemberEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(topic.responseTasks(0, 1, 2, 3)) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()), @@ -1782,6 +1794,7 @@ public void testStaticMemberRejoinUpdatesUserEndpointInformationEpoch( .setMemberEpoch(bumpedEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(topic.responseTasks(0, 1, 2)) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()) @@ -1814,6 +1827,7 @@ public void testStaticMemberRejoinUpdatesUserEndpointInformationEpoch( .setMemberEpoch(bumpedEpoch) .setHeartbeatIntervalMs(5000) .setTaskOffsetIntervalMs(60000) + .setAcceptableRecoveryLag(10000) .setActiveTasks(topic.responseTasks(0, 1, 2)) .setWarmupTasks(List.of()) .setStandbyTasks(List.of()) From cf970554a3692a26cf60f3d50b0bf96cdef97214 Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Tue, 9 Jun 2026 23:30:32 +0900 Subject: [PATCH 15/16] Fix broken test after rebase. --- .../coordinator/group/GroupCoordinatorServiceTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorServiceTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorServiceTest.java index 9ffbf69a16a63..23c12223af05e 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorServiceTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorServiceTest.java @@ -2110,7 +2110,7 @@ public void testStreamsGroupDescribe() throws InterruptedException, ExecutionExc int partitionCount = 2; service.startup(() -> partitionCount); @SuppressWarnings("unchecked") - ArgumentCaptor>> readOperationCaptor = + ArgumentCaptor> readOperationCaptor = ArgumentCaptor.forClass(CoordinatorRuntime.CoordinatorReadOperation.class); StreamsGroupDescribeResponseData.DescribedGroup describedGroup1 = new StreamsGroupDescribeResponseData.DescribedGroup() @@ -2126,9 +2126,9 @@ public void testStreamsGroupDescribe() throws InterruptedException, ExecutionExc ArgumentMatchers.eq("streams-group-describe"), ArgumentMatchers.eq(new TopicPartition("__consumer_offsets", 0)), readOperationCaptor.capture() - )).thenReturn(CompletableFuture.completedFuture(List.of(describedGroup1))); + )).thenReturn(CompletableFuture.completedFuture(new StreamsGroupDescribeResult(List.of(describedGroup1), Map.of()))); - CompletableFuture> describedGroupFuture = new CompletableFuture<>(); + CompletableFuture describedGroupFuture = new CompletableFuture<>(); when(runtime.scheduleReadOperation( ArgumentMatchers.eq("streams-group-describe"), ArgumentMatchers.eq(new TopicPartition("__consumer_offsets", 1)), @@ -2139,7 +2139,7 @@ public void testStreamsGroupDescribe() throws InterruptedException, ExecutionExc service.streamsGroupDescribe(requestContext(ApiKeys.STREAMS_GROUP_DESCRIBE), Arrays.asList("group-id-1", "group-id-2")); assertFalse(future.isDone()); - describedGroupFuture.complete(List.of(describedGroup2)); + describedGroupFuture.complete(new StreamsGroupDescribeResult(List.of(describedGroup2), Map.of())); assertEquals(expectedDescribedGroups, future.get()); // Validate that the captured read operations, on the first and the second partition From a10b596de8c106e02292eb0284fb64a89253af9f Mon Sep 17 00:00:00 2001 From: chickenchickenlove Date: Tue, 9 Jun 2026 23:38:23 +0900 Subject: [PATCH 16/16] Preserve Streams static member state on temporary leave --- .../coordinator/group/GroupMetadataManager.java | 13 +------------ ...msGroupStaticMemberGroupMetadataManagerTest.java | 6 ++++-- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java index 1c4a277f27772..ac4b38d53c61a 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java @@ -4513,20 +4513,9 @@ private CoordinatorResult stream StreamsGroup group, StreamsGroupMember member ) { - // A static member leaving with epoch -2 may later be replaced with a new - // member id for the same instance id. Since we clear the pending revocations - // and reset the assigned task epochs for that replacement, keeping the member - // in UNREVOKED_TASKS would leave an inconsistent state: there are no pending - // revocations left to acknowledge, but reconciliation would still treat the - // member as waiting for revocation acknowledgement. Move it back to STABLE so - // the rejoining static member can be reconciled from the reset assignment. - org.apache.kafka.coordinator.group.streams.MemberState nextState = - member.state() == org.apache.kafka.coordinator.group.streams.MemberState.UNREVOKED_TASKS ? - org.apache.kafka.coordinator.group.streams.MemberState.STABLE : - member.state(); + // TODO: https://issues.apache.org/jira/browse/KAFKA-20680 StreamsGroupMember leavingStaticMember = new StreamsGroupMember.Builder(member) .setMemberEpoch(LEAVE_GROUP_STATIC_MEMBER_EPOCH) - .setState(nextState) .setTasksPendingRevocation(TasksTupleWithEpochs.EMPTY) .resetAssignedTasksEpochsToZero() .build(); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java index a91db518653e1..073a613b5d519 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/StreamsGroupStaticMemberGroupMetadataManagerTest.java @@ -987,6 +987,7 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnrevokedSta StreamsGroupMember expectedMember = streamsGroupMemberBuilderWithDefaults(memberId, instanceId) .setMemberEpoch(leaveEpoch) + .setState(MemberState.UNREVOKED_TASKS) .setPreviousMemberEpoch(memberEpoch - 1) .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) .build(); @@ -1005,7 +1006,7 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnrevokedSta List.of(StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, expectedMember)), result.records() ); - assertEquals(MemberState.STABLE, context.streamsGroupMemberState(groupId, memberId)); + assertEquals(MemberState.UNREVOKED_TASKS, context.streamsGroupMemberState(groupId, memberId)); assertEquals(groupEpoch, context.groupMetadataManager.streamsGroup(groupId).groupEpoch()); } @@ -1269,11 +1270,12 @@ public void testStaticMemberLeaveWithLeaveGroupStaticMemberEpochFromUnrevokedSta StreamsCoordinatorRecordHelpers.newStreamsGroupCurrentAssignmentRecord(groupId, streamsGroupMemberBuilderWithDefaults(leavingMemberId, instanceId) .setMemberEpoch(leaveEpoch) + .setState(MemberState.UNREVOKED_TASKS) .setPreviousMemberEpoch(memberEpoch - 1) .setAssignedTasks(resetAssignedTasksEpochsToZero(assignedTasks)) .build())); assertRecordsEquals(expectedRecordsTriggeredByLeave, leaveResult.records()); - assertEquals(MemberState.STABLE, context.streamsGroupMemberState(groupId, leavingMemberId)); + assertEquals(MemberState.UNREVOKED_TASKS, context.streamsGroupMemberState(groupId, leavingMemberId)); // Waiting member send a heartbeat expecting get unreleased tasks. CoordinatorResult waitingMemberResult = context.streamsGroupHeartbeat(