From 5b5d00f3ddbfdb902af72ef6491ae614ba5a6ec7 Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Tue, 18 Apr 2023 00:53:02 +0800 Subject: [PATCH 01/24] add PartitionMigrateTimeCost and revise MetricSensor#fetch --- .../astraea/app/web/BalancerHandlerTest.java | 2 +- .../org/astraea/common/admin/ClusterBean.java | 4 +- .../astraea/common/cost/BrokerInputCost.java | 3 +- .../astraea/common/cost/BrokerOutputCost.java | 3 +- .../java/org/astraea/common/cost/CpuCost.java | 2 +- .../org/astraea/common/cost/LoadCost.java | 2 +- .../org/astraea/common/cost/MemoryCost.java | 2 +- .../astraea/common/cost/MigrationCost.java | 58 +++++- .../org/astraea/common/cost/NetworkCost.java | 4 +- .../astraea/common/cost/NodeMetricsCost.java | 2 +- .../common/cost/PartitionMigrateTimeCost.java | 164 +++++++++++++++ .../common/cost/ReplicaLeaderCost.java | 3 +- .../common/cost/utils/ClusterInfoSensor.java | 2 +- .../common/metrics/broker/HasRate.java | 4 + .../metrics/collector/MetricSensor.java | 7 +- .../common/metrics/collector/MetricStore.java | 4 +- .../astraea/common/cost/ClusterCostTest.java | 2 +- .../org/astraea/common/cost/MoveCostTest.java | 7 +- .../common/cost/NodeLatencyCostTest.java | 2 +- .../common/cost/NodeThroughputCostTest.java | 2 +- .../cost/PartitionMigrateTimeCostTest.java | 187 ++++++++++++++++++ .../metrics/collector/MetricSensorTest.java | 26 +-- 22 files changed, 455 insertions(+), 37 deletions(-) create mode 100644 common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java create mode 100644 common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java diff --git a/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java b/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java index 0aed00deaa..662c3857c0 100644 --- a/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java +++ b/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java @@ -1221,7 +1221,7 @@ public SensorAndCost(Configuration configuration) { @Override public Optional metricSensor() { - return Optional.of((c, ignored) -> List.of(HostMetrics.jvmMemory(c))); + return Optional.of((ignore, c, ignored) -> List.of(HostMetrics.jvmMemory(c))); } @Override diff --git a/common/src/main/java/org/astraea/common/admin/ClusterBean.java b/common/src/main/java/org/astraea/common/admin/ClusterBean.java index 46387a3bd0..5844d0101a 100644 --- a/common/src/main/java/org/astraea/common/admin/ClusterBean.java +++ b/common/src/main/java/org/astraea/common/admin/ClusterBean.java @@ -50,7 +50,9 @@ static ClusterBean of(Map clients, MetricSensor sensor) { Map.Entry::getKey, entry -> sensor.fetch( - MBeanClient.of(entry.getValue().beans(BeanQuery.all())), EMPTY)))); + entry.getKey(), + MBeanClient.of(entry.getValue().beans(BeanQuery.all())), + EMPTY)))); } static ClusterBean masked(ClusterBean clusterBean, Predicate nodeFilter) { diff --git a/common/src/main/java/org/astraea/common/cost/BrokerInputCost.java b/common/src/main/java/org/astraea/common/cost/BrokerInputCost.java index a5d7fffdef..845b95209a 100644 --- a/common/src/main/java/org/astraea/common/cost/BrokerInputCost.java +++ b/common/src/main/java/org/astraea/common/cost/BrokerInputCost.java @@ -46,7 +46,8 @@ public BrokerCost brokerCost(ClusterInfo clusterInfo, ClusterBean clusterBean) { @Override public Optional metricSensor() { return Optional.of( - (client, ignored) -> List.of(ServerMetrics.BrokerTopic.BYTES_IN_PER_SEC.fetch(client))); + (ignore, client, ignored) -> + List.of(ServerMetrics.BrokerTopic.BYTES_IN_PER_SEC.fetch(client))); } @Override diff --git a/common/src/main/java/org/astraea/common/cost/BrokerOutputCost.java b/common/src/main/java/org/astraea/common/cost/BrokerOutputCost.java index c40c2faacc..1a37f36b47 100644 --- a/common/src/main/java/org/astraea/common/cost/BrokerOutputCost.java +++ b/common/src/main/java/org/astraea/common/cost/BrokerOutputCost.java @@ -45,7 +45,8 @@ public BrokerCost brokerCost(ClusterInfo clusterInfo, ClusterBean clusterBean) { @Override public Optional metricSensor() { return Optional.of( - (client, ignored) -> List.of(ServerMetrics.BrokerTopic.BYTES_OUT_PER_SEC.fetch(client))); + (ignore, client, ignored) -> + List.of(ServerMetrics.BrokerTopic.BYTES_OUT_PER_SEC.fetch(client))); } @Override diff --git a/common/src/main/java/org/astraea/common/cost/CpuCost.java b/common/src/main/java/org/astraea/common/cost/CpuCost.java index 30dcba53a9..2a4969f947 100644 --- a/common/src/main/java/org/astraea/common/cost/CpuCost.java +++ b/common/src/main/java/org/astraea/common/cost/CpuCost.java @@ -57,7 +57,7 @@ public BrokerCost brokerCost(ClusterInfo clusterInfo, ClusterBean clusterBean) { @Override public Optional metricSensor() { - return Optional.of((client, ignored) -> List.of(HostMetrics.operatingSystem(client))); + return Optional.of((ignore, client, ignored) -> List.of(HostMetrics.operatingSystem(client))); } @Override diff --git a/common/src/main/java/org/astraea/common/cost/LoadCost.java b/common/src/main/java/org/astraea/common/cost/LoadCost.java index 4d778354a5..ee739e268f 100644 --- a/common/src/main/java/org/astraea/common/cost/LoadCost.java +++ b/common/src/main/java/org/astraea/common/cost/LoadCost.java @@ -150,7 +150,7 @@ Map computeLoad(Map> allBea @Override public Optional metricSensor() { return Optional.of( - (client, ignored) -> + (ignore, client, ignored) -> List.of( ServerMetrics.BrokerTopic.BYTES_IN_PER_SEC.fetch(client), ServerMetrics.BrokerTopic.BYTES_OUT_PER_SEC.fetch(client))); diff --git a/common/src/main/java/org/astraea/common/cost/MemoryCost.java b/common/src/main/java/org/astraea/common/cost/MemoryCost.java index 7316e74970..cf62d7710a 100644 --- a/common/src/main/java/org/astraea/common/cost/MemoryCost.java +++ b/common/src/main/java/org/astraea/common/cost/MemoryCost.java @@ -57,7 +57,7 @@ public BrokerCost brokerCost(ClusterInfo clusterInfo, ClusterBean clusterBean) { @Override public Optional metricSensor() { - return Optional.of((client, ignored) -> List.of(HostMetrics.jvmMemory(client))); + return Optional.of((ignore, client, ignored) -> List.of(HostMetrics.jvmMemory(client))); } @Override diff --git a/common/src/main/java/org/astraea/common/cost/MigrationCost.java b/common/src/main/java/org/astraea/common/cost/MigrationCost.java index 764abf2d83..4b747b3221 100644 --- a/common/src/main/java/org/astraea/common/cost/MigrationCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrationCost.java @@ -16,12 +16,15 @@ */ package org.astraea.common.cost; +import static org.astraea.common.cost.PartitionMigrateTimeCost.brokerMaxRate; + import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.astraea.common.admin.ClusterBean; import org.astraea.common.admin.ClusterInfo; import org.astraea.common.admin.NodeInfo; import org.astraea.common.admin.Replica; @@ -29,7 +32,6 @@ public class MigrationCost { public final String name; - public final Map brokerCosts; public static final String TO_SYNC_BYTES = "record size to sync (bytes)"; public static final String TO_FETCH_BYTES = "record size to fetch (bytes)"; @@ -69,6 +71,60 @@ static Map replicaLeaderChanged(ClusterInfo before, ClusterInfo a return changedReplicaNumber(before, after, Replica::isLeader); } + /** + * @param before the ClusterInfo before migrated replicas + * @param after the ClusterInfo after migrated replicas + * @param clusterBean cluster metrics + * @return estimated migrated time required by all brokers (seconds) + */ + public static Map brokerMigrationTime( + ClusterInfo before, ClusterInfo after, ClusterBean clusterBean) { + var brokerInRate = + brokerMaxRate(before, clusterBean, PartitionMigrateTimeCost.MaxReplicationInRateBean.class); + var brokerOutRate = + brokerMaxRate( + before, clusterBean, PartitionMigrateTimeCost.MaxReplicationOutRateBean.class); + var brokerMigrateInTime = + MigrationCost.recordSizeToFetch(before, after).entrySet().stream() + .map( + brokerSize -> + Map.entry( + brokerSize.getKey(), + brokerSize.getValue() + / brokerInRate + .get(brokerSize.getKey()) + .orElseThrow( + () -> + new RuntimeException( + "No any metric for broker" + brokerSize.getKey())))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + var brokerMigrateOutTime = + MigrationCost.recordSizeToSync(before, after).entrySet().stream() + .map( + brokerSize -> + Map.entry( + brokerSize.getKey(), + brokerSize.getValue() + / brokerOutRate + .get(brokerSize.getKey()) + .orElseThrow( + () -> + new RuntimeException( + "No any metric for broker" + brokerSize.getKey())))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + return Stream.concat(before.nodes().stream(), after.nodes().stream()) + .distinct() + .map( + nodeInfo -> + Map.entry( + nodeInfo.id(), + (long) + Math.max( + brokerMigrateInTime.get(nodeInfo.id()), + brokerMigrateOutTime.get(nodeInfo.id())))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + /** * @param before the ClusterInfo before migrated replicas * @param after the ClusterInfo after migrated replicas diff --git a/common/src/main/java/org/astraea/common/cost/NetworkCost.java b/common/src/main/java/org/astraea/common/cost/NetworkCost.java index d5213ecd15..e7a0d6ea74 100644 --- a/common/src/main/java/org/astraea/common/cost/NetworkCost.java +++ b/common/src/main/java/org/astraea/common/cost/NetworkCost.java @@ -196,13 +196,13 @@ public Optional metricSensor() { // obtain the replica info, so we intentionally sample log size but never use it. // https://github.com/skiptests/astraea/pull/1240#discussion_r1044487473 return Optional.of( - (client, clusterBean) -> + (identity, client, clusterBean) -> Stream.of( List.of(HostMetrics.jvmMemory(client)), ServerMetrics.Topic.BYTES_IN_PER_SEC.fetch(client), ServerMetrics.Topic.BYTES_OUT_PER_SEC.fetch(client), LogMetrics.Log.SIZE.fetch(client), - clusterInfoSensor.fetch(client, clusterBean)) + clusterInfoSensor.fetch(identity, client, clusterBean)) .flatMap(Collection::stream) .collect(Collectors.toUnmodifiableList())); } diff --git a/common/src/main/java/org/astraea/common/cost/NodeMetricsCost.java b/common/src/main/java/org/astraea/common/cost/NodeMetricsCost.java index a64d5e41a1..44c298098b 100644 --- a/common/src/main/java/org/astraea/common/cost/NodeMetricsCost.java +++ b/common/src/main/java/org/astraea/common/cost/NodeMetricsCost.java @@ -72,6 +72,6 @@ public BrokerCost brokerCost(ClusterInfo clusterInfo, ClusterBean clusterBean) { @Override public Optional metricSensor() { - return Optional.of((client, clusterBean) -> ProducerMetrics.node(client)); + return Optional.of((ignore, client, clusterBean) -> ProducerMetrics.node(client)); } } diff --git a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java new file mode 100644 index 0000000000..4cc6c517e8 --- /dev/null +++ b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.astraea.common.cost; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalDouble; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.astraea.common.Configuration; +import org.astraea.common.admin.ClusterBean; +import org.astraea.common.admin.ClusterInfo; +import org.astraea.common.metrics.BeanObject; +import org.astraea.common.metrics.HasBeanObject; +import org.astraea.common.metrics.Sensor; +import org.astraea.common.metrics.broker.HasMeter; +import org.astraea.common.metrics.broker.HasRate; +import org.astraea.common.metrics.broker.ServerMetrics; +import org.astraea.common.metrics.collector.MetricSensor; +import org.astraea.common.metrics.stats.Max; + +/** MoveCost: more max write rate change -> higher migrate cost. */ +public class PartitionMigrateTimeCost implements HasMoveCost { + private static final String REPLICATION_IN_RATE = "replication_in_rate"; + private static final String REPLICATION_OUT_RATE = "replication_out_rate"; + public static final String MAX_MIGRATE_TIME_KEY = "max.migrated.time.limit"; + static final Map> maxBrokerReplicationInRate = new HashMap<>(); + static final Map> maxBrokerReplicationOutRate = new HashMap<>(); + // metrics windows size + private final long maxMigrateTime; + + public PartitionMigrateTimeCost(Configuration config) { + this.maxMigrateTime = + config.string(MAX_MIGRATE_TIME_KEY).map(Long::parseLong).orElse(Long.MAX_VALUE); + } + + @Override + public Optional metricSensor() { + return Optional.of( + (identity, client, clusterBean) -> { + var metrics = + clusterBean.all().values().stream() + .flatMap(Collection::stream) + .distinct() + .collect(Collectors.toCollection(ArrayList::new)); + var newInMetrics = ServerMetrics.BrokerTopic.REPLICATION_BYTES_IN_PER_SEC.fetch(client); + var newOutMetrics = ServerMetrics.BrokerTopic.REPLICATION_BYTES_OUT_PER_SEC.fetch(client); + var current = Duration.ofMillis(System.currentTimeMillis()); + var maxInRateSensor = + maxBrokerReplicationInRate.computeIfAbsent( + identity, + ignore -> + Sensor.builder().addStat(REPLICATION_IN_RATE, Max.of()).build()); + var maxOutRateSensor = + maxBrokerReplicationOutRate.computeIfAbsent( + identity, + ignore -> + Sensor.builder().addStat(REPLICATION_OUT_RATE, Max.of()).build()); + maxInRateSensor.record(newInMetrics.oneMinuteRate()); + maxOutRateSensor.record(newOutMetrics.oneMinuteRate()); + var inRate = maxInRateSensor.measure(REPLICATION_IN_RATE); + var outRate = maxOutRateSensor.measure(REPLICATION_OUT_RATE); + metrics.add( + (MaxReplicationInRateBean) + () -> + new BeanObject( + newInMetrics.beanObject().domainName(), + newInMetrics.beanObject().properties(), + Map.of(HasRate.ONE_MIN_RATE_KEY, inRate), + current.toMillis())); + metrics.add( + (MaxReplicationOutRateBean) + () -> + new BeanObject( + newOutMetrics.beanObject().domainName(), + newOutMetrics.beanObject().properties(), + Map.of(HasRate.ONE_MIN_RATE_KEY, outRate), + current.toMillis())); + return metrics; + }); + } + + public static Map brokerMaxRate( + ClusterInfo clusterInfo, + ClusterBean clusterBean, + Class statisticMetrics) { + return clusterInfo.brokers().stream() + .map( + broker -> + Map.entry( + broker.id(), + clusterBean.all().getOrDefault(broker.id(), List.of()).stream() + .filter(x -> statisticMetrics.isAssignableFrom(x.getClass())) + .mapToDouble(x -> ((HasMeter) x).oneMinuteRate()) + .max())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + public Map brokerMigratedTime( + Map needToMigrated, Map brokerRate) { + return needToMigrated.entrySet().stream() + .map( + brokerSize -> + Map.entry( + brokerSize.getKey(), + brokerSize.getValue() + / brokerRate + .get(brokerSize.getKey()) + .orElseThrow( + () -> + new NoSufficientMetricsException( + this, + Duration.ofSeconds(1), + "No metric for broker" + brokerSize.getKey())))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + @Override + public MoveCost moveCost(ClusterInfo before, ClusterInfo after, ClusterBean clusterBean) { + var brokerInRate = brokerMaxRate(before, clusterBean, MaxReplicationInRateBean.class); + var brokerOutRate = brokerMaxRate(before, clusterBean, MaxReplicationOutRateBean.class); + var needToMigrateIn = MigrationCost.recordSizeToFetch(before, after); + var needToMigrateOut = MigrationCost.recordSizeToSync(before, after); + + var brokerMigrateInTime = brokerMigratedTime(needToMigrateIn, brokerInRate); + var brokerMigrateOutTime = brokerMigratedTime(needToMigrateOut, brokerOutRate); + var maxMigrateTime = + Stream.concat(before.nodes().stream(), after.nodes().stream()) + .distinct() + .map( + nodeInfo -> + Math.max( + brokerMigrateInTime.get(nodeInfo.id()), + brokerMigrateOutTime.get(nodeInfo.id()))) + .max(Comparator.comparing(Function.identity())) + .orElse(0.0); + return () -> maxMigrateTime > this.maxMigrateTime; + } + + public interface MaxReplicationInRateBean extends HasMeter {} + + public interface MaxReplicationOutRateBean extends HasMeter {} +} diff --git a/common/src/main/java/org/astraea/common/cost/ReplicaLeaderCost.java b/common/src/main/java/org/astraea/common/cost/ReplicaLeaderCost.java index f755f73a97..cc91864ebe 100644 --- a/common/src/main/java/org/astraea/common/cost/ReplicaLeaderCost.java +++ b/common/src/main/java/org/astraea/common/cost/ReplicaLeaderCost.java @@ -71,7 +71,8 @@ static Map leaderCount(ClusterInfo clusterInfo) { @Override public Optional metricSensor() { return Optional.of( - (client, ignored) -> List.of(ServerMetrics.ReplicaManager.LEADER_COUNT.fetch(client))); + (ignore, client, ignored) -> + List.of(ServerMetrics.ReplicaManager.LEADER_COUNT.fetch(client))); } public Configuration config() { diff --git a/common/src/main/java/org/astraea/common/cost/utils/ClusterInfoSensor.java b/common/src/main/java/org/astraea/common/cost/utils/ClusterInfoSensor.java index ef6c408230..e39311860a 100644 --- a/common/src/main/java/org/astraea/common/cost/utils/ClusterInfoSensor.java +++ b/common/src/main/java/org/astraea/common/cost/utils/ClusterInfoSensor.java @@ -38,7 +38,7 @@ public class ClusterInfoSensor implements MetricSensor { @Override - public List fetch(MBeanClient client, ClusterBean bean) { + public List fetch(int identity, MBeanClient client, ClusterBean bean) { return Stream.of( List.of(ServerMetrics.KafkaServer.CLUSTER_ID.fetch(client)), LogMetrics.Log.SIZE.fetch(client), diff --git a/common/src/main/java/org/astraea/common/metrics/broker/HasRate.java b/common/src/main/java/org/astraea/common/metrics/broker/HasRate.java index 780690506c..021e0be81b 100644 --- a/common/src/main/java/org/astraea/common/metrics/broker/HasRate.java +++ b/common/src/main/java/org/astraea/common/metrics/broker/HasRate.java @@ -20,6 +20,10 @@ import org.astraea.common.metrics.HasBeanObject; public interface HasRate extends HasBeanObject { + String ONE_MIN_RATE_KEY = "OneMinuteRate"; + String FIVE_MINUTE_RATE = "FiveMinuteRate"; + String FIFTEEN_MINUTE_RATE = "FifteenMinuteRate"; + default double meanRate() { return (double) beanObject().attributes().get("MeanRate"); } diff --git a/common/src/main/java/org/astraea/common/metrics/collector/MetricSensor.java b/common/src/main/java/org/astraea/common/metrics/collector/MetricSensor.java index 2afedac949..54d31f09d8 100644 --- a/common/src/main/java/org/astraea/common/metrics/collector/MetricSensor.java +++ b/common/src/main/java/org/astraea/common/metrics/collector/MetricSensor.java @@ -48,12 +48,12 @@ static Optional of( Collection metricSensors, Consumer exceptionHandler) { if (metricSensors.isEmpty()) return Optional.empty(); return Optional.of( - (client, clusterBean) -> + (identity, client, clusterBean) -> metricSensors.stream() .flatMap( ms -> { try { - return ms.fetch(client, clusterBean).stream(); + return ms.fetch(identity, client, clusterBean).stream(); } catch (Exception ex) { exceptionHandler.accept(ex); return Stream.empty(); @@ -66,9 +66,10 @@ static Optional of( * fetch metrics from remote/local mbean server. Or the implementation can generate custom metrics * according to existent cluster bean * + * @param identity * @param client mbean client (don't close it!) * @param bean current cluster bean * @return java metrics */ - Collection fetch(MBeanClient client, ClusterBean bean); + Collection fetch(int identity, MBeanClient client, ClusterBean bean); } diff --git a/common/src/main/java/org/astraea/common/metrics/collector/MetricStore.java b/common/src/main/java/org/astraea/common/metrics/collector/MetricStore.java index 0be5bbaf4f..b27807dd68 100644 --- a/common/src/main/java/org/astraea/common/metrics/collector/MetricStore.java +++ b/common/src/main/java/org/astraea/common/metrics/collector/MetricStore.java @@ -117,7 +117,7 @@ class Builder { () -> Map.of( (MetricSensor) - (client, bean) -> + (identity, client, bean) -> client.beans(BeanQuery.all()).stream() .map(bs -> (HasBeanObject) () -> bs) .collect(Collectors.toUnmodifiableList()), @@ -236,7 +236,7 @@ private MetricStoreImpl( try { beans .computeIfAbsent(id, ignored -> new ConcurrentLinkedQueue<>()) - .addAll(sensor.fetch(client, clusterBean)); + .addAll(sensor.fetch(id, client, clusterBean)); } catch (Exception e) { errorHandler.accept(id, e); } diff --git a/common/src/test/java/org/astraea/common/cost/ClusterCostTest.java b/common/src/test/java/org/astraea/common/cost/ClusterCostTest.java index b62684474c..8ecb03427b 100644 --- a/common/src/test/java/org/astraea/common/cost/ClusterCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/ClusterCostTest.java @@ -63,7 +63,7 @@ void testSensor() { var mergeCost = HasClusterCost.of(Map.of(cost1, 1.0, cost2, 1.0)); var metrics = mergeCost.metricSensor().stream() - .map(x -> x.fetch(MBeanClient.of(SERVICE.jmxServiceURL()), ClusterBean.EMPTY)) + .map(x -> x.fetch(-1, MBeanClient.of(SERVICE.jmxServiceURL()), ClusterBean.EMPTY)) .collect(Collectors.toSet()); Assertions.assertTrue( metrics.iterator().next().stream() diff --git a/common/src/test/java/org/astraea/common/cost/MoveCostTest.java b/common/src/test/java/org/astraea/common/cost/MoveCostTest.java index 798592eea8..f4ac6f1553 100644 --- a/common/src/test/java/org/astraea/common/cost/MoveCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/MoveCostTest.java @@ -49,7 +49,7 @@ void testSensor() { var mergeCost = HasMoveCost.of(List.of(cost1, cost2)); var metrics = mergeCost.metricSensor().stream() - .map(x -> x.fetch(MBeanClient.of(SERVICE.jmxServiceURL()), ClusterBean.EMPTY)) + .map(x -> x.fetch(-1, MBeanClient.of(SERVICE.jmxServiceURL()), ClusterBean.EMPTY)) .collect(Collectors.toSet()); Assertions.assertEquals(3, metrics.iterator().next().size()); Assertions.assertTrue( @@ -84,9 +84,10 @@ class FakeCf implements HasMoveCost { public Optional metricSensor() { return MetricSensor.of( List.of( - (c, ignored) -> + (ignore, c, ignored) -> List.of(ServerMetrics.BrokerTopic.REPLICATION_BYTES_IN_PER_SEC.fetch(c)), - (c, ignored) -> List.of(ServerMetrics.BrokerTopic.BYTES_IN_PER_SEC.fetch(c)))); + (ignore, c, ignored) -> + List.of(ServerMetrics.BrokerTopic.BYTES_IN_PER_SEC.fetch(c)))); } @Override diff --git a/common/src/test/java/org/astraea/common/cost/NodeLatencyCostTest.java b/common/src/test/java/org/astraea/common/cost/NodeLatencyCostTest.java index 43cf8a0f76..7fd88cf1f9 100644 --- a/common/src/test/java/org/astraea/common/cost/NodeLatencyCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/NodeLatencyCostTest.java @@ -109,7 +109,7 @@ void testSensor() { new BeanObject("a", Map.of("node-id", "node-10"), Map.of()), new BeanObject("a", Map.of("node-id", "node-10"), Map.of()), new BeanObject("a", Map.of("node-id", "node-11"), Map.of()))); - var result = function.metricSensor().get().fetch(client, ClusterBean.EMPTY); + var result = function.metricSensor().get().fetch(-1, client, ClusterBean.EMPTY); Assertions.assertEquals(3, result.size()); } } diff --git a/common/src/test/java/org/astraea/common/cost/NodeThroughputCostTest.java b/common/src/test/java/org/astraea/common/cost/NodeThroughputCostTest.java index 58f1f2d8bb..c10b01249b 100644 --- a/common/src/test/java/org/astraea/common/cost/NodeThroughputCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/NodeThroughputCostTest.java @@ -78,7 +78,7 @@ void testSensor() { var bean = new BeanObject("aaa", Map.of("node-id", "node-1"), Map.of()); var client = Mockito.mock(MBeanClient.class); Mockito.when(client.beans(Mockito.any())).thenReturn(List.of(bean)); - var result = sensor.fetch(client, ClusterBean.EMPTY); + var result = sensor.fetch(-1, client, ClusterBean.EMPTY); Assertions.assertEquals(1, result.size()); Assertions.assertEquals(bean, result.iterator().next().beanObject()); } diff --git a/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java b/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java new file mode 100644 index 0000000000..8faedad984 --- /dev/null +++ b/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.astraea.common.cost; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.kafka.common.Node; +import org.astraea.common.Configuration; +import org.astraea.common.admin.Broker; +import org.astraea.common.admin.ClusterBean; +import org.astraea.common.admin.ClusterInfo; +import org.astraea.common.admin.NodeInfo; +import org.astraea.common.admin.Replica; +import org.astraea.common.metrics.BeanObject; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class PartitionMigrateTimeCostTest { + + private static final BeanObject inBean0 = + new BeanObject("domain", Map.of(), Map.of("OneMinuteRate", 1000.0)); + private static final BeanObject outBean0 = + new BeanObject("domain", Map.of(), Map.of("OneMinuteRate", 1500.0)); + private static final BeanObject inBean1 = + new BeanObject("domain", Map.of(), Map.of("OneMinuteRate", 2000.0)); + private static final BeanObject outBean1 = + new BeanObject("domain", Map.of(), Map.of("OneMinuteRate", 2500.0)); + private static final BeanObject inBean2 = + new BeanObject("domain", Map.of(), Map.of("OneMinuteRate", 3000.0)); + private static final BeanObject outBean2 = + new BeanObject("domain", Map.of(), Map.of("OneMinuteRate", 3500.0)); + + @Test + void testMigratedCost() { + // before(partition-broker): p10-1, p11-2, p12-0, p12-0 + // after(partition-broker): p10-0, p11-1, p12-2, p12-0 + // p10:777, p11:700, p12:500 + List brokers = + before().stream() + .map(Replica::nodeInfo) + .distinct() + .map( + nodeInfo -> + Broker.of( + false, + new Node(nodeInfo.id(), "", nodeInfo.port()), + Map.of(), + Map.of(), + List.of())) + .collect(Collectors.toList()); + var before = of(before(), brokers); + var after = of(after(), brokers); + var migrationCost = MigrationCost.brokerMigrationTime(before, after, clusterBean()); + Assertions.assertEquals(Math.max(10000000 / 1000, 30000000 / 1500), migrationCost.get(0)); + Assertions.assertEquals(Math.max(20000000 / 2000, 10000000 / 2500), migrationCost.get(1)); + Assertions.assertEquals(Math.max(30000000 / 3000, 20000000 / 3500), migrationCost.get(2)); + } + + @Test + void testMostCost() { + List brokers = + before().stream() + .map(Replica::nodeInfo) + .distinct() + .map( + nodeInfo -> + Broker.of( + false, + new Node(nodeInfo.id(), "", nodeInfo.port()), + Map.of(), + Map.of(), + List.of())) + .collect(Collectors.toList()); + var before = of(before(), brokers); + var after = of(after(), brokers); + var timeLimit = + Configuration.of(Map.of(PartitionMigrateTimeCost.MAX_MIGRATE_TIME_KEY, "20000")); + var overFlowTimeLimit = + Configuration.of(Map.of(PartitionMigrateTimeCost.MAX_MIGRATE_TIME_KEY, "19999")); + var cf = new PartitionMigrateTimeCost(timeLimit); + var overFlowCf = new PartitionMigrateTimeCost(overFlowTimeLimit); + var moveCost = cf.moveCost(before, after, clusterBean()); + var overflowCost = overFlowCf.moveCost(before, after, clusterBean()); + Assertions.assertFalse(moveCost.overflow()); + Assertions.assertTrue(overflowCost.overflow()); + } + + public static ClusterInfo of(List replicas, List nodeInfos) { + return ClusterInfo.of("fake", nodeInfos, Map.of(), replicas); + } + + private List after() { + return List.of( + Replica.builder() + .topic("t") + .partition(10) + .isLeader(true) + .size(10000000) + .nodeInfo(NodeInfo.of(1, "", -1)) + .build(), + Replica.builder() + .topic("t") + .partition(11) + .isLeader(true) + .size(20000000) + .nodeInfo(NodeInfo.of(2, "", -1)) + .build(), + Replica.builder() + .topic("t") + .partition(12) + .isLeader(true) + .size(30000000) + .nodeInfo(NodeInfo.of(0, "", -1)) + .build(), + Replica.builder() + .topic("t") + .partition(12) + .isLeader(false) + .size(30000000) + .nodeInfo(NodeInfo.of(0, "", -1)) + .build()); + } + + private List before() { + return List.of( + Replica.builder() + .topic("t") + .partition(10) + .isLeader(true) + .size(10000000) + .nodeInfo(NodeInfo.of(0, "", -1)) + .build(), + Replica.builder() + .topic("t") + .partition(11) + .isLeader(true) + .size(20000000) + .nodeInfo(NodeInfo.of(1, "", -1)) + .build(), + Replica.builder() + .topic("t") + .partition(12) + .isLeader(true) + .size(30000000) + .nodeInfo(NodeInfo.of(2, "", -1)) + .build(), + Replica.builder() + .topic("t") + .partition(12) + .isLeader(false) + .size(30000000) + .nodeInfo(NodeInfo.of(0, "", -1)) + .build()); + } + + private static ClusterBean clusterBean() { + return ClusterBean.of( + Map.of( + 0, + List.of( + (PartitionMigrateTimeCost.MaxReplicationInRateBean) () -> inBean0, + (PartitionMigrateTimeCost.MaxReplicationOutRateBean) () -> outBean0), + 1, + List.of( + (PartitionMigrateTimeCost.MaxReplicationInRateBean) () -> inBean1, + (PartitionMigrateTimeCost.MaxReplicationOutRateBean) () -> outBean1), + 2, + List.of( + (PartitionMigrateTimeCost.MaxReplicationInRateBean) () -> inBean2, + (PartitionMigrateTimeCost.MaxReplicationOutRateBean) () -> outBean2))); + } +} diff --git a/common/src/test/java/org/astraea/common/metrics/collector/MetricSensorTest.java b/common/src/test/java/org/astraea/common/metrics/collector/MetricSensorTest.java index 123239caea..cf9a9ded0f 100644 --- a/common/src/test/java/org/astraea/common/metrics/collector/MetricSensorTest.java +++ b/common/src/test/java/org/astraea/common/metrics/collector/MetricSensorTest.java @@ -31,13 +31,13 @@ public class MetricSensorTest { @Test void testMultipleSensors() { var mbean0 = Mockito.mock(HasBeanObject.class); - MetricSensor metricSensor0 = (client, ignored) -> List.of(mbean0); + MetricSensor metricSensor0 = (ignore, client, ignored) -> List.of(mbean0); var mbean1 = Mockito.mock(HasBeanObject.class); - MetricSensor metricSensor1 = (client, ignored) -> List.of(mbean1); + MetricSensor metricSensor1 = (ignore, client, ignored) -> List.of(mbean1); var sensor = MetricSensor.of(List.of(metricSensor0, metricSensor1)).get(); - var result = sensor.fetch(Mockito.mock(MBeanClient.class), ClusterBean.EMPTY); + var result = sensor.fetch(-1, Mockito.mock(MBeanClient.class), ClusterBean.EMPTY); Assertions.assertEquals(2, result.size()); Assertions.assertTrue(result.contains(mbean0)); @@ -52,9 +52,9 @@ void testEmpty() { @Test void testNoSwallowException() { var result = List.of(Mockito.mock(HasBeanObject.class)); - MetricSensor goodMetricSensor = (client, ignored) -> result; + MetricSensor goodMetricSensor = (ignore, client, ignored) -> result; MetricSensor badMetricSensor = - (client, ignored) -> { + (ignore, client, ignored) -> { throw new RuntimeException("xxx"); }; var sensor = @@ -68,33 +68,33 @@ void testNoSwallowException() { .get(); Assertions.assertThrows( RuntimeException.class, - () -> sensor.fetch(Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); + () -> sensor.fetch(-1, Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); } @Test void testSensorsWithExceptionHandler() { var mbean0 = Mockito.mock(HasBeanObject.class); - MetricSensor metricSensor0 = (client, ignored) -> List.of(mbean0); + MetricSensor metricSensor0 = (ignore, client, ignored) -> List.of(mbean0); MetricSensor metricSensor1 = - (client, ignored) -> { + (ignore, client, ignored) -> { throw new NoSuchElementException(); }; MetricSensor metricSensor2 = - (client, ignored) -> { + (ignore, client, ignored) -> { throw new RuntimeException(); }; var sensor = MetricSensor.of(List.of(metricSensor0, metricSensor1)).get(); Assertions.assertDoesNotThrow( - () -> sensor.fetch(Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); + () -> sensor.fetch(-1, Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); Assertions.assertEquals( - 1, sensor.fetch(Mockito.mock(MBeanClient.class), ClusterBean.EMPTY).size()); + 1, sensor.fetch(-1, Mockito.mock(MBeanClient.class), ClusterBean.EMPTY).size()); Assertions.assertDoesNotThrow( () -> MetricSensor.of(List.of(metricSensor0, metricSensor2)) .get() - .fetch(Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); + .fetch(-1, Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); Assertions.assertThrows( NoSuchElementException.class, () -> @@ -104,6 +104,6 @@ void testSensorsWithExceptionHandler() { if (e instanceof NoSuchElementException) throw new NoSuchElementException(); }) .get() - .fetch(Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); + .fetch(-1, Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); } } From 05b838e4eb4c742a8fb4361aba8409507f527b88 Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Fri, 21 Apr 2023 19:17:51 +0800 Subject: [PATCH 02/24] fix issues --- .../app/publisher/MetricPublisher.java | 1 + .../org/astraea/app/web/BalancerHandler.java | 4 ++- .../java/org/astraea/app/web/BeanHandler.java | 2 +- .../astraea/app/web/BalancerHandlerTest.java | 2 +- .../org/astraea/common/admin/ClusterBean.java | 3 +- .../org/astraea/common/assignor/Assignor.java | 3 +- .../astraea/common/cost/BrokerInputCost.java | 3 +- .../astraea/common/cost/BrokerOutputCost.java | 3 +- .../java/org/astraea/common/cost/CpuCost.java | 2 +- .../org/astraea/common/cost/LoadCost.java | 2 +- .../org/astraea/common/cost/MemoryCost.java | 2 +- .../org/astraea/common/cost/NetworkCost.java | 4 +-- .../astraea/common/cost/NodeMetricsCost.java | 2 +- .../common/cost/PartitionMigrateTimeCost.java | 6 ++-- .../common/cost/ReplicaLeaderCost.java | 3 +- .../common/cost/utils/ClusterInfoSensor.java | 2 +- .../astraea/common/metrics/MBeanClient.java | 27 +++++++++++++---- .../metrics/collector/MetricSensor.java | 7 ++--- .../common/metrics/collector/MetricStore.java | 6 ++-- .../SmoothWeightRoundRobinPartitioner.java | 3 +- .../partitioner/StrictCostPartitioner.java | 3 +- .../common/cost/BrokerInputCostTest.java | 3 +- .../astraea/common/cost/ClusterCostTest.java | 2 +- .../org/astraea/common/cost/MoveCostTest.java | 7 ++--- .../common/cost/NodeLatencyCostTest.java | 2 +- .../common/cost/NodeThroughputCostTest.java | 2 +- .../metrics/BeanQueryIntegratedTest.java | 2 +- .../common/metrics/MBeanClientTest.java | 30 +++++++++---------- .../metrics/collector/MetricSensorTest.java | 26 ++++++++-------- .../main/java/org/astraea/gui/Context.java | 4 +-- 30 files changed, 93 insertions(+), 75 deletions(-) diff --git a/app/src/main/java/org/astraea/app/publisher/MetricPublisher.java b/app/src/main/java/org/astraea/app/publisher/MetricPublisher.java index 976ca23bb9..5c9a6f5660 100644 --- a/app/src/main/java/org/astraea/app/publisher/MetricPublisher.java +++ b/app/src/main/java/org/astraea/app/publisher/MetricPublisher.java @@ -59,6 +59,7 @@ static void execute(Arguments arguments) { NodeInfo::id, node -> MBeanClient.jndi( + node.id(), node.host(), arguments.idToJmxPort().apply(node.id())))))) .fetchBeanDelay(arguments.period) diff --git a/app/src/main/java/org/astraea/app/web/BalancerHandler.java b/app/src/main/java/org/astraea/app/web/BalancerHandler.java index 75c1ad93c2..65ae211107 100644 --- a/app/src/main/java/org/astraea/app/web/BalancerHandler.java +++ b/app/src/main/java/org/astraea/app/web/BalancerHandler.java @@ -78,7 +78,9 @@ class BalancerHandler implements Handler, AutoCloseable { .collect( Collectors.toUnmodifiableMap( NodeInfo::id, - b -> MBeanClient.jndi(b.host(), jmxPortMapper.apply(b.id()))))); + b -> + MBeanClient.jndi( + b.id(), b.host(), jmxPortMapper.apply(b.id()))))); this.metricStore = MetricStore.builder() .beanExpiration(Duration.ofSeconds(90)) diff --git a/app/src/main/java/org/astraea/app/web/BeanHandler.java b/app/src/main/java/org/astraea/app/web/BeanHandler.java index 26dd4edc9f..f2c019d7a4 100644 --- a/app/src/main/java/org/astraea/app/web/BeanHandler.java +++ b/app/src/main/java/org/astraea/app/web/BeanHandler.java @@ -46,7 +46,7 @@ public CompletionStage get(Channel channel) { .map( b -> { try (var client = - MBeanClient.jndi(b.host(), jmxPorts.apply(b.id()))) { + MBeanClient.jndi(b.id(), b.host(), jmxPorts.apply(b.id()))) { return new NodeBean( b.host(), client.beans(builder.build()).stream() diff --git a/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java b/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java index 662c3857c0..0aed00deaa 100644 --- a/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java +++ b/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java @@ -1221,7 +1221,7 @@ public SensorAndCost(Configuration configuration) { @Override public Optional metricSensor() { - return Optional.of((ignore, c, ignored) -> List.of(HostMetrics.jvmMemory(c))); + return Optional.of((c, ignored) -> List.of(HostMetrics.jvmMemory(c))); } @Override diff --git a/common/src/main/java/org/astraea/common/admin/ClusterBean.java b/common/src/main/java/org/astraea/common/admin/ClusterBean.java index 5844d0101a..79bd4bbb2a 100644 --- a/common/src/main/java/org/astraea/common/admin/ClusterBean.java +++ b/common/src/main/java/org/astraea/common/admin/ClusterBean.java @@ -50,8 +50,7 @@ static ClusterBean of(Map clients, MetricSensor sensor) { Map.Entry::getKey, entry -> sensor.fetch( - entry.getKey(), - MBeanClient.of(entry.getValue().beans(BeanQuery.all())), + MBeanClient.of(entry.getKey(), entry.getValue().beans(BeanQuery.all())), EMPTY)))); } diff --git a/common/src/main/java/org/astraea/common/assignor/Assignor.java b/common/src/main/java/org/astraea/common/assignor/Assignor.java index a1c063a1e7..90b88ad747 100644 --- a/common/src/main/java/org/astraea/common/assignor/Assignor.java +++ b/common/src/main/java/org/astraea/common/assignor/Assignor.java @@ -163,7 +163,8 @@ public final void configure(Map configs) { brokers.forEach( b -> map.put( - b.id(), MBeanClient.jndi(b.host(), jmxPortGetter.apply(b.id())))); + b.id(), + MBeanClient.jndi(b.id(), b.host(), jmxPortGetter.apply(b.id())))); // add local client to fetch consumer metrics map.put(-1, MBeanClient.local()); return Collections.unmodifiableMap(map); diff --git a/common/src/main/java/org/astraea/common/cost/BrokerInputCost.java b/common/src/main/java/org/astraea/common/cost/BrokerInputCost.java index 845b95209a..a5d7fffdef 100644 --- a/common/src/main/java/org/astraea/common/cost/BrokerInputCost.java +++ b/common/src/main/java/org/astraea/common/cost/BrokerInputCost.java @@ -46,8 +46,7 @@ public BrokerCost brokerCost(ClusterInfo clusterInfo, ClusterBean clusterBean) { @Override public Optional metricSensor() { return Optional.of( - (ignore, client, ignored) -> - List.of(ServerMetrics.BrokerTopic.BYTES_IN_PER_SEC.fetch(client))); + (client, ignored) -> List.of(ServerMetrics.BrokerTopic.BYTES_IN_PER_SEC.fetch(client))); } @Override diff --git a/common/src/main/java/org/astraea/common/cost/BrokerOutputCost.java b/common/src/main/java/org/astraea/common/cost/BrokerOutputCost.java index 1a37f36b47..c40c2faacc 100644 --- a/common/src/main/java/org/astraea/common/cost/BrokerOutputCost.java +++ b/common/src/main/java/org/astraea/common/cost/BrokerOutputCost.java @@ -45,8 +45,7 @@ public BrokerCost brokerCost(ClusterInfo clusterInfo, ClusterBean clusterBean) { @Override public Optional metricSensor() { return Optional.of( - (ignore, client, ignored) -> - List.of(ServerMetrics.BrokerTopic.BYTES_OUT_PER_SEC.fetch(client))); + (client, ignored) -> List.of(ServerMetrics.BrokerTopic.BYTES_OUT_PER_SEC.fetch(client))); } @Override diff --git a/common/src/main/java/org/astraea/common/cost/CpuCost.java b/common/src/main/java/org/astraea/common/cost/CpuCost.java index 2a4969f947..30dcba53a9 100644 --- a/common/src/main/java/org/astraea/common/cost/CpuCost.java +++ b/common/src/main/java/org/astraea/common/cost/CpuCost.java @@ -57,7 +57,7 @@ public BrokerCost brokerCost(ClusterInfo clusterInfo, ClusterBean clusterBean) { @Override public Optional metricSensor() { - return Optional.of((ignore, client, ignored) -> List.of(HostMetrics.operatingSystem(client))); + return Optional.of((client, ignored) -> List.of(HostMetrics.operatingSystem(client))); } @Override diff --git a/common/src/main/java/org/astraea/common/cost/LoadCost.java b/common/src/main/java/org/astraea/common/cost/LoadCost.java index ee739e268f..4d778354a5 100644 --- a/common/src/main/java/org/astraea/common/cost/LoadCost.java +++ b/common/src/main/java/org/astraea/common/cost/LoadCost.java @@ -150,7 +150,7 @@ Map computeLoad(Map> allBea @Override public Optional metricSensor() { return Optional.of( - (ignore, client, ignored) -> + (client, ignored) -> List.of( ServerMetrics.BrokerTopic.BYTES_IN_PER_SEC.fetch(client), ServerMetrics.BrokerTopic.BYTES_OUT_PER_SEC.fetch(client))); diff --git a/common/src/main/java/org/astraea/common/cost/MemoryCost.java b/common/src/main/java/org/astraea/common/cost/MemoryCost.java index cf62d7710a..7316e74970 100644 --- a/common/src/main/java/org/astraea/common/cost/MemoryCost.java +++ b/common/src/main/java/org/astraea/common/cost/MemoryCost.java @@ -57,7 +57,7 @@ public BrokerCost brokerCost(ClusterInfo clusterInfo, ClusterBean clusterBean) { @Override public Optional metricSensor() { - return Optional.of((ignore, client, ignored) -> List.of(HostMetrics.jvmMemory(client))); + return Optional.of((client, ignored) -> List.of(HostMetrics.jvmMemory(client))); } @Override diff --git a/common/src/main/java/org/astraea/common/cost/NetworkCost.java b/common/src/main/java/org/astraea/common/cost/NetworkCost.java index e7a0d6ea74..d5213ecd15 100644 --- a/common/src/main/java/org/astraea/common/cost/NetworkCost.java +++ b/common/src/main/java/org/astraea/common/cost/NetworkCost.java @@ -196,13 +196,13 @@ public Optional metricSensor() { // obtain the replica info, so we intentionally sample log size but never use it. // https://github.com/skiptests/astraea/pull/1240#discussion_r1044487473 return Optional.of( - (identity, client, clusterBean) -> + (client, clusterBean) -> Stream.of( List.of(HostMetrics.jvmMemory(client)), ServerMetrics.Topic.BYTES_IN_PER_SEC.fetch(client), ServerMetrics.Topic.BYTES_OUT_PER_SEC.fetch(client), LogMetrics.Log.SIZE.fetch(client), - clusterInfoSensor.fetch(identity, client, clusterBean)) + clusterInfoSensor.fetch(client, clusterBean)) .flatMap(Collection::stream) .collect(Collectors.toUnmodifiableList())); } diff --git a/common/src/main/java/org/astraea/common/cost/NodeMetricsCost.java b/common/src/main/java/org/astraea/common/cost/NodeMetricsCost.java index 44c298098b..a64d5e41a1 100644 --- a/common/src/main/java/org/astraea/common/cost/NodeMetricsCost.java +++ b/common/src/main/java/org/astraea/common/cost/NodeMetricsCost.java @@ -72,6 +72,6 @@ public BrokerCost brokerCost(ClusterInfo clusterInfo, ClusterBean clusterBean) { @Override public Optional metricSensor() { - return Optional.of((ignore, client, clusterBean) -> ProducerMetrics.node(client)); + return Optional.of((client, clusterBean) -> ProducerMetrics.node(client)); } } diff --git a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java index 4cc6c517e8..f4d74d5c3c 100644 --- a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java +++ b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java @@ -58,7 +58,7 @@ public PartitionMigrateTimeCost(Configuration config) { @Override public Optional metricSensor() { return Optional.of( - (identity, client, clusterBean) -> { + (client, clusterBean) -> { var metrics = clusterBean.all().values().stream() .flatMap(Collection::stream) @@ -69,12 +69,12 @@ public Optional metricSensor() { var current = Duration.ofMillis(System.currentTimeMillis()); var maxInRateSensor = maxBrokerReplicationInRate.computeIfAbsent( - identity, + client.identity(), ignore -> Sensor.builder().addStat(REPLICATION_IN_RATE, Max.of()).build()); var maxOutRateSensor = maxBrokerReplicationOutRate.computeIfAbsent( - identity, + client.identity(), ignore -> Sensor.builder().addStat(REPLICATION_OUT_RATE, Max.of()).build()); maxInRateSensor.record(newInMetrics.oneMinuteRate()); diff --git a/common/src/main/java/org/astraea/common/cost/ReplicaLeaderCost.java b/common/src/main/java/org/astraea/common/cost/ReplicaLeaderCost.java index cc91864ebe..f755f73a97 100644 --- a/common/src/main/java/org/astraea/common/cost/ReplicaLeaderCost.java +++ b/common/src/main/java/org/astraea/common/cost/ReplicaLeaderCost.java @@ -71,8 +71,7 @@ static Map leaderCount(ClusterInfo clusterInfo) { @Override public Optional metricSensor() { return Optional.of( - (ignore, client, ignored) -> - List.of(ServerMetrics.ReplicaManager.LEADER_COUNT.fetch(client))); + (client, ignored) -> List.of(ServerMetrics.ReplicaManager.LEADER_COUNT.fetch(client))); } public Configuration config() { diff --git a/common/src/main/java/org/astraea/common/cost/utils/ClusterInfoSensor.java b/common/src/main/java/org/astraea/common/cost/utils/ClusterInfoSensor.java index e39311860a..ef6c408230 100644 --- a/common/src/main/java/org/astraea/common/cost/utils/ClusterInfoSensor.java +++ b/common/src/main/java/org/astraea/common/cost/utils/ClusterInfoSensor.java @@ -38,7 +38,7 @@ public class ClusterInfoSensor implements MetricSensor { @Override - public List fetch(int identity, MBeanClient client, ClusterBean bean) { + public List fetch(MBeanClient client, ClusterBean bean) { return Stream.of( List.of(ServerMetrics.KafkaServer.CLUSTER_ID.fetch(client)), LogMetrics.Log.SIZE.fetch(client), diff --git a/common/src/main/java/org/astraea/common/metrics/MBeanClient.java b/common/src/main/java/org/astraea/common/metrics/MBeanClient.java index a0e1fbcdd2..9f017daf47 100644 --- a/common/src/main/java/org/astraea/common/metrics/MBeanClient.java +++ b/common/src/main/java/org/astraea/common/metrics/MBeanClient.java @@ -56,7 +56,7 @@ * } */ public interface MBeanClient extends AutoCloseable { - static MBeanClient of(Collection objs) { + static MBeanClient of(int identity, Collection objs) { return new MBeanClient() { @Override @@ -95,6 +95,11 @@ public Collection beans( .matches())) .collect(Collectors.toUnmodifiableList()); } + + @Override + public int identity() { + return identity; + } }; } @@ -103,9 +108,10 @@ public Collection beans( * @param port the port of jmx server * @return a mbean client using JNDI to lookup metrics. */ - static MBeanClient jndi(String host, int port) { + static MBeanClient jndi(int identity, String host, int port) { try { return of( + identity, new JMXServiceURL( String.format( "service:jmx:rmi://%s:%s/jndi/rmi://%s:%s/jmxrmi", host, port, host, port))); @@ -114,11 +120,12 @@ static MBeanClient jndi(String host, int port) { } } - static MBeanClient of(JMXServiceURL jmxServiceURL) { + static MBeanClient of(int identity, JMXServiceURL jmxServiceURL) { return Utils.packException( () -> { var jmxConnector = JMXConnectorFactory.connect(jmxServiceURL); return new BasicMBeanClient( + identity, jmxConnector.getMBeanServerConnection(), jmxServiceURL.getHost(), jmxServiceURL.getPort()) { @@ -131,7 +138,8 @@ public void close() { } static MBeanClient local() { - return new BasicMBeanClient(ManagementFactory.getPlatformMBeanServer(), Utils.hostname(), -1); + return new BasicMBeanClient( + -1, ManagementFactory.getPlatformMBeanServer(), Utils.hostname(), -1); } /** @@ -168,17 +176,21 @@ default Collection beans(BeanQuery beanQuery) { */ Collection beans(BeanQuery beanQuery, Consumer errorHandle); + int identity(); + @Override default void close() {} class BasicMBeanClient implements MBeanClient { private final MBeanServerConnection connection; + final int identity; final String host; final int port; - BasicMBeanClient(MBeanServerConnection connection, String host, int port) { + BasicMBeanClient(int identity, MBeanServerConnection connection, String host, int port) { + this.identity = identity; this.connection = connection; this.host = host; this.port = port; @@ -261,6 +273,11 @@ public Collection beans( .collect(Collectors.toUnmodifiableList())); } + @Override + public int identity() { + return identity; + } + /** * Returns the list of domains in which any MBean is currently registered. * diff --git a/common/src/main/java/org/astraea/common/metrics/collector/MetricSensor.java b/common/src/main/java/org/astraea/common/metrics/collector/MetricSensor.java index 54d31f09d8..2afedac949 100644 --- a/common/src/main/java/org/astraea/common/metrics/collector/MetricSensor.java +++ b/common/src/main/java/org/astraea/common/metrics/collector/MetricSensor.java @@ -48,12 +48,12 @@ static Optional of( Collection metricSensors, Consumer exceptionHandler) { if (metricSensors.isEmpty()) return Optional.empty(); return Optional.of( - (identity, client, clusterBean) -> + (client, clusterBean) -> metricSensors.stream() .flatMap( ms -> { try { - return ms.fetch(identity, client, clusterBean).stream(); + return ms.fetch(client, clusterBean).stream(); } catch (Exception ex) { exceptionHandler.accept(ex); return Stream.empty(); @@ -66,10 +66,9 @@ static Optional of( * fetch metrics from remote/local mbean server. Or the implementation can generate custom metrics * according to existent cluster bean * - * @param identity * @param client mbean client (don't close it!) * @param bean current cluster bean * @return java metrics */ - Collection fetch(int identity, MBeanClient client, ClusterBean bean); + Collection fetch(MBeanClient client, ClusterBean bean); } diff --git a/common/src/main/java/org/astraea/common/metrics/collector/MetricStore.java b/common/src/main/java/org/astraea/common/metrics/collector/MetricStore.java index b27807dd68..7e7104b55a 100644 --- a/common/src/main/java/org/astraea/common/metrics/collector/MetricStore.java +++ b/common/src/main/java/org/astraea/common/metrics/collector/MetricStore.java @@ -117,7 +117,7 @@ class Builder { () -> Map.of( (MetricSensor) - (identity, client, bean) -> + (client, bean) -> client.beans(BeanQuery.all()).stream() .map(bs -> (HasBeanObject) () -> bs) .collect(Collectors.toUnmodifiableList()), @@ -229,14 +229,14 @@ private MetricStoreImpl( lastSensors = sensorsSupplier.get(); allBeans.forEach( (id, bs) -> { - var client = MBeanClient.of(bs); + var client = MBeanClient.of(id, bs); var clusterBean = clusterBean(); lastSensors.forEach( (sensor, errorHandler) -> { try { beans .computeIfAbsent(id, ignored -> new ConcurrentLinkedQueue<>()) - .addAll(sensor.fetch(id, client, clusterBean)); + .addAll(sensor.fetch(client, clusterBean)); } catch (Exception e) { errorHandler.accept(id, e); } diff --git a/common/src/main/java/org/astraea/common/partitioner/SmoothWeightRoundRobinPartitioner.java b/common/src/main/java/org/astraea/common/partitioner/SmoothWeightRoundRobinPartitioner.java index 3bf7f44e8c..8b7f95b55e 100644 --- a/common/src/main/java/org/astraea/common/partitioner/SmoothWeightRoundRobinPartitioner.java +++ b/common/src/main/java/org/astraea/common/partitioner/SmoothWeightRoundRobinPartitioner.java @@ -128,7 +128,8 @@ void configure( brokers.forEach( b -> map.put( - b.id(), MBeanClient.jndi(b.host(), jmxPortGetter.apply(b.id())))); + b.id(), + MBeanClient.jndi(b.id(), b.host(), jmxPortGetter.apply(b.id())))); // add local client to fetch consumer metrics map.put(-1, MBeanClient.local()); return Collections.unmodifiableMap(map); diff --git a/common/src/main/java/org/astraea/common/partitioner/StrictCostPartitioner.java b/common/src/main/java/org/astraea/common/partitioner/StrictCostPartitioner.java index 8792ac6d04..6e423fff65 100644 --- a/common/src/main/java/org/astraea/common/partitioner/StrictCostPartitioner.java +++ b/common/src/main/java/org/astraea/common/partitioner/StrictCostPartitioner.java @@ -149,7 +149,8 @@ public void configure(Configuration config) { brokers.forEach( b -> map.put( - b.id(), MBeanClient.jndi(b.host(), jmxPortGetter.apply(b.id())))); + b.id(), + MBeanClient.jndi(b.id(), b.host(), jmxPortGetter.apply(b.id())))); // add local client to fetch consumer metrics map.put(-1, MBeanClient.local()); return Collections.unmodifiableMap(map); diff --git a/common/src/test/java/org/astraea/common/cost/BrokerInputCostTest.java b/common/src/test/java/org/astraea/common/cost/BrokerInputCostTest.java index 9ef2cde6ae..db217456d1 100644 --- a/common/src/test/java/org/astraea/common/cost/BrokerInputCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/BrokerInputCostTest.java @@ -64,7 +64,8 @@ void testSensor() { } var f = new BrokerInputCost(); var clusterBean = - ClusterBean.of(Map.of(0, MBeanClient.of(SERVICE.jmxServiceURL())), f.metricSensor().get()); + ClusterBean.of( + Map.of(0, MBeanClient.of(0, SERVICE.jmxServiceURL())), f.metricSensor().get()); Assertions.assertNotEquals( 0, clusterBean.brokerMetrics(0, ServerMetrics.BrokerTopic.Meter.class).count()); diff --git a/common/src/test/java/org/astraea/common/cost/ClusterCostTest.java b/common/src/test/java/org/astraea/common/cost/ClusterCostTest.java index 8ecb03427b..6b6541d234 100644 --- a/common/src/test/java/org/astraea/common/cost/ClusterCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/ClusterCostTest.java @@ -63,7 +63,7 @@ void testSensor() { var mergeCost = HasClusterCost.of(Map.of(cost1, 1.0, cost2, 1.0)); var metrics = mergeCost.metricSensor().stream() - .map(x -> x.fetch(-1, MBeanClient.of(SERVICE.jmxServiceURL()), ClusterBean.EMPTY)) + .map(x -> x.fetch(MBeanClient.of(-1, SERVICE.jmxServiceURL()), ClusterBean.EMPTY)) .collect(Collectors.toSet()); Assertions.assertTrue( metrics.iterator().next().stream() diff --git a/common/src/test/java/org/astraea/common/cost/MoveCostTest.java b/common/src/test/java/org/astraea/common/cost/MoveCostTest.java index f4ac6f1553..1ca528e117 100644 --- a/common/src/test/java/org/astraea/common/cost/MoveCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/MoveCostTest.java @@ -49,7 +49,7 @@ void testSensor() { var mergeCost = HasMoveCost.of(List.of(cost1, cost2)); var metrics = mergeCost.metricSensor().stream() - .map(x -> x.fetch(-1, MBeanClient.of(SERVICE.jmxServiceURL()), ClusterBean.EMPTY)) + .map(x -> x.fetch(MBeanClient.of(-1, SERVICE.jmxServiceURL()), ClusterBean.EMPTY)) .collect(Collectors.toSet()); Assertions.assertEquals(3, metrics.iterator().next().size()); Assertions.assertTrue( @@ -84,10 +84,9 @@ class FakeCf implements HasMoveCost { public Optional metricSensor() { return MetricSensor.of( List.of( - (ignore, c, ignored) -> + (c, ignored) -> List.of(ServerMetrics.BrokerTopic.REPLICATION_BYTES_IN_PER_SEC.fetch(c)), - (ignore, c, ignored) -> - List.of(ServerMetrics.BrokerTopic.BYTES_IN_PER_SEC.fetch(c)))); + (c, ignored) -> List.of(ServerMetrics.BrokerTopic.BYTES_IN_PER_SEC.fetch(c)))); } @Override diff --git a/common/src/test/java/org/astraea/common/cost/NodeLatencyCostTest.java b/common/src/test/java/org/astraea/common/cost/NodeLatencyCostTest.java index 7fd88cf1f9..43cf8a0f76 100644 --- a/common/src/test/java/org/astraea/common/cost/NodeLatencyCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/NodeLatencyCostTest.java @@ -109,7 +109,7 @@ void testSensor() { new BeanObject("a", Map.of("node-id", "node-10"), Map.of()), new BeanObject("a", Map.of("node-id", "node-10"), Map.of()), new BeanObject("a", Map.of("node-id", "node-11"), Map.of()))); - var result = function.metricSensor().get().fetch(-1, client, ClusterBean.EMPTY); + var result = function.metricSensor().get().fetch(client, ClusterBean.EMPTY); Assertions.assertEquals(3, result.size()); } } diff --git a/common/src/test/java/org/astraea/common/cost/NodeThroughputCostTest.java b/common/src/test/java/org/astraea/common/cost/NodeThroughputCostTest.java index c10b01249b..58f1f2d8bb 100644 --- a/common/src/test/java/org/astraea/common/cost/NodeThroughputCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/NodeThroughputCostTest.java @@ -78,7 +78,7 @@ void testSensor() { var bean = new BeanObject("aaa", Map.of("node-id", "node-1"), Map.of()); var client = Mockito.mock(MBeanClient.class); Mockito.when(client.beans(Mockito.any())).thenReturn(List.of(bean)); - var result = sensor.fetch(-1, client, ClusterBean.EMPTY); + var result = sensor.fetch(client, ClusterBean.EMPTY); Assertions.assertEquals(1, result.size()); Assertions.assertEquals(bean, result.iterator().next().beanObject()); } diff --git a/common/src/test/java/org/astraea/common/metrics/BeanQueryIntegratedTest.java b/common/src/test/java/org/astraea/common/metrics/BeanQueryIntegratedTest.java index ba67cae671..ca81dc5091 100644 --- a/common/src/test/java/org/astraea/common/metrics/BeanQueryIntegratedTest.java +++ b/common/src/test/java/org/astraea/common/metrics/BeanQueryIntegratedTest.java @@ -35,7 +35,7 @@ static void closeService() { @Test void testAllBuiltInQueries() { - try (var client = MBeanClient.of(SERVICE.jmxServiceURL())) { + try (var client = MBeanClient.of(-1, SERVICE.jmxServiceURL())) { var exist = new HashSet>(); MetricFetcher.QUERIES.forEach( q -> diff --git a/common/src/test/java/org/astraea/common/metrics/MBeanClientTest.java b/common/src/test/java/org/astraea/common/metrics/MBeanClientTest.java index da60325638..c3f2155fde 100644 --- a/common/src/test/java/org/astraea/common/metrics/MBeanClientTest.java +++ b/common/src/test/java/org/astraea/common/metrics/MBeanClientTest.java @@ -101,7 +101,7 @@ void tearDown() throws IOException { @Test void testFetchAttributes() { // arrange - try (var client = MBeanClient.of(jmxServer.getAddress())) { + try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { BeanQuery beanQuery = BeanQuery.builder().domainName("java.lang").property("type", "Memory").build(); @@ -118,7 +118,7 @@ void testFetchAttributes() { @Test void testFetchMbeanWithMultipleProperties() { // arrange - try (var client = MBeanClient.of(jmxServer.getAddress())) { + try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { BeanQuery query1 = BeanQuery.builder() .domainName("java.lang") @@ -160,7 +160,7 @@ void testFetchSelectedAttributes() AttributeNotFoundException, MBeanException { // arrange - try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(jmxServer.getAddress())) { + try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(-1, jmxServer.getAddress())) { BeanQuery beanQuery = BeanQuery.builder().domainName("java.lang").property("type", "Memory").build(); List selectedAttribute = List.of("HeapMemoryUsage"); @@ -178,7 +178,7 @@ void testFetchSelectedAttributes() @Test void testQueryBeans() { // arrange 1 query beans - try (var client = MBeanClient.of(jmxServer.getAddress())) { + try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { BeanQuery beanQuery = BeanQuery.builder().domainName("java.lang").property("type", "C*").build(); @@ -216,7 +216,7 @@ void testQueryBeans() { @Test void testQueryNonExistsBeans() { // arrange - try (var client = MBeanClient.of(jmxServer.getAddress())) { + try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { BeanQuery beanQuery = BeanQuery.builder().domainName("java.lang").property("type", "Something").build(); @@ -231,7 +231,7 @@ void testQueryNonExistsBeans() { @Test void testFetchNonExistsBeans() { // arrange - try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(jmxServer.getAddress())) { + try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(-1, jmxServer.getAddress())) { BeanQuery beanQuery = BeanQuery.builder().domainName("java.lang").property("type", "Something").build(); @@ -246,7 +246,7 @@ void testFetchNonExistsBeans() { @Test void testCloseOnceMore() { // arrange - var client = MBeanClient.of(jmxServer.getAddress()); + var client = MBeanClient.of(-1, jmxServer.getAddress()); // act client.close(); @@ -262,7 +262,7 @@ void testCloseOnceMore() { @Test void testGetAllMBeans() { // arrange - try (var client = MBeanClient.of(jmxServer.getAddress())) { + try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { // act Collection beanObjects = client.beans(BeanQuery.all()); @@ -276,7 +276,7 @@ void testGetAllMBeans() { @Test void testGetAllMBeansUnderSpecificDomainName() { // arrange - try (var client = MBeanClient.of(jmxServer.getAddress())) { + try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { // act Collection beanObjects = client.beans(BeanQuery.all("java.lang")); @@ -290,7 +290,7 @@ void testGetAllMBeansUnderSpecificDomainName() { @Test void testGetAllMBeansUnderSpecificDomainNamePattern() { // arrange - try (var client = MBeanClient.of(jmxServer.getAddress())) { + try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { // act Collection beanObjects = client.beans(BeanQuery.all("java.*")); @@ -303,7 +303,7 @@ void testGetAllMBeansUnderSpecificDomainNamePattern() { @Test void testUsePropertyListPatternForRemote() { - testUsePropertyListPattern(MBeanClient.of(jmxServer.getAddress())); + testUsePropertyListPattern(MBeanClient.of(-1, jmxServer.getAddress())); } @Test @@ -366,7 +366,7 @@ private void testUsePropertyListPattern(MBeanClient client) { @Test void testListDomains() { // arrange - try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(jmxServer.getAddress())) { + try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(-1, jmxServer.getAddress())) { // act List domains = client.domains(); @@ -380,7 +380,7 @@ void testListDomains() { @Test void testHostAndPort() { // arrange - try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(jmxServer.getAddress())) { + try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(-1, jmxServer.getAddress())) { assertEquals(jmxServer.getAddress().getHost(), client.host); assertEquals(jmxServer.getAddress().getPort(), client.port); } @@ -396,7 +396,7 @@ void testCustomMBeanWith500Attributes() throws Exception { register(objectName0, customMBean0); - try (var client = MBeanClient.of(jmxServer.getAddress())) { + try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { // act Collection all = @@ -423,7 +423,7 @@ void testWith100CustomMBeans() throws Exception { register(objectName, mbean); } - try (var client = MBeanClient.of(jmxServer.getAddress())) { + try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { // act Collection all = diff --git a/common/src/test/java/org/astraea/common/metrics/collector/MetricSensorTest.java b/common/src/test/java/org/astraea/common/metrics/collector/MetricSensorTest.java index cf9a9ded0f..123239caea 100644 --- a/common/src/test/java/org/astraea/common/metrics/collector/MetricSensorTest.java +++ b/common/src/test/java/org/astraea/common/metrics/collector/MetricSensorTest.java @@ -31,13 +31,13 @@ public class MetricSensorTest { @Test void testMultipleSensors() { var mbean0 = Mockito.mock(HasBeanObject.class); - MetricSensor metricSensor0 = (ignore, client, ignored) -> List.of(mbean0); + MetricSensor metricSensor0 = (client, ignored) -> List.of(mbean0); var mbean1 = Mockito.mock(HasBeanObject.class); - MetricSensor metricSensor1 = (ignore, client, ignored) -> List.of(mbean1); + MetricSensor metricSensor1 = (client, ignored) -> List.of(mbean1); var sensor = MetricSensor.of(List.of(metricSensor0, metricSensor1)).get(); - var result = sensor.fetch(-1, Mockito.mock(MBeanClient.class), ClusterBean.EMPTY); + var result = sensor.fetch(Mockito.mock(MBeanClient.class), ClusterBean.EMPTY); Assertions.assertEquals(2, result.size()); Assertions.assertTrue(result.contains(mbean0)); @@ -52,9 +52,9 @@ void testEmpty() { @Test void testNoSwallowException() { var result = List.of(Mockito.mock(HasBeanObject.class)); - MetricSensor goodMetricSensor = (ignore, client, ignored) -> result; + MetricSensor goodMetricSensor = (client, ignored) -> result; MetricSensor badMetricSensor = - (ignore, client, ignored) -> { + (client, ignored) -> { throw new RuntimeException("xxx"); }; var sensor = @@ -68,33 +68,33 @@ void testNoSwallowException() { .get(); Assertions.assertThrows( RuntimeException.class, - () -> sensor.fetch(-1, Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); + () -> sensor.fetch(Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); } @Test void testSensorsWithExceptionHandler() { var mbean0 = Mockito.mock(HasBeanObject.class); - MetricSensor metricSensor0 = (ignore, client, ignored) -> List.of(mbean0); + MetricSensor metricSensor0 = (client, ignored) -> List.of(mbean0); MetricSensor metricSensor1 = - (ignore, client, ignored) -> { + (client, ignored) -> { throw new NoSuchElementException(); }; MetricSensor metricSensor2 = - (ignore, client, ignored) -> { + (client, ignored) -> { throw new RuntimeException(); }; var sensor = MetricSensor.of(List.of(metricSensor0, metricSensor1)).get(); Assertions.assertDoesNotThrow( - () -> sensor.fetch(-1, Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); + () -> sensor.fetch(Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); Assertions.assertEquals( - 1, sensor.fetch(-1, Mockito.mock(MBeanClient.class), ClusterBean.EMPTY).size()); + 1, sensor.fetch(Mockito.mock(MBeanClient.class), ClusterBean.EMPTY).size()); Assertions.assertDoesNotThrow( () -> MetricSensor.of(List.of(metricSensor0, metricSensor2)) .get() - .fetch(-1, Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); + .fetch(Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); Assertions.assertThrows( NoSuchElementException.class, () -> @@ -104,6 +104,6 @@ void testSensorsWithExceptionHandler() { if (e instanceof NoSuchElementException) throw new NoSuchElementException(); }) .get() - .fetch(-1, Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); + .fetch(Mockito.mock(MBeanClient.class), ClusterBean.EMPTY)); } } diff --git a/gui/src/main/java/org/astraea/gui/Context.java b/gui/src/main/java/org/astraea/gui/Context.java index ead99959c9..843dbac814 100644 --- a/gui/src/main/java/org/astraea/gui/Context.java +++ b/gui/src/main/java/org/astraea/gui/Context.java @@ -78,7 +78,7 @@ public Map addBrokerClients(List nodeInfos) { nodeInfos.forEach( n -> brokerClients.clients.computeIfAbsent( - n.id(), ignored -> MBeanClient.jndi(n.host(), brokerClients.jmxPort))); + n.id(), ignored -> MBeanClient.jndi(n.id(), n.host(), brokerClients.jmxPort))); return Map.copyOf(brokerClients.clients); } @@ -88,7 +88,7 @@ public Map addWorkerClients(Set hostnames) { hostnames.forEach( n -> workerClients.clients.computeIfAbsent( - n, ignored -> MBeanClient.jndi(n, workerClients.jmxPort))); + n, ignored -> MBeanClient.jndi(-1, n, workerClients.jmxPort))); return Map.copyOf(workerClients.clients); } From 43b1a5fb77b82bffa7e1e2faf06769ecc17c1189 Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Fri, 21 Apr 2023 20:53:40 +0800 Subject: [PATCH 03/24] fix bug --- .../test/java/org/astraea/common/metrics/MetricsTestUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/test/java/org/astraea/common/metrics/MetricsTestUtils.java b/common/src/test/java/org/astraea/common/metrics/MetricsTestUtils.java index 1054e63c71..9433d80edc 100644 --- a/common/src/test/java/org/astraea/common/metrics/MetricsTestUtils.java +++ b/common/src/test/java/org/astraea/common/metrics/MetricsTestUtils.java @@ -52,7 +52,7 @@ public static ClusterBean clusterBean(Map clients, MetricS Map.Entry::getKey, entry -> sensor.fetch( - MBeanClient.of(entry.getValue().beans(BeanQuery.all())), + MBeanClient.of(entry.getKey(), entry.getValue().beans(BeanQuery.all())), ClusterBean.EMPTY)))); } From 749b63bcc0a43a8b41b4b57b99da5caa62e40f2e Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Mon, 24 Apr 2023 00:36:16 +0800 Subject: [PATCH 04/24] fix issues --- .../app/publisher/MetricPublisher.java | 1 - .../java/org/astraea/app/web/BeanHandler.java | 2 +- .../org/astraea/common/assignor/Assignor.java | 3 +- .../astraea/common/metrics/MBeanClient.java | 20 +++++++++++-- .../SmoothWeightRoundRobinPartitioner.java | 3 +- .../partitioner/StrictCostPartitioner.java | 3 +- .../common/cost/BrokerInputCostTest.java | 2 +- .../astraea/common/cost/ClusterCostTest.java | 2 +- .../org/astraea/common/cost/MoveCostTest.java | 2 +- .../metrics/BeanQueryIntegratedTest.java | 2 +- .../common/metrics/MBeanClientTest.java | 30 +++++++++---------- .../main/java/org/astraea/gui/Context.java | 4 +-- 12 files changed, 43 insertions(+), 31 deletions(-) diff --git a/app/src/main/java/org/astraea/app/publisher/MetricPublisher.java b/app/src/main/java/org/astraea/app/publisher/MetricPublisher.java index 5c9a6f5660..976ca23bb9 100644 --- a/app/src/main/java/org/astraea/app/publisher/MetricPublisher.java +++ b/app/src/main/java/org/astraea/app/publisher/MetricPublisher.java @@ -59,7 +59,6 @@ static void execute(Arguments arguments) { NodeInfo::id, node -> MBeanClient.jndi( - node.id(), node.host(), arguments.idToJmxPort().apply(node.id())))))) .fetchBeanDelay(arguments.period) diff --git a/app/src/main/java/org/astraea/app/web/BeanHandler.java b/app/src/main/java/org/astraea/app/web/BeanHandler.java index f2c019d7a4..26dd4edc9f 100644 --- a/app/src/main/java/org/astraea/app/web/BeanHandler.java +++ b/app/src/main/java/org/astraea/app/web/BeanHandler.java @@ -46,7 +46,7 @@ public CompletionStage get(Channel channel) { .map( b -> { try (var client = - MBeanClient.jndi(b.id(), b.host(), jmxPorts.apply(b.id()))) { + MBeanClient.jndi(b.host(), jmxPorts.apply(b.id()))) { return new NodeBean( b.host(), client.beans(builder.build()).stream() diff --git a/common/src/main/java/org/astraea/common/assignor/Assignor.java b/common/src/main/java/org/astraea/common/assignor/Assignor.java index 90b88ad747..a1c063a1e7 100644 --- a/common/src/main/java/org/astraea/common/assignor/Assignor.java +++ b/common/src/main/java/org/astraea/common/assignor/Assignor.java @@ -163,8 +163,7 @@ public final void configure(Map configs) { brokers.forEach( b -> map.put( - b.id(), - MBeanClient.jndi(b.id(), b.host(), jmxPortGetter.apply(b.id())))); + b.id(), MBeanClient.jndi(b.host(), jmxPortGetter.apply(b.id())))); // add local client to fetch consumer metrics map.put(-1, MBeanClient.local()); return Collections.unmodifiableMap(map); diff --git a/common/src/main/java/org/astraea/common/metrics/MBeanClient.java b/common/src/main/java/org/astraea/common/metrics/MBeanClient.java index 9f017daf47..7279109f56 100644 --- a/common/src/main/java/org/astraea/common/metrics/MBeanClient.java +++ b/common/src/main/java/org/astraea/common/metrics/MBeanClient.java @@ -56,6 +56,11 @@ * } */ public interface MBeanClient extends AutoCloseable { + + static MBeanClient of(Collection objs) { + return of(-1, objs); + } + static MBeanClient of(int identity, Collection objs) { return new MBeanClient() { @@ -103,6 +108,10 @@ public int identity() { }; } + static MBeanClient jndi(String host, int port) { + return jndi(-1, host, port); + } + /** * @param host the address of jmx server * @param port the port of jmx server @@ -120,6 +129,10 @@ static MBeanClient jndi(int identity, String host, int port) { } } + static MBeanClient of(JMXServiceURL jmxServiceURL) { + return of(-1, jmxServiceURL); + } + static MBeanClient of(int identity, JMXServiceURL jmxServiceURL) { return Utils.packException( () -> { @@ -138,8 +151,7 @@ public void close() { } static MBeanClient local() { - return new BasicMBeanClient( - -1, ManagementFactory.getPlatformMBeanServer(), Utils.hostname(), -1); + return new BasicMBeanClient(ManagementFactory.getPlatformMBeanServer(), Utils.hostname(), -1); } /** @@ -189,6 +201,10 @@ class BasicMBeanClient implements MBeanClient { final int port; + BasicMBeanClient(MBeanServerConnection connection, String host, int port) { + this(-1, connection, host, port); + } + BasicMBeanClient(int identity, MBeanServerConnection connection, String host, int port) { this.identity = identity; this.connection = connection; diff --git a/common/src/main/java/org/astraea/common/partitioner/SmoothWeightRoundRobinPartitioner.java b/common/src/main/java/org/astraea/common/partitioner/SmoothWeightRoundRobinPartitioner.java index 8b7f95b55e..3bf7f44e8c 100644 --- a/common/src/main/java/org/astraea/common/partitioner/SmoothWeightRoundRobinPartitioner.java +++ b/common/src/main/java/org/astraea/common/partitioner/SmoothWeightRoundRobinPartitioner.java @@ -128,8 +128,7 @@ void configure( brokers.forEach( b -> map.put( - b.id(), - MBeanClient.jndi(b.id(), b.host(), jmxPortGetter.apply(b.id())))); + b.id(), MBeanClient.jndi(b.host(), jmxPortGetter.apply(b.id())))); // add local client to fetch consumer metrics map.put(-1, MBeanClient.local()); return Collections.unmodifiableMap(map); diff --git a/common/src/main/java/org/astraea/common/partitioner/StrictCostPartitioner.java b/common/src/main/java/org/astraea/common/partitioner/StrictCostPartitioner.java index 6e423fff65..8792ac6d04 100644 --- a/common/src/main/java/org/astraea/common/partitioner/StrictCostPartitioner.java +++ b/common/src/main/java/org/astraea/common/partitioner/StrictCostPartitioner.java @@ -149,8 +149,7 @@ public void configure(Configuration config) { brokers.forEach( b -> map.put( - b.id(), - MBeanClient.jndi(b.id(), b.host(), jmxPortGetter.apply(b.id())))); + b.id(), MBeanClient.jndi(b.host(), jmxPortGetter.apply(b.id())))); // add local client to fetch consumer metrics map.put(-1, MBeanClient.local()); return Collections.unmodifiableMap(map); diff --git a/common/src/test/java/org/astraea/common/cost/BrokerInputCostTest.java b/common/src/test/java/org/astraea/common/cost/BrokerInputCostTest.java index 45dd78a8f0..3ec4a1b943 100644 --- a/common/src/test/java/org/astraea/common/cost/BrokerInputCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/BrokerInputCostTest.java @@ -66,7 +66,7 @@ void testSensor() { var f = new BrokerInputCost(); var clusterBean = MetricsTestUtils.clusterBean( - Map.of(0, MBeanClient.of(0, SERVICE.jmxServiceURL())), f.metricSensor().get()); + Map.of(0, MBeanClient.of(SERVICE.jmxServiceURL())), f.metricSensor().get()); Assertions.assertNotEquals( 0, clusterBean.brokerMetrics(0, ServerMetrics.BrokerTopic.Meter.class).count()); diff --git a/common/src/test/java/org/astraea/common/cost/ClusterCostTest.java b/common/src/test/java/org/astraea/common/cost/ClusterCostTest.java index 6b6541d234..b62684474c 100644 --- a/common/src/test/java/org/astraea/common/cost/ClusterCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/ClusterCostTest.java @@ -63,7 +63,7 @@ void testSensor() { var mergeCost = HasClusterCost.of(Map.of(cost1, 1.0, cost2, 1.0)); var metrics = mergeCost.metricSensor().stream() - .map(x -> x.fetch(MBeanClient.of(-1, SERVICE.jmxServiceURL()), ClusterBean.EMPTY)) + .map(x -> x.fetch(MBeanClient.of(SERVICE.jmxServiceURL()), ClusterBean.EMPTY)) .collect(Collectors.toSet()); Assertions.assertTrue( metrics.iterator().next().stream() diff --git a/common/src/test/java/org/astraea/common/cost/MoveCostTest.java b/common/src/test/java/org/astraea/common/cost/MoveCostTest.java index 1ca528e117..798592eea8 100644 --- a/common/src/test/java/org/astraea/common/cost/MoveCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/MoveCostTest.java @@ -49,7 +49,7 @@ void testSensor() { var mergeCost = HasMoveCost.of(List.of(cost1, cost2)); var metrics = mergeCost.metricSensor().stream() - .map(x -> x.fetch(MBeanClient.of(-1, SERVICE.jmxServiceURL()), ClusterBean.EMPTY)) + .map(x -> x.fetch(MBeanClient.of(SERVICE.jmxServiceURL()), ClusterBean.EMPTY)) .collect(Collectors.toSet()); Assertions.assertEquals(3, metrics.iterator().next().size()); Assertions.assertTrue( diff --git a/common/src/test/java/org/astraea/common/metrics/BeanQueryIntegratedTest.java b/common/src/test/java/org/astraea/common/metrics/BeanQueryIntegratedTest.java index ca81dc5091..ba67cae671 100644 --- a/common/src/test/java/org/astraea/common/metrics/BeanQueryIntegratedTest.java +++ b/common/src/test/java/org/astraea/common/metrics/BeanQueryIntegratedTest.java @@ -35,7 +35,7 @@ static void closeService() { @Test void testAllBuiltInQueries() { - try (var client = MBeanClient.of(-1, SERVICE.jmxServiceURL())) { + try (var client = MBeanClient.of(SERVICE.jmxServiceURL())) { var exist = new HashSet>(); MetricFetcher.QUERIES.forEach( q -> diff --git a/common/src/test/java/org/astraea/common/metrics/MBeanClientTest.java b/common/src/test/java/org/astraea/common/metrics/MBeanClientTest.java index 2852dc68d2..2edccd30ca 100644 --- a/common/src/test/java/org/astraea/common/metrics/MBeanClientTest.java +++ b/common/src/test/java/org/astraea/common/metrics/MBeanClientTest.java @@ -109,7 +109,7 @@ void tearDown() throws IOException { @Test void testFetchAttributes() { // arrange - try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { + try (var client = MBeanClient.of(jmxServer.getAddress())) { BeanQuery beanQuery = BeanQuery.builder().domainName("java.lang").property("type", "Memory").build(); @@ -126,7 +126,7 @@ void testFetchAttributes() { @Test void testFetchMbeanWithMultipleProperties() { // arrange - try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { + try (var client = MBeanClient.of(jmxServer.getAddress())) { BeanQuery query1 = BeanQuery.builder() .domainName("java.lang") @@ -168,7 +168,7 @@ void testFetchSelectedAttributes() AttributeNotFoundException, MBeanException { // arrange - try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(-1, jmxServer.getAddress())) { + try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(jmxServer.getAddress())) { BeanQuery beanQuery = BeanQuery.builder().domainName("java.lang").property("type", "Memory").build(); List selectedAttribute = List.of("HeapMemoryUsage"); @@ -186,7 +186,7 @@ void testFetchSelectedAttributes() @Test void testQueryBeans() { // arrange 1 query beans - try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { + try (var client = MBeanClient.of(jmxServer.getAddress())) { BeanQuery beanQuery = BeanQuery.builder().domainName("java.lang").property("type", "C*").build(); @@ -224,7 +224,7 @@ void testQueryBeans() { @Test void testQueryNonExistsBeans() { // arrange - try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { + try (var client = MBeanClient.of(jmxServer.getAddress())) { BeanQuery beanQuery = BeanQuery.builder().domainName("java.lang").property("type", "Something").build(); @@ -239,7 +239,7 @@ void testQueryNonExistsBeans() { @Test void testFetchNonExistsBeans() { // arrange - try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(-1, jmxServer.getAddress())) { + try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(jmxServer.getAddress())) { BeanQuery beanQuery = BeanQuery.builder().domainName("java.lang").property("type", "Something").build(); @@ -254,7 +254,7 @@ void testFetchNonExistsBeans() { @Test void testCloseOnceMore() { // arrange - var client = MBeanClient.of(-1, jmxServer.getAddress()); + var client = MBeanClient.of(jmxServer.getAddress()); // act client.close(); @@ -270,7 +270,7 @@ void testCloseOnceMore() { @Test void testGetAllMBeans() { // arrange - try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { + try (var client = MBeanClient.of(jmxServer.getAddress())) { // act Collection beanObjects = client.beans(BeanQuery.all()); @@ -284,7 +284,7 @@ void testGetAllMBeans() { @Test void testGetAllMBeansUnderSpecificDomainName() { // arrange - try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { + try (var client = MBeanClient.of(jmxServer.getAddress())) { // act Collection beanObjects = client.beans(BeanQuery.all("java.lang")); @@ -298,7 +298,7 @@ void testGetAllMBeansUnderSpecificDomainName() { @Test void testGetAllMBeansUnderSpecificDomainNamePattern() { // arrange - try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { + try (var client = MBeanClient.of(jmxServer.getAddress())) { // act Collection beanObjects = client.beans(BeanQuery.all("java.*")); @@ -311,7 +311,7 @@ void testGetAllMBeansUnderSpecificDomainNamePattern() { @Test void testUsePropertyListPatternForRemote() { - testUsePropertyListPattern(MBeanClient.of(-1, jmxServer.getAddress())); + testUsePropertyListPattern(MBeanClient.of(jmxServer.getAddress())); } @Test @@ -374,7 +374,7 @@ private void testUsePropertyListPattern(MBeanClient client) { @Test void testListDomains() { // arrange - try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(-1, jmxServer.getAddress())) { + try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(jmxServer.getAddress())) { // act List domains = client.domains(); @@ -388,7 +388,7 @@ void testListDomains() { @Test void testHostAndPort() { // arrange - try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(-1, jmxServer.getAddress())) { + try (var client = (MBeanClient.BasicMBeanClient) MBeanClient.of(jmxServer.getAddress())) { assertEquals(jmxServer.getAddress().getHost(), client.host); assertEquals(jmxServer.getAddress().getPort(), client.port); } @@ -404,7 +404,7 @@ void testCustomMBeanWith500Attributes() throws Exception { register(objectName0, customMBean0); - try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { + try (var client = MBeanClient.of(jmxServer.getAddress())) { // act Collection all = @@ -431,7 +431,7 @@ void testWith100CustomMBeans() throws Exception { register(objectName, mbean); } - try (var client = MBeanClient.of(-1, jmxServer.getAddress())) { + try (var client = MBeanClient.of(jmxServer.getAddress())) { // act Collection all = diff --git a/gui/src/main/java/org/astraea/gui/Context.java b/gui/src/main/java/org/astraea/gui/Context.java index 843dbac814..ead99959c9 100644 --- a/gui/src/main/java/org/astraea/gui/Context.java +++ b/gui/src/main/java/org/astraea/gui/Context.java @@ -78,7 +78,7 @@ public Map addBrokerClients(List nodeInfos) { nodeInfos.forEach( n -> brokerClients.clients.computeIfAbsent( - n.id(), ignored -> MBeanClient.jndi(n.id(), n.host(), brokerClients.jmxPort))); + n.id(), ignored -> MBeanClient.jndi(n.host(), brokerClients.jmxPort))); return Map.copyOf(brokerClients.clients); } @@ -88,7 +88,7 @@ public Map addWorkerClients(Set hostnames) { hostnames.forEach( n -> workerClients.clients.computeIfAbsent( - n, ignored -> MBeanClient.jndi(-1, n, workerClients.jmxPort))); + n, ignored -> MBeanClient.jndi(n, workerClients.jmxPort))); return Map.copyOf(workerClients.clients); } From 262a407df6e1f2f31e85a950e5ebc7f9b895c51b Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Sat, 29 Apr 2023 20:35:20 +0800 Subject: [PATCH 05/24] update docs --- docs/web_server/web_api_balancer_chinese.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/web_server/web_api_balancer_chinese.md b/docs/web_server/web_api_balancer_chinese.md index dc13d978c8..7de1b969dd 100644 --- a/docs/web_server/web_api_balancer_chinese.md +++ b/docs/web_server/web_api_balancer_chinese.md @@ -32,12 +32,14 @@ POST /balancer costConfig: -| config key | config value | -| --------------------------- | ----------------------------- | -| max.migrated.size | 設定最大可搬移的資料量 | -| max.migrated.leader.number | 設定最大可搬移的leader 數量 | -| max.migrated.replica.number | 設定最大可搬移的replica 數量 | -| max.migrated.leader.size | 設定最大可搬移的leader 資料量 | +| config key | config value | +| --------------------------- | --------------------------------- | +| max.migrated.size | 設定最大可搬移的資料量 | +| max.migrated.leader.number | 設定最大可搬移的leader 數量 | +| max.migrated.replica.number | 設定最大可搬移的replica 數量 | +| max.migrated.leader.size | 設定最大可搬移的leader 資料量 | +| max.migrated.time.limit | 設定最大可接受的leader 資料量 | +| max.migrated.time.limit | 設定最大可接受的partition搬移時間 | 目前支援的 Cost Function From 33967dd94c5565a88e788534330f3382f156b383 Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Mon, 1 May 2023 02:01:54 +0800 Subject: [PATCH 06/24] fix issues --- .../org/astraea/app/web/BalancerHandler.java | 3 +- .../org/astraea/app/web/SensorHandler.java | 3 +- .../astraea/common/cost/MigrationCost.java | 27 +++++- .../common/cost/PartitionMigrateTimeCost.java | 91 +++++-------------- 4 files changed, 51 insertions(+), 73 deletions(-) diff --git a/app/src/main/java/org/astraea/app/web/BalancerHandler.java b/app/src/main/java/org/astraea/app/web/BalancerHandler.java index 16875dd7e7..c18b96840c 100644 --- a/app/src/main/java/org/astraea/app/web/BalancerHandler.java +++ b/app/src/main/java/org/astraea/app/web/BalancerHandler.java @@ -242,7 +242,8 @@ static class BalancerPostRequest implements Request { "org.astraea.common.cost.ReplicaLeaderCost", "org.astraea.common.cost.RecordSizeCost", "org.astraea.common.cost.ReplicaNumberCost", - "org.astraea.common.cost.ReplicaLeaderSizeCost"); + "org.astraea.common.cost.ReplicaLeaderSizeCost", + "org.astraea.common.cost.PartitionMigrateTimeCost"); HasClusterCost clusterCost() { if (clusterCosts.isEmpty()) diff --git a/app/src/main/java/org/astraea/app/web/SensorHandler.java b/app/src/main/java/org/astraea/app/web/SensorHandler.java index 425d2f15cc..c59922b9e2 100644 --- a/app/src/main/java/org/astraea/app/web/SensorHandler.java +++ b/app/src/main/java/org/astraea/app/web/SensorHandler.java @@ -32,7 +32,8 @@ public class SensorHandler implements Handler { private static final Set DEFAULT_COSTS = Set.of( "org.astraea.common.cost.ReplicaLeaderCost", - "org.astraea.common.cost.NetworkIngressCost"); + "org.astraea.common.cost.NetworkIngressCost", + "org.astraea.common.cost.PartitionMigrateTimeCost"); SensorHandler(Sensors sensors) { this.sensors = sensors; diff --git a/common/src/main/java/org/astraea/common/cost/MigrationCost.java b/common/src/main/java/org/astraea/common/cost/MigrationCost.java index f7cebbfecd..490a11af21 100644 --- a/common/src/main/java/org/astraea/common/cost/MigrationCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrationCost.java @@ -84,17 +84,33 @@ static Map replicaLeaderChanged(ClusterInfo before, ClusterInfo a public static Map brokerMigrationTime( ClusterInfo before, ClusterInfo after, ClusterBean clusterBean) { var brokerInRate = - brokerMaxRate(before, clusterBean, PartitionMigrateTimeCost.MaxReplicationInRateBean.class); + before.nodes().stream() + .collect( + Collectors.toMap( + NodeInfo::id, + nodeInfo -> + brokerMaxRate( + nodeInfo.id(), + clusterBean, + PartitionMigrateTimeCost.MaxReplicationInRateBean.class))); var brokerOutRate = - brokerMaxRate( - before, clusterBean, PartitionMigrateTimeCost.MaxReplicationOutRateBean.class); + before.nodes().stream() + .collect( + Collectors.toMap( + NodeInfo::id, + nodeInfo -> + brokerMaxRate( + nodeInfo.id(), + clusterBean, + PartitionMigrateTimeCost.MaxReplicationOutRateBean.class))); var brokerMigrateInTime = MigrationCost.recordSizeToFetch(before, after).entrySet().stream() .map( brokerSize -> Map.entry( brokerSize.getKey(), - brokerSize.getValue() / brokerInRate.get(brokerSize.getKey()).orElse(0))) + brokerSize.getValue() + / brokerInRate.get(brokerSize.getKey()).orElse(Long.MAX_VALUE))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); var brokerMigrateOutTime = MigrationCost.recordSizeToSync(before, after).entrySet().stream() @@ -102,7 +118,8 @@ public static Map brokerMigrationTime( brokerSize -> Map.entry( brokerSize.getKey(), - brokerSize.getValue() / brokerOutRate.get(brokerSize.getKey()).orElse(0))) + brokerSize.getValue() + / brokerOutRate.get(brokerSize.getKey()).orElse(Long.MAX_VALUE))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); return Stream.concat(before.nodes().stream(), after.nodes().stream()) .distinct() diff --git a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java index a5c1e6eadb..8afe32202d 100644 --- a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java +++ b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java @@ -18,15 +18,12 @@ import java.time.Duration; import java.util.ArrayList; -import java.util.Collection; import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalDouble; import java.util.function.Function; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.astraea.common.Configuration; import org.astraea.common.admin.ClusterInfo; @@ -45,8 +42,6 @@ public class PartitionMigrateTimeCost implements HasMoveCost { private static final String REPLICATION_IN_RATE = "replication_in_rate"; private static final String REPLICATION_OUT_RATE = "replication_out_rate"; public static final String MAX_MIGRATE_TIME_KEY = "max.migrated.time.limit"; - static final Map> maxBrokerReplicationInRate = new HashMap<>(); - static final Map> maxBrokerReplicationOutRate = new HashMap<>(); // metrics windows size private final long maxMigrateTime; @@ -59,35 +54,36 @@ public PartitionMigrateTimeCost(Configuration config) { public Optional metricSensor() { return Optional.of( (client, clusterBean) -> { - var metrics = - clusterBean.all().values().stream() - .flatMap(Collection::stream) - .distinct() - .collect(Collectors.toCollection(ArrayList::new)); + var oldInRate = + brokerMaxRate( + client.identity(), + clusterBean, + PartitionMigrateTimeCost.MaxReplicationInRateBean.class); + var oldOutRate = + brokerMaxRate( + client.identity(), + clusterBean, + PartitionMigrateTimeCost.MaxReplicationOutRateBean.class); var newInMetrics = ServerMetrics.BrokerTopic.REPLICATION_BYTES_IN_PER_SEC.fetch(client); var newOutMetrics = ServerMetrics.BrokerTopic.REPLICATION_BYTES_OUT_PER_SEC.fetch(client); var current = Duration.ofMillis(System.currentTimeMillis()); var maxInRateSensor = - maxBrokerReplicationInRate.computeIfAbsent( - client.identity(), - ignore -> - Sensor.builder().addStat(REPLICATION_IN_RATE, Max.of()).build()); + Sensor.builder().addStat(REPLICATION_IN_RATE, Max.of()).build(); var maxOutRateSensor = - maxBrokerReplicationOutRate.computeIfAbsent( - client.identity(), - ignore -> - Sensor.builder().addStat(REPLICATION_OUT_RATE, Max.of()).build()); + Sensor.builder().addStat(REPLICATION_OUT_RATE, Max.of()).build(); maxInRateSensor.record(newInMetrics.oneMinuteRate()); maxOutRateSensor.record(newOutMetrics.oneMinuteRate()); var inRate = maxInRateSensor.measure(REPLICATION_IN_RATE); var outRate = maxOutRateSensor.measure(REPLICATION_OUT_RATE); + + var metrics = new ArrayList(); metrics.add( (MaxReplicationInRateBean) () -> new BeanObject( newInMetrics.beanObject().domainName(), newInMetrics.beanObject().properties(), - Map.of(HasRate.ONE_MIN_RATE_KEY, inRate), + Map.of(HasRate.ONE_MIN_RATE_KEY, Math.max(oldInRate.orElse(0), inRate)), current.toMillis())); metrics.add( (MaxReplicationOutRateBean) @@ -95,66 +91,29 @@ public Optional metricSensor() { new BeanObject( newOutMetrics.beanObject().domainName(), newOutMetrics.beanObject().properties(), - Map.of(HasRate.ONE_MIN_RATE_KEY, outRate), + Map.of(HasRate.ONE_MIN_RATE_KEY, Math.max(oldOutRate.orElse(0), outRate)), current.toMillis())); return metrics; }); } - public static Map brokerMaxRate( - ClusterInfo clusterInfo, - ClusterBean clusterBean, - Class statisticMetrics) { - return clusterInfo.brokers().stream() - .map( - broker -> - Map.entry( - broker.id(), - clusterBean.all().getOrDefault(broker.id(), List.of()).stream() - .filter(x -> statisticMetrics.isAssignableFrom(x.getClass())) - .mapToDouble(x -> ((HasMeter) x).oneMinuteRate()) - .max())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - public Map brokerMigratedTime( - Map needToMigrated, Map brokerRate) { - return needToMigrated.entrySet().stream() - .map( - brokerSize -> - Map.entry( - brokerSize.getKey(), - brokerSize.getValue() - / brokerRate - .get(brokerSize.getKey()) - .orElseThrow( - () -> - new NoSufficientMetricsException( - this, - Duration.ofSeconds(1), - "No metric for broker" + brokerSize.getKey())))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + public static OptionalDouble brokerMaxRate( + int identity, ClusterBean clusterBean, Class statisticMetrics) { + return clusterBean.all().getOrDefault(identity, List.of()).stream() + .filter(x -> statisticMetrics.isAssignableFrom(x.getClass())) + .mapToDouble(x -> ((HasMeter) x).oneMinuteRate()) + .max(); } @Override public MoveCost moveCost(ClusterInfo before, ClusterInfo after, ClusterBean clusterBean) { - var brokerInRate = brokerMaxRate(before, clusterBean, MaxReplicationInRateBean.class); - var brokerOutRate = brokerMaxRate(before, clusterBean, MaxReplicationOutRateBean.class); - var needToMigrateIn = MigrationCost.recordSizeToFetch(before, after); - var needToMigrateOut = MigrationCost.recordSizeToSync(before, after); - - var brokerMigrateInTime = brokerMigratedTime(needToMigrateIn, brokerInRate); - var brokerMigrateOutTime = brokerMigratedTime(needToMigrateOut, brokerOutRate); + var brokerMigrateTime = MigrationCost.brokerMigrationTime(before, after, clusterBean); var maxMigrateTime = Stream.concat(before.nodes().stream(), after.nodes().stream()) .distinct() - .map( - nodeInfo -> - Math.max( - brokerMigrateInTime.get(nodeInfo.id()), - brokerMigrateOutTime.get(nodeInfo.id()))) + .map(nodeInfo -> brokerMigrateTime.get(nodeInfo.id())) .max(Comparator.comparing(Function.identity())) - .orElse(0.0); + .orElse(Long.MAX_VALUE); return () -> maxMigrateTime > this.maxMigrateTime; } From e4e5981a1100bbe14e1f78626f860303a8dfea23 Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Mon, 1 May 2023 23:52:14 +0800 Subject: [PATCH 07/24] fix issue --- .../main/java/org/astraea/common/cost/MigrationCost.java | 6 ++---- .../org/astraea/common/cost/PartitionMigrateTimeCost.java | 5 +---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/common/src/main/java/org/astraea/common/cost/MigrationCost.java b/common/src/main/java/org/astraea/common/cost/MigrationCost.java index 012bd40d76..dec8e61006 100644 --- a/common/src/main/java/org/astraea/common/cost/MigrationCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrationCost.java @@ -116,8 +116,7 @@ public static Map brokerMigrationTime( brokerSize -> Map.entry( brokerSize.getKey(), - brokerSize.getValue() - / brokerInRate.get(brokerSize.getKey()).orElse(Long.MAX_VALUE))) + brokerSize.getValue() / brokerInRate.get(brokerSize.getKey()).orElse(0))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); var brokerMigrateOutTime = MigrationCost.recordSizeToSync(before, after).entrySet().stream() @@ -125,8 +124,7 @@ public static Map brokerMigrationTime( brokerSize -> Map.entry( brokerSize.getKey(), - brokerSize.getValue() - / brokerOutRate.get(brokerSize.getKey()).orElse(Long.MAX_VALUE))) + brokerSize.getValue() / brokerOutRate.get(brokerSize.getKey()).orElse(0))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); return Stream.concat(before.nodes().stream(), after.nodes().stream()) .distinct() diff --git a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java index 8afe32202d..6f4724add3 100644 --- a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java +++ b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java @@ -24,7 +24,6 @@ import java.util.Optional; import java.util.OptionalDouble; import java.util.function.Function; -import java.util.stream.Stream; import org.astraea.common.Configuration; import org.astraea.common.admin.ClusterInfo; import org.astraea.common.metrics.BeanObject; @@ -109,9 +108,7 @@ public static OptionalDouble brokerMaxRate( public MoveCost moveCost(ClusterInfo before, ClusterInfo after, ClusterBean clusterBean) { var brokerMigrateTime = MigrationCost.brokerMigrationTime(before, after, clusterBean); var maxMigrateTime = - Stream.concat(before.nodes().stream(), after.nodes().stream()) - .distinct() - .map(nodeInfo -> brokerMigrateTime.get(nodeInfo.id())) + brokerMigrateTime.values().stream() .max(Comparator.comparing(Function.identity())) .orElse(Long.MAX_VALUE); return () -> maxMigrateTime > this.maxMigrateTime; From efb3216d4245ad2c8f5ca351ff0af47342460cda Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Thu, 4 May 2023 16:39:53 +0800 Subject: [PATCH 08/24] fix issues --- .../java/org/astraea/common/admin/Broker.java | 3 +- .../common/cost/PartitionMigrateTimeCost.java | 12 ++--- .../common/metrics/broker/HasRate.java | 3 -- .../cost/PartitionMigrateTimeCostTest.java | 49 +++++++------------ 4 files changed, 26 insertions(+), 41 deletions(-) diff --git a/common/src/main/java/org/astraea/common/admin/Broker.java b/common/src/main/java/org/astraea/common/admin/Broker.java index 7bc6767038..18072e3268 100644 --- a/common/src/main/java/org/astraea/common/admin/Broker.java +++ b/common/src/main/java/org/astraea/common/admin/Broker.java @@ -100,8 +100,7 @@ public int hashCode() { @Override public boolean equals(Object other) { - if (other instanceof NodeInfo) { - var node = (NodeInfo) other; + if (other instanceof NodeInfo node) { return id() == node.id() && port() == node.port() && host().equals(node.host()); } return false; diff --git a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java index 6f4724add3..a9d6fd5180 100644 --- a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java +++ b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java @@ -19,7 +19,6 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Comparator; -import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalDouble; @@ -31,7 +30,6 @@ import org.astraea.common.metrics.HasBeanObject; import org.astraea.common.metrics.Sensor; import org.astraea.common.metrics.broker.HasMeter; -import org.astraea.common.metrics.broker.HasRate; import org.astraea.common.metrics.broker.ServerMetrics; import org.astraea.common.metrics.collector.MetricSensor; import org.astraea.common.metrics.stats.Max; @@ -41,6 +39,8 @@ public class PartitionMigrateTimeCost implements HasMoveCost { private static final String REPLICATION_IN_RATE = "replication_in_rate"; private static final String REPLICATION_OUT_RATE = "replication_out_rate"; public static final String MAX_MIGRATE_TIME_KEY = "max.migrated.time.limit"; + public static final String STATISTICS_RATE_KEY = "statistics.rate.key"; + // metrics windows size private final long maxMigrateTime; @@ -82,7 +82,7 @@ public Optional metricSensor() { new BeanObject( newInMetrics.beanObject().domainName(), newInMetrics.beanObject().properties(), - Map.of(HasRate.ONE_MIN_RATE_KEY, Math.max(oldInRate.orElse(0), inRate)), + Map.of(STATISTICS_RATE_KEY, Math.max(oldInRate.orElse(0), inRate)), current.toMillis())); metrics.add( (MaxReplicationOutRateBean) @@ -90,7 +90,7 @@ public Optional metricSensor() { new BeanObject( newOutMetrics.beanObject().domainName(), newOutMetrics.beanObject().properties(), - Map.of(HasRate.ONE_MIN_RATE_KEY, Math.max(oldOutRate.orElse(0), outRate)), + Map.of(STATISTICS_RATE_KEY, Math.max(oldOutRate.orElse(0), outRate)), current.toMillis())); return metrics; }); @@ -98,8 +98,8 @@ public Optional metricSensor() { public static OptionalDouble brokerMaxRate( int identity, ClusterBean clusterBean, Class statisticMetrics) { - return clusterBean.all().getOrDefault(identity, List.of()).stream() - .filter(x -> statisticMetrics.isAssignableFrom(x.getClass())) + return clusterBean + .brokerMetrics(identity, statisticMetrics) .mapToDouble(x -> ((HasMeter) x).oneMinuteRate()) .max(); } diff --git a/common/src/main/java/org/astraea/common/metrics/broker/HasRate.java b/common/src/main/java/org/astraea/common/metrics/broker/HasRate.java index 021e0be81b..622514b5f1 100644 --- a/common/src/main/java/org/astraea/common/metrics/broker/HasRate.java +++ b/common/src/main/java/org/astraea/common/metrics/broker/HasRate.java @@ -20,9 +20,6 @@ import org.astraea.common.metrics.HasBeanObject; public interface HasRate extends HasBeanObject { - String ONE_MIN_RATE_KEY = "OneMinuteRate"; - String FIVE_MINUTE_RATE = "FiveMinuteRate"; - String FIFTEEN_MINUTE_RATE = "FifteenMinuteRate"; default double meanRate() { return (double) beanObject().attributes().get("MeanRate"); diff --git a/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java b/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java index 8bbbf64b26..049ba177e0 100644 --- a/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java @@ -50,44 +50,33 @@ void testMigratedCost() { // before(partition-broker): p10-1, p11-2, p12-0, p12-0 // after(partition-broker): p10-0, p11-1, p12-2, p12-0 // p10:777, p11:700, p12:500 - List brokers = - before().stream() - .map(Replica::nodeInfo) - .distinct() - .map( - nodeInfo -> - Broker.of( - false, - new Node(nodeInfo.id(), "", nodeInfo.port()), - Map.of(), - Map.of(), - List.of())) - .collect(Collectors.toList()); - var before = of(before(), brokers); - var after = of(after(), brokers); + var before = of(before(), brokers()); + var after = of(after(), brokers()); var migrationCost = MigrationCost.brokerMigrationTime(before, after, clusterBean()); Assertions.assertEquals(Math.max(10000000 / 1000, 30000000 / 1500), migrationCost.get(0)); Assertions.assertEquals(Math.max(20000000 / 2000, 10000000 / 2500), migrationCost.get(1)); Assertions.assertEquals(Math.max(30000000 / 3000, 20000000 / 3500), migrationCost.get(2)); } + private List brokers() { + return before().stream() + .map(Replica::nodeInfo) + .distinct() + .map( + nodeInfo -> + Broker.of( + false, + new Node(nodeInfo.id(), "", nodeInfo.port()), + Map.of(), + Map.of(), + List.of())) + .collect(Collectors.toList()); + } + @Test void testMostCost() { - List brokers = - before().stream() - .map(Replica::nodeInfo) - .distinct() - .map( - nodeInfo -> - Broker.of( - false, - new Node(nodeInfo.id(), "", nodeInfo.port()), - Map.of(), - Map.of(), - List.of())) - .collect(Collectors.toList()); - var before = of(before(), brokers); - var after = of(after(), brokers); + var before = of(before(), brokers()); + var after = of(after(), brokers()); var timeLimit = Configuration.of(Map.of(PartitionMigrateTimeCost.MAX_MIGRATE_TIME_KEY, "20000")); var overFlowTimeLimit = From 7452921b3218d7e056f212e5344197d6d23c57af Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Tue, 9 May 2023 23:44:51 +0800 Subject: [PATCH 09/24] update --- .../java/org/astraea/common/admin/Broker.java | 16 ---------------- .../org/astraea/common/cost/MigrationCost.java | 8 ++++---- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/common/src/main/java/org/astraea/common/admin/Broker.java b/common/src/main/java/org/astraea/common/admin/Broker.java index 18072e3268..822cd3ed40 100644 --- a/common/src/main/java/org/astraea/common/admin/Broker.java +++ b/common/src/main/java/org/astraea/common/admin/Broker.java @@ -19,7 +19,6 @@ import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import org.apache.kafka.common.requests.DescribeLogDirsResponse; @@ -91,21 +90,6 @@ public Map orphanPartitionSizes() { .collect(Collectors.toUnmodifiableSet()); return new Broker() { - private final int hashCode = Objects.hash(id(), host(), port()); - - @Override - public int hashCode() { - return hashCode; - } - - @Override - public boolean equals(Object other) { - if (other instanceof NodeInfo node) { - return id() == node.id() && port() == node.port() && host().equals(node.host()); - } - return false; - } - @Override public String host() { return nodeInfo.host(); diff --git a/common/src/main/java/org/astraea/common/cost/MigrationCost.java b/common/src/main/java/org/astraea/common/cost/MigrationCost.java index dec8e61006..14bcab45a0 100644 --- a/common/src/main/java/org/astraea/common/cost/MigrationCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrationCost.java @@ -127,15 +127,15 @@ public static Map brokerMigrationTime( brokerSize.getValue() / brokerOutRate.get(brokerSize.getKey()).orElse(0))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); return Stream.concat(before.nodes().stream(), after.nodes().stream()) + .map(NodeInfo::id) .distinct() .map( - nodeInfo -> + nodeId -> Map.entry( - nodeInfo.id(), + nodeId, (long) Math.max( - brokerMigrateInTime.get(nodeInfo.id()), - brokerMigrateOutTime.get(nodeInfo.id())))) + brokerMigrateInTime.get(nodeId), brokerMigrateOutTime.get(nodeId)))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } From ea28b651919145bfdbc6aa7eb45b900a3816114a Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Wed, 10 May 2023 21:27:23 +0800 Subject: [PATCH 10/24] fix issue --- .../java/org/astraea/common/admin/Broker.java | 1 - .../org/astraea/common/balancer/Balancer.java | 40 ++++--------------- .../astraea/common/cost/MigrationCost.java | 11 +++-- .../common/cost/PartitionMigrateTimeCost.java | 10 ++--- 4 files changed, 16 insertions(+), 46 deletions(-) diff --git a/common/src/main/java/org/astraea/common/admin/Broker.java b/common/src/main/java/org/astraea/common/admin/Broker.java index 822cd3ed40..c4ea2d763a 100644 --- a/common/src/main/java/org/astraea/common/admin/Broker.java +++ b/common/src/main/java/org/astraea/common/admin/Broker.java @@ -89,7 +89,6 @@ public Map orphanPartitionSizes() { .map(p -> TopicPartition.of(topic.name(), p.partition()))) .collect(Collectors.toUnmodifiableSet()); return new Broker() { - @Override public String host() { return nodeInfo.host(); diff --git a/common/src/main/java/org/astraea/common/balancer/Balancer.java b/common/src/main/java/org/astraea/common/balancer/Balancer.java index a2be5851df..3118c76dad 100644 --- a/common/src/main/java/org/astraea/common/balancer/Balancer.java +++ b/common/src/main/java/org/astraea/common/balancer/Balancer.java @@ -33,48 +33,24 @@ public interface Balancer { */ Optional offer(AlgorithmConfig config); - class Plan { - private final ClusterBean clusterBean; - private final ClusterInfo initialClusterInfo; - private final ClusterCost initialClusterCost; - - private final ClusterInfo proposal; - private final ClusterCost proposalClusterCost; - - public Plan( - ClusterBean clusterBean, - ClusterInfo initialClusterInfo, - ClusterCost initialClusterCost, - ClusterInfo proposal, - ClusterCost proposalClusterCost) { - this.clusterBean = clusterBean; - this.initialClusterInfo = initialClusterInfo; - this.initialClusterCost = initialClusterCost; - this.proposal = proposal; - this.proposalClusterCost = proposalClusterCost; - } - - public ClusterBean clusterBean() { - return clusterBean; - } - - public ClusterInfo initialClusterInfo() { - return initialClusterInfo; - } + record Plan( + ClusterBean clusterBean, + ClusterInfo initialClusterInfo, + ClusterCost initialClusterCost, + ClusterInfo proposal, + ClusterCost proposalClusterCost) { /** * The {@link ClusterCost} score of the original {@link ClusterInfo} when this plan is start * generating. */ + @Override public ClusterCost initialClusterCost() { return initialClusterCost; } - public ClusterInfo proposal() { - return proposal; - } - /** The {@link ClusterCost} score of the proposed new allocation. */ + @Override public ClusterCost proposalClusterCost() { return proposalClusterCost; } diff --git a/common/src/main/java/org/astraea/common/cost/MigrationCost.java b/common/src/main/java/org/astraea/common/cost/MigrationCost.java index 14bcab45a0..b90d080bab 100644 --- a/common/src/main/java/org/astraea/common/cost/MigrationCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrationCost.java @@ -129,14 +129,13 @@ public static Map brokerMigrationTime( return Stream.concat(before.nodes().stream(), after.nodes().stream()) .map(NodeInfo::id) .distinct() - .map( - nodeId -> - Map.entry( - nodeId, + .collect( + Collectors.toMap( + nodeId -> nodeId, + nodeId -> (long) Math.max( - brokerMigrateInTime.get(nodeId), brokerMigrateOutTime.get(nodeId)))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + brokerMigrateInTime.get(nodeId), brokerMigrateOutTime.get(nodeId)))); } /** diff --git a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java index 218c6a8b89..4d874763a5 100644 --- a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java +++ b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java @@ -17,8 +17,8 @@ package org.astraea.common.cost; import java.time.Duration; -import java.util.ArrayList; import java.util.Comparator; +import java.util.List; import java.util.Map; import java.util.OptionalDouble; import java.util.function.Function; @@ -71,17 +71,14 @@ public MetricSensor metricSensor() { maxOutRateSensor.record(newOutMetrics.oneMinuteRate()); var inRate = maxInRateSensor.measure(REPLICATION_IN_RATE); var outRate = maxOutRateSensor.measure(REPLICATION_OUT_RATE); - - var metrics = new ArrayList(); - metrics.add( + return List.of( (MaxReplicationInRateBean) () -> new BeanObject( newInMetrics.beanObject().domainName(), newInMetrics.beanObject().properties(), Map.of(STATISTICS_RATE_KEY, Math.max(oldInRate.orElse(0), inRate)), - current.toMillis())); - metrics.add( + current.toMillis()), (MaxReplicationOutRateBean) () -> new BeanObject( @@ -89,7 +86,6 @@ public MetricSensor metricSensor() { newOutMetrics.beanObject().properties(), Map.of(STATISTICS_RATE_KEY, Math.max(oldOutRate.orElse(0), outRate)), current.toMillis())); - return metrics; }; } From 1ad789283ce16b5194e648dff7e9bb97836dfa13 Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Thu, 11 May 2023 20:53:33 +0800 Subject: [PATCH 11/24] fix issues --- .../org/astraea/common/balancer/Balancer.java | 18 +--------- .../astraea/common/cost/MigrationCost.java | 34 +++++++++---------- .../common/cost/PartitionMigrateTimeCost.java | 14 ++++---- .../common/metrics/broker/HasRate.java | 1 - .../cost/PartitionMigrateTimeCostTest.java | 20 +++++++---- 5 files changed, 38 insertions(+), 49 deletions(-) diff --git a/common/src/main/java/org/astraea/common/balancer/Balancer.java b/common/src/main/java/org/astraea/common/balancer/Balancer.java index 3118c76dad..489b299118 100644 --- a/common/src/main/java/org/astraea/common/balancer/Balancer.java +++ b/common/src/main/java/org/astraea/common/balancer/Balancer.java @@ -38,23 +38,7 @@ record Plan( ClusterInfo initialClusterInfo, ClusterCost initialClusterCost, ClusterInfo proposal, - ClusterCost proposalClusterCost) { - - /** - * The {@link ClusterCost} score of the original {@link ClusterInfo} when this plan is start - * generating. - */ - @Override - public ClusterCost initialClusterCost() { - return initialClusterCost; - } - - /** The {@link ClusterCost} score of the proposed new allocation. */ - @Override - public ClusterCost proposalClusterCost() { - return proposalClusterCost; - } - } + ClusterCost proposalClusterCost) {} /** The official implementation of {@link Balancer}. */ enum Official implements EnumInfo { diff --git a/common/src/main/java/org/astraea/common/cost/MigrationCost.java b/common/src/main/java/org/astraea/common/cost/MigrationCost.java index b90d080bab..9ff9f213ec 100644 --- a/common/src/main/java/org/astraea/common/cost/MigrationCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrationCost.java @@ -51,7 +51,8 @@ public static List migrationCosts( new MigrationCost(TO_SYNC_BYTES, migrateInBytes), new MigrationCost(TO_FETCH_BYTES, migrateOutBytes), new MigrationCost(CHANGED_REPLICAS, migrateReplicaNum), - new MigrationCost(PARTITION_MIGRATED_TIME, brokerMigrationTime(before, after, clusterBean)), + new MigrationCost( + PARTITION_MIGRATED_TIME, brokerMigrationSecond(before, after, clusterBean)), new MigrationCost(REPLICA_LEADERS_TO_ADDED, migrateInLeader), new MigrationCost(REPLICA_LEADERS_TO_REMOVE, migrateOutLeader), new MigrationCost(CHANGED_REPLICAS, migrateReplicaNum)); @@ -88,7 +89,7 @@ static Map replicaLeaderToRemove(ClusterInfo before, ClusterInfo * @param clusterBean cluster metrics * @return estimated migrated time required by all brokers (seconds) */ - public static Map brokerMigrationTime( + public static Map brokerMigrationSecond( ClusterInfo before, ClusterInfo after, ClusterBean clusterBean) { var brokerInRate = before.nodes().stream() @@ -110,22 +111,20 @@ public static Map brokerMigrationTime( nodeInfo.id(), clusterBean, PartitionMigrateTimeCost.MaxReplicationOutRateBean.class))); - var brokerMigrateInTime = + var brokerMigrateInSecond = MigrationCost.recordSizeToFetch(before, after).entrySet().stream() - .map( - brokerSize -> - Map.entry( - brokerSize.getKey(), - brokerSize.getValue() / brokerInRate.get(brokerSize.getKey()).orElse(0))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - var brokerMigrateOutTime = + .collect( + Collectors.toMap( + Map.Entry::getKey, + brokerSize -> + brokerSize.getValue() / brokerInRate.get(brokerSize.getKey()).orElse(0))); + var brokerMigrateOutSecond = MigrationCost.recordSizeToSync(before, after).entrySet().stream() - .map( - brokerSize -> - Map.entry( - brokerSize.getKey(), - brokerSize.getValue() / brokerOutRate.get(brokerSize.getKey()).orElse(0))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + .collect( + Collectors.toMap( + Map.Entry::getKey, + brokerSize -> + brokerSize.getValue() / brokerOutRate.get(brokerSize.getKey()).orElse(0))); return Stream.concat(before.nodes().stream(), after.nodes().stream()) .map(NodeInfo::id) .distinct() @@ -135,7 +134,8 @@ public static Map brokerMigrationTime( nodeId -> (long) Math.max( - brokerMigrateInTime.get(nodeId), brokerMigrateOutTime.get(nodeId)))); + brokerMigrateInSecond.get(nodeId), + brokerMigrateOutSecond.get(nodeId)))); } /** diff --git a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java index 4d874763a5..deba94a029 100644 --- a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java +++ b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java @@ -41,11 +41,11 @@ public class PartitionMigrateTimeCost implements HasMoveCost { public static final String STATISTICS_RATE_KEY = "statistics.rate.key"; // metrics windows size - private final long maxMigrateTime; + private final Duration maxMigrateTime; public PartitionMigrateTimeCost(Configuration config) { this.maxMigrateTime = - config.string(MAX_MIGRATE_TIME_KEY).map(Long::parseLong).orElse(Long.MAX_VALUE); + config.duration(MAX_MIGRATE_TIME_KEY).orElse(Duration.ofSeconds(Long.MAX_VALUE)); } @Override @@ -93,18 +93,18 @@ public static OptionalDouble brokerMaxRate( int identity, ClusterBean clusterBean, Class statisticMetrics) { return clusterBean .brokerMetrics(identity, statisticMetrics) - .mapToDouble(x -> ((HasMeter) x).oneMinuteRate()) + .mapToDouble(x -> ((double) x.beanObject().attributes().get(STATISTICS_RATE_KEY))) .max(); } @Override public MoveCost moveCost(ClusterInfo before, ClusterInfo after, ClusterBean clusterBean) { - var brokerMigrateTime = MigrationCost.brokerMigrationTime(before, after, clusterBean); - var maxMigrateTime = - brokerMigrateTime.values().stream() + var brokerMigrateSecond = MigrationCost.brokerMigrationSecond(before, after, clusterBean); + var planMigrateSecond = + brokerMigrateSecond.values().stream() .max(Comparator.comparing(Function.identity())) .orElse(Long.MAX_VALUE); - return () -> maxMigrateTime > this.maxMigrateTime; + return () -> planMigrateSecond > this.maxMigrateTime.getSeconds(); } public interface MaxReplicationInRateBean extends HasMeter {} diff --git a/common/src/main/java/org/astraea/common/metrics/broker/HasRate.java b/common/src/main/java/org/astraea/common/metrics/broker/HasRate.java index 622514b5f1..780690506c 100644 --- a/common/src/main/java/org/astraea/common/metrics/broker/HasRate.java +++ b/common/src/main/java/org/astraea/common/metrics/broker/HasRate.java @@ -20,7 +20,6 @@ import org.astraea.common.metrics.HasBeanObject; public interface HasRate extends HasBeanObject { - default double meanRate() { return (double) beanObject().attributes().get("MeanRate"); } diff --git a/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java b/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java index 049ba177e0..31a66d6cb2 100644 --- a/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java @@ -33,17 +33,23 @@ class PartitionMigrateTimeCostTest { private static final BeanObject inBean0 = - new BeanObject("domain", Map.of(), Map.of("OneMinuteRate", 1000.0)); + new BeanObject( + "domain", Map.of(), Map.of(PartitionMigrateTimeCost.STATISTICS_RATE_KEY, 1000.0)); private static final BeanObject outBean0 = - new BeanObject("domain", Map.of(), Map.of("OneMinuteRate", 1500.0)); + new BeanObject( + "domain", Map.of(), Map.of(PartitionMigrateTimeCost.STATISTICS_RATE_KEY, 1500.0)); private static final BeanObject inBean1 = - new BeanObject("domain", Map.of(), Map.of("OneMinuteRate", 2000.0)); + new BeanObject( + "domain", Map.of(), Map.of(PartitionMigrateTimeCost.STATISTICS_RATE_KEY, 2000.0)); private static final BeanObject outBean1 = - new BeanObject("domain", Map.of(), Map.of("OneMinuteRate", 2500.0)); + new BeanObject( + "domain", Map.of(), Map.of(PartitionMigrateTimeCost.STATISTICS_RATE_KEY, 2500.0)); private static final BeanObject inBean2 = - new BeanObject("domain", Map.of(), Map.of("OneMinuteRate", 3000.0)); + new BeanObject( + "domain", Map.of(), Map.of(PartitionMigrateTimeCost.STATISTICS_RATE_KEY, 3000.0)); private static final BeanObject outBean2 = - new BeanObject("domain", Map.of(), Map.of("OneMinuteRate", 3500.0)); + new BeanObject( + "domain", Map.of(), Map.of(PartitionMigrateTimeCost.STATISTICS_RATE_KEY, 3500.0)); @Test void testMigratedCost() { @@ -52,7 +58,7 @@ void testMigratedCost() { // p10:777, p11:700, p12:500 var before = of(before(), brokers()); var after = of(after(), brokers()); - var migrationCost = MigrationCost.brokerMigrationTime(before, after, clusterBean()); + var migrationCost = MigrationCost.brokerMigrationSecond(before, after, clusterBean()); Assertions.assertEquals(Math.max(10000000 / 1000, 30000000 / 1500), migrationCost.get(0)); Assertions.assertEquals(Math.max(20000000 / 2000, 10000000 / 2500), migrationCost.get(1)); Assertions.assertEquals(Math.max(30000000 / 3000, 20000000 / 3500), migrationCost.get(2)); From e4ac803c72c30d7597ff3f2f3d0b06bff437d7f2 Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Mon, 15 May 2023 16:45:22 +0800 Subject: [PATCH 12/24] update docs --- docs/web_server/web_api_balancer_chinese.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/web_server/web_api_balancer_chinese.md b/docs/web_server/web_api_balancer_chinese.md index 126b0f15f8..e6b4cc4b70 100644 --- a/docs/web_server/web_api_balancer_chinese.md +++ b/docs/web_server/web_api_balancer_chinese.md @@ -270,4 +270,4 @@ JSON Response 範例 > 3. 同一筆優化計劃所產生分數能夠反映叢集的好壞，即 `scoreA` > `scoreB` 有定義，其代表 B 負載分佈比 A 負載分佈在此特定情境下更好。 -目前此 endpoint 僅能查詢負載優化計劃是否完成，如想知道更細部的搬移進度，可考慮使用 [Web Service Reassignments API](web_api_reassignments_chinese.md) 查詢。 +目前此 endpoint 僅能查詢負載優化計劃是否完成，如想知道更細部的搬移進度，可考慮使用 [Web Service Reassignments API](web_api_reassignments_chinese.md) 查詢。 \ No newline at end of file From f4e129703f0051bbb7a790980918474d4c7447d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=AB=E7=A5=A5=E9=88=9E?= Date: Tue, 16 May 2023 22:54:50 +0800 Subject: [PATCH 13/24] fix issues --- .../common/cost/PartitionMigrateTimeCost.java | 30 +++++++++++-------- .../org/astraea/common/metrics/Sensor.java | 8 +++++ .../common/metrics/broker/HasMaxRate.java | 25 ++++++++++++++++ 3 files changed, 50 insertions(+), 13 deletions(-) create mode 100644 common/src/main/java/org/astraea/common/metrics/broker/HasMaxRate.java diff --git a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java index deba94a029..4b6fdf8b6e 100644 --- a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java +++ b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java @@ -28,7 +28,7 @@ import org.astraea.common.metrics.ClusterBean; import org.astraea.common.metrics.HasBeanObject; import org.astraea.common.metrics.Sensor; -import org.astraea.common.metrics.broker.HasMeter; +import org.astraea.common.metrics.broker.HasMaxRate; import org.astraea.common.metrics.broker.ServerMetrics; import org.astraea.common.metrics.collector.MetricSensor; import org.astraea.common.metrics.stats.Max; @@ -37,7 +37,7 @@ public class PartitionMigrateTimeCost implements HasMoveCost { private static final String REPLICATION_IN_RATE = "replication_in_rate"; private static final String REPLICATION_OUT_RATE = "replication_out_rate"; - public static final String MAX_MIGRATE_TIME_KEY = "max.migrated.time.limit"; + static final String MAX_MIGRATE_TIME_KEY = "max.migrated.time.limit"; public static final String STATISTICS_RATE_KEY = "statistics.rate.key"; // metrics windows size @@ -64,13 +64,17 @@ public MetricSensor metricSensor() { var newInMetrics = ServerMetrics.BrokerTopic.REPLICATION_BYTES_IN_PER_SEC.fetch(client); var newOutMetrics = ServerMetrics.BrokerTopic.REPLICATION_BYTES_OUT_PER_SEC.fetch(client); var current = Duration.ofMillis(System.currentTimeMillis()); - var maxInRateSensor = Sensor.builder().addStat(REPLICATION_IN_RATE, Max.of()).build(); - var maxOutRateSensor = - Sensor.builder().addStat(REPLICATION_OUT_RATE, Max.of()).build(); - maxInRateSensor.record(newInMetrics.oneMinuteRate()); - maxOutRateSensor.record(newOutMetrics.oneMinuteRate()); - var inRate = maxInRateSensor.measure(REPLICATION_IN_RATE); - var outRate = maxOutRateSensor.measure(REPLICATION_OUT_RATE); + var maxRateSensor = + Sensor.builder() + .addStats( + Map.of( + REPLICATION_IN_RATE, Max.of(), + REPLICATION_OUT_RATE, Max.of())) + .build(); + maxRateSensor.record(REPLICATION_IN_RATE, newInMetrics.oneMinuteRate()); + maxRateSensor.record(REPLICATION_OUT_RATE, newOutMetrics.oneMinuteRate()); + var inRate = maxRateSensor.measure(REPLICATION_IN_RATE); + var outRate = maxRateSensor.measure(REPLICATION_OUT_RATE); return List.of( (MaxReplicationInRateBean) () -> @@ -89,11 +93,11 @@ public MetricSensor metricSensor() { }; } - public static OptionalDouble brokerMaxRate( + static OptionalDouble brokerMaxRate( int identity, ClusterBean clusterBean, Class statisticMetrics) { return clusterBean .brokerMetrics(identity, statisticMetrics) - .mapToDouble(x -> ((double) x.beanObject().attributes().get(STATISTICS_RATE_KEY))) + .mapToDouble(b -> ((HasMaxRate) b).maxRate()) .max(); } @@ -107,7 +111,7 @@ public MoveCost moveCost(ClusterInfo before, ClusterInfo after, ClusterBean clus return () -> planMigrateSecond > this.maxMigrateTime.getSeconds(); } - public interface MaxReplicationInRateBean extends HasMeter {} + public interface MaxReplicationInRateBean extends HasMaxRate {} - public interface MaxReplicationOutRateBean extends HasMeter {} + public interface MaxReplicationOutRateBean extends HasMaxRate {} } diff --git a/common/src/main/java/org/astraea/common/metrics/Sensor.java b/common/src/main/java/org/astraea/common/metrics/Sensor.java index 5119d96340..e3153d1e78 100644 --- a/common/src/main/java/org/astraea/common/metrics/Sensor.java +++ b/common/src/main/java/org/astraea/common/metrics/Sensor.java @@ -30,6 +30,9 @@ static Builder builder() { /** Record the new get data */ void record(V value); + /** Record the new get data for given `metricName` */ + void record(String statName, V value); + /** * Get the statistic by the given `metricName`. * @@ -69,6 +72,11 @@ public synchronized void record(V value) { stats.values().forEach(stat -> stat.record(value)); } + @Override + public void record(String statName, V value) { + stats.get(statName).record(value); + } + @Override public V measure(String statName) { return stats.get(statName).measure(); diff --git a/common/src/main/java/org/astraea/common/metrics/broker/HasMaxRate.java b/common/src/main/java/org/astraea/common/metrics/broker/HasMaxRate.java new file mode 100644 index 0000000000..7a2ab8cb10 --- /dev/null +++ b/common/src/main/java/org/astraea/common/metrics/broker/HasMaxRate.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.astraea.common.metrics.broker; + +import org.astraea.common.metrics.HasBeanObject; + +public interface HasMaxRate extends HasBeanObject { + default double maxRate() { + return (double) beanObject().attributes().get("statistics.rate.key"); + } +} From 9257e724e992bbc6af3d636f8a65498dea5b17c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=AB=E7=A5=A5=E9=88=9E?= Date: Tue, 16 May 2023 23:07:58 +0800 Subject: [PATCH 14/24] move brokerMaxRate() to MigrationCost --- .../java/org/astraea/common/cost/MigrationCost.java | 13 +++++++++++-- .../common/cost/PartitionMigrateTimeCost.java | 12 ++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/common/src/main/java/org/astraea/common/cost/MigrationCost.java b/common/src/main/java/org/astraea/common/cost/MigrationCost.java index 9ff9f213ec..1d6ad20e08 100644 --- a/common/src/main/java/org/astraea/common/cost/MigrationCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrationCost.java @@ -16,10 +16,9 @@ */ package org.astraea.common.cost; -import static org.astraea.common.cost.PartitionMigrateTimeCost.brokerMaxRate; - import java.util.List; import java.util.Map; +import java.util.OptionalDouble; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -28,6 +27,8 @@ import org.astraea.common.admin.NodeInfo; import org.astraea.common.admin.Replica; import org.astraea.common.metrics.ClusterBean; +import org.astraea.common.metrics.HasBeanObject; +import org.astraea.common.metrics.broker.HasMaxRate; public class MigrationCost { @@ -138,6 +139,14 @@ public static Map brokerMigrationSecond( brokerMigrateOutSecond.get(nodeId)))); } + static OptionalDouble brokerMaxRate( + int identity, ClusterBean clusterBean, Class statisticMetrics) { + return clusterBean + .brokerMetrics(identity, statisticMetrics) + .mapToDouble(b -> ((HasMaxRate) b).maxRate()) + .max(); + } + /** * @param before the ClusterInfo before migrated replicas * @param after the ClusterInfo after migrated replicas diff --git a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java index 4b6fdf8b6e..de58591d6c 100644 --- a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java +++ b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java @@ -16,17 +16,17 @@ */ package org.astraea.common.cost; +import static org.astraea.common.cost.MigrationCost.brokerMaxRate; + import java.time.Duration; import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.OptionalDouble; import java.util.function.Function; import org.astraea.common.Configuration; import org.astraea.common.admin.ClusterInfo; import org.astraea.common.metrics.BeanObject; import org.astraea.common.metrics.ClusterBean; -import org.astraea.common.metrics.HasBeanObject; import org.astraea.common.metrics.Sensor; import org.astraea.common.metrics.broker.HasMaxRate; import org.astraea.common.metrics.broker.ServerMetrics; @@ -93,14 +93,6 @@ public MetricSensor metricSensor() { }; } - static OptionalDouble brokerMaxRate( - int identity, ClusterBean clusterBean, Class statisticMetrics) { - return clusterBean - .brokerMetrics(identity, statisticMetrics) - .mapToDouble(b -> ((HasMaxRate) b).maxRate()) - .max(); - } - @Override public MoveCost moveCost(ClusterInfo before, ClusterInfo after, ClusterBean clusterBean) { var brokerMigrateSecond = MigrationCost.brokerMigrationSecond(before, after, clusterBean); From 6df0a153e7b85c33802536440ee876acbb23657e Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Thu, 18 May 2023 10:12:37 +0800 Subject: [PATCH 15/24] fix conflict --- .../org/astraea/common/cost/PartitionMigrateTimeCostTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java b/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java index 31a66d6cb2..93a542cc52 100644 --- a/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java @@ -84,9 +84,9 @@ void testMostCost() { var before = of(before(), brokers()); var after = of(after(), brokers()); var timeLimit = - Configuration.of(Map.of(PartitionMigrateTimeCost.MAX_MIGRATE_TIME_KEY, "20000")); + new Configuration(Map.of(PartitionMigrateTimeCost.MAX_MIGRATE_TIME_KEY, "20000")); var overFlowTimeLimit = - Configuration.of(Map.of(PartitionMigrateTimeCost.MAX_MIGRATE_TIME_KEY, "19999")); + new Configuration(Map.of(PartitionMigrateTimeCost.MAX_MIGRATE_TIME_KEY, "19999")); var cf = new PartitionMigrateTimeCost(timeLimit); var overFlowCf = new PartitionMigrateTimeCost(overFlowTimeLimit); var moveCost = cf.moveCost(before, after, clusterBean()); From 848f4e93de6853f17b3b30725ec0daffa3c9b51a Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Thu, 18 May 2023 14:21:46 +0800 Subject: [PATCH 16/24] fix tests --- app/src/test/java/org/astraea/app/web/SensorHandlerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/test/java/org/astraea/app/web/SensorHandlerTest.java b/app/src/test/java/org/astraea/app/web/SensorHandlerTest.java index bce5c14fdd..070266e9f9 100644 --- a/app/src/test/java/org/astraea/app/web/SensorHandlerTest.java +++ b/app/src/test/java/org/astraea/app/web/SensorHandlerTest.java @@ -38,7 +38,7 @@ void testBeans() { Assertions.assertInstanceOf( SensorHandler.Response.class, defaultCostHandler.get(Channel.EMPTY).toCompletableFuture().join()); - Assertions.assertEquals(2, defaultCostResponse.costs.size()); + Assertions.assertEquals(3, defaultCostResponse.costs.size()); var changedCostResponse = Assertions.assertInstanceOf( From e3dd72e77c5248a6b3232484391e2076fd2e9932 Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Mon, 22 May 2023 21:27:36 +0800 Subject: [PATCH 17/24] fix issue --- .../astraea/common/cost/MigrationCost.java | 4 ++++ .../common/cost/PartitionMigrateTimeCost.java | 22 ++++++++++++++----- .../cost/PartitionMigrateTimeCostTest.java | 12 +++++----- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/common/src/main/java/org/astraea/common/cost/MigrationCost.java b/common/src/main/java/org/astraea/common/cost/MigrationCost.java index 5083fced65..46df3bb27c 100644 --- a/common/src/main/java/org/astraea/common/cost/MigrationCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrationCost.java @@ -126,6 +126,10 @@ public static Map brokerMigrationSecond( Map.Entry::getKey, brokerSize -> brokerSize.getValue() / brokerOutRate.get(brokerSize.getKey()).orElse(0))); + brokerInRate.forEach( + (b, rate) -> System.out.println("broker: " + b + " inRate: " + rate.orElse(0))); + brokerOutRate.forEach( + (b, rate) -> System.out.println("broker: " + b + " OutRate: " + rate.orElse(0))); return Stream.concat(before.nodes().stream(), after.nodes().stream()) .map(NodeInfo::id) .distinct() diff --git a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java index de58591d6c..86f3665740 100644 --- a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java +++ b/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java @@ -76,20 +76,20 @@ public MetricSensor metricSensor() { var inRate = maxRateSensor.measure(REPLICATION_IN_RATE); var outRate = maxRateSensor.measure(REPLICATION_OUT_RATE); return List.of( - (MaxReplicationInRateBean) + new MaxReplicationInRateBean( () -> new BeanObject( newInMetrics.beanObject().domainName(), newInMetrics.beanObject().properties(), Map.of(STATISTICS_RATE_KEY, Math.max(oldInRate.orElse(0), inRate)), - current.toMillis()), - (MaxReplicationOutRateBean) + current.toMillis())), + new MaxReplicationOutRateBean( () -> new BeanObject( newOutMetrics.beanObject().domainName(), newOutMetrics.beanObject().properties(), Map.of(STATISTICS_RATE_KEY, Math.max(oldOutRate.orElse(0), outRate)), - current.toMillis())); + current.toMillis()))); }; } @@ -103,7 +103,17 @@ public MoveCost moveCost(ClusterInfo before, ClusterInfo after, ClusterBean clus return () -> planMigrateSecond > this.maxMigrateTime.getSeconds(); } - public interface MaxReplicationInRateBean extends HasMaxRate {} + public record MaxReplicationInRateBean(HasMaxRate hasMaxRate) implements HasMaxRate { + @Override + public BeanObject beanObject() { + return hasMaxRate.beanObject(); + } + } - public interface MaxReplicationOutRateBean extends HasMaxRate {} + public record MaxReplicationOutRateBean(HasMaxRate hasMaxRate) implements HasMaxRate { + @Override + public BeanObject beanObject() { + return hasMaxRate.beanObject(); + } + } } diff --git a/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java b/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java index 93a542cc52..52e4a2fa44 100644 --- a/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java @@ -168,15 +168,15 @@ private static ClusterBean clusterBean() { Map.of( 0, List.of( - (PartitionMigrateTimeCost.MaxReplicationInRateBean) () -> inBean0, - (PartitionMigrateTimeCost.MaxReplicationOutRateBean) () -> outBean0), + new PartitionMigrateTimeCost.MaxReplicationInRateBean(() -> inBean0), + new PartitionMigrateTimeCost.MaxReplicationOutRateBean(() -> outBean0)), 1, List.of( - (PartitionMigrateTimeCost.MaxReplicationInRateBean) () -> inBean1, - (PartitionMigrateTimeCost.MaxReplicationOutRateBean) () -> outBean1), + new PartitionMigrateTimeCost.MaxReplicationInRateBean(() -> inBean1), + new PartitionMigrateTimeCost.MaxReplicationOutRateBean(() -> outBean1)), 2, List.of( - (PartitionMigrateTimeCost.MaxReplicationInRateBean) () -> inBean2, - (PartitionMigrateTimeCost.MaxReplicationOutRateBean) () -> outBean2))); + new PartitionMigrateTimeCost.MaxReplicationInRateBean(() -> inBean2), + new PartitionMigrateTimeCost.MaxReplicationOutRateBean(() -> outBean2)))); } } From dbae033af9bb69f2f6040f1ad7707121c8af9455 Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Mon, 22 May 2023 21:33:36 +0800 Subject: [PATCH 18/24] merge main --- .../org/astraea/common/balancer/algorithms/GreedyBalancer.java | 1 + .../astraea/common/balancer/algorithms/SingleStepBalancer.java | 1 + 2 files changed, 2 insertions(+) diff --git a/common/src/main/java/org/astraea/common/balancer/algorithms/GreedyBalancer.java b/common/src/main/java/org/astraea/common/balancer/algorithms/GreedyBalancer.java index 67136915a3..1892040546 100644 --- a/common/src/main/java/org/astraea/common/balancer/algorithms/GreedyBalancer.java +++ b/common/src/main/java/org/astraea/common/balancer/algorithms/GreedyBalancer.java @@ -239,6 +239,7 @@ public Optional offer(AlgorithmConfig config) { .overflow()) { return Optional.of( new Plan( + config.clusterBean(), config.clusterInfo(), config.clusterCostFunction().clusterCost(config.clusterInfo(), clusterBean), currentClusterInfo, diff --git a/common/src/main/java/org/astraea/common/balancer/algorithms/SingleStepBalancer.java b/common/src/main/java/org/astraea/common/balancer/algorithms/SingleStepBalancer.java index 378816217e..e28ab7343b 100644 --- a/common/src/main/java/org/astraea/common/balancer/algorithms/SingleStepBalancer.java +++ b/common/src/main/java/org/astraea/common/balancer/algorithms/SingleStepBalancer.java @@ -140,6 +140,7 @@ public Optional offer(AlgorithmConfig config) { .overflow()) { return Optional.of( new Plan( + config.clusterBean(), config.clusterInfo(), config.clusterCostFunction().clusterCost(config.clusterInfo(), clusterBean), currentClusterInfo, From ebbc82b8863893c88c06f812cc599e05c3959dbc Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Mon, 22 May 2023 23:10:15 +0800 Subject: [PATCH 19/24] remove reduant code --- .../src/main/java/org/astraea/common/cost/MigrationCost.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/common/src/main/java/org/astraea/common/cost/MigrationCost.java b/common/src/main/java/org/astraea/common/cost/MigrationCost.java index 46df3bb27c..5083fced65 100644 --- a/common/src/main/java/org/astraea/common/cost/MigrationCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrationCost.java @@ -126,10 +126,6 @@ public static Map brokerMigrationSecond( Map.Entry::getKey, brokerSize -> brokerSize.getValue() / brokerOutRate.get(brokerSize.getKey()).orElse(0))); - brokerInRate.forEach( - (b, rate) -> System.out.println("broker: " + b + " inRate: " + rate.orElse(0))); - brokerOutRate.forEach( - (b, rate) -> System.out.println("broker: " + b + " OutRate: " + rate.orElse(0))); return Stream.concat(before.nodes().stream(), after.nodes().stream()) .map(NodeInfo::id) .distinct() From 85ce98d1d09ac0a6014169be48784fa361494b21 Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Thu, 25 May 2023 22:57:41 +0800 Subject: [PATCH 20/24] fix bugs --- .../astraea/app/web/BalancerHandlerTest.java | 4 ++-- .../astraea/common/cost/MigrationCost.java | 22 +++++++++++++------ .../astraea/common/cost/CostUtilsTest.java | 4 ++-- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java b/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java index f6727111cb..f33cd18550 100644 --- a/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java +++ b/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java @@ -23,7 +23,7 @@ import static org.astraea.common.balancer.BalancerConsole.TaskPhase.Searched; import static org.astraea.common.cost.MigrationCost.REPLICA_LEADERS_TO_ADDED; import static org.astraea.common.cost.MigrationCost.REPLICA_LEADERS_TO_REMOVE; -import static org.astraea.common.cost.MigrationCost.TO_FETCH_BYTES; +import static org.astraea.common.cost.MigrationCost.TO_FETCHED_BYTES; import static org.astraea.common.cost.MigrationCost.TO_SYNC_BYTES; import java.nio.charset.StandardCharsets; @@ -322,7 +322,7 @@ void testMoveCost(String leaderLimit, String sizeLimit) { migrationCost -> { switch (migrationCost.name) { case TO_SYNC_BYTES: - case TO_FETCH_BYTES: + case TO_FETCHED_BYTES: Assertions.assertTrue( migrationCost.brokerCosts.values().stream().mapToLong(Long::intValue).sum() <= DataSize.of(sizeLimit).bytes()); diff --git a/common/src/main/java/org/astraea/common/cost/MigrationCost.java b/common/src/main/java/org/astraea/common/cost/MigrationCost.java index 5083fced65..4a0d6cc697 100644 --- a/common/src/main/java/org/astraea/common/cost/MigrationCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrationCost.java @@ -35,7 +35,7 @@ public class MigrationCost { public final String name; public final Map brokerCosts; public static final String TO_SYNC_BYTES = "record size to sync (bytes)"; - public static final String TO_FETCH_BYTES = "record size to fetch (bytes)"; + public static final String TO_FETCHED_BYTES = "record size to fetched (bytes)"; public static final String REPLICA_LEADERS_TO_ADDED = "leader number to add"; public static final String REPLICA_LEADERS_TO_REMOVE = "leader number to remove"; public static final String CHANGED_REPLICAS = "changed replicas"; @@ -44,13 +44,13 @@ public class MigrationCost { public static List migrationCosts( ClusterInfo before, ClusterInfo after, ClusterBean clusterBean) { var migrateInBytes = recordSizeToSync(before, after); - var migrateOutBytes = recordSizeToFetch(before, after); + var migrateOutBytes = recordSizeToFetched(before, after); var migrateReplicaNum = replicaNumChanged(before, after); var migrateInLeader = replicaLeaderToAdd(before, after); var migrateOutLeader = replicaLeaderToRemove(before, after); return List.of( new MigrationCost(TO_SYNC_BYTES, migrateInBytes), - new MigrationCost(TO_FETCH_BYTES, migrateOutBytes), + new MigrationCost(TO_FETCHED_BYTES, migrateOutBytes), new MigrationCost(CHANGED_REPLICAS, migrateReplicaNum), new MigrationCost( PARTITION_MIGRATED_TIME, brokerMigrationSecond(before, after, clusterBean)), @@ -64,7 +64,7 @@ public MigrationCost(String name, Map brokerCosts) { this.brokerCosts = brokerCosts; } - static Map recordSizeToFetch(ClusterInfo before, ClusterInfo after) { + static Map recordSizeToFetched(ClusterInfo before, ClusterInfo after) { return migratedChanged(before, after, true, (ignore) -> true, Replica::size); } @@ -113,14 +113,14 @@ public static Map brokerMigrationSecond( clusterBean, PartitionMigrateTimeCost.MaxReplicationOutRateBean.class))); var brokerMigrateInSecond = - MigrationCost.recordSizeToFetch(before, after).entrySet().stream() + MigrationCost.recordSizeToSync(before, after).entrySet().stream() .collect( Collectors.toMap( Map.Entry::getKey, brokerSize -> brokerSize.getValue() / brokerInRate.get(brokerSize.getKey()).orElse(0))); var brokerMigrateOutSecond = - MigrationCost.recordSizeToSync(before, after).entrySet().stream() + MigrationCost.recordSizeToFetched(before, after).entrySet().stream() .collect( Collectors.toMap( Map.Entry::getKey, @@ -170,7 +170,14 @@ private static Map migratedChanged( p -> dest.replicas(p).stream() .filter(predicate) - .filter(r -> !source.replicas(p).contains(r))) + .filter( + r -> + source.replicas(p).stream() + .noneMatch( + sourceReplica -> + sourceReplica + .topicPartitionReplica() + .equals(r.topicPartitionReplica())))) .map( r -> { if (migrateOut) return dest.replicaLeader(r.topicPartition()).orElse(r); @@ -181,6 +188,7 @@ private static Map migratedChanged( r -> r.nodeInfo().id(), Collectors.mapping( Function.identity(), Collectors.summingLong(replicaFunction::apply)))); + return Stream.concat(dest.nodes().stream(), source.nodes().stream()) .map(NodeInfo::id) .distinct() diff --git a/common/src/test/java/org/astraea/common/cost/CostUtilsTest.java b/common/src/test/java/org/astraea/common/cost/CostUtilsTest.java index e086242e57..81c990aa9d 100644 --- a/common/src/test/java/org/astraea/common/cost/CostUtilsTest.java +++ b/common/src/test/java/org/astraea/common/cost/CostUtilsTest.java @@ -17,7 +17,7 @@ package org.astraea.common.cost; import static org.astraea.common.cost.CostUtils.changedRecordSizeOverflow; -import static org.astraea.common.cost.MigrationCost.recordSizeToFetch; +import static org.astraea.common.cost.MigrationCost.recordSizeToFetched; import static org.astraea.common.cost.MigrationCost.recordSizeToSync; import java.util.List; @@ -43,7 +43,7 @@ void testChangedRecordSizeOverflow() { Assertions.assertEquals(1000, moveInResult.get(1)); Assertions.assertEquals(100 + 500, moveInResult.get(2)); - var moveOutResult = recordSizeToFetch(beforeClusterInfo(), afterClusterInfo()); + var moveOutResult = recordSizeToFetched(beforeClusterInfo(), afterClusterInfo()); Assertions.assertEquals(3, moveOutResult.size()); Assertions.assertEquals(100 + 500, moveOutResult.get(0)); Assertions.assertEquals(0, moveOutResult.get(1)); From 52987693102c3668967c9ed10a82d0d9c97b1a90 Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Sat, 27 May 2023 22:43:20 +0800 Subject: [PATCH 21/24] fix issues --- .../astraea/app/web/BalancerHandlerTest.java | 4 +- ...rateTimeCost.java => MigrateTimeCost.java} | 37 ++++++----------- .../astraea/common/cost/MigrationCost.java | 18 ++++---- .../org/astraea/common/metrics/Sensor.java | 8 ---- .../astraea/common/cost/CostUtilsTest.java | 4 +- ...CostTest.java => MigrateTimeCostTest.java} | 41 ++++++++----------- 6 files changed, 43 insertions(+), 69 deletions(-) rename common/src/main/java/org/astraea/common/cost/{PartitionMigrateTimeCost.java => MigrateTimeCost.java} (73%) rename common/src/test/java/org/astraea/common/cost/{PartitionMigrateTimeCostTest.java => MigrateTimeCostTest.java} (76%) diff --git a/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java b/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java index 50c5ff493b..53725255ed 100644 --- a/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java +++ b/app/src/test/java/org/astraea/app/web/BalancerHandlerTest.java @@ -23,7 +23,7 @@ import static org.astraea.common.balancer.BalancerConsole.TaskPhase.Searched; import static org.astraea.common.cost.MigrationCost.REPLICA_LEADERS_TO_ADDED; import static org.astraea.common.cost.MigrationCost.REPLICA_LEADERS_TO_REMOVE; -import static org.astraea.common.cost.MigrationCost.TO_FETCHED_BYTES; +import static org.astraea.common.cost.MigrationCost.TO_FETCH_BYTES; import static org.astraea.common.cost.MigrationCost.TO_SYNC_BYTES; import java.nio.charset.StandardCharsets; @@ -322,7 +322,7 @@ void testMoveCost(String leaderLimit, String sizeLimit) { migrationCost -> { switch (migrationCost.name) { case TO_SYNC_BYTES: - case TO_FETCHED_BYTES: + case TO_FETCH_BYTES: Assertions.assertTrue( migrationCost.brokerCosts.values().stream().mapToLong(Long::intValue).sum() <= DataSize.of(sizeLimit).bytes()); diff --git a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java b/common/src/main/java/org/astraea/common/cost/MigrateTimeCost.java similarity index 73% rename from common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java rename to common/src/main/java/org/astraea/common/cost/MigrateTimeCost.java index 86f3665740..5c14fe9af3 100644 --- a/common/src/main/java/org/astraea/common/cost/PartitionMigrateTimeCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrateTimeCost.java @@ -34,16 +34,14 @@ import org.astraea.common.metrics.stats.Max; /** MoveCost: more max write rate change -> higher migrate cost. */ -public class PartitionMigrateTimeCost implements HasMoveCost { - private static final String REPLICATION_IN_RATE = "replication_in_rate"; - private static final String REPLICATION_OUT_RATE = "replication_out_rate"; +public class MigrateTimeCost implements HasMoveCost { + private static final String REPLICATION_RATE = "replication_rate"; static final String MAX_MIGRATE_TIME_KEY = "max.migrated.time.limit"; - public static final String STATISTICS_RATE_KEY = "statistics.rate.key"; // metrics windows size private final Duration maxMigrateTime; - public PartitionMigrateTimeCost(Configuration config) { + public MigrateTimeCost(Configuration config) { this.maxMigrateTime = config.duration(MAX_MIGRATE_TIME_KEY).orElse(Duration.ofSeconds(Long.MAX_VALUE)); } @@ -53,42 +51,33 @@ public MetricSensor metricSensor() { return (client, clusterBean) -> { var oldInRate = brokerMaxRate( - client.identity(), - clusterBean, - PartitionMigrateTimeCost.MaxReplicationInRateBean.class); + client.identity(), clusterBean, MigrateTimeCost.MaxReplicationInRateBean.class); var oldOutRate = brokerMaxRate( - client.identity(), - clusterBean, - PartitionMigrateTimeCost.MaxReplicationOutRateBean.class); + client.identity(), clusterBean, MigrateTimeCost.MaxReplicationOutRateBean.class); var newInMetrics = ServerMetrics.BrokerTopic.REPLICATION_BYTES_IN_PER_SEC.fetch(client); var newOutMetrics = ServerMetrics.BrokerTopic.REPLICATION_BYTES_OUT_PER_SEC.fetch(client); var current = Duration.ofMillis(System.currentTimeMillis()); - var maxRateSensor = - Sensor.builder() - .addStats( - Map.of( - REPLICATION_IN_RATE, Max.of(), - REPLICATION_OUT_RATE, Max.of())) - .build(); - maxRateSensor.record(REPLICATION_IN_RATE, newInMetrics.oneMinuteRate()); - maxRateSensor.record(REPLICATION_OUT_RATE, newOutMetrics.oneMinuteRate()); - var inRate = maxRateSensor.measure(REPLICATION_IN_RATE); - var outRate = maxRateSensor.measure(REPLICATION_OUT_RATE); + var maxInRateSensor = Sensor.builder().addStat(REPLICATION_RATE, Max.of()).build(); + var maxOutRateSensor = Sensor.builder().addStat(REPLICATION_RATE, Max.of()).build(); + maxInRateSensor.record(newInMetrics.oneMinuteRate()); + maxOutRateSensor.record(newOutMetrics.oneMinuteRate()); + var inRate = maxInRateSensor.measure(REPLICATION_RATE); + var outRate = maxOutRateSensor.measure(REPLICATION_RATE); return List.of( new MaxReplicationInRateBean( () -> new BeanObject( newInMetrics.beanObject().domainName(), newInMetrics.beanObject().properties(), - Map.of(STATISTICS_RATE_KEY, Math.max(oldInRate.orElse(0), inRate)), + Map.of(REPLICATION_RATE, Math.max(oldInRate.orElse(0), inRate)), current.toMillis())), new MaxReplicationOutRateBean( () -> new BeanObject( newOutMetrics.beanObject().domainName(), newOutMetrics.beanObject().properties(), - Map.of(STATISTICS_RATE_KEY, Math.max(oldOutRate.orElse(0), outRate)), + Map.of(REPLICATION_RATE, Math.max(oldOutRate.orElse(0), outRate)), current.toMillis()))); }; } diff --git a/common/src/main/java/org/astraea/common/cost/MigrationCost.java b/common/src/main/java/org/astraea/common/cost/MigrationCost.java index 4f21b3d333..3211f6eb39 100644 --- a/common/src/main/java/org/astraea/common/cost/MigrationCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrationCost.java @@ -35,25 +35,25 @@ public class MigrationCost { public final String name; public final Map brokerCosts; public static final String TO_SYNC_BYTES = "record size to sync (bytes)"; - public static final String TO_FETCHED_BYTES = "record size to fetched (bytes)"; + public static final String TO_FETCH_BYTES = "record size to fetch (bytes)"; public static final String REPLICA_LEADERS_TO_ADDED = "leader number to add"; public static final String REPLICA_LEADERS_TO_REMOVE = "leader number to remove"; public static final String CHANGED_REPLICAS = "changed replicas"; - public static final String PARTITION_MIGRATED_TIME = "partition migrated time"; + public static final String MIGRATION_ELAPSED_TIME = "migration elapsed time"; public static List migrationCosts( ClusterInfo before, ClusterInfo after, ClusterBean clusterBean) { var migrateInBytes = recordSizeToSync(before, after); - var migrateOutBytes = recordSizeToFetched(before, after); + var migrateOutBytes = recordSizeToFetch(before, after); var migrateReplicaNum = replicaNumChanged(before, after); var migrateInLeader = replicaLeaderToAdd(before, after); var migrateOutLeader = replicaLeaderToRemove(before, after); return List.of( new MigrationCost(TO_SYNC_BYTES, migrateInBytes), - new MigrationCost(TO_FETCHED_BYTES, migrateOutBytes), + new MigrationCost(TO_FETCH_BYTES, migrateOutBytes), new MigrationCost(CHANGED_REPLICAS, migrateReplicaNum), new MigrationCost( - PARTITION_MIGRATED_TIME, brokerMigrationSecond(before, after, clusterBean)), + MIGRATION_ELAPSED_TIME, brokerMigrationSecond(before, after, clusterBean)), new MigrationCost(REPLICA_LEADERS_TO_ADDED, migrateInLeader), new MigrationCost(REPLICA_LEADERS_TO_REMOVE, migrateOutLeader), new MigrationCost(CHANGED_REPLICAS, migrateReplicaNum)); @@ -64,7 +64,7 @@ public MigrationCost(String name, Map brokerCosts) { this.brokerCosts = brokerCosts; } - static Map recordSizeToFetched(ClusterInfo before, ClusterInfo after) { + static Map recordSizeToFetch(ClusterInfo before, ClusterInfo after) { return migratedChanged(before, after, true, (ignore) -> true, Replica::size); } @@ -101,7 +101,7 @@ public static Map brokerMigrationSecond( brokerMaxRate( nodeInfo.id(), clusterBean, - PartitionMigrateTimeCost.MaxReplicationInRateBean.class))); + MigrateTimeCost.MaxReplicationInRateBean.class))); var brokerOutRate = before.brokers().stream() .collect( @@ -111,7 +111,7 @@ public static Map brokerMigrationSecond( brokerMaxRate( nodeInfo.id(), clusterBean, - PartitionMigrateTimeCost.MaxReplicationOutRateBean.class))); + MigrateTimeCost.MaxReplicationOutRateBean.class))); var brokerMigrateInSecond = MigrationCost.recordSizeToSync(before, after).entrySet().stream() .collect( @@ -120,7 +120,7 @@ public static Map brokerMigrationSecond( brokerSize -> brokerSize.getValue() / brokerInRate.get(brokerSize.getKey()).orElse(0))); var brokerMigrateOutSecond = - MigrationCost.recordSizeToFetched(before, after).entrySet().stream() + MigrationCost.recordSizeToFetch(before, after).entrySet().stream() .collect( Collectors.toMap( Map.Entry::getKey, diff --git a/common/src/main/java/org/astraea/common/metrics/Sensor.java b/common/src/main/java/org/astraea/common/metrics/Sensor.java index e3153d1e78..5119d96340 100644 --- a/common/src/main/java/org/astraea/common/metrics/Sensor.java +++ b/common/src/main/java/org/astraea/common/metrics/Sensor.java @@ -30,9 +30,6 @@ static Builder builder() { /** Record the new get data */ void record(V value); - /** Record the new get data for given `metricName` */ - void record(String statName, V value); - /** * Get the statistic by the given `metricName`. * @@ -72,11 +69,6 @@ public synchronized void record(V value) { stats.values().forEach(stat -> stat.record(value)); } - @Override - public void record(String statName, V value) { - stats.get(statName).record(value); - } - @Override public V measure(String statName) { return stats.get(statName).measure(); diff --git a/common/src/test/java/org/astraea/common/cost/CostUtilsTest.java b/common/src/test/java/org/astraea/common/cost/CostUtilsTest.java index 2e07d8779e..db1817e725 100644 --- a/common/src/test/java/org/astraea/common/cost/CostUtilsTest.java +++ b/common/src/test/java/org/astraea/common/cost/CostUtilsTest.java @@ -17,7 +17,7 @@ package org.astraea.common.cost; import static org.astraea.common.cost.CostUtils.changedRecordSizeOverflow; -import static org.astraea.common.cost.MigrationCost.recordSizeToFetched; +import static org.astraea.common.cost.MigrationCost.recordSizeToFetch; import static org.astraea.common.cost.MigrationCost.recordSizeToSync; import java.util.List; @@ -42,7 +42,7 @@ void testChangedRecordSizeOverflow() { Assertions.assertEquals(1000, moveInResult.get(1)); Assertions.assertEquals(100 + 500, moveInResult.get(2)); - var moveOutResult = recordSizeToFetched(beforeClusterInfo(), afterClusterInfo()); + var moveOutResult = recordSizeToFetch(beforeClusterInfo(), afterClusterInfo()); Assertions.assertEquals(3, moveOutResult.size()); Assertions.assertEquals(100 + 500, moveOutResult.get(0)); Assertions.assertEquals(0, moveOutResult.get(1)); diff --git a/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java b/common/src/test/java/org/astraea/common/cost/MigrateTimeCostTest.java similarity index 76% rename from common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java rename to common/src/test/java/org/astraea/common/cost/MigrateTimeCostTest.java index c39e068ceb..6030af4726 100644 --- a/common/src/test/java/org/astraea/common/cost/PartitionMigrateTimeCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/MigrateTimeCostTest.java @@ -29,26 +29,20 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -class PartitionMigrateTimeCostTest { +class MigrateTimeCostTest { private static final BeanObject inBean0 = - new BeanObject( - "domain", Map.of(), Map.of(PartitionMigrateTimeCost.STATISTICS_RATE_KEY, 1000.0)); + new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.STATISTICS_RATE_KEY, 1000.0)); private static final BeanObject outBean0 = - new BeanObject( - "domain", Map.of(), Map.of(PartitionMigrateTimeCost.STATISTICS_RATE_KEY, 1500.0)); + new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.STATISTICS_RATE_KEY, 1500.0)); private static final BeanObject inBean1 = - new BeanObject( - "domain", Map.of(), Map.of(PartitionMigrateTimeCost.STATISTICS_RATE_KEY, 2000.0)); + new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.STATISTICS_RATE_KEY, 2000.0)); private static final BeanObject outBean1 = - new BeanObject( - "domain", Map.of(), Map.of(PartitionMigrateTimeCost.STATISTICS_RATE_KEY, 2500.0)); + new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.STATISTICS_RATE_KEY, 2500.0)); private static final BeanObject inBean2 = - new BeanObject( - "domain", Map.of(), Map.of(PartitionMigrateTimeCost.STATISTICS_RATE_KEY, 3000.0)); + new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.STATISTICS_RATE_KEY, 3000.0)); private static final BeanObject outBean2 = - new BeanObject( - "domain", Map.of(), Map.of(PartitionMigrateTimeCost.STATISTICS_RATE_KEY, 3500.0)); + new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.STATISTICS_RATE_KEY, 3500.0)); @Test void testMigratedCost() { @@ -85,12 +79,11 @@ private List brokers() { void testMostCost() { var before = of(before(), brokers()); var after = of(after(), brokers()); - var timeLimit = - new Configuration(Map.of(PartitionMigrateTimeCost.MAX_MIGRATE_TIME_KEY, "20000")); + var timeLimit = new Configuration(Map.of(MigrateTimeCost.MAX_MIGRATE_TIME_KEY, "20000")); var overFlowTimeLimit = - new Configuration(Map.of(PartitionMigrateTimeCost.MAX_MIGRATE_TIME_KEY, "19999")); - var cf = new PartitionMigrateTimeCost(timeLimit); - var overFlowCf = new PartitionMigrateTimeCost(overFlowTimeLimit); + new Configuration(Map.of(MigrateTimeCost.MAX_MIGRATE_TIME_KEY, "19999")); + var cf = new MigrateTimeCost(timeLimit); + var overFlowCf = new MigrateTimeCost(overFlowTimeLimit); var moveCost = cf.moveCost(before, after, clusterBean()); var overflowCost = overFlowCf.moveCost(before, after, clusterBean()); Assertions.assertFalse(moveCost.overflow()); @@ -170,15 +163,15 @@ private static ClusterBean clusterBean() { Map.of( 0, List.of( - new PartitionMigrateTimeCost.MaxReplicationInRateBean(() -> inBean0), - new PartitionMigrateTimeCost.MaxReplicationOutRateBean(() -> outBean0)), + new MigrateTimeCost.MaxReplicationInRateBean(() -> inBean0), + new MigrateTimeCost.MaxReplicationOutRateBean(() -> outBean0)), 1, List.of( - new PartitionMigrateTimeCost.MaxReplicationInRateBean(() -> inBean1), - new PartitionMigrateTimeCost.MaxReplicationOutRateBean(() -> outBean1)), + new MigrateTimeCost.MaxReplicationInRateBean(() -> inBean1), + new MigrateTimeCost.MaxReplicationOutRateBean(() -> outBean1)), 2, List.of( - new PartitionMigrateTimeCost.MaxReplicationInRateBean(() -> inBean2), - new PartitionMigrateTimeCost.MaxReplicationOutRateBean(() -> outBean2)))); + new MigrateTimeCost.MaxReplicationInRateBean(() -> inBean2), + new MigrateTimeCost.MaxReplicationOutRateBean(() -> outBean2)))); } } From 3731482c66f4b824744917eb63a4b5e797d3fe2e Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Sat, 27 May 2023 23:17:40 +0800 Subject: [PATCH 22/24] fix bugs --- .../org/astraea/common/cost/MigrateTimeCost.java | 2 +- .../astraea/common/metrics/broker/HasMaxRate.java | 2 +- .../org/astraea/common/cost/MigrateTimeCostTest.java | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/common/src/main/java/org/astraea/common/cost/MigrateTimeCost.java b/common/src/main/java/org/astraea/common/cost/MigrateTimeCost.java index 5c14fe9af3..d38db7c5c4 100644 --- a/common/src/main/java/org/astraea/common/cost/MigrateTimeCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrateTimeCost.java @@ -35,7 +35,7 @@ /** MoveCost: more max write rate change -> higher migrate cost. */ public class MigrateTimeCost implements HasMoveCost { - private static final String REPLICATION_RATE = "replication_rate"; + static final String REPLICATION_RATE = "replication_rate"; static final String MAX_MIGRATE_TIME_KEY = "max.migrated.time.limit"; // metrics windows size diff --git a/common/src/main/java/org/astraea/common/metrics/broker/HasMaxRate.java b/common/src/main/java/org/astraea/common/metrics/broker/HasMaxRate.java index 7a2ab8cb10..77e7e3d87b 100644 --- a/common/src/main/java/org/astraea/common/metrics/broker/HasMaxRate.java +++ b/common/src/main/java/org/astraea/common/metrics/broker/HasMaxRate.java @@ -20,6 +20,6 @@ public interface HasMaxRate extends HasBeanObject { default double maxRate() { - return (double) beanObject().attributes().get("statistics.rate.key"); + return (double) beanObject().attributes().get("replication_rate"); } } diff --git a/common/src/test/java/org/astraea/common/cost/MigrateTimeCostTest.java b/common/src/test/java/org/astraea/common/cost/MigrateTimeCostTest.java index 6030af4726..35cb3a1870 100644 --- a/common/src/test/java/org/astraea/common/cost/MigrateTimeCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/MigrateTimeCostTest.java @@ -32,17 +32,17 @@ class MigrateTimeCostTest { private static final BeanObject inBean0 = - new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.STATISTICS_RATE_KEY, 1000.0)); + new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.REPLICATION_RATE, 1000.0)); private static final BeanObject outBean0 = - new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.STATISTICS_RATE_KEY, 1500.0)); + new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.REPLICATION_RATE, 1500.0)); private static final BeanObject inBean1 = - new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.STATISTICS_RATE_KEY, 2000.0)); + new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.REPLICATION_RATE, 2000.0)); private static final BeanObject outBean1 = - new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.STATISTICS_RATE_KEY, 2500.0)); + new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.REPLICATION_RATE, 2500.0)); private static final BeanObject inBean2 = - new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.STATISTICS_RATE_KEY, 3000.0)); + new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.REPLICATION_RATE, 3000.0)); private static final BeanObject outBean2 = - new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.STATISTICS_RATE_KEY, 3500.0)); + new BeanObject("domain", Map.of(), Map.of(MigrateTimeCost.REPLICATION_RATE, 3500.0)); @Test void testMigratedCost() { From 3e006c61bb684dd592dc842e7bbe3ad775e81ea5 Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Tue, 30 May 2023 21:45:35 +0800 Subject: [PATCH 23/24] fix bug and merge main --- .../astraea/common/cost/MigrationCost.java | 14 ++++------ .../common/cost/MigrateTimeCostTest.java | 28 +++++++------------ 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/common/src/main/java/org/astraea/common/cost/MigrationCost.java b/common/src/main/java/org/astraea/common/cost/MigrationCost.java index b9e8c1ac66..ecd1f0bf71 100644 --- a/common/src/main/java/org/astraea/common/cost/MigrationCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrationCost.java @@ -171,13 +171,7 @@ private static Map migratedChanged( dest.replicas(p).stream() .filter(predicate) .filter( - r -> - source.replicas(p).stream() - .noneMatch( - sourceReplica -> - sourceReplica - .topicPartitionReplica() - .equals(r.topicPartitionReplica())))) + r -> source.replicas(p).stream().noneMatch(x -> checkoutSameTPR(r, x)))) .map( r -> { if (migrateOut) return dest.replicaLeader(r.topicPartition()).orElse(r); @@ -185,7 +179,7 @@ private static Map migratedChanged( }) .collect( Collectors.groupingBy( - r -> r.brokerId(), + Replica::brokerId, Collectors.mapping( Function.identity(), Collectors.summingLong(replicaFunction::apply)))); return Stream.concat(dest.brokers().stream(), source.brokers().stream()) @@ -195,6 +189,10 @@ private static Map migratedChanged( .collect(Collectors.toMap(Function.identity(), n -> cost.getOrDefault(n, 0L))); } + private static boolean checkoutSameTPR(Replica r1, Replica r2) { + return r1.topicPartitionReplica().equals(r2.topicPartitionReplica()); + } + private static Map changedReplicaNumber(ClusterInfo before, ClusterInfo after) { return Stream.concat(before.brokers().stream(), after.brokers().stream()) .map(Broker::id) diff --git a/common/src/test/java/org/astraea/common/cost/MigrateTimeCostTest.java b/common/src/test/java/org/astraea/common/cost/MigrateTimeCostTest.java index 35cb3a1870..a724ce3119 100644 --- a/common/src/test/java/org/astraea/common/cost/MigrateTimeCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/MigrateTimeCostTest.java @@ -19,7 +19,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import org.apache.kafka.common.Node; import org.astraea.common.Configuration; import org.astraea.common.admin.Broker; import org.astraea.common.admin.ClusterInfo; @@ -62,16 +61,9 @@ void testMigratedCost() { private List brokers() { return before().stream() - .map(Replica::broker) + .map(Replica::brokerId) .distinct() - .map( - nodeInfo -> - Broker.of( - false, - new Node(nodeInfo.id(), "", nodeInfo.port()), - Map.of(), - Map.of(), - List.of())) + .map(nodeId -> Broker.of(nodeId, "", -1)) .collect(Collectors.toList()); } @@ -101,28 +93,28 @@ private List after() { .partition(10) .isLeader(true) .size(10000000) - .broker(Broker.of(1, "", -1)) + .brokerId(1) .build(), Replica.builder() .topic("t") .partition(11) .isLeader(true) .size(20000000) - .broker(Broker.of(2, "", -1)) + .brokerId(2) .build(), Replica.builder() .topic("t") .partition(12) .isLeader(true) .size(30000000) - .broker(Broker.of(0, "", -1)) + .brokerId(0) .build(), Replica.builder() .topic("t") .partition(12) .isLeader(false) .size(30000000) - .broker(Broker.of(1, "", -1)) + .brokerId(1) .build()); } @@ -133,28 +125,28 @@ private List before() { .partition(10) .isLeader(true) .size(10000000) - .broker(Broker.of(0, "", -1)) + .brokerId(0) .build(), Replica.builder() .topic("t") .partition(11) .isLeader(true) .size(20000000) - .broker(Broker.of(1, "", -1)) + .brokerId(1) .build(), Replica.builder() .topic("t") .partition(12) .isLeader(true) .size(30000000) - .broker(Broker.of(2, "", -1)) + .brokerId(2) .build(), Replica.builder() .topic("t") .partition(12) .isLeader(false) .size(30000000) - .broker(Broker.of(0, "", -1)) + .brokerId(0) .build()); } From bd59c891cb6c1b7158b936b39b2c0aea743851a3 Mon Sep 17 00:00:00 2001 From: qoo332001 Date: Tue, 30 May 2023 23:06:23 +0800 Subject: [PATCH 24/24] fix issue --- .../org/astraea/common/balancer/Balancer.java | 5 ++++ .../astraea/common/cost/MigrateTimeCost.java | 30 +++++++++---------- .../astraea/common/cost/MigrationCost.java | 15 ++++++---- .../common/cost/MigrateTimeCostTest.java | 12 ++++---- 4 files changed, 34 insertions(+), 28 deletions(-) diff --git a/common/src/main/java/org/astraea/common/balancer/Balancer.java b/common/src/main/java/org/astraea/common/balancer/Balancer.java index 489b299118..49da6f29bd 100644 --- a/common/src/main/java/org/astraea/common/balancer/Balancer.java +++ b/common/src/main/java/org/astraea/common/balancer/Balancer.java @@ -33,6 +33,11 @@ public interface Balancer { */ Optional offer(AlgorithmConfig config); + /** + * @param initialClusterCost The {@link ClusterCost} score of the original {@link ClusterInfo} + * when this plan is start generating. + * @param proposalClusterCost The {@link ClusterCost} score of the proposed new allocation. + */ record Plan( ClusterBean clusterBean, ClusterInfo initialClusterInfo, diff --git a/common/src/main/java/org/astraea/common/cost/MigrateTimeCost.java b/common/src/main/java/org/astraea/common/cost/MigrateTimeCost.java index d38db7c5c4..e4c715fd81 100644 --- a/common/src/main/java/org/astraea/common/cost/MigrateTimeCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrateTimeCost.java @@ -66,19 +66,17 @@ public MetricSensor metricSensor() { var outRate = maxOutRateSensor.measure(REPLICATION_RATE); return List.of( new MaxReplicationInRateBean( - () -> - new BeanObject( - newInMetrics.beanObject().domainName(), - newInMetrics.beanObject().properties(), - Map.of(REPLICATION_RATE, Math.max(oldInRate.orElse(0), inRate)), - current.toMillis())), + new BeanObject( + newInMetrics.beanObject().domainName(), + newInMetrics.beanObject().properties(), + Map.of(REPLICATION_RATE, Math.max(oldInRate.orElse(0.0), inRate)), + current.toMillis())), new MaxReplicationOutRateBean( - () -> - new BeanObject( - newOutMetrics.beanObject().domainName(), - newOutMetrics.beanObject().properties(), - Map.of(REPLICATION_RATE, Math.max(oldOutRate.orElse(0), outRate)), - current.toMillis()))); + new BeanObject( + newOutMetrics.beanObject().domainName(), + newOutMetrics.beanObject().properties(), + Map.of(REPLICATION_RATE, Math.max(oldOutRate.orElse(0.0), outRate)), + current.toMillis()))); }; } @@ -92,17 +90,17 @@ public MoveCost moveCost(ClusterInfo before, ClusterInfo after, ClusterBean clus return () -> planMigrateSecond > this.maxMigrateTime.getSeconds(); } - public record MaxReplicationInRateBean(HasMaxRate hasMaxRate) implements HasMaxRate { + public record MaxReplicationInRateBean(BeanObject beanObject) implements HasMaxRate { @Override public BeanObject beanObject() { - return hasMaxRate.beanObject(); + return beanObject; } } - public record MaxReplicationOutRateBean(HasMaxRate hasMaxRate) implements HasMaxRate { + public record MaxReplicationOutRateBean(BeanObject beanObject) implements HasMaxRate { @Override public BeanObject beanObject() { - return hasMaxRate.beanObject(); + return beanObject; } } } diff --git a/common/src/main/java/org/astraea/common/cost/MigrationCost.java b/common/src/main/java/org/astraea/common/cost/MigrationCost.java index ecd1f0bf71..1978bccbd8 100644 --- a/common/src/main/java/org/astraea/common/cost/MigrationCost.java +++ b/common/src/main/java/org/astraea/common/cost/MigrationCost.java @@ -16,9 +16,10 @@ */ package org.astraea.common.cost; +import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.OptionalDouble; +import java.util.Optional; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -118,14 +119,15 @@ public static Map brokerMigrationSecond( Collectors.toMap( Map.Entry::getKey, brokerSize -> - brokerSize.getValue() / brokerInRate.get(brokerSize.getKey()).orElse(0))); + brokerSize.getValue() / brokerInRate.get(brokerSize.getKey()).orElse(0.0))); var brokerMigrateOutSecond = MigrationCost.recordSizeToFetch(before, after).entrySet().stream() .collect( Collectors.toMap( Map.Entry::getKey, brokerSize -> - brokerSize.getValue() / brokerOutRate.get(brokerSize.getKey()).orElse(0))); + brokerSize.getValue() + / brokerOutRate.get(brokerSize.getKey()).orElse(0.0))); return Stream.concat(before.brokers().stream(), after.brokers().stream()) .map(Broker::id) .distinct() @@ -139,12 +141,13 @@ public static Map brokerMigrationSecond( brokerMigrateOutSecond.get(nodeId)))); } - static OptionalDouble brokerMaxRate( + static Optional brokerMaxRate( int identity, ClusterBean clusterBean, Class statisticMetrics) { return clusterBean .brokerMetrics(identity, statisticMetrics) - .mapToDouble(b -> ((HasMaxRate) b).maxRate()) - .max(); + .filter(b -> b instanceof HasMaxRate) + .map(b -> ((HasMaxRate) b).maxRate()) + .max(Comparator.naturalOrder()); } /** diff --git a/common/src/test/java/org/astraea/common/cost/MigrateTimeCostTest.java b/common/src/test/java/org/astraea/common/cost/MigrateTimeCostTest.java index a724ce3119..843a941527 100644 --- a/common/src/test/java/org/astraea/common/cost/MigrateTimeCostTest.java +++ b/common/src/test/java/org/astraea/common/cost/MigrateTimeCostTest.java @@ -155,15 +155,15 @@ private static ClusterBean clusterBean() { Map.of( 0, List.of( - new MigrateTimeCost.MaxReplicationInRateBean(() -> inBean0), - new MigrateTimeCost.MaxReplicationOutRateBean(() -> outBean0)), + new MigrateTimeCost.MaxReplicationInRateBean(inBean0), + new MigrateTimeCost.MaxReplicationOutRateBean(outBean0)), 1, List.of( - new MigrateTimeCost.MaxReplicationInRateBean(() -> inBean1), - new MigrateTimeCost.MaxReplicationOutRateBean(() -> outBean1)), + new MigrateTimeCost.MaxReplicationInRateBean(inBean1), + new MigrateTimeCost.MaxReplicationOutRateBean(outBean1)), 2, List.of( - new MigrateTimeCost.MaxReplicationInRateBean(() -> inBean2), - new MigrateTimeCost.MaxReplicationOutRateBean(() -> outBean2)))); + new MigrateTimeCost.MaxReplicationInRateBean(inBean2), + new MigrateTimeCost.MaxReplicationOutRateBean(outBean2)))); } }