diff --git a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java index d26fb7adb56a9..604bf97dbc8af 100644 --- a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java @@ -17,15 +17,19 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterApplier; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.Transport; +import org.elasticsearch.transport.TransportConnectionListener; import org.elasticsearch.transport.TransportService; import java.util.ArrayList; @@ -188,6 +192,7 @@ public String toString() { @Override protected void doStart() { + transportService.addConnectionListener(new ConnectionChangeListener()); final ConnectionChecker connectionChecker = new ConnectionChecker(); this.connectionChecker = connectionChecker; connectionChecker.scheduleNextCheck(); @@ -209,12 +214,36 @@ public void reconnectToNodes(DiscoveryNodes discoveryNodes, Runnable onCompletio }); } - private class ConnectionTarget { + // exposed for testing + protected ConnectionTarget connectionTargetForNode(DiscoveryNode node) { + synchronized (mutex) { + return targetsByNode.get(node); + } + } + + /** + * Time of disconnect in absolute time ({@link ThreadPool#absoluteTimeInMillis()}), + * and disconnect-causing exception, if any + */ + record DisconnectionHistory(long disconnectTimeMillis, @Nullable Exception disconnectCause) { + public long getDisconnectTimeMillis() { + return disconnectTimeMillis; + } + + public Exception getDisconnectCause() { + return disconnectCause; + } + } + + protected class ConnectionTarget { private final DiscoveryNode discoveryNode; private final AtomicInteger consecutiveFailureCount = new AtomicInteger(); private final AtomicReference connectionRef = new AtomicReference<>(); + // access is synchronized by the service mutex + protected DisconnectionHistory disconnectionHistory = null; + // all access to these fields is synchronized private List pendingRefs; private boolean connectionInProgress; @@ -347,4 +376,67 @@ public String toString() { } } } + + /** + * Receives connection/disconnection events from the transport, and records them in per-node DisconnectionHistory + * structures for logging network issues. DisconnectionHistory records are stored their node's ConnectionTarget. + * + * Network issues (that this listener monitors for) occur whenever a reconnection to a node succeeds, + * and it has the same ephemeral ID as it did during the last connection; this happens when a connection event + * occurs, and its ConnectionTarget entry has a previous DisconnectionHistory stored. + */ + private class ConnectionChangeListener implements TransportConnectionListener { + @Override + public void onNodeConnected(DiscoveryNode node, Transport.Connection connection) { + DisconnectionHistory disconnectionHistory = null; + synchronized (mutex) { + ConnectionTarget connectionTarget = targetsByNode.get(node); + if (connectionTarget != null) { + disconnectionHistory = connectionTarget.disconnectionHistory; + connectionTarget.disconnectionHistory = null; + } + } + + if (disconnectionHistory != null) { + long millisSinceDisconnect = threadPool.absoluteTimeInMillis() - disconnectionHistory.disconnectTimeMillis; + if (disconnectionHistory.disconnectCause != null) { + logger.warn( + () -> format( + """ + reopened transport connection to node [%s] \ + which disconnected exceptionally [%dms] ago but did not \ + restart, so the disconnection is unexpected; \ + see [%s] for troubleshooting guidance""", + node.descriptionWithoutAttributes(), + millisSinceDisconnect, + ReferenceDocs.NETWORK_DISCONNECT_TROUBLESHOOTING + ), + disconnectionHistory.disconnectCause + ); + } else { + logger.warn( + """ + reopened transport connection to node [{}] \ + which disconnected gracefully [{}ms] ago but did not \ + restart, so the disconnection is unexpected; \ + see [{}] for troubleshooting guidance""", + node.descriptionWithoutAttributes(), + millisSinceDisconnect, + ReferenceDocs.NETWORK_DISCONNECT_TROUBLESHOOTING + ); + } + } + } + + @Override + public void onNodeDisconnected(DiscoveryNode node, @Nullable Exception closeException) { + DisconnectionHistory disconnectionHistory = new DisconnectionHistory(threadPool.absoluteTimeInMillis(), closeException); + synchronized (mutex) { + ConnectionTarget connectionTarget = targetsByNode.get(node); + if (connectionTarget != null) { + connectionTarget.disconnectionHistory = disconnectionHistory; + } + } + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java b/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java index 37ed3a7e739fd..1990c70a7f089 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.CheckedRunnable; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.monitor.NodeHealthService; import org.elasticsearch.monitor.StatusInfo; @@ -33,7 +34,6 @@ import org.elasticsearch.transport.AbstractTransportRequest; import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.ReceiveTimeoutTransportException; -import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportConnectionListener; import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequestOptions; @@ -137,7 +137,7 @@ public FollowersChecker( ); transportService.addConnectionListener(new TransportConnectionListener() { @Override - public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { + public void onNodeDisconnected(DiscoveryNode node, @Nullable Exception closeException) { handleDisconnectedNode(node); } }); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java index bbf0884cb4c26..6c39014747077 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java @@ -32,7 +32,6 @@ import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.NodeDisconnectedException; import org.elasticsearch.transport.ReceiveTimeoutTransportException; -import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportConnectionListener; import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequestOptions; @@ -124,7 +123,7 @@ public class LeaderChecker { transportService.addConnectionListener(new TransportConnectionListener() { @Override - public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { + public void onNodeDisconnected(DiscoveryNode node, @Nullable Exception closeException) { handleDisconnectedNode(node); } }); diff --git a/server/src/main/java/org/elasticsearch/transport/ClusterConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ClusterConnectionManager.java index 5d13b10f5a6a4..49658b1c1f8c6 100644 --- a/server/src/main/java/org/elasticsearch/transport/ClusterConnectionManager.java +++ b/server/src/main/java/org/elasticsearch/transport/ClusterConnectionManager.java @@ -229,11 +229,23 @@ private void connectToNodeOrRetry( try { connectionListener.onNodeConnected(node, conn); } finally { - conn.addCloseListener(ActionListener.running(() -> { - connectedNodes.remove(node, conn); - connectionListener.onNodeDisconnected(node, conn); - managerRefs.decRef(); - })); + conn.addCloseListener(new ActionListener() { + @Override + public void onResponse(Void ignored) { + handleClose(null); + } + + @Override + public void onFailure(Exception e) { + handleClose(e); + } + + void handleClose(@Nullable Exception e) { + connectedNodes.remove(node, conn); + connectionListener.onNodeDisconnected(node, e); + managerRefs.decRef(); + } + }); conn.addCloseListener(ActionListener.running(() -> { if (connectingRefCounter.hasReferences() == false) { diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java index daccac3fbe2cb..bc11ca0e8699f 100644 --- a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java +++ b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java @@ -60,9 +60,9 @@ final class DelegatingNodeConnectionListener implements TransportConnectionListe private final CopyOnWriteArrayList listeners = new CopyOnWriteArrayList<>(); @Override - public void onNodeDisconnected(DiscoveryNode key, Transport.Connection connection) { + public void onNodeDisconnected(DiscoveryNode key, @Nullable Exception closeException) { for (TransportConnectionListener listener : listeners) { - listener.onNodeDisconnected(key, connection); + listener.onNodeDisconnected(key, closeException); } } diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionManager.java index 8bc5771485f6d..c27d9cf69a905 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionManager.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionManager.java @@ -61,7 +61,7 @@ public void onNodeConnected(DiscoveryNode node, Transport.Connection connection) } @Override - public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { + public void onNodeDisconnected(DiscoveryNode node, @Nullable Exception closeException) { removeConnectedNode(node); } }); diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java index 248ca4313cefb..a715797b97977 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.threadpool.ThreadPool; @@ -339,7 +340,7 @@ boolean shouldRebuildConnection(Settings newSettings) { protected abstract ConnectionStrategy strategyType(); @Override - public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { + public void onNodeDisconnected(DiscoveryNode node, @Nullable Exception closeException) { if (shouldOpenMoreConnections()) { // try to reconnect and fill up the slot of the disconnected node connect( diff --git a/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java b/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java index 92796f826fc3a..470014eb7a676 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportConnectionListener.java @@ -10,6 +10,7 @@ package org.elasticsearch.transport; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.core.Nullable; /** * A listener interface that allows to react on transport events. All methods may be @@ -38,5 +39,5 @@ default void onNodeConnected(DiscoveryNode node, Transport.Connection connection /** * Called once a node connection is closed and unregistered. */ - default void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) {} + default void onNodeDisconnected(DiscoveryNode node, @Nullable Exception closeException) {} } diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java index 7ab9b8611b8c4..3c5dc6b39292c 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java @@ -52,6 +52,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; @@ -831,7 +832,7 @@ public void testCCSRemoteReduceWithDisconnectedRemoteClusters() throws Exception CountDownLatch disconnectedLatch = new CountDownLatch(numDisconnectedClusters); RemoteClusterServiceTests.addConnectionListener(remoteClusterService, new TransportConnectionListener() { @Override - public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { + public void onNodeDisconnected(DiscoveryNode node, @Nullable Exception closeException) { if (disconnectedNodes.remove(node)) { disconnectedLatch.countDown(); } @@ -1134,7 +1135,7 @@ public void testCollectSearchShards() throws Exception { CountDownLatch disconnectedLatch = new CountDownLatch(numDisconnectedClusters); RemoteClusterServiceTests.addConnectionListener(remoteClusterService, new TransportConnectionListener() { @Override - public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { + public void onNodeDisconnected(DiscoveryNode node, @Nullable Exception closeException) { if (disconnectedNodes.remove(node)) { disconnectedLatch.countDown(); } diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java index 8963bb10d6573..8d4d0548bad19 100644 --- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.SubscribableListener; +import org.elasticsearch.cluster.NodeConnectionsService.ConnectionTarget; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; @@ -30,6 +31,7 @@ import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue; import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.CheckedRunnable; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; @@ -48,6 +50,7 @@ import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.TransportStats; +import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; @@ -74,6 +77,7 @@ public class NodeConnectionsServiceTests extends ESTestCase { private ThreadPool threadPool; private TransportService transportService; private Map> nodeConnectionBlocks; + private Map nodeCloseExceptions; private List generateNodes() { List nodes = new ArrayList<>(); @@ -245,13 +249,111 @@ public String toString() { assertConnectedExactlyToNodes(transportService, targetNodes); } + public void testDisconnectionHistory() { + final Settings.Builder settings = Settings.builder(); + settings.put(CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.getKey(), "100ms"); + + final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(); + final ThreadPool threadPool = deterministicTaskQueue.getThreadPool(); + + MockTransport transport = new MockTransport(deterministicTaskQueue.getThreadPool()); + TestTransportService transportService = new TestTransportService(transport, deterministicTaskQueue.getThreadPool()); + transportService.start(); + transportService.acceptIncomingRequests(); + + final NodeConnectionsService service = new NodeConnectionsService(settings.build(), threadPool, transportService); + service.start(); + + final DiscoveryNode noClose = DiscoveryNodeUtils.create("noClose"); + final DiscoveryNode gracefulClose = DiscoveryNodeUtils.create("gracefulClose"); + final DiscoveryNode exceptionalClose = DiscoveryNodeUtils.create("exceptionalClose"); + + nodeCloseExceptions.put(exceptionalClose, new RuntimeException()); + + final AtomicBoolean connectionCompleted = new AtomicBoolean(); + DiscoveryNodes nodes = DiscoveryNodes.builder().add(noClose).add(gracefulClose).add(exceptionalClose).build(); + + service.connectToNodes(nodes, () -> connectionCompleted.set(true)); + deterministicTaskQueue.runAllRunnableTasks(); + assertTrue(connectionCompleted.get()); + + assertNullDisconnectionHistory(service, noClose); + assertNullDisconnectionHistory(service, gracefulClose); + assertNullDisconnectionHistory(service, exceptionalClose); + + try (var mockLog = MockLog.capture(NodeConnectionsService.class)) { + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "reconnect after graceful close", + NodeConnectionsService.class.getCanonicalName(), + Level.WARN, + "reopened transport connection to node [" + + gracefulClose.descriptionWithoutAttributes() + + "] which disconnected gracefully [*ms] ago " + + "but did not restart, so the disconnection is unexpected; " + + "see [https://www.elastic.co/docs/*] for troubleshooting guidance" + ) + ); + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "reconnect after exceptional close", + NodeConnectionsService.class.getCanonicalName(), + Level.WARN, + "reopened transport connection to node [" + + exceptionalClose.descriptionWithoutAttributes() + + "] which disconnected exceptionally [*ms] ago " + + "but did not restart, so the disconnection is unexpected; " + + "see [https://www.elastic.co/docs/*] for troubleshooting guidance" + ) + ); + transportService.disconnectFromNode(gracefulClose); + transportService.disconnectFromNode(exceptionalClose); + + // check disconnection history set after close + assertNullDisconnectionHistory(service, noClose); + assertDisconnectionHistoryDetails(service, threadPool, gracefulClose, null); + assertDisconnectionHistoryDetails(service, threadPool, exceptionalClose, RuntimeException.class); + + runTasksUntil(deterministicTaskQueue, 200); + + // check on reconnect -- disconnection history is reset + assertNullDisconnectionHistory(service, noClose); + assertNullDisconnectionHistory(service, gracefulClose); + assertNullDisconnectionHistory(service, exceptionalClose); + + mockLog.assertAllExpectationsMatched(); + } + } + + private void assertNullDisconnectionHistory(NodeConnectionsService service, DiscoveryNode node) { + ConnectionTarget nodeTarget = service.connectionTargetForNode(node); + assertNull(nodeTarget.disconnectionHistory); + } + + private void assertDisconnectionHistoryDetails( + NodeConnectionsService service, + ThreadPool threadPool, + DiscoveryNode node, + @Nullable Class disconnectCauseClass + ) { + ConnectionTarget nodeTarget = service.connectionTargetForNode(node); + assertNotNull(nodeTarget.disconnectionHistory); + assertTrue(threadPool.absoluteTimeInMillis() - nodeTarget.disconnectionHistory.getDisconnectTimeMillis() >= 0); + assertTrue(threadPool.absoluteTimeInMillis() - nodeTarget.disconnectionHistory.getDisconnectTimeMillis() <= 200); + if (disconnectCauseClass != null) { + assertThat(nodeTarget.disconnectionHistory.getDisconnectCause(), Matchers.isA(disconnectCauseClass)); + } else { + assertNull(nodeTarget.disconnectionHistory.getDisconnectCause()); + } + } + public void testOnlyBlocksOnConnectionsToNewNodes() throws Exception { final NodeConnectionsService service = new NodeConnectionsService(Settings.EMPTY, threadPool, transportService); final AtomicReference> disconnectListenerRef = new AtomicReference<>(); transportService.addConnectionListener(new TransportConnectionListener() { @Override - public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { + public void onNodeDisconnected(DiscoveryNode node, @Nullable Exception closeException) { final ActionListener disconnectListener = disconnectListenerRef.getAndSet(null); if (disconnectListener != null) { disconnectListener.onResponse(node); @@ -525,6 +627,7 @@ public void setUp() throws Exception { ThreadPool threadPool = new TestThreadPool(getClass().getName()); this.threadPool = threadPool; nodeConnectionBlocks = newConcurrentMap(); + nodeCloseExceptions = newConcurrentMap(); transportService = new TestTransportService(new MockTransport(threadPool), threadPool); transportService.start(); transportService.acceptIncomingRequests(); @@ -643,7 +746,12 @@ public void addCloseListener(ActionListener listener1) { @Override public void close() { - closeListener.onResponse(null); + Exception closeException = nodeCloseExceptions.get(node); + if (closeException != null) { + closeListener.onFailure(closeException); + } else { + closeListener.onResponse(null); + } } @Override diff --git a/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java b/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java index 0a446e999c3c3..1e735de2651a5 100644 --- a/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java +++ b/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.util.concurrent.RunOnce; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.AbstractRefCounted; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; @@ -101,7 +102,7 @@ public void onNodeConnected(DiscoveryNode node, Transport.Connection connection) } @Override - public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { + public void onNodeDisconnected(DiscoveryNode node, @Nullable Exception closeException) { nodeDisconnectedCount.incrementAndGet(); } }); @@ -658,7 +659,7 @@ public void onNodeConnected(DiscoveryNode node, Transport.Connection connection) } @Override - public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { + public void onNodeDisconnected(DiscoveryNode node, @Nullable Exception closeException) { nodeDisconnectedCount.incrementAndGet(); } }); @@ -698,7 +699,7 @@ public void onNodeConnected(DiscoveryNode node, Transport.Connection connection) } @Override - public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { + public void onNodeDisconnected(DiscoveryNode node, @Nullable Exception closeException) { nodeDisconnectedCount.incrementAndGet(); } }); diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index 2fba7b29d0aa3..8a9cd74511e06 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -202,7 +202,7 @@ public void onNodeConnected(DiscoveryNode node, Transport.Connection connection) } @Override - public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { + public void onNodeDisconnected(DiscoveryNode node, @Nullable Exception closeException) { fail("disconnect should not be called " + node); } }; @@ -924,7 +924,7 @@ public void onNodeConnected(DiscoveryNode node, Transport.Connection connection) } @Override - public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { + public void onNodeDisconnected(DiscoveryNode node, @Nullable Exception closeException) { latch.countDown(); } }; @@ -2118,7 +2118,7 @@ public void onNodeConnected(DiscoveryNode node, Transport.Connection connection) } @Override - public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connection) { + public void onNodeDisconnected(DiscoveryNode node, @Nullable Exception closeException) { fail("disconnect should not be called " + node); } };