Release node name on graceful shutdown (Issue #9734)

ydb/core/driver_lib/run/run.cpp

@@ -65,6 +65,7 @@
 
 #include <ydb/core/mind/local.h>
 #include <ydb/core/mind/tenant_pool.h>
+#include <ydb/core/mind/node_broker.h>
 #include <ydb/core/base/hive.h>
 
 #include <ydb/core/base/tablet_resolver.h>
@@ -1772,6 +1773,27 @@ void TKikimrRunner::KikimrStart() {
 void TKikimrRunner::KikimrStop(bool graceful) {
     Y_UNUSED(graceful);
 
+    bool enableReleaseNodeNameOnGracefulShutdown = AppData->FeatureFlags.GetEnableReleaseNodeNameOnGracefulShutdown();
+
+    if (enableReleaseNodeNameOnGracefulShutdown) {
+        using namespace NKikimr::NNodeBroker;
+        using TEvent = TEvNodeBroker::TEvGracefulShutdownRequest;
+
+        const ui32 nodeId = ActorSystem->NodeId;
+        bool isDynamicNode = AppData->DynamicNameserviceConfig->MinDynamicNodeId <= nodeId;
+
+        if (isDynamicNode) {
+            NTabletPipe::TClientConfig pipeConfig;
+            pipeConfig.RetryPolicy = {.RetryLimitCount = 10};
+            auto pipe = NTabletPipe::CreateClient({}, MakeNodeBrokerID(), pipeConfig);
+            TActorId nodeBrokerPipe = ActorSystem->Register(pipe);
+            THolder<TEvent> event = MakeHolder<TEvent>();
+            event->Record.SetNodeId(nodeId);
+
+            NTabletPipe::SendData({}, nodeBrokerPipe, event.Release());
+        }
+    }
+
     if (EnabledGrpcService) {
         ActorSystem->Send(new IEventHandle(NGRpcService::CreateGrpcPublisherServiceActorId(), {}, new TEvents::TEvPoisonPill));
     }

ydb/core/mind/node_broker.cpp

@@ -846,6 +846,18 @@ void TNodeBroker::DbUpdateNodeLocation(const TNodeInfo &node,
     db.Table<T>().Key(node.NodeId).Update<T::Location>(node.Location.GetSerializedLocation());
 }
 
+void TNodeBroker::DbReleaseSlotIndex(const TNodeInfo &node,
+                                       TTransactionContext &txc) 
+{
+
+    LOG_DEBUG_S(TActorContext::AsActorContext(), NKikimrServices::NODE_BROKER,
+                "Release slot index (" << node.SlotIndex << ") node " << node.IdString() );
+    NIceDb::TNiceDb db(txc.DB);
+    using T = Schema::Nodes;
+    db.Table<T>().Key(node.NodeId)
+        .UpdateToNull<T::SlotIndex>();
+}
+
 void TNodeBroker::DbUpdateNodeAuthorizedByCertificate(const TNodeInfo &node,
                                        TTransactionContext &txc)
 {
@@ -1016,6 +1028,14 @@ void TNodeBroker::Handle(TEvNodeBroker::TEvRegistrationRequest::TPtr &ev,
     ctx.RegisterWithSameMailbox(new TResolveTenantActor(ev, SelfId()));
 }
 
+void TNodeBroker::Handle(TEvNodeBroker::TEvGracefulShutdownRequest::TPtr &ev,
+                         const TActorContext &ctx) {
+    LOG_TRACE_S(ctx, NKikimrServices::NODE_BROKER, "Handle TEvNodeBroker::TEvGracefulShutdownRequest"
+        << ": request# " << ev->Get()->Record.ShortDebugString());   
+    TabletCounters->Cumulative()[COUNTER_GRACEFUL_SHUTDOWN_REQUESTS].Increment(1);
+    ProcessTx(CreateTxGracefulShutdown(ev), ctx);                     
+}
+
 void TNodeBroker::Handle(TEvNodeBroker::TEvExtendLeaseRequest::TPtr &ev,
                          const TActorContext &ctx)
 {

ydb/core/mind/node_broker.h

@@ -95,6 +95,10 @@ struct TEvNodeBroker {
         EvSetConfigRequest,
         EvSetConfigResponse,
 
+        // decommission
+        EvGracefulShutdownRequest,
+        EvGracefulShutdownResponse,
+
         // TODO: remove
         // internal
         //EvNodeExpire = EvListNodes + 512,
@@ -125,6 +129,11 @@ struct TEvNodeBroker {
                                                     EvRegistrationRequest> {
     };
 
+    struct TEvGracefulShutdownRequest : public TEventPB<TEvGracefulShutdownRequest,
+                                                    NKikimrNodeBroker::TGracefulShutdownRequest,
+                                                    EvGracefulShutdownRequest> {
+    };
+
     struct TEvExtendLeaseRequest : public TEventPB<TEvExtendLeaseRequest,
                                                    NKikimrNodeBroker::TExtendLeaseRequest,
                                                    EvExtendLeaseRequest> {
@@ -156,6 +165,11 @@ struct TEvNodeBroker {
                                                      EvRegistrationResponse> {
     };
 
+    struct TEvGracefulShutdownResponse : public TEventPB<TEvGracefulShutdownResponse,
+                                                     NKikimrNodeBroker::TGracefulShutdownResponse,
+                                                     EvGracefulShutdownResponse> {
+    };
+
     struct TEvExtendLeaseResponse : public TEventPB<TEvExtendLeaseResponse,
                                                     NKikimrNodeBroker::TExtendLeaseResponse,
                                                     EvExtendLeaseResponse> {

ydb/core/mind/node_broker__graceful_shutdown.cpp

@@ -0,0 +1,64 @@
+#include "node_broker_impl.h"
+
+#include <ydb/core/protos/counters_node_broker.pb.h>
+
+namespace NKikimr::NNodeBroker {
+
+using namespace NKikimrNodeBroker;
+
+class TNodeBroker::TTxGracefulShutdown : public TTransactionBase<TNodeBroker> {
+public:
+    TTxGracefulShutdown(TNodeBroker *self, TEvNodeBroker::TEvGracefulShutdownRequest::TPtr &ev)
+        : TBase(self)
+        , Event(ev)
+    {
+    }
+
+    TTxType GetTxType() const override { return TXTYPE_GRACESFUL_SHUTDOWN; }
+
+    bool Execute(TTransactionContext &txc, const TActorContext &ctx) override
+    {
+        const auto& rec = Event->Get()->Record;
+        const auto nodeId = rec.GetNodeId();
+
+        LOG_DEBUG_S(ctx, NKikimrServices::NODE_BROKER,
+                    "TTxGracefulShutdown Execute. Graceful Shutdown request from " << nodeId << " ");
+
+        Response = MakeHolder<TEvNodeBroker::TEvGracefulShutdownResponse>();
+        const auto it = Self->Nodes.find(nodeId);
+
+        if (it != Self->Nodes.end()) {
+            auto& node = it->second;
+            Self->SlotIndexesPools[node.ServicedSubDomain].Release(node.SlotIndex.value());
+            Self->DbReleaseSlotIndex(node, txc);
+            node.SlotIndex.reset();
+
+            Response->Record.MutableStatus()->SetCode(TStatus::OK);
+
+            return true;
+        }
+
+        Response->Record.MutableStatus()->SetCode(TStatus::ERROR);
+        Response->Record.MutableStatus()->SetReason(TStringBuilder() << "Cannot find node " << nodeId);
+
+        return true;
+    }
+
+    void Complete(const TActorContext &ctx) override
+    {
+        LOG_DEBUG(ctx, NKikimrServices::NODE_BROKER, "TTxGracefulShutdown Complete");
+        ctx.Send(Event->Sender, Response.Release());
+        Self->TxCompleted(this, ctx);
+    }
+
+private:
+    TEvNodeBroker::TEvGracefulShutdownRequest::TPtr Event;
+    THolder<TEvNodeBroker::TEvGracefulShutdownResponse> Response;
+};
+
+ITransaction *TNodeBroker::CreateTxGracefulShutdown(TEvNodeBroker::TEvGracefulShutdownRequest::TPtr &ev)
+{
+    return new TTxGracefulShutdown(this, ev);
+}
+
+} // NKikimr::NNodeBroker

ydb/core/mind/node_broker_impl.h

@@ -148,6 +148,7 @@ class TNodeBroker : public TActor<TNodeBroker>
     class TTxInitScheme;
     class TTxLoadState;
     class TTxRegisterNode;
+    class TTxGracefulShutdown;
     class TTxUpdateConfig;
     class TTxUpdateConfigSubscription;
     class TTxUpdateEpoch;
@@ -156,6 +157,7 @@ class TNodeBroker : public TActor<TNodeBroker>
     ITransaction *CreateTxInitScheme();
     ITransaction *CreateTxLoadState();
     ITransaction *CreateTxRegisterNode(TEvPrivate::TEvResolvedRegistrationRequest::TPtr &ev);
+    ITransaction *CreateTxGracefulShutdown(TEvNodeBroker::TEvGracefulShutdownRequest::TPtr &ev);
     ITransaction *CreateTxUpdateConfig(TEvConsole::TEvConfigNotificationRequest::TPtr &ev);
     ITransaction *CreateTxUpdateConfig(TEvNodeBroker::TEvSetConfigRequest::TPtr &ev);
     ITransaction *CreateTxUpdateConfigSubscription(TEvConsole::TEvReplaceConfigSubscriptionsResponse::TPtr &ev);
@@ -202,6 +204,7 @@ class TNodeBroker : public TActor<TNodeBroker>
             HFuncTraced(TEvNodeBroker::TEvListNodes, Handle);
             HFuncTraced(TEvNodeBroker::TEvResolveNode, Handle);
             HFuncTraced(TEvNodeBroker::TEvRegistrationRequest, Handle);
+            HFuncTraced(TEvNodeBroker::TEvGracefulShutdownRequest, Handle);
             HFuncTraced(TEvNodeBroker::TEvExtendLeaseRequest, Handle);
             HFuncTraced(TEvNodeBroker::TEvCompactTables, Handle);
             HFuncTraced(TEvNodeBroker::TEvGetConfigRequest, Handle);
@@ -288,9 +291,10 @@ class TNodeBroker : public TActor<TNodeBroker>
                            TTransactionContext &txc);
     void DbUpdateNodeLocation(const TNodeInfo &node,
                               TTransactionContext &txc);
+    void DbReleaseSlotIndex(const TNodeInfo &node,
+                            TTransactionContext &txc);
     void DbUpdateNodeAuthorizedByCertificate(const TNodeInfo &node,
                               TTransactionContext &txc);
-
     void Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &ev,
                 const TActorContext &ctx);
     void Handle(TEvConsole::TEvReplaceConfigSubscriptionsResponse::TPtr &ev,
@@ -301,6 +305,8 @@ class TNodeBroker : public TActor<TNodeBroker>
                 const TActorContext &ctx);
     void Handle(TEvNodeBroker::TEvRegistrationRequest::TPtr &ev,
                 const TActorContext &ctx);
+    void Handle(TEvNodeBroker::TEvGracefulShutdownRequest::TPtr &ev,
+                const TActorContext &ctx);
     void Handle(TEvNodeBroker::TEvExtendLeaseRequest::TPtr &ev,
                 const TActorContext &ctx);
     void Handle(TEvNodeBroker::TEvCompactTables::TPtr &ev,

ydb/core/mind/node_broker_ut.cpp

@@ -370,6 +370,26 @@ void CheckRegistration(TTestActorRuntime &runtime,
                       false, path, Nothing(), name);
 }
 
+THolder<TEvNodeBroker::TEvGracefulShutdownRequest>
+MakeEventGracefulShutdown (ui32 nodeId) 
+{
+    auto eventGracefulShutdown = MakeHolder<TEvNodeBroker::TEvGracefulShutdownRequest>();
+    eventGracefulShutdown->Record.SetNodeId(nodeId);
+    return eventGracefulShutdown;
+}
+
+void CheckGracefulShutdown(TTestActorRuntime &runtime,
+                           TActorId sender,
+                           ui32 nodeId)
+{   
+    auto eventGracefulShutdown = MakeEventGracefulShutdown(nodeId);
+    TAutoPtr<IEventHandle> handle;
+    runtime.SendToPipe(MakeNodeBrokerID(), sender, eventGracefulShutdown.Release(), 0, GetPipeConfigWithRetries());
+    auto replyGracefulShutdown = runtime.GrabEdgeEventRethrow<TEvNodeBroker::TEvGracefulShutdownResponse>(handle);
+
+    UNIT_ASSERT_VALUES_EQUAL(replyGracefulShutdown->Record.GetStatus().GetCode(), TStatus::OK);
+}
+
 NKikimrNodeBroker::TEpoch GetEpoch(TTestActorRuntime &runtime,
                                    TActorId sender)
 {
@@ -1795,4 +1815,24 @@ Y_UNIT_TEST_SUITE(TSlotIndexesPoolTest) {
     }
 }
 
+Y_UNIT_TEST_SUITE(GracefulShutdown) {
+    Y_UNIT_TEST(TTxGracefulShutdown) {
+        TTestBasicRuntime runtime(8, false);
+        Setup(runtime, 4);
+        TActorId sender = runtime.AllocateEdgeActor();
+
+        auto epoch = GetEpoch(runtime, sender);
+
+        CheckRegistration(runtime, sender, "host1", 1001, "host1.yandex.net", "1.2.3.4",
+                          1, 2, 3, 4, TStatus::OK, NODE1, epoch.GetNextEnd(),
+                          false, DOMAIN_NAME, {}, "slot-0");
+
+        CheckGracefulShutdown(runtime, sender, NODE1);    
+
+        CheckRegistration(runtime, sender, "host2", 1001, "host2.yandex.net", "1.2.3.5",
+                          1, 2, 3, 5, TStatus::OK, NODE2, epoch.GetNextEnd(),
+                          false, DOMAIN_NAME, {}, "slot-0");
+    }
+}
+
 } // NKikimr

ydb/core/mind/ya.make

@@ -14,6 +14,7 @@ SRCS(
     lease_holder.h
     local.cpp
     local.h
+    node_broker__graceful_shutdown.cpp
     node_broker.cpp
     node_broker.h
     node_broker_impl.h

ydb/core/protos/counters_node_broker.proto

@@ -16,6 +16,7 @@ enum ECumulativeCounters {
     COUNTER_RESOLVE_NODE_REQUESTS = 1      [(CounterOpts) = {Name: "ResolveNodeRequests"}];
     COUNTER_REGISTRATION_REQUESTS = 2      [(CounterOpts) = {Name: "RegistrationRequests"}];
     COUNTER_EXTEND_LEASE_REQUESTS = 3      [(CounterOpts) = {Name: "ExtendLeaseRequests"}];
+    COUNTER_GRACEFUL_SHUTDOWN_REQUESTS = 4 [(CounterOpts) = {Name: "GracefulShutdownRequests"}];
 }
 
 enum EPercentileCounters {
@@ -52,4 +53,5 @@ enum ETxTypes {
     TXTYPE_UPDATE_CONFIG = 4                [(TxTypeOpts) = {Name: "TTxUpdateConfig"}];
     TXTYPE_UPDATE_CONFIG_SUBSCRIPTION = 5   [(TxTypeOpts) = {Name: "TTxUpdateConfigSubscription"}];
     TXTYPE_UPDATE_EPOCH = 6                 [(TxTypeOpts) = {Name: "TTxUpdateEpoch"}];
+    TXTYPE_GRACESFUL_SHUTDOWN = 7           [(TxTypeOpts) = {Name: "TTxGracefulShutdown"}];
 }

ydb/core/protos/feature_flags.proto

@@ -177,4 +177,5 @@ message TFeatureFlags {
     optional bool EnableDriveSerialsDiscovery = 152 [default = false];
     optional bool EnableSeparateDiskSpaceQuotas = 153 [default = false];
     optional bool EnableAntlr4Parser = 154 [default = false];
+    optional bool EnableReleaseNodeNameOnGracefulShutdown = 155 [default = false];
 }

ydb/core/protos/node_broker.proto

@@ -79,6 +79,14 @@ message TRegistrationResponse {
   optional uint64 ScopePathId = 4;
 }
 
+message TGracefulShutdownRequest {
+  optional uint32 NodeId = 1;
+}
+
+message TGracefulShutdownResponse {
+  optional TStatus Status = 1;
+}
+
 message TExtendLeaseRequest {
   optional uint32 NodeId = 1;
 }