[NBS-6247]: Extend TDurableClient retry logic with optional whitelist

efimov-ivan-yteam · efimov-ivan-yteam · commit feec364727c5 · 2025-10-09T15:02:57.000Z
diff --git a/cloud/blockstore/config/client.proto b/cloud/blockstore/config/client.proto
@@ -10,6 +10,91 @@ package NCloud.NBlockStore.NProto;
 
 option go_package = "github.com/ydb-platform/nbs/cloud/blockstore/config";
 
+////////////////////////////////////////////////////////////////////////////////
+// Well-known error codes.
+
+enum EWellKnownErrorCode
+{
+    S_OK = 0x00000000;
+    S_FALSE = 0x00000001;
+    S_ALREADY = 0x00000002;
+
+    E_FAIL = -0x80000000; // 0x80000000
+    E_ARGUMENT = -0x7FFFFFFF; // 0x80000001
+    E_REJECTED = -0x7FFFFFFE; // 0x80000002
+
+    reserved                      -0x7FFFFFFD; // 0x80000003
+
+    E_INVALID_STATE = -0x7FFFFFFC; // 0x80000004
+    E_TIMEOUT = -0x7FFFFFFB; // 0x80000005
+    E_NOT_FOUND = -0x7FFFFFFA; // 0x80000006
+    E_UNAUTHORIZED = -0x7FFFFFF9; // 0x80000007
+    E_NOT_IMPLEMENTED = -0x7FFFFFF8; // 0x80000008
+    E_ABORTED = -0x7FFFFFF7; // 0x80000009
+    E_TRY_AGAIN = -0x7FFFFFF6; // 0x8000000A
+    E_IO = -0x7FFFFFF5; // 0x8000000B
+    E_CANCELLED = -0x7FFFFFF4; // 0x8000000C
+    E_IO_SILENT = -0x7FFFFFF3; // 0x8000000D
+    E_RETRY_TIMEOUT = -0x7FFFFFF2; // 0x8000000E
+    E_PRECONDITION_FAILED = -0x7FFFFFF1; // 0x8000000F
+
+    E_GRPC_CANCELLED = -0x7FFDFFFF; // 0x80020001
+    E_GRPC_UNKNOWN = -0x7FFDFFFE; // 0x80020002
+    E_GRPC_INVALID_ARGUMENT = -0x7FFDFFFD; // 0x80020003
+    E_GRPC_DEADLINE_EXCEEDED = -0x7FFDFFFC; // 0x80020004
+    E_GRPC_NOT_FOUND = -0x7FFDFFFB; // 0x80020005
+    E_GRPC_ALREADY_EXISTS = -0x7FFDFFFA; // 0x80020006
+    E_GRPC_PERMISSION_DENIED = -0x7FFDFFF9; // 0x80020007
+    E_GRPC_RESOURCE_EXHAUSTED = -0x7FFDFFF8; // 0x80020008
+    E_GRPC_FAILED_PRECONDITION = -0x7FFDFFF7; // 0x80020009
+    E_GRPC_ABORTED = -0x7FFDFFF6; // 0x8002000A
+    E_GRPC_OUT_OF_RANGE = -0x7FFDFFF5; // 0x8002000B
+    E_GRPC_UNIMPLEMENTED = -0x7FFDFFF4; // 0x8002000C
+    E_GRPC_INTERNAL = -0x7FFDFFF3; // 0x8002000D
+    E_GRPC_UNAVAILABLE = -0x7FFDFFF2; // 0x8002000E
+    E_GRPC_DATA_LOSS = -0x7FFDFFF1; // 0x8002000F
+    E_GRPC_UNAUTHENTICATED = -0x7FFDFFF0; // 0x80020010
+
+    E_BS_INVALID_SESSION = -0x7FFAFFFF; // 0x80050001
+    E_BS_OUT_OF_SPACE = -0x7FFAFFFE; // 0x80050002
+    E_BS_THROTTLED = -0x7FFAFFFD; // 0x80050003
+    E_BS_RESOURCE_EXHAUSTED = -0x7FFAFFFC; // 0x80050004
+    E_BS_DISK_ALLOCATION_FAILED = -0x7FFAFFFB; // 0x80050005
+    E_BS_MOUNT_CONFLICT = -0x7FFAFFFA; // 0x80050006
+
+    E_FS_IO = -0x7FF90000; // 0x80070000
+    E_FS_PERM = -0x7FF8FFFF; // 0x80070001
+    E_FS_NOENT = -0x7FF8FFFE; // 0x80070002
+    E_FS_NXIO = -0x7FF8FFFD; // 0x80070003
+    E_FS_ACCESS = -0x7FF8FFFC; // 0x80070004
+    E_FS_EXIST = -0x7FF8FFFB; // 0x80070005
+    E_FS_XDEV = -0x7FF8FFFA; // 0x80070006
+    E_FS_NODEV = -0x7FF8FFF9; // 0x80070007
+    E_FS_NOTDIR = -0x7FF8FFF8; // 0x80070008
+    E_FS_ISDIR = -0x7FF8FFF7; // 0x80070009
+    E_FS_INVAL = -0x7FF8FFF6; // 0x8007000A
+    E_FS_FBIG = -0x7FF8FFF5; // 0x8007000B
+    E_FS_NOSPC = -0x7FF8FFF4; // 0x8007000C
+    E_FS_ROFS = -0x7FF8FFF3; // 0x8007000D
+    E_FS_MLINK = -0x7FF8FFF2; // 0x8007000E
+    E_FS_NAMETOOLONG = -0x7FF8FFF1; // 0x8007000F
+    E_FS_NOTEMPTY = -0x7FF8FFF0; // 0x80070010
+    E_FS_DQUOT = -0x7FF8FFEF; // 0x80070011
+    E_FS_STALE = -0x7FF8FFEE; // 0x80070012
+    E_FS_REMOTE = -0x7FF8FFED; // 0x80070013
+    E_FS_BADHANDLE = -0x7FF8FFEC; // 0x80070014
+    E_FS_NOTSUPP = -0x7FF8FFEB; // 0x80070015
+
+    E_FS_WOULDBLOCK = -0x7FF8FFE2; // 0x8007001E
+    E_FS_NOLCK = -0x7FF8FFE1; // 0x8007001F
+
+    E_FS_INVALID_SESSION = -0x7FF8FF9C; // 0x80070064
+    E_FS_OUT_OF_SPACE = -0x7FF8FF9B; // 0x80070065
+    E_FS_THROTTLED = -0x7FF8FF9A; // 0x80070066
+
+    E_RDMA_UNAVAILABLE = -0x7FF7FFFF; // 0x80080001
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 message TClientMediaKindThrottlingConfig
@@ -191,6 +276,18 @@ message TClientConfig
 
     // First retry timeout for YDB based disks (in milliseconds).
     optional uint32 YDBBasedDiskInitialRetryTimeout = 44;
+
+    // Optional whitelist for request errors to be passed up without retry in DurableClient.
+    // If empty, whitelist mechanism is disabled and default
+    // "EErrorKind::ErrorRetriable"-based predicate is used.
+    // Applied for disks with IsReliableMediaKind == true.
+    repeated EWellKnownErrorCode NonRetriableErrorsForReliableMedia = 45;
+
+    // Optional whitelist for request errors to be passed up without retry in DurableClient.
+    // If empty, whitelist mechanism is disabled and default
+    // "EErrorKind::ErrorRetriable"-based predicate is used.
+    // Applied for disks with IsReliableMediaKind == false.
+    repeated EWellKnownErrorCode NonRetriableErrorsForUnreliableMedia = 46;
 }
 
 ////////////////////////////////////////////////////////////////////////////////
diff --git a/cloud/blockstore/libs/client/config.cpp b/cloud/blockstore/libs/client/config.cpp
@@ -43,8 +43,10 @@ TDuration MSeconds(ui64 x)
     xxx(ConnectionErrorMaxRetryTimeout, TDuration,      MSeconds(100)         )\
     xxx(GrpcReconnectBackoff,           TDuration,      MSeconds(100)         )\
     xxx(DiskRegistryBasedDiskInitialRetryTimeout,  TDuration,  MSeconds(500)  )\
-    xxx(YDBBasedDiskInitialRetryTimeout,           TDuration,  MSeconds(500)  )\
-                                                                               \
+    xxx(YDBBasedDiskInitialRetryTimeout,           TDuration,  MSeconds(500)  )          \
+    xxx(NonRetriableErrorsForReliableMedia, TVector<ui32>, {})              \
+    xxx(NonRetriableErrorsForUnreliableMedia, TVector<ui32>, {})            \
+                                                                            \
     xxx(MemoryQuotaBytes,       ui32,             0                           )\
     xxx(SecurePort,             ui32,             0                           )\
     xxx(RootCertsFile,          TString,          {}                          )\
@@ -89,7 +91,7 @@ TTarget ConvertValue(TSource value)
 }
 
 template <>
-TDuration ConvertValue<TDuration, ui32>(ui32 value)
+TDuration ConvertValue<TDuration, const ui32&>(const ui32& value)
 {
     return TDuration::MilliSeconds(value);
 }
@@ -102,9 +104,27 @@ ConvertValue<TRequestThresholds, TProtoRequestThresholds>(
     return ConvertRequestThresholds(value);
 }
 
+template <>
+TVector<ui32>
+ConvertValue<TVector<ui32>, const google::protobuf::RepeatedField<int>&>(
+    const google::protobuf::RepeatedField<int>& value)
+{
+    TVector<ui32> v;
+    for (const auto& x: value) {
+        v.push_back(static_cast<ui32>(x));
+    }
+    return v;
+}
+
+template <typename T>
+bool IsEmpty(const google::protobuf::RepeatedField<T>& value)
+{
+    return value.empty();
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
-IOutputStream& operator <<(IOutputStream& out, NProto::EClientIpcType ipcType)
+IOutputStream& operator <<(IOutputStream& out, const NProto::EClientIpcType& ipcType)
 {
     switch (ipcType) {
         case NProto::IPC_GRPC:
@@ -124,6 +144,22 @@ IOutputStream& operator <<(IOutputStream& out, NProto::EClientIpcType ipcType)
     }
 }
 
+template <typename T>
+void DumpImpl(const T& t, IOutputStream& os)
+{
+    os << t;
+}
+
+void DumpImpl(const TVector<ui32>& value, IOutputStream& os)
+{
+    for (size_t i = 0; i < value.size(); ++i) {
+        if (i) {
+            os << ",";
+        }
+        os << value[i];
+    }
+}
+
 }   // namespace
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -136,12 +172,42 @@ TClientAppConfig::TClientAppConfig(NProto::TClientAppConfig appConfig)
     , IamConfig(AppConfig.GetIamConfig())
 {}
 
-#define BLOCKSTORE_CONFIG_GETTER(name, type, ...)                              \
-type TClientAppConfig::Get##name() const                                       \
-{                                                                              \
-    auto has = ClientConfig.Has##name();                                       \
-    return has ? ConvertValue<type>(ClientConfig.Get##name()) : Default##name; \
-}                                                                              \
+#define DECLARE_FIELD_CHECKER(name, type, ...)                               \
+    template <typename TProtoConfig, typename = void>                        \
+    struct Has##name##Method: std::false_type                                \
+    {                                                                        \
+    };                                                                       \
+                                                                             \
+    template <typename TProtoConfig>                                         \
+    struct Has##name##Method<                                                \
+        TProtoConfig,                                                        \
+        std::void_t<decltype(std::declval<TProtoConfig>().Has##name())>>     \
+        : std::true_type                                                     \
+    {                                                                        \
+    };                                                                       \
+                                                                             \
+    template <typename TProtoConfig, typename TProtoValue>                   \
+    bool IsEmpty##name(const TProtoConfig& config, const TProtoValue& value) \
+    {                                                                        \
+        if constexpr (Has##name##Method<TProtoConfig>::value) {              \
+            return !config.Has##name();                                      \
+        } else {                                                             \
+            return IsEmpty(value);                                           \
+        }                                                                    \
+    }
+
+BLOCKSTORE_CLIENT_CONFIG(DECLARE_FIELD_CHECKER)
+
+#undef DECLARE_FIELD_CHECKER
+
+#define BLOCKSTORE_CONFIG_GETTER(name, type, ...)                \
+    type TClientAppConfig::Get##name() const                     \
+    {                                                            \
+        const auto& value = ClientConfig.Get##name();            \
+        return IsEmpty##name(ClientConfig, value)                \
+                   ? Default##name                               \
+                   : ConvertValue<type, decltype(value)>(value); \
+    }
 // BLOCKSTORE_CONFIG_GETTER
 
 BLOCKSTORE_CLIENT_CONFIG(BLOCKSTORE_CONFIG_GETTER)
@@ -154,9 +220,11 @@ TRequestThresholds TClientAppConfig::GetRequestThresholds() const {
 
 void TClientAppConfig::Dump(IOutputStream& out) const
 {
-#define BLOCKSTORE_CONFIG_DUMP(name, ...)                                      \
-    out << #name << ": " << Get##name() << Endl;                               \
-// BLOCKSTORE_CONFIG_DUMP
+#define BLOCKSTORE_CONFIG_DUMP(name, ...) \
+    out << #name << ": ";                 \
+    DumpImpl(Get##name(), out);           \
+    out << Endl;                          \
+    // BLOCKSTORE_CONFIG_DUMP
 
     BLOCKSTORE_CLIENT_CONFIG(BLOCKSTORE_CONFIG_DUMP);
 
@@ -168,7 +236,7 @@ void TClientAppConfig::DumpHtml(IOutputStream& out) const
 #define BLOCKSTORE_CONFIG_DUMP(name, ...)                                      \
     TABLER() {                                                                 \
         TABLED() { out << #name; }                                             \
-        TABLED() { out << Get##name(); }                                       \
+        TABLED() { DumpImpl(Get##name(), out); }                               \
     }                                                                          \
 // BLOCKSTORE_CONFIG_DUMP
 
diff --git a/cloud/blockstore/libs/client/config.h b/cloud/blockstore/libs/client/config.h
@@ -3,14 +3,14 @@
 #include "public.h"
 
 #include <cloud/blockstore/config/client.pb.h>
-
 #include <cloud/blockstore/libs/diagnostics/dumpable.h>
 
 #include <cloud/storage/core/libs/diagnostics/trace_reader.h>
 #include <cloud/storage/core/libs/iam/iface/config.h>
 
 #include <util/datetime/base.h>
 #include <util/generic/string.h>
+#include <util/generic/vector.h>
 #include <util/stream/output.h>
 
 namespace NCloud::NBlockStore::NClient {
@@ -60,6 +60,8 @@ class TClientAppConfig
     TDuration GetGrpcReconnectBackoff() const;
     TDuration GetDiskRegistryBasedDiskInitialRetryTimeout() const;
     TDuration GetYDBBasedDiskInitialRetryTimeout() const;
+    TVector<ui32> GetNonRetriableErrorsForReliableMedia() const;
+    TVector<ui32> GetNonRetriableErrorsForUnreliableMedia() const;
     ui32 GetMemoryQuotaBytes() const;
     ui32 GetSecurePort() const;
     bool GetSkipCertVerification() const;
diff --git a/cloud/blockstore/libs/client/durable.cpp b/cloud/blockstore/libs/client/durable.cpp
@@ -478,20 +478,42 @@ class TRetryPolicy final: public IRetryPolicy
 private:
     TClientAppConfigPtr Config;
     TDuration InitialRetryTimeout;
+    bool MediaIsReliable;
+    TVector<ui32> NeverRetriableWithErrorListPolicy;
 
 public:
-    TRetryPolicy(TClientAppConfigPtr config, TDuration initialRetryTimeout)
+    TRetryPolicy(
+        TClientAppConfigPtr config,
+        TDuration initialRetryTimeout,
+        bool mediaIsReliable)
         : Config(std::move(config))
         , InitialRetryTimeout(initialRetryTimeout)
+        , MediaIsReliable(mediaIsReliable)
+        , NeverRetriableWithErrorListPolicy({E_BS_INVALID_SESSION})
     {}
 
     TRetrySpec ShouldRetry(
         TRetryState& state,
         const NProto::TError& error) override
     {
         TRetrySpec spec;
-        spec.IsRetriableError =
-            GetErrorKind(error) == EErrorKind::ErrorRetriable;
+
+        auto nonRetriableErrorsList =
+            MediaIsReliable ? Config->GetNonRetriableErrorsForReliableMedia()
+                            : Config->GetNonRetriableErrorsForUnreliableMedia();
+
+        if (nonRetriableErrorsList.empty()) {
+            spec.IsRetriableError =
+                GetErrorKind(error) == EErrorKind::ErrorRetriable;
+        } else {
+            spec.IsRetriableError =
+                std::ranges::find(
+                    NeverRetriableWithErrorListPolicy,
+                    error.GetCode()) ==
+                    NeverRetriableWithErrorListPolicy.end() &&
+                std::ranges::find(nonRetriableErrorsList, error.GetCode()) ==
+                    nonRetriableErrorsList.end();
+        }
 
         if (!spec.IsRetriableError ||
             TInstant::Now() - state.Started >= Config->GetRetryTimeout())
@@ -546,7 +568,8 @@ IRetryPolicyPtr CreateRetryPolicy(
 
     return std::make_shared<TRetryPolicy>(
         std::move(config),
-        initialRetryTimeout);
+        initialRetryTimeout,
+        IsReliableMediaKind(mediaKind.value_or(NProto::STORAGE_MEDIA_DEFAULT)));
 }
 
 IBlockStorePtr CreateDurableClient(
diff --git a/cloud/blockstore/libs/client/durable_ut.cpp b/cloud/blockstore/libs/client/durable_ut.cpp
