Chaos for DiskAgent (#4511)

The disk agent has learned to respond to some requests with errors and
at the same time spoil the writted data.

Author: drbasic

cloud/blockstore/config/disk.proto

@@ -31,6 +31,15 @@ enum EDiskAgentBackendType
     DISK_AGENT_BACKEND_IO_URING_NULL = 4;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+// The policy of critical events reporting when chaos generates an error.
+
+enum EChaosCritEventReportingPolicy
+{
+    CCERP_FIRST_ERROR = 0;
+    CCERP_ALL_ERRORS = 1;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 message TMemoryDeviceArgs
@@ -168,6 +177,26 @@ message TDiskAgentThrottlingConfig
 
 ////////////////////////////////////////////////////////////////////////////////
 
+message TChaosConfig
+{
+    // The policy of critical events reporting when chaos generates an error.
+    optional EChaosCritEventReportingPolicy CritEventReportingPolicy = 1;
+
+    // The probability of getting an error on the request start and finish.
+    optional double ChaosProbability = 2;
+
+    // Error codes that the chaos will generate. When empty E_REJECTED is used.
+    repeated uint32 ErrorCodes = 3;
+
+    // The probability of data corruption when responding with an error.
+    optional double DataDamageProbability = 4;
+
+    // The probability of a response immediately upon receipt of the request.
+    optional double ImmediateReplyProbability = 5;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
 message TDiskAgentConfig
 {
     optional bool Enabled = 1;
@@ -319,6 +348,9 @@ message TDiskAgentConfig
     // Enables data integrity validation for the DR-based disks.
     // See "EnableDataIntegrityClient" in server.proto for more details.
     optional bool EnableDataIntegrityValidationForDrBasedDisks = 44;
+
+    // DANGER! It is used to generate errors during chaos testing.
+    optional TChaosConfig ChaosConfig = 45;
 }
 
 ////////////////////////////////////////////////////////////////////////////////

cloud/blockstore/libs/diagnostics/critical_events.h

@@ -87,6 +87,7 @@ using TCritEventParams = TVector<std::pair<TStringBuf, TValue>>;
     xxx(DiskAgentSessionCacheRestoreError)                                     \
     xxx(DiskAgentSessionCacheUpdateError)                                      \
     xxx(UnexpectedIdentifierRepetition)                                        \
+    xxx(ChaosGeneratedError)                                                   \
 // BLOCKSTORE_DISK_AGENT_CRITICAL_EVENTS
 
 #define BLOCKSTORE_IMPOSSIBLE_EVENTS(xxx)                                      \

cloud/blockstore/libs/service/request.cpp

@@ -26,6 +26,21 @@ const TString& GetBlockStoreRequestName(EBlockStoreRequest request)
     return Unknown;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+
+#define BLOCKSTORE_DECLARE_METHOD(name, ...)                     \
+    template <>                                                  \
+    TString GetBlockStoreRequestName<NProto::T##name##Request>() \
+    {                                                            \
+        return #name;                                            \
+    }
+
+BLOCKSTORE_SERVICE(BLOCKSTORE_DECLARE_METHOD)
+
+#undef BLOCKSTORE_DECLARE_METHOD
+
+////////////////////////////////////////////////////////////////////////////////
+
 TString GetSysRequestName(ESysRequestType requestType)
 {
     return ToString(requestType);

cloud/blockstore/libs/service/request.h

@@ -174,6 +174,9 @@ constexpr size_t BlockStoreRequestsCount = (size_t)EBlockStoreRequest::MAX;
 
 const TString& GetBlockStoreRequestName(EBlockStoreRequest requestType);
 
+template<typename TRequest>
+TString GetBlockStoreRequestName();
+
 enum class ESysRequestType
 {
     Compaction = 10000,

cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_io.cpp

@@ -177,7 +177,14 @@ void TDiskAgentActor::PerformIO(
     auto* actorSystem = ctx.ActorSystem();
     auto replyFrom = ctx.SelfID;
 
-    auto replySuccess = [=] (auto result) {
+    auto reply = [actorSystem,
+                  replyFrom,
+                  requestInfo,
+                  range,
+                  volumeRequestId,
+                  deviceUUID,
+                  started](auto result)
+    {
         Reply<TMethod>(
             *actorSystem,
             replyFrom,
@@ -189,18 +196,6 @@ void TDiskAgentActor::PerformIO(
             started);
     };
 
-    auto replyError = [=] (auto code, auto message) {
-        Reply<TMethod>(
-            *actorSystem,
-            replyFrom,
-            *requestInfo,
-            MakeError(code, std::move(message)),
-            range,
-            volumeRequestId,
-            deviceUUID,
-            started);
-    };
-
     LOG_TRACE(ctx, TBlockStoreComponents::DISK_AGENT,
         "%s [%s / %s]",
         TMethod::Name,
@@ -221,7 +216,7 @@ void TDiskAgentActor::PerformIO(
                  {"isWrite", IsWriteDeviceMethod<TMethod>},
                  {"isRdma", 0}});
         }
-        replyError(E_REJECTED, "Secure erase in progress");
+        reply(MakeError(E_REJECTED, "Secure erase in progress"));
         return;
     }
 
@@ -232,9 +227,10 @@ void TDiskAgentActor::PerformIO(
             std::invoke(operation, *State, ctx.Now(), std::move(record));
 
         result.Subscribe(
-            [=] (auto future) {
+            [reply, deviceUUID, clientId, actorSystem](auto future)
+            {
                 try {
-                    replySuccess(future.ExtractValue());
+                    reply(future.ExtractValue());
                 } catch (...) {
                     auto [code, message] = HandleException(
                         *actorSystem,
@@ -243,7 +239,7 @@ void TDiskAgentActor::PerformIO(
                         deviceUUID,
                         clientId);
 
-                    replyError(code, message);
+                    reply(MakeError(code, message));
                 }
             });
     } catch (...) {
@@ -254,7 +250,7 @@ void TDiskAgentActor::PerformIO(
             deviceUUID,
             clientId);
 
-        replyError(code, message);
+        reply(MakeError(code, message));
     }
 }
 

cloud/blockstore/libs/storage/disk_agent/model/bootstrap.cpp

@@ -1,5 +1,6 @@
 #include "bootstrap.h"
 
+#include "chaos_storage_provider.h"
 #include "config.h"
 
 #include <cloud/blockstore/libs/nvme/nvme.h>
@@ -108,11 +109,14 @@ IStorageProviderPtr CreateStorageProvider(
     NServer::IFileIOServiceProviderPtr provider,
     NNvme::INvmeManagerPtr nvmeManager)
 {
+    IStorageProviderPtr result;
+
     switch (config.GetBackend()) {
-        case NProto::DISK_AGENT_BACKEND_SPDK:
+        case NProto::DISK_AGENT_BACKEND_SPDK: {
             break;
-        case NProto::DISK_AGENT_BACKEND_AIO:
-            return CreateLocalStorageProvider(
+        }
+        case NProto::DISK_AGENT_BACKEND_AIO: {
+            result = CreateLocalStorageProvider(
                 std::move(provider),
                 std::move(nvmeManager),
                 {
@@ -122,11 +126,15 @@ IStorageProviderPtr CreateStorageProvider(
                     .EnableDataIntegrityValidation =
                         config.GetEnableDataIntegrityValidationForDrBasedDisks(),
                 });
-        case NProto::DISK_AGENT_BACKEND_NULL:
-            return NServer::CreateNullStorageProvider();
+            break;
+        }
+        case NProto::DISK_AGENT_BACKEND_NULL: {
+            result = NServer::CreateNullStorageProvider();
+            break;
+        }
         case NProto::DISK_AGENT_BACKEND_IO_URING:
-        case NProto::DISK_AGENT_BACKEND_IO_URING_NULL:
-            return CreateLocalStorageProvider(
+        case NProto::DISK_AGENT_BACKEND_IO_URING_NULL: {
+            result = CreateLocalStorageProvider(
                 std::move(provider),
                 std::move(nvmeManager),
                 {
@@ -137,9 +145,17 @@ IStorageProviderPtr CreateStorageProvider(
                     .EnableDataIntegrityValidation =
                         config.GetEnableDataIntegrityValidationForDrBasedDisks(),
                 });
+            break;
+        }
     }
 
-    return nullptr;
+    if (result && config.HasChaosConfig()) {
+        result = NServer::CreateChaosStorageProvider(
+            std::move(result),
+            config.GetChaosConfig());
+    }
+
+    return result;
 }
 
 NNvme::INvmeManagerPtr CreateNvmeManager(const TDiskAgentConfig& config)