Skip to content

Commit aee1801

Browse files
stanislav-shchetininstanislav-shchetinin
authored
Added EnableAutoDropping flag (#17758)
1 parent 69674ea commit aee1801

File tree

4 files changed

+115
-8
lines changed

4 files changed

+115
-8
lines changed

‎ydb/core/protos/feature_flags.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,4 +206,5 @@ message TFeatureFlags {
206206
optional bool SwitchToConfigV1 = 180 [default = false];
207207
optional bool EnableEncryptedExport = 181 [default = false];
208208
optional bool EnableAlterDatabase = 182 [default = false];
209+
optional bool EnableExportAutoDropping = 183 [default = false];
209210
}

‎ydb/core/tx/schemeshard/schemeshard_export__create.cpp

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1158,7 +1158,11 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase
11581158
Self->PersistExportItemState(db, exportInfo, itemIdx);
11591159

11601160
if (AllOf(exportInfo->Items, &TExportInfo::TItem::IsDone)) {
1161-
PrepareAutoDropping(Self, exportInfo, db);
1161+
if (!AppData()->FeatureFlags.GetEnableExportAutoDropping()) {
1162+
EndExport(exportInfo, EState::Done, db);
1163+
} else {
1164+
PrepareAutoDropping(Self, exportInfo, db);
1165+
}
11621166
}
11631167
} else if (exportInfo->State == EState::Cancellation) {
11641168
item.State = EState::Cancelled;
@@ -1358,7 +1362,12 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase
13581362
}
13591363
}
13601364
if (!itemHasIssues && AllOf(exportInfo->Items, &TExportInfo::TItem::IsDone)) {
1361-
PrepareAutoDropping(Self, exportInfo, db);
1365+
if (!AppData()->FeatureFlags.GetEnableExportAutoDropping()) {
1366+
exportInfo->State = EState::Done;
1367+
exportInfo->EndTime = TAppData::TimeProvider->Now();
1368+
} else {
1369+
PrepareAutoDropping(Self, exportInfo, db);
1370+
}
13621371
}
13631372

13641373
Self->PersistExportItemState(db, exportInfo, itemIdx);

‎ydb/core/tx/schemeshard/ut_export/ut_export.cpp

Lines changed: 65 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ namespace {
3737
void Run(TTestBasicRuntime& runtime, TTestEnv& env, const std::variant<TVector<TString>, TTablesWithAttrs>& tablesVar, const TString& request,
3838
Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS,
3939
const TString& dbName = "/MyRoot", bool serverless = false, const TString& userSID = "", const TString& peerName = "",
40-
const TVector<TString>& cdcStreams = {}) {
40+
const TVector<TString>& cdcStreams = {}, bool checkAutoDropping = false) {
4141

4242
TTablesWithAttrs tables;
4343

@@ -150,6 +150,40 @@ namespace {
150150
const ui64 exportId = txId;
151151
TestGetExport(runtime, schemeshardId, exportId, dbName, expectedStatus);
152152

153+
if (!runtime.GetAppData().FeatureFlags.GetEnableExportAutoDropping() && checkAutoDropping) {
154+
auto desc = DescribePath(runtime, "/MyRoot");
155+
Cerr << "desc: " << desc.GetPathDescription().ChildrenSize()<< Endl;
156+
UNIT_ASSERT(desc.GetPathDescription().ChildrenSize() > 1);
157+
158+
bool foundExportDir = false;
159+
bool foundOriginalTable = false;
160+
161+
for (size_t i = 0; i < desc.GetPathDescription().ChildrenSize(); ++i) {
162+
const auto& child = desc.GetPathDescription().GetChildren(i);
163+
const auto& name = child.GetName();
164+
165+
if (name.StartsWith("Table")) {
166+
foundOriginalTable = true;
167+
} else if (name.StartsWith("export-")) {
168+
foundExportDir = true;
169+
auto exportDirDesc = DescribePath(runtime, "/MyRoot/" + name);
170+
UNIT_ASSERT(exportDirDesc.GetPathDescription().ChildrenSize() >= 1);
171+
UNIT_ASSERT_EQUAL(exportDirDesc.GetPathDescription().GetChildren(0).GetName(), "0");
172+
}
173+
}
174+
175+
UNIT_ASSERT(foundExportDir);
176+
UNIT_ASSERT(foundOriginalTable);
177+
} else if (checkAutoDropping) {
178+
auto desc = DescribePath(runtime, "/MyRoot");
179+
Cerr << "desc: " << desc.GetPathDescription().ChildrenSize()<< Endl;
180+
for (size_t i = 0; i < desc.GetPathDescription().ChildrenSize(); ++i) {
181+
const auto& child = desc.GetPathDescription().GetChildren(i);
182+
const auto& name = child.GetName();
183+
UNIT_ASSERT(!name.StartsWith("export-"));
184+
}
185+
}
186+
153187
TestForgetExport(runtime, schemeshardId, ++txId, dbName, exportId);
154188
env.TestWaitNotification(runtime, exportId, schemeshardId);
155189

@@ -1201,6 +1235,7 @@ partitioning_settings {
12011235
ui64 txId = 100;
12021236

12031237
THashSet<ui64> statsCollected;
1238+
Runtime().GetAppData().FeatureFlags.SetEnableExportAutoDropping(true);
12041239
Runtime().SetObserverFunc([&](TAutoPtr<IEventHandle>& ev) {
12051240
if (ev->GetTypeRewrite() == TEvDataShard::EvPeriodicTableStats) {
12061241
statsCollected.insert(ev->Get<TEvDataShard::TEvPeriodicTableStats>()->Record.GetDatashardId());
@@ -1286,7 +1321,7 @@ partitioning_settings {
12861321
Y_UNIT_TEST(CheckItemProgress) {
12871322
Env(); // Init test env
12881323
ui64 txId = 100;
1289-
1324+
Runtime().GetAppData().FeatureFlags.SetEnableExportAutoDropping(true);
12901325
TBlockEvents<NKikimr::NWrappers::NExternalStorage::TEvPutObjectRequest> blockPartition01(Runtime(), [](auto&& ev) {
12911326
return ev->Get()->Request.GetKey() == "/data_01.csv";
12921327
});
@@ -2773,17 +2808,41 @@ attributes {
27732808
}
27742809
)", S3Port());
27752810

2811+
Env();
2812+
Runtime().GetAppData().FeatureFlags.SetEnableExportAutoDropping(true);
2813+
27762814
Run(Runtime(), Env(), TVector<TString>{
27772815
R"(
27782816
Name: "Table"
27792817
Columns { Name: "key" Type: "Utf8" }
27802818
Columns { Name: "value" Type: "Utf8" }
27812819
KeyColumnNames: ["key"]
27822820
)",
2783-
}, request, Ydb::StatusIds::SUCCESS, "/MyRoot");
2821+
}, request, Ydb::StatusIds::SUCCESS, "/MyRoot", false, "", "", {}, true);
2822+
}
2823+
2824+
Y_UNIT_TEST(DisableAutoDropping) {
2825+
auto request = Sprintf(R"(
2826+
ExportToS3Settings {
2827+
endpoint: "localhost:%d"
2828+
scheme: HTTP
2829+
items {
2830+
source_path: "/MyRoot/Table"
2831+
destination_prefix: ""
2832+
}
2833+
}
2834+
)", S3Port());
2835+
2836+
Env();
2837+
Runtime().GetAppData().FeatureFlags.SetEnableExportAutoDropping(false);
27842838

2785-
auto desc = DescribePath(Runtime(), "/MyRoot");
2786-
UNIT_ASSERT_EQUAL(desc.GetPathDescription().ChildrenSize(), 1);
2787-
UNIT_ASSERT_EQUAL(desc.GetPathDescription().GetChildren(0).GetName(), "Table");
2839+
Run(Runtime(), Env(), TVector<TString>{
2840+
R"(
2841+
Name: "Table"
2842+
Columns { Name: "key" Type: "Utf8" }
2843+
Columns { Name: "value" Type: "Utf8" }
2844+
KeyColumnNames: ["key"]
2845+
)",
2846+
}, request, Ydb::StatusIds::SUCCESS, "/MyRoot", false, "", "", {}, true);
27882847
}
27892848
}

‎ydb/core/tx/schemeshard/ut_export_reboots_s3/ut_export_reboots_s3.cpp

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -598,6 +598,7 @@ Y_UNIT_TEST_SUITE(TExportToS3WithRebootsTests) {
598598

599599
t.Run([&](TTestActorRuntime& runtime, bool& activeZone) {
600600
runtime.SetLogPriority(NKikimrServices::EXPORT, NActors::NLog::PRI_TRACE);
601+
runtime.GetAppData().FeatureFlags.SetEnableExportAutoDropping(true);
601602
{
602603
TInactiveZone inactive(activeZone);
603604
CreateSchemeObjects(t, runtime, {
@@ -620,4 +621,41 @@ Y_UNIT_TEST_SUITE(TExportToS3WithRebootsTests) {
620621
}
621622
});
622623
}
624+
625+
Y_UNIT_TEST(ShouldDisableAutoDropping) {
626+
TPortManager portManager;
627+
const ui16 port = portManager.GetPort();
628+
629+
TTestWithReboots t;
630+
TS3Mock s3Mock({}, TS3Mock::TSettings(port));
631+
UNIT_ASSERT(s3Mock.Start());
632+
633+
t.Run([&](TTestActorRuntime& runtime, bool& activeZone) {
634+
runtime.SetLogPriority(NKikimrServices::EXPORT, NActors::NLog::PRI_TRACE);
635+
runtime.GetAppData().FeatureFlags.SetEnableExportAutoDropping(false);
636+
{
637+
TInactiveZone inactive(activeZone);
638+
CreateSchemeObjects(t, runtime, {
639+
TTestData::Table()
640+
});
641+
642+
TestExport(runtime, ++t.TxId, "/MyRoot", Sprintf(TTestData::Request().data(), port));
643+
}
644+
645+
const ui64 exportId = t.TxId;
646+
t.TestEnv->TestWaitNotification(runtime, exportId);
647+
648+
{
649+
TInactiveZone inactive(activeZone);
650+
TestGetExport(runtime, exportId, "/MyRoot");
651+
TestRmDir(runtime, ++t.TxId, "/MyRoot", "DirA");
652+
auto desc = DescribePath(runtime, "/MyRoot");
653+
UNIT_ASSERT_EQUAL(desc.GetPathDescription().ChildrenSize(), 2);
654+
const auto namesVector = {desc.GetPathDescription().GetChildren(0).GetName(),
655+
desc.GetPathDescription().GetChildren(1).GetName()};
656+
UNIT_ASSERT(IsIn(namesVector, "Table"));
657+
UNIT_ASSERT(IsIn(namesVector, "export-1003"));
658+
}
659+
});
660+
}
623661
}

0 commit comments

Comments
 (0)