Skip to content

Commit 3945e09

Browse files
shnikdshnikd
authored
Support partition policy for Index Table in SHOW CREATE TABLE (#17759)
1 parent 1c6e1f1 commit 3945e09

File tree

4 files changed

+243
-0
lines changed

4 files changed

+243
-0
lines changed

‎ydb/core/sys_view/show_create/create_table_formatter.cpp

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@ using namespace NKikimrSchemeOp;
1818
using namespace Ydb::Table;
1919
using namespace NYdb;
2020

21+
namespace {
22+
const ui64 defaultSizeToSplit = 2ul << 30; // 2048 Mb
23+
}
24+
2125
void TCreateTableFormatter::FormatValue(NYdb::TValueParser& parser, bool isPartition, TString del) {
2226
TGuard<NMiniKQL::TScopedAlloc> guard(Alloc);
2327
switch (parser.GetKind()) {
@@ -434,6 +438,20 @@ TFormatResult TCreateTableFormatter::Format(const TString& tablePath, const TStr
434438
}
435439
}
436440

441+
if (!tableDesc.GetTableIndexes().empty()) {
442+
try {
443+
for (const auto& indexDesc: tableDesc.GetTableIndexes()) {
444+
if (indexDesc.GetType() != NKikimrSchemeOp::EIndexType::EIndexTypeGlobalVectorKmeansTree) {
445+
FormatIndexImplTable(tablePath, indexDesc.GetName(), indexDesc.GetIndexImplTableDescriptions(0));
446+
}
447+
}
448+
} catch (const TFormatFail& ex) {
449+
return TFormatResult(ex.Status, ex.Error);
450+
} catch (const yexception& e) {
451+
return TFormatResult(Ydb::StatusIds::INTERNAL_ERROR, e.what());
452+
}
453+
}
454+
437455
TString statement = Stream.Str();
438456
TString formattedStatement;
439457
NYql::TIssues issues;
@@ -1048,6 +1066,68 @@ void TCreateTableFormatter::Format(const TString& tablePath, const NKikimrScheme
10481066
Stream << ";";
10491067
}
10501068

1069+
void TCreateTableFormatter::FormatIndexImplTable(const TString& tablePath, const TString& indexName, const NKikimrSchemeOp::TTableDescription& indexImplDesc) {
1070+
if (!indexImplDesc.HasPartitionConfig() || !indexImplDesc.GetPartitionConfig().HasPartitioningPolicy()) {
1071+
return;
1072+
}
1073+
1074+
const auto& policy = indexImplDesc.GetPartitionConfig().GetPartitioningPolicy();
1075+
1076+
ui32 shardsToCreate = NSchemeShard::TTableInfo::ShardsToCreate(indexImplDesc);
1077+
1078+
bool printed = false;
1079+
if ((policy.HasSizeToSplit() && (policy.GetSizeToSplit() != defaultSizeToSplit)) || policy.HasSplitByLoadSettings()
1080+
|| (policy.HasMinPartitionsCount() && policy.GetMinPartitionsCount() && policy.GetMinPartitionsCount() != shardsToCreate)
1081+
|| (policy.HasMaxPartitionsCount() && policy.GetMaxPartitionsCount())) {
1082+
printed = true;
1083+
}
1084+
if (!printed) {
1085+
return;
1086+
}
1087+
1088+
Stream << "ALTER TABLE ";
1089+
EscapeName(tablePath, Stream);
1090+
Stream << " ALTER INDEX ";
1091+
EscapeName(indexName, Stream);
1092+
1093+
Stream << " SET (\n";
1094+
1095+
TString del = "";
1096+
if (policy.HasSizeToSplit()) {
1097+
if (policy.GetSizeToSplit()) {
1098+
if (policy.GetSizeToSplit() != defaultSizeToSplit) {
1099+
Stream << "\tAUTO_PARTITIONING_BY_SIZE = ENABLED,\n";
1100+
auto partitionBySize = policy.GetSizeToSplit() / (1 << 20);
1101+
Stream << "\tAUTO_PARTITIONING_PARTITION_SIZE_MB = " << partitionBySize;
1102+
del = ",\n";
1103+
}
1104+
} else {
1105+
Stream << "\tAUTO_PARTITIONING_BY_SIZE = DISABLED";
1106+
del = ",\n";
1107+
}
1108+
}
1109+
1110+
if (policy.HasSplitByLoadSettings()) {
1111+
if (policy.GetSplitByLoadSettings().GetEnabled()) {
1112+
Stream << del << "\tAUTO_PARTITIONING_BY_LOAD = ENABLED";
1113+
} else {
1114+
Stream << del << "\tAUTO_PARTITIONING_BY_LOAD = DISABLED";
1115+
}
1116+
del = ",\n";
1117+
}
1118+
1119+
if (policy.HasMinPartitionsCount() && policy.GetMinPartitionsCount() && policy.GetMinPartitionsCount() != shardsToCreate) {
1120+
Stream << del << "\tAUTO_PARTITIONING_MIN_PARTITIONS_COUNT = " << policy.GetMinPartitionsCount();
1121+
del = ",\n";
1122+
}
1123+
1124+
if (policy.HasMaxPartitionsCount() && policy.GetMaxPartitionsCount()) {
1125+
Stream << del << "\tAUTO_PARTITIONING_MAX_PARTITIONS_COUNT = " << policy.GetMaxPartitionsCount();
1126+
}
1127+
1128+
Stream << "\n);";
1129+
}
1130+
10511131

10521132
TFormatResult TCreateTableFormatter::Format(const TString& tablePath, const TColumnTableDescription& tableDesc, bool temporary) {
10531133
Stream.Clear();

‎ydb/core/sys_view/show_create/create_table_formatter.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ class TCreateTableFormatter {
4343
void Format(const TString& tablePath, const NKikimrSchemeOp::TCdcStreamDescription& cdcStream,
4444
const THashMap<TString, THolder<NKikimrSchemeOp::TPersQueueGroupDescription>>& persQueues, ui32 firstColumnTypeId);
4545
void Format(const TString& fullTablePath, const NKikimrSchemeOp::TSequenceDescription& sequence, const THashMap<TPathId, THolder<NSequenceProxy::TEvSequenceProxy::TEvGetSequenceResult>>& sequences);
46+
void FormatIndexImplTable(const TString& tablePath, const TString& indexName, const NKikimrSchemeOp::TTableDescription& indexImplDesc);
4647

4748
void Format(const Ydb::Table::TableIndex& index);
4849
bool Format(const Ydb::Table::ExplicitPartitions& explicitPartitions, TString& del, bool needWith);

‎ydb/core/sys_view/show_create/show_create.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,10 @@ class TShowCreate : public TScanActorBase<TShowCreate> {
155155
if (PathType == "Table") {
156156
record->MutableOptions()->SetReturnBoundaries(true);
157157
record->MutableOptions()->SetShowPrivateTable(false);
158+
record->MutableOptions()->SetReturnBoundaries(true);
159+
record->MutableOptions()->SetShowPrivateTable(true);
160+
record->MutableOptions()->SetReturnIndexTableBoundaries(true);
161+
record->MutableOptions()->SetReturnPartitionConfig(true);
158162
}
159163

160164
Send(MakeTxProxyID(), navigateRequest.release());

‎ydb/core/sys_view/ut_kqp.cpp

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1868,6 +1868,164 @@ ALTER SEQUENCE `/Root/test_show_create/_serial_column_Value1` START WITH 101 INC
18681868
);
18691869
}
18701870

1871+
Y_UNIT_TEST(ShowCreateTablePartitionPolicyIndexTable) {
1872+
TTestEnv env(1, 4, {.StoragePools = 3, .ShowCreateTable = true});
1873+
1874+
env.GetServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_DEBUG);
1875+
env.GetServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPILE_SERVICE, NActors::NLog::PRI_DEBUG);
1876+
env.GetServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_YQL, NActors::NLog::PRI_TRACE);
1877+
env.GetServer().GetRuntime()->SetLogPriority(NKikimrServices::SYSTEM_VIEWS, NActors::NLog::PRI_DEBUG);
1878+
1879+
TShowCreateChecker checker(env);
1880+
1881+
checker.CheckShowCreateTable(R"(
1882+
CREATE TABLE test_show_create (
1883+
Key1 Int64 NOT NULL,
1884+
Key2 Utf8 NOT NULL,
1885+
Key3 PgInt2 NOT NULL,
1886+
Value1 Utf8,
1887+
Value2 Bool,
1888+
Value3 String,
1889+
INDEX Index1 GLOBAL SYNC ON (Key2, Value1, Value2),
1890+
PRIMARY KEY (Key1, Key2, Key3)
1891+
);
1892+
ALTER TABLE test_show_create ALTER INDEX Index1 SET (
1893+
AUTO_PARTITIONING_BY_LOAD = ENABLED,
1894+
AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 5000
1895+
);
1896+
)", "test_show_create",
1897+
R"(CREATE TABLE `test_show_create` (
1898+
`Key1` Int64 NOT NULL,
1899+
`Key2` Utf8 NOT NULL,
1900+
`Key3` pgint2 NOT NULL,
1901+
`Value1` Utf8,
1902+
`Value2` Bool,
1903+
`Value3` String,
1904+
INDEX `Index1` GLOBAL SYNC ON (`Key2`, `Value1`, `Value2`),
1905+
PRIMARY KEY (`Key1`, `Key2`, `Key3`)
1906+
);
1907+
1908+
ALTER TABLE `test_show_create`
1909+
ALTER INDEX `Index1` SET (AUTO_PARTITIONING_BY_LOAD = ENABLED, AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 5000)
1910+
;
1911+
)"
1912+
);
1913+
1914+
checker.CheckShowCreateTable(R"(
1915+
CREATE TABLE test_show_create (
1916+
Key1 Int64 NOT NULL DEFAULT -100,
1917+
Key2 Utf8 NOT NULL,
1918+
Key3 BigSerial NOT NULL,
1919+
Value1 Utf8 FAMILY Family1,
1920+
Value2 Bool FAMILY Family2,
1921+
Value3 String FAMILY Family2,
1922+
INDEX Index1 GLOBAL ASYNC ON (Key2, Value1, Value2),
1923+
INDEX Index2 GLOBAL ASYNC ON (Key3, Value2) COVER (Value1, Value3),
1924+
PRIMARY KEY (Key1, Key2, Key3),
1925+
FAMILY Family1 (
1926+
DATA = "test0",
1927+
COMPRESSION = "off"
1928+
),
1929+
FAMILY Family2 (
1930+
DATA = "test1",
1931+
COMPRESSION = "lz4"
1932+
)
1933+
) WITH (
1934+
AUTO_PARTITIONING_PARTITION_SIZE_MB = 1000
1935+
);
1936+
ALTER TABLE test_show_create ALTER INDEX Index1 SET (
1937+
AUTO_PARTITIONING_BY_LOAD = ENABLED,
1938+
AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 2000
1939+
);
1940+
ALTER TABLE test_show_create ALTER INDEX Index2 SET (
1941+
AUTO_PARTITIONING_BY_SIZE = ENABLED,
1942+
AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 100
1943+
);
1944+
)", "test_show_create",
1945+
R"(CREATE TABLE `test_show_create` (
1946+
`Key1` Int64 NOT NULL DEFAULT -100,
1947+
`Key2` Utf8 NOT NULL,
1948+
`Key3` Serial8 NOT NULL,
1949+
`Value1` Utf8 FAMILY `Family1`,
1950+
`Value2` Bool FAMILY `Family2`,
1951+
`Value3` String FAMILY `Family2`,
1952+
INDEX `Index1` GLOBAL ASYNC ON (`Key2`, `Value1`, `Value2`),
1953+
INDEX `Index2` GLOBAL ASYNC ON (`Key3`, `Value2`) COVER (`Value1`, `Value3`),
1954+
FAMILY `Family1` (DATA = 'test0', COMPRESSION = 'off'),
1955+
FAMILY `Family2` (DATA = 'test1', COMPRESSION = 'lz4'),
1956+
PRIMARY KEY (`Key1`, `Key2`, `Key3`)
1957+
)
1958+
WITH (
1959+
AUTO_PARTITIONING_BY_SIZE = ENABLED,
1960+
AUTO_PARTITIONING_PARTITION_SIZE_MB = 1000
1961+
);
1962+
1963+
ALTER TABLE `test_show_create`
1964+
ALTER INDEX `Index1` SET (AUTO_PARTITIONING_BY_LOAD = ENABLED, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 2000)
1965+
;
1966+
1967+
ALTER TABLE `test_show_create`
1968+
ALTER INDEX `Index2` SET (AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 100)
1969+
;
1970+
)"
1971+
);
1972+
1973+
checker.CheckShowCreateTable(R"(
1974+
CREATE TABLE test_show_create (
1975+
Key1 Int64 NOT NULL,
1976+
Key2 Utf8 NOT NULL,
1977+
Key3 PgInt2 NOT NULL,
1978+
Value1 Utf8,
1979+
Value2 Bool,
1980+
Value3 String,
1981+
INDEX Index1 GLOBAL SYNC ON (Key2, Value1, Value2),
1982+
INDEX Index2 GLOBAL ASYNC ON (Key3, Value1) COVER (Value2, Value3),
1983+
INDEX Index3 GLOBAL SYNC ON (Key1, Key2, Value1),
1984+
PRIMARY KEY (Key1, Key2, Key3)
1985+
);
1986+
ALTER TABLE test_show_create ALTER INDEX Index1 SET (
1987+
AUTO_PARTITIONING_BY_LOAD = ENABLED,
1988+
AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 5000,
1989+
AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 1000
1990+
);
1991+
ALTER TABLE test_show_create ALTER INDEX Index2 SET (
1992+
AUTO_PARTITIONING_BY_SIZE = ENABLED,
1993+
AUTO_PARTITIONING_PARTITION_SIZE_MB = 10000,
1994+
AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 2700
1995+
);
1996+
ALTER TABLE test_show_create ALTER INDEX Index3 SET (
1997+
AUTO_PARTITIONING_BY_SIZE = DISABLED,
1998+
AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 3500
1999+
);
2000+
)", "test_show_create",
2001+
R"(CREATE TABLE `test_show_create` (
2002+
`Key1` Int64 NOT NULL,
2003+
`Key2` Utf8 NOT NULL,
2004+
`Key3` pgint2 NOT NULL,
2005+
`Value1` Utf8,
2006+
`Value2` Bool,
2007+
`Value3` String,
2008+
INDEX `Index1` GLOBAL SYNC ON (`Key2`, `Value1`, `Value2`),
2009+
INDEX `Index2` GLOBAL ASYNC ON (`Key3`, `Value1`) COVER (`Value2`, `Value3`),
2010+
INDEX `Index3` GLOBAL SYNC ON (`Key1`, `Key2`, `Value1`),
2011+
PRIMARY KEY (`Key1`, `Key2`, `Key3`)
2012+
);
2013+
2014+
ALTER TABLE `test_show_create`
2015+
ALTER INDEX `Index1` SET (AUTO_PARTITIONING_BY_LOAD = ENABLED, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 1000, AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 5000)
2016+
;
2017+
2018+
ALTER TABLE `test_show_create`
2019+
ALTER INDEX `Index2` SET (AUTO_PARTITIONING_BY_SIZE = ENABLED, AUTO_PARTITIONING_PARTITION_SIZE_MB = 10000, AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 2700)
2020+
;
2021+
2022+
ALTER TABLE `test_show_create`
2023+
ALTER INDEX `Index3` SET (AUTO_PARTITIONING_BY_SIZE = DISABLED, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 3500)
2024+
;
2025+
)"
2026+
);
2027+
}
2028+
18712029
Y_UNIT_TEST(Nodes) {
18722030
TTestEnv env;
18732031
CreateTenantsAndTables(env, false);

0 commit comments

Comments
 (0)