Skip to content

Support partition policy for Index Table in SHOW CREATE TABLE #17759

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 6, 2025
Merged

Support partition policy for Index Table in SHOW CREATE TABLE #17759

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4fe7990
Initial commit
shnikd Apr 26, 2025
5513975
Fixes
shnikd May 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions ydb/core/sys_view/show_create/create_table_formatter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ using namespace NKikimrSchemeOp;
using namespace Ydb::Table;
using namespace NYdb;

namespace {
const ui64 defaultSizeToSplit = 2ul << 30; // 2048 Mb
}

void TCreateTableFormatter::FormatValue(NYdb::TValueParser& parser, bool isPartition, TString del) {
TGuard<NMiniKQL::TScopedAlloc> guard(Alloc);
switch (parser.GetKind()) {
Expand Down Expand Up @@ -434,6 +438,20 @@ TFormatResult TCreateTableFormatter::Format(const TString& tablePath, const TStr
}
}

if (!tableDesc.GetTableIndexes().empty()) {
try {
for (const auto& indexDesc: tableDesc.GetTableIndexes()) {
if (indexDesc.GetType() != NKikimrSchemeOp::EIndexType::EIndexTypeGlobalVectorKmeansTree) {
FormatIndexImplTable(tablePath, indexDesc.GetName(), indexDesc.GetIndexImplTableDescriptions(0));
}
}
} catch (const TFormatFail& ex) {
return TFormatResult(ex.Status, ex.Error);
} catch (const yexception& e) {
return TFormatResult(Ydb::StatusIds::INTERNAL_ERROR, e.what());
}
}

TString statement = Stream.Str();
TString formattedStatement;
NYql::TIssues issues;
Expand Down Expand Up @@ -1048,6 +1066,68 @@ void TCreateTableFormatter::Format(const TString& tablePath, const NKikimrScheme
Stream << ";";
}

void TCreateTableFormatter::FormatIndexImplTable(const TString& tablePath, const TString& indexName, const NKikimrSchemeOp::TTableDescription& indexImplDesc) {
if (!indexImplDesc.HasPartitionConfig() || !indexImplDesc.GetPartitionConfig().HasPartitioningPolicy()) {
return;
}

const auto& policy = indexImplDesc.GetPartitionConfig().GetPartitioningPolicy();

ui32 shardsToCreate = NSchemeShard::TTableInfo::ShardsToCreate(indexImplDesc);

bool printed = false;
if ((policy.HasSizeToSplit() && (policy.GetSizeToSplit() != defaultSizeToSplit)) || policy.HasSplitByLoadSettings()
|| (policy.HasMinPartitionsCount() && policy.GetMinPartitionsCount() && policy.GetMinPartitionsCount() != shardsToCreate)
|| (policy.HasMaxPartitionsCount() && policy.GetMaxPartitionsCount())) {
printed = true;
}
if (!printed) {
return;
}

Stream << "ALTER TABLE ";
EscapeName(tablePath, Stream);
Stream << " ALTER INDEX ";
EscapeName(indexName, Stream);

Stream << " SET (\n";

TString del = "";
Copy link
Member

@gridnevvvit gridnevvvit Apr 26, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

весь код ниже в этом cpp файле похож на дублирование. этого лучше избежать

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Индексная таблица по умолчанию создается с автопартицированием по размеру, тут вывод настроек отличается от обычных таблиц.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

так можно не дублировать и учесть и этот случай тоже?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

В принципе можно вынести в общий случай, но тут дублирование 15 строчек, выглядит как усложнение кода, особенно если будут добавляться другие дефолтные значения для настроек в индексных таблицах

if (policy.HasSizeToSplit()) {
if (policy.GetSizeToSplit()) {
if (policy.GetSizeToSplit() != defaultSizeToSplit) {
Stream << "\tAUTO_PARTITIONING_BY_SIZE = ENABLED,\n";
auto partitionBySize = policy.GetSizeToSplit() / (1 << 20);
Stream << "\tAUTO_PARTITIONING_PARTITION_SIZE_MB = " << partitionBySize;
del = ",\n";
}
} else {
Stream << "\tAUTO_PARTITIONING_BY_SIZE = DISABLED";
del = ",\n";
}
}

if (policy.HasSplitByLoadSettings()) {
if (policy.GetSplitByLoadSettings().GetEnabled()) {
Stream << del << "\tAUTO_PARTITIONING_BY_LOAD = ENABLED";
} else {
Stream << del << "\tAUTO_PARTITIONING_BY_LOAD = DISABLED";
}
del = ",\n";
}

if (policy.HasMinPartitionsCount() && policy.GetMinPartitionsCount() && policy.GetMinPartitionsCount() != shardsToCreate) {
Stream << del << "\tAUTO_PARTITIONING_MIN_PARTITIONS_COUNT = " << policy.GetMinPartitionsCount();
del = ",\n";
}

if (policy.HasMaxPartitionsCount() && policy.GetMaxPartitionsCount()) {
Stream << del << "\tAUTO_PARTITIONING_MAX_PARTITIONS_COUNT = " << policy.GetMaxPartitionsCount();
}

Stream << "\n);";
}


TFormatResult TCreateTableFormatter::Format(const TString& tablePath, const TColumnTableDescription& tableDesc, bool temporary) {
Stream.Clear();
Expand Down
1 change: 1 addition & 0 deletions ydb/core/sys_view/show_create/create_table_formatter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ class TCreateTableFormatter {
void Format(const TString& tablePath, const NKikimrSchemeOp::TCdcStreamDescription& cdcStream,
const THashMap<TString, THolder<NKikimrSchemeOp::TPersQueueGroupDescription>>& persQueues, ui32 firstColumnTypeId);
void Format(const TString& fullTablePath, const NKikimrSchemeOp::TSequenceDescription& sequence, const THashMap<TPathId, THolder<NSequenceProxy::TEvSequenceProxy::TEvGetSequenceResult>>& sequences);
void FormatIndexImplTable(const TString& tablePath, const TString& indexName, const NKikimrSchemeOp::TTableDescription& indexImplDesc);

void Format(const Ydb::Table::TableIndex& index);
bool Format(const Ydb::Table::ExplicitPartitions& explicitPartitions, TString& del, bool needWith);
Expand Down
4 changes: 4 additions & 0 deletions ydb/core/sys_view/show_create/show_create.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,10 @@ class TShowCreate : public TScanActorBase<TShowCreate> {
if (PathType == "Table") {
record->MutableOptions()->SetReturnBoundaries(true);
record->MutableOptions()->SetShowPrivateTable(false);
record->MutableOptions()->SetReturnBoundaries(true);
record->MutableOptions()->SetShowPrivateTable(true);
record->MutableOptions()->SetReturnIndexTableBoundaries(true);
record->MutableOptions()->SetReturnPartitionConfig(true);
}

Send(MakeTxProxyID(), navigateRequest.release());
Expand Down
158 changes: 158 additions & 0 deletions ydb/core/sys_view/ut_kqp.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1868,6 +1868,164 @@ ALTER SEQUENCE `/Root/test_show_create/_serial_column_Value1` START WITH 101 INC
);
}

Y_UNIT_TEST(ShowCreateTablePartitionPolicyIndexTable) {
TTestEnv env(1, 4, {.StoragePools = 3, .ShowCreateTable = true});

env.GetServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_EXECUTER, NActors::NLog::PRI_DEBUG);
env.GetServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_COMPILE_SERVICE, NActors::NLog::PRI_DEBUG);
env.GetServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_YQL, NActors::NLog::PRI_TRACE);
env.GetServer().GetRuntime()->SetLogPriority(NKikimrServices::SYSTEM_VIEWS, NActors::NLog::PRI_DEBUG);

TShowCreateChecker checker(env);

checker.CheckShowCreateTable(R"(
CREATE TABLE test_show_create (
Key1 Int64 NOT NULL,
Key2 Utf8 NOT NULL,
Key3 PgInt2 NOT NULL,
Value1 Utf8,
Value2 Bool,
Value3 String,
INDEX Index1 GLOBAL SYNC ON (Key2, Value1, Value2),
PRIMARY KEY (Key1, Key2, Key3)
);
ALTER TABLE test_show_create ALTER INDEX Index1 SET (
AUTO_PARTITIONING_BY_LOAD = ENABLED,
AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 5000
);
)", "test_show_create",
R"(CREATE TABLE `test_show_create` (
`Key1` Int64 NOT NULL,
`Key2` Utf8 NOT NULL,
`Key3` pgint2 NOT NULL,
`Value1` Utf8,
`Value2` Bool,
`Value3` String,
INDEX `Index1` GLOBAL SYNC ON (`Key2`, `Value1`, `Value2`),
PRIMARY KEY (`Key1`, `Key2`, `Key3`)
);

ALTER TABLE `test_show_create`
ALTER INDEX `Index1` SET (AUTO_PARTITIONING_BY_LOAD = ENABLED, AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 5000)
;
)"
);

checker.CheckShowCreateTable(R"(
CREATE TABLE test_show_create (
Key1 Int64 NOT NULL DEFAULT -100,
Key2 Utf8 NOT NULL,
Key3 BigSerial NOT NULL,
Value1 Utf8 FAMILY Family1,
Value2 Bool FAMILY Family2,
Value3 String FAMILY Family2,
INDEX Index1 GLOBAL ASYNC ON (Key2, Value1, Value2),
INDEX Index2 GLOBAL ASYNC ON (Key3, Value2) COVER (Value1, Value3),
PRIMARY KEY (Key1, Key2, Key3),
FAMILY Family1 (
DATA = "test0",
COMPRESSION = "off"
),
FAMILY Family2 (
DATA = "test1",
COMPRESSION = "lz4"
)
) WITH (
AUTO_PARTITIONING_PARTITION_SIZE_MB = 1000
);
ALTER TABLE test_show_create ALTER INDEX Index1 SET (
AUTO_PARTITIONING_BY_LOAD = ENABLED,
AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 2000
);
ALTER TABLE test_show_create ALTER INDEX Index2 SET (
AUTO_PARTITIONING_BY_SIZE = ENABLED,
AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 100
);
)", "test_show_create",
R"(CREATE TABLE `test_show_create` (
`Key1` Int64 NOT NULL DEFAULT -100,
`Key2` Utf8 NOT NULL,
`Key3` Serial8 NOT NULL,
`Value1` Utf8 FAMILY `Family1`,
`Value2` Bool FAMILY `Family2`,
`Value3` String FAMILY `Family2`,
INDEX `Index1` GLOBAL ASYNC ON (`Key2`, `Value1`, `Value2`),
INDEX `Index2` GLOBAL ASYNC ON (`Key3`, `Value2`) COVER (`Value1`, `Value3`),
FAMILY `Family1` (DATA = 'test0', COMPRESSION = 'off'),
FAMILY `Family2` (DATA = 'test1', COMPRESSION = 'lz4'),
PRIMARY KEY (`Key1`, `Key2`, `Key3`)
)
WITH (
AUTO_PARTITIONING_BY_SIZE = ENABLED,
AUTO_PARTITIONING_PARTITION_SIZE_MB = 1000
);

ALTER TABLE `test_show_create`
ALTER INDEX `Index1` SET (AUTO_PARTITIONING_BY_LOAD = ENABLED, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 2000)
;

ALTER TABLE `test_show_create`
ALTER INDEX `Index2` SET (AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 100)
;
)"
);

checker.CheckShowCreateTable(R"(
CREATE TABLE test_show_create (
Key1 Int64 NOT NULL,
Key2 Utf8 NOT NULL,
Key3 PgInt2 NOT NULL,
Value1 Utf8,
Value2 Bool,
Value3 String,
INDEX Index1 GLOBAL SYNC ON (Key2, Value1, Value2),
INDEX Index2 GLOBAL ASYNC ON (Key3, Value1) COVER (Value2, Value3),
INDEX Index3 GLOBAL SYNC ON (Key1, Key2, Value1),
PRIMARY KEY (Key1, Key2, Key3)
);
ALTER TABLE test_show_create ALTER INDEX Index1 SET (
AUTO_PARTITIONING_BY_LOAD = ENABLED,
AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 5000,
AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 1000
);
ALTER TABLE test_show_create ALTER INDEX Index2 SET (
AUTO_PARTITIONING_BY_SIZE = ENABLED,
AUTO_PARTITIONING_PARTITION_SIZE_MB = 10000,
AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 2700
);
ALTER TABLE test_show_create ALTER INDEX Index3 SET (
AUTO_PARTITIONING_BY_SIZE = DISABLED,
AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 3500
);
)", "test_show_create",
R"(CREATE TABLE `test_show_create` (
`Key1` Int64 NOT NULL,
`Key2` Utf8 NOT NULL,
`Key3` pgint2 NOT NULL,
`Value1` Utf8,
`Value2` Bool,
`Value3` String,
INDEX `Index1` GLOBAL SYNC ON (`Key2`, `Value1`, `Value2`),
INDEX `Index2` GLOBAL ASYNC ON (`Key3`, `Value1`) COVER (`Value2`, `Value3`),
INDEX `Index3` GLOBAL SYNC ON (`Key1`, `Key2`, `Value1`),
PRIMARY KEY (`Key1`, `Key2`, `Key3`)
);

ALTER TABLE `test_show_create`
ALTER INDEX `Index1` SET (AUTO_PARTITIONING_BY_LOAD = ENABLED, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 1000, AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 5000)
;

ALTER TABLE `test_show_create`
ALTER INDEX `Index2` SET (AUTO_PARTITIONING_BY_SIZE = ENABLED, AUTO_PARTITIONING_PARTITION_SIZE_MB = 10000, AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = 2700)
;

ALTER TABLE `test_show_create`
ALTER INDEX `Index3` SET (AUTO_PARTITIONING_BY_SIZE = DISABLED, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 3500)
;
)"
);
}

Y_UNIT_TEST(Nodes) {
TTestEnv env;
CreateTenantsAndTables(env, false);
Expand Down
Loading