diff --git a/snuba/datasets/configuration/events_analytics_platform/storages/eap_item_co_occurring_attrs.yaml b/snuba/datasets/configuration/events_analytics_platform/storages/eap_item_co_occurring_attrs.yaml index 39e0babca2..d422e73782 100644 --- a/snuba/datasets/configuration/events_analytics_platform/storages/eap_item_co_occurring_attrs.yaml +++ b/snuba/datasets/configuration/events_analytics_platform/storages/eap_item_co_occurring_attrs.yaml @@ -25,5 +25,5 @@ schema: ] local_table_name: eap_item_co_occurring_attrs_1_local dist_table_name: eap_item_co_occurring_attrs_1_dist - partition_format: [date] + partition_format: [retention_days, date] allocation_policies: [] diff --git a/snuba/datasets/configuration/events_analytics_platform/storages/eap_items.yaml b/snuba/datasets/configuration/events_analytics_platform/storages/eap_items.yaml index e7d32f3867..f521925b85 100644 --- a/snuba/datasets/configuration/events_analytics_platform/storages/eap_items.yaml +++ b/snuba/datasets/configuration/events_analytics_platform/storages/eap_items.yaml @@ -108,7 +108,7 @@ schema: ] local_table_name: eap_items_1_local dist_table_name: eap_items_1_dist - partition_format: [date] + partition_format: [retention_days, date] query_processors: - processor: UniqInSelectAndHavingProcessor @@ -138,6 +138,7 @@ deletion_settings: is_enabled: 1 max_rows_to_delete: 2000000 bulk_delete_only: true + partition_column: timestamp tables: - eap_items_1_local - eap_items_1_downsample_8_local diff --git a/snuba/datasets/configuration/events_analytics_platform/storages/eap_items_downsample_512.yaml b/snuba/datasets/configuration/events_analytics_platform/storages/eap_items_downsample_512.yaml index 8e5e877010..c333a87df3 100644 --- a/snuba/datasets/configuration/events_analytics_platform/storages/eap_items_downsample_512.yaml +++ b/snuba/datasets/configuration/events_analytics_platform/storages/eap_items_downsample_512.yaml @@ -107,7 +107,7 @@ schema: ] local_table_name: eap_items_1_downsample_512_local dist_table_name: eap_items_1_downsample_512_dist - partition_format: [date] + partition_format: [retention_days, date] query_processors: - processor: UniqInSelectAndHavingProcessor diff --git a/snuba/datasets/configuration/events_analytics_platform/storages/eap_items_downsample_64.yaml b/snuba/datasets/configuration/events_analytics_platform/storages/eap_items_downsample_64.yaml index 0b2573c28f..9b2b8ff3df 100644 --- a/snuba/datasets/configuration/events_analytics_platform/storages/eap_items_downsample_64.yaml +++ b/snuba/datasets/configuration/events_analytics_platform/storages/eap_items_downsample_64.yaml @@ -108,7 +108,7 @@ schema: ] local_table_name: eap_items_1_downsample_64_local dist_table_name: eap_items_1_downsample_64_dist - partition_format: [date] + partition_format: [retention_days, date] query_processors: - processor: UniqInSelectAndHavingProcessor diff --git a/snuba/datasets/configuration/events_analytics_platform/storages/eap_items_downsample_8.yaml b/snuba/datasets/configuration/events_analytics_platform/storages/eap_items_downsample_8.yaml index 6b4cea2805..84d680f655 100644 --- a/snuba/datasets/configuration/events_analytics_platform/storages/eap_items_downsample_8.yaml +++ b/snuba/datasets/configuration/events_analytics_platform/storages/eap_items_downsample_8.yaml @@ -107,7 +107,7 @@ schema: ] local_table_name: eap_items_1_downsample_8_local dist_table_name: eap_items_1_downsample_8_dist - partition_format: [date] + partition_format: [retention_days, date] query_processors: - processor: UniqInSelectAndHavingProcessor diff --git a/snuba/datasets/configuration/issues/storages/search_issues.yaml b/snuba/datasets/configuration/issues/storages/search_issues.yaml index 8c94760a23..83225d4c2d 100644 --- a/snuba/datasets/configuration/issues/storages/search_issues.yaml +++ b/snuba/datasets/configuration/issues/storages/search_issues.yaml @@ -68,6 +68,7 @@ schema: ] local_table_name: search_issues_local_v2 dist_table_name: search_issues_dist_v2 + partition_format: [retention_days, date] allocation_policies: - name: ConcurrentRateLimitAllocationPolicy @@ -143,6 +144,7 @@ query_processors: deletion_settings: is_enabled: 1 max_rows_to_delete: 2000000 + partition_column: client_timestamp tables: - search_issues_local_v2 allowed_columns: diff --git a/snuba/datasets/configuration/json_schema.py b/snuba/datasets/configuration/json_schema.py index ca8befa3cc..88c5063a5b 100644 --- a/snuba/datasets/configuration/json_schema.py +++ b/snuba/datasets/configuration/json_schema.py @@ -622,6 +622,10 @@ def registered_class_array_schema( "items": {"type": "string"}, }, }, + "partition_column": { + "type": "string", + "description": "Column used for partition splitting in lightweight deletes. When set, deletes can be split by toMonday(partition_column) to reduce per-mutation CPU.", + }, }, "required": ["is_enabled", "tables"], "additionalProperties": False, diff --git a/snuba/datasets/deletion_settings.py b/snuba/datasets/deletion_settings.py index 9c9ca432ae..67e7567837 100644 --- a/snuba/datasets/deletion_settings.py +++ b/snuba/datasets/deletion_settings.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Dict, List, Sequence +from typing import Dict, List, Optional, Sequence from sentry_protos.snuba.v1.request_common_pb2 import TraceItemType @@ -16,6 +16,7 @@ class DeletionSettings: allowed_columns: Sequence[str] = field(default_factory=list) max_rows_to_delete: int = MAX_ROWS_TO_DELETE_DEFAULT allowed_attributes_by_item_type: Dict[str, List[str]] = field(default_factory=dict) + partition_column: Optional[str] = None def get_trace_item_type_name(item_type: int) -> str: