Skip to content

Commit 98ffcdb

Browse files
DrewKimballDrewKimball
committed
opt: add session setting to control new norm rule
This commit adds a session setting `optimizer_use_exists_filter_hoist_rule` to toggle the new optimizer rule `HoistUnboundFilterFromExistsSubquery`. It if off by default in this commit, while the next will switch it on only on master. Informs #146000 Release note: None
1 parent 4c38b14 commit 98ffcdb

File tree

16 files changed

+304
-250
lines changed

16 files changed

+304
-250
lines changed

‎pkg/sql/exec_util.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4140,6 +4140,10 @@ func (m *sessionDataMutator) SetPropagateAdmissionHeaderToLeafTransactions(val b
41404140
m.data.PropagateAdmissionHeaderToLeafTransactions = val
41414141
}
41424142

4143+
func (m *sessionDataMutator) SetOptimizerUseExistsFilterHoistRule(val bool) {
4144+
m.data.OptimizerUseExistsFilterHoistRule = val
4145+
}
4146+
41434147
// Utility functions related to scrubbing sensitive information on SQL Stats.
41444148

41454149
// quantizeCounts ensures that the Count field in the

‎pkg/sql/logictest/testdata/logic_test/information_schema

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4026,6 +4026,7 @@ optimizer_push_limit_into_project_filtered_scan on
40264026
optimizer_push_offset_into_index_join on
40274027
optimizer_use_conditional_hoist_fix on
40284028
optimizer_use_delete_range_fast_path on
4029+
optimizer_use_exists_filter_hoist_rule off
40294030
optimizer_use_forecasts on
40304031
optimizer_use_histograms on
40314032
optimizer_use_improved_computed_column_filters_derivation on

‎pkg/sql/logictest/testdata/logic_test/pg_catalog

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3035,6 +3035,7 @@ optimizer_push_limit_into_project_filtered_scan on N
30353035
optimizer_push_offset_into_index_join on NULL NULL NULL string
30363036
optimizer_use_conditional_hoist_fix on NULL NULL NULL string
30373037
optimizer_use_delete_range_fast_path on NULL NULL NULL string
3038+
optimizer_use_exists_filter_hoist_rule off NULL NULL NULL string
30383039
optimizer_use_forecasts on NULL NULL NULL string
30393040
optimizer_use_histograms on NULL NULL NULL string
30403041
optimizer_use_improved_computed_column_filters_derivation on NULL NULL NULL string
@@ -3255,6 +3256,7 @@ optimizer_push_limit_into_project_filtered_scan on N
32553256
optimizer_push_offset_into_index_join on NULL user NULL on on
32563257
optimizer_use_conditional_hoist_fix on NULL user NULL on on
32573258
optimizer_use_delete_range_fast_path on NULL user NULL on on
3259+
optimizer_use_exists_filter_hoist_rule off NULL user NULL off off
32583260
optimizer_use_forecasts on NULL user NULL on on
32593261
optimizer_use_histograms on NULL user NULL on on
32603262
optimizer_use_improved_computed_column_filters_derivation on NULL user NULL on on
@@ -3474,6 +3476,7 @@ optimizer_push_limit_into_project_filtered_scan NULL NULL NULL
34743476
optimizer_push_offset_into_index_join NULL NULL NULL NULL NULL
34753477
optimizer_use_conditional_hoist_fix NULL NULL NULL NULL NULL
34763478
optimizer_use_delete_range_fast_path NULL NULL NULL NULL NULL
3479+
optimizer_use_exists_filter_hoist_rule NULL NULL NULL NULL NULL
34773480
optimizer_use_forecasts NULL NULL NULL NULL NULL
34783481
optimizer_use_histograms NULL NULL NULL NULL NULL
34793482
optimizer_use_improved_computed_column_filters_derivation NULL NULL NULL NULL NULL

‎pkg/sql/logictest/testdata/logic_test/show_source

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ optimizer_push_limit_into_project_filtered_scan on
151151
optimizer_push_offset_into_index_join on
152152
optimizer_use_conditional_hoist_fix on
153153
optimizer_use_delete_range_fast_path on
154+
optimizer_use_exists_filter_hoist_rule off
154155
optimizer_use_forecasts on
155156
optimizer_use_histograms on
156157
optimizer_use_improved_computed_column_filters_derivation on

‎pkg/sql/opt/memo/memo.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ type Memo struct {
206206
useInsertFastPath bool
207207
internal bool
208208
usePre_25_2VariadicBuiltins bool
209+
useExistsFilterHoistRule bool
209210

210211
// txnIsoLevel is the isolation level under which the plan was created. This
211212
// affects the planning of some locking operations, so it must be included in
@@ -309,6 +310,7 @@ func (m *Memo) Init(ctx context.Context, evalCtx *eval.Context) {
309310
useInsertFastPath: evalCtx.SessionData().InsertFastPath,
310311
internal: evalCtx.SessionData().Internal,
311312
usePre_25_2VariadicBuiltins: evalCtx.SessionData().UsePre_25_2VariadicBuiltins,
313+
useExistsFilterHoistRule: evalCtx.SessionData().OptimizerUseExistsFilterHoistRule,
312314
txnIsoLevel: evalCtx.TxnIsoLevel,
313315
}
314316
m.metadata.Init()
@@ -485,6 +487,7 @@ func (m *Memo) IsStale(
485487
m.useInsertFastPath != evalCtx.SessionData().InsertFastPath ||
486488
m.internal != evalCtx.SessionData().Internal ||
487489
m.usePre_25_2VariadicBuiltins != evalCtx.SessionData().UsePre_25_2VariadicBuiltins ||
490+
m.useExistsFilterHoistRule != evalCtx.SessionData().OptimizerUseExistsFilterHoistRule ||
488491
m.txnIsoLevel != evalCtx.TxnIsoLevel {
489492
return true, nil
490493
}

‎pkg/sql/opt/memo/memo_test.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -574,6 +574,11 @@ func TestMemoIsStale(t *testing.T) {
574574
evalCtx.SessionData().UsePre_25_2VariadicBuiltins = false
575575
notStale()
576576

577+
evalCtx.SessionData().OptimizerUseExistsFilterHoistRule = true
578+
stale()
579+
evalCtx.SessionData().OptimizerUseExistsFilterHoistRule = false
580+
notStale()
581+
577582
// User no longer has access to view.
578583
catalog.View(tree.NewTableNameWithSchema("t", catconstants.PublicSchemaName, "abcview")).Revoked = true
579584
_, err = o.Memo().IsStale(ctx, &evalCtx, catalog)

‎pkg/sql/opt/memo/testdata/logprops/join

Lines changed: 29 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -996,38 +996,39 @@ project
996996
opt
997997
SELECT * FROM (SELECT count(*) cnt FROM xysd) WHERE EXISTS(SELECT * FROM uv WHERE cnt=1)
998998
----
999-
select
999+
project
10001000
├── columns: cnt:7(int!null)
10011001
├── cardinality: [0 - 1]
10021002
├── key: ()
10031003
├── fd: ()-->(7)
1004-
├── scalar-group-by
1005-
│ ├── columns: count_rows:7(int!null)
1006-
│ ├── cardinality: [1 - 1]
1007-
│ ├── key: ()
1008-
│ ├── fd: ()-->(7)
1009-
│ ├── prune: (7)
1010-
│ ├── scan xysd@xysd_s_d_key
1011-
│ └── aggregations
1012-
│ └── count-rows [as=count_rows:7, type=int]
1013-
└── filters
1014-
├── coalesce [type=bool, subquery]
1015-
│ ├── subquery [type=bool]
1016-
│ │ └── project
1017-
│ │ ├── columns: column14:14(bool!null)
1018-
│ │ ├── cardinality: [0 - 1]
1019-
│ │ ├── key: ()
1020-
│ │ ├── fd: ()-->(14)
1021-
│ │ ├── prune: (14)
1022-
│ │ ├── scan uv
1023-
│ │ │ ├── limit: 1
1024-
│ │ │ └── key: ()
1025-
│ │ └── projections
1026-
│ │ └── true [as=column14:14, type=bool]
1027-
│ └── false [type=bool]
1028-
└── eq [type=bool, outer=(7), constraints=(/7: [/1 - /1]; tight), fd=()-->(7)]
1029-
├── variable: count_rows:7 [type=int]
1030-
└── const: 1 [type=int]
1004+
└── inner-join (cross)
1005+
├── columns: count_rows:7(int!null)
1006+
├── cardinality: [0 - 1]
1007+
├── multiplicity: left-rows(zero-or-one), right-rows(zero-or-one)
1008+
├── key: ()
1009+
├── fd: ()-->(7)
1010+
├── select
1011+
│ ├── columns: count_rows:7(int!null)
1012+
│ ├── cardinality: [0 - 1]
1013+
│ ├── key: ()
1014+
│ ├── fd: ()-->(7)
1015+
│ ├── scalar-group-by
1016+
│ │ ├── columns: count_rows:7(int!null)
1017+
│ │ ├── cardinality: [1 - 1]
1018+
│ │ ├── key: ()
1019+
│ │ ├── fd: ()-->(7)
1020+
│ │ ├── prune: (7)
1021+
│ │ ├── scan xysd@xysd_s_d_key
1022+
│ │ └── aggregations
1023+
│ │ └── count-rows [as=count_rows:7, type=int]
1024+
│ └── filters
1025+
│ └── eq [type=bool, outer=(7), constraints=(/7: [/1 - /1]; tight), fd=()-->(7)]
1026+
│ ├── variable: count_rows:7 [type=int]
1027+
│ └── const: 1 [type=int]
1028+
├── scan uv
1029+
│ ├── limit: 1
1030+
│ └── key: ()
1031+
└── filters (true)
10311032

10321033
# Maximum cardinality of the right input is propagated to the SemiJoin when
10331034
# right rows are guaranteed at most one match each over the join filters.

‎pkg/sql/opt/memo/testdata/logprops/limit

Lines changed: 76 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -247,83 +247,83 @@ WHERE t1.b IN (
247247
)
248248
ORDER BY t1.a ASC;
249249
----
250-
project
251-
├── columns: "?column?":24(int!null) [hidden: t1.a:1(int!null)]
250+
sort
251+
├── columns: "?column?":23(int!null) [hidden: t1.a:1(int!null)]
252252
├── key: (1)
253-
├── fd: ()-->(24)
254-
├── ordering: +1 opt(24) [actual: +1]
255-
├── prune: (1,24)
256-
├── interesting orderings: (+1 opt(24))
257-
├── select
258-
│ ├── columns: t1.a:1(int!null) t1.b:2(int!null)
259-
│ ├── key: (1)
260-
│ ├── fd: ()-->(2)
261-
│ ├── ordering: +1 opt(2) [actual: +1]
262-
│ ├── prune: (1)
263-
│ ├── interesting orderings: (+1 opt(2))
264-
│ ├── scan t65038 [as=t1]
265-
│ │ ├── columns: t1.a:1(int!null) t1.b:2(int)
266-
│ │ ├── key: (1)
267-
│ │ ├── fd: (1)-->(2)
268-
│ │ ├── ordering: +1
269-
│ │ ├── prune: (1,2)
270-
│ │ └── interesting orderings: (+1)
271-
│ └── filters
272-
│ ├── coalesce [type=bool, subquery]
273-
│ │ ├── subquery [type=bool]
274-
│ │ │ └── project
275-
│ │ │ ├── columns: column23:23(bool!null)
276-
│ │ │ ├── cardinality: [0 - 1]
277-
│ │ │ ├── key: ()
278-
│ │ │ ├── fd: ()-->(23)
279-
│ │ │ ├── prune: (23)
280-
│ │ │ ├── top-k
281-
│ │ │ │ ├── columns: t2.a:11(int!null) t2.b:12(int!null) t2.c:13(int!null) t3.a:16(int!null) t3.b:17(int!null) t3.c:18(int!null)
282-
│ │ │ │ ├── internal-ordering: +(12|17)
283-
│ │ │ │ ├── k: 1
284-
│ │ │ │ ├── cardinality: [0 - 1]
285-
│ │ │ │ ├── key: ()
286-
│ │ │ │ ├── fd: ()-->(11-13,16-18), (11)==(16), (16)==(11), (13)==(18), (18)==(13), (12)==(17), (17)==(12)
287-
│ │ │ │ ├── interesting orderings: (+(12|17))
288-
│ │ │ │ └── inner-join (cross)
289-
│ │ │ │ ├── columns: t2.a:11(int!null) t2.b:12(int!null) t2.c:13(int!null) t3.a:16(int!null) t3.b:17(int!null) t3.c:18(int!null)
290-
│ │ │ │ ├── fd: (11)-->(12,13), (16)-->(17,18), (11)==(16), (16)==(11), (13)==(18), (18)==(13), (12)==(17), (17)==(12)
291-
│ │ │ │ ├── interesting orderings: (+11) (+16)
292-
│ │ │ │ ├── scan t65038
293-
│ │ │ │ │ └── unfiltered-cols: (6-10)
294-
│ │ │ │ ├── inner-join (merge)
295-
│ │ │ │ │ ├── columns: t2.a:11(int!null) t2.b:12(int!null) t2.c:13(int!null) t3.a:16(int!null) t3.b:17(int!null) t3.c:18(int!null)
296-
│ │ │ │ │ ├── left ordering: +11,+12,+13
297-
│ │ │ │ │ ├── right ordering: +16,+17,+18
298-
│ │ │ │ │ ├── key: (16)
299-
│ │ │ │ │ ├── fd: (11)-->(12,13), (16)-->(17,18), (11)==(16), (16)==(11), (13)==(18), (18)==(13), (12)==(17), (17)==(12)
300-
│ │ │ │ │ ├── interesting orderings: (+11) (+16)
301-
│ │ │ │ │ ├── scan t65038 [as=t2]
302-
│ │ │ │ │ │ ├── columns: t2.a:11(int!null) t2.b:12(int) t2.c:13(int)
303-
│ │ │ │ │ │ ├── key: (11)
304-
│ │ │ │ │ │ ├── fd: (11)-->(12,13)
305-
│ │ │ │ │ │ ├── ordering: +11
306-
│ │ │ │ │ │ ├── prune: (11-13)
307-
│ │ │ │ │ │ ├── interesting orderings: (+11)
308-
│ │ │ │ │ │ └── unfiltered-cols: (11-15)
309-
│ │ │ │ │ ├── scan t65038 [as=t3]
310-
│ │ │ │ │ │ ├── columns: t3.a:16(int!null) t3.b:17(int) t3.c:18(int)
311-
│ │ │ │ │ │ ├── key: (16)
312-
│ │ │ │ │ │ ├── fd: (16)-->(17,18)
313-
│ │ │ │ │ │ ├── ordering: +16
314-
│ │ │ │ │ │ ├── prune: (16-18)
315-
│ │ │ │ │ │ ├── interesting orderings: (+16)
316-
│ │ │ │ │ │ └── unfiltered-cols: (16-20)
317-
│ │ │ │ │ └── filters (true)
318-
│ │ │ │ └── filters (true)
319-
│ │ │ └── projections
320-
│ │ │ └── true [as=column23:23, type=bool]
321-
│ │ └── false [type=bool]
322-
│ └── eq [type=bool, outer=(2), constraints=(/2: [/1 - /1]; tight), fd=()-->(2)]
323-
│ ├── variable: t1.b:2 [type=int]
324-
│ └── const: 1 [type=int]
325-
└── projections
326-
└── const: 1 [as="?column?":24, type=int]
253+
├── fd: ()-->(23)
254+
├── ordering: +1 opt(23) [actual: +1]
255+
├── prune: (1,23)
256+
├── interesting orderings: (+1 opt(23))
257+
└── project
258+
├── columns: "?column?":23(int!null) t1.a:1(int!null)
259+
├── key: (1)
260+
├── fd: ()-->(23)
261+
├── prune: (1,23)
262+
├── interesting orderings: (+1 opt(23))
263+
├── semi-join (cross)
264+
│ ├── columns: t1.a:1(int!null) t1.b:2(int!null)
265+
│ ├── key: (1)
266+
│ ├── fd: ()-->(2)
267+
│ ├── prune: (1)
268+
│ ├── interesting orderings: (+1 opt(2))
269+
│ ├── select
270+
│ │ ├── columns: t1.a:1(int!null) t1.b:2(int!null)
271+
│ │ ├── key: (1)
272+
│ │ ├── fd: ()-->(2)
273+
│ │ ├── prune: (1)
274+
│ │ ├── interesting orderings: (+1 opt(2))
275+
│ │ ├── scan t65038 [as=t1]
276+
│ │ │ ├── columns: t1.a:1(int!null) t1.b:2(int)
277+
│ │ │ ├── key: (1)
278+
│ │ │ ├── fd: (1)-->(2)
279+
│ │ │ ├── prune: (1,2)
280+
│ │ │ └── interesting orderings: (+1)
281+
│ │ └── filters
282+
│ │ └── eq [type=bool, outer=(2), constraints=(/2: [/1 - /1]; tight), fd=()-->(2)]
283+
│ │ ├── variable: t1.b:2 [type=int]
284+
│ │ └── const: 1 [type=int]
285+
│ ├── top-k
286+
│ │ ├── columns: t2.a:11(int!null) t2.b:12(int!null) t2.c:13(int!null) t3.a:16(int!null) t3.b:17(int!null) t3.c:18(int!null)
287+
│ │ ├── internal-ordering: +(12|17)
288+
│ │ ├── k: 1
289+
│ │ ├── cardinality: [0 - 1]
290+
│ │ ├── key: ()
291+
│ │ ├── fd: ()-->(11-13,16-18), (11)==(16), (16)==(11), (13)==(18), (18)==(13), (12)==(17), (17)==(12)
292+
│ │ ├── interesting orderings: (+(12|17))
293+
│ │ └── inner-join (cross)
294+
│ │ ├── columns: t2.a:11(int!null) t2.b:12(int!null) t2.c:13(int!null) t3.a:16(int!null) t3.b:17(int!null) t3.c:18(int!null)
295+
│ │ ├── fd: (11)-->(12,13), (16)-->(17,18), (11)==(16), (16)==(11), (13)==(18), (18)==(13), (12)==(17), (17)==(12)
296+
│ │ ├── interesting orderings: (+11) (+16)
297+
│ │ ├── scan t65038
298+
│ │ │ └── unfiltered-cols: (6-10)
299+
│ │ ├── inner-join (merge)
300+
│ │ │ ├── columns: t2.a:11(int!null) t2.b:12(int!null) t2.c:13(int!null) t3.a:16(int!null) t3.b:17(int!null) t3.c:18(int!null)
301+
│ │ │ ├── left ordering: +11,+12,+13
302+
│ │ │ ├── right ordering: +16,+17,+18
303+
│ │ │ ├── key: (16)
304+
│ │ │ ├── fd: (11)-->(12,13), (16)-->(17,18), (11)==(16), (16)==(11), (13)==(18), (18)==(13), (12)==(17), (17)==(12)
305+
│ │ │ ├── interesting orderings: (+11) (+16)
306+
│ │ │ ├── scan t65038 [as=t2]
307+
│ │ │ │ ├── columns: t2.a:11(int!null) t2.b:12(int) t2.c:13(int)
308+
│ │ │ │ ├── key: (11)
309+
│ │ │ │ ├── fd: (11)-->(12,13)
310+
│ │ │ │ ├── ordering: +11
311+
│ │ │ │ ├── prune: (11-13)
312+
│ │ │ │ ├── interesting orderings: (+11)
313+
│ │ │ │ └── unfiltered-cols: (11-15)
314+
│ │ │ ├── scan t65038 [as=t3]
315+
│ │ │ │ ├── columns: t3.a:16(int!null) t3.b:17(int) t3.c:18(int)
316+
│ │ │ │ ├── key: (16)
317+
│ │ │ │ ├── fd: (16)-->(17,18)
318+
│ │ │ │ ├── ordering: +16
319+
│ │ │ │ ├── prune: (16-18)
320+
│ │ │ │ ├── interesting orderings: (+16)
321+
│ │ │ │ └── unfiltered-cols: (16-20)
322+
│ │ │ └── filters (true)
323+
│ │ └── filters (true)
324+
│ └── filters (true)
325+
└── projections
326+
└── const: 1 [as="?column?":23, type=int]
327327

328328
opt
329329
SELECT * FROM xyzs ORDER BY y DESC LIMIT 10

‎pkg/sql/opt/norm/decorrelate_funcs.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1566,3 +1566,9 @@ func (c *CustomFuncs) MakeAnyNotNullScalarGroupBy(input memo.RelExpr) memo.RelEx
15661566
memo.EmptyGroupingPrivate,
15671567
)
15681568
}
1569+
1570+
// CanHoistUnboundFilterFromExistsSubquery returns true if the
1571+
// HoistUnboundFilterFromExistsSubquery rule is enabled by session-setting.
1572+
func (c *CustomFuncs) CanHoistUnboundFilterFromExistsSubquery() bool {
1573+
return c.f.evalCtx.SessionData().OptimizerUseExistsFilterHoistRule
1574+
}

‎pkg/sql/opt/norm/rules/decorrelate.opt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -917,7 +917,7 @@
917917
# apply to the filter which was previously hidden inside the subquery.
918918
[HoistUnboundFilterFromExistsSubquery, Normalize]
919919
(Select
920-
$input:*
920+
$input:* & (CanHoistUnboundFilterFromExistsSubquery)
921921
$filters:[
922922
...
923923
$item:(FiltersItem
@@ -963,7 +963,7 @@
963963
# HoistUnboundFilterFromExistsSubquery, but it applies to a join filter.
964964
[HoistUnboundJoinFilterFromExistsSubquery, Normalize]
965965
(Select
966-
$input:*
966+
$input:* & (CanHoistUnboundFilterFromExistsSubquery)
967967
$filters:[
968968
...
969969
$item:(FiltersItem

0 commit comments

Comments
 (0)