From 83578afb9df376cb357ed369e5b159fd5cc98735 Mon Sep 17 00:00:00 2001 From: trouze Date: Fri, 16 May 2025 13:34:53 -0500 Subject: [PATCH 1/6] feat(core/dbt/graph/selector_methods.py): update ConfigSelectorMethod to select from all configurable nodes --- core/dbt/graph/selector_methods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index 41a9cc0d51c..fe4d36ceb3c 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -536,7 +536,7 @@ def search( # search sources is kind of useless now source configs only have # 'enabled', which you can't really filter on anyway, but maybe we'll # add more someday, so search them anyway. - for unique_id, node in self.configurable_nodes(included_nodes): + for unique_id, node in self.all_nodes(included_nodes): try: value = _getattr_descend(node.config, parts) except AttributeError: From 3481c3cc45c6c496bd4046d94812e50fc40e2fec Mon Sep 17 00:00:00 2001 From: trouze Date: Fri, 16 May 2025 13:36:14 -0500 Subject: [PATCH 2/6] feat(core/dbt/graph/selector_methods.py): remove configurable_nodes method from SelectorMethod class --- core/dbt/graph/selector_methods.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index fe4d36ceb3c..cee54939489 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -205,11 +205,6 @@ def all_nodes( self.saved_query_nodes(included_nodes), ) - def configurable_nodes( - self, included_nodes: Set[UniqueId] - ) -> Iterator[Tuple[UniqueId, ResultNode]]: - yield from chain(self.parsed_nodes(included_nodes), self.source_nodes(included_nodes)) - def non_source_nodes( self, included_nodes: Set[UniqueId], From 5d37b84f8b261e3c43f0109dda2b3360335cbf5f Mon Sep 17 00:00:00 2001 From: trouze Date: Fri, 27 Jun 2025 09:42:02 -0500 Subject: [PATCH 3/6] fix(core/dbt/graph): remove unused ResultNode from selector_methods --- .changes/unreleased/Features-20250627-094000.yaml | 6 ++++++ core/dbt/graph/selector_methods.py | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Features-20250627-094000.yaml diff --git a/.changes/unreleased/Features-20250627-094000.yaml b/.changes/unreleased/Features-20250627-094000.yaml new file mode 100644 index 00000000000..f7160c1457c --- /dev/null +++ b/.changes/unreleased/Features-20250627-094000.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Let ConfigSelectorMethod search through all nodes to return matching selections +time: 2025-06-27T09:40:00.485654-05:00 +custom: + Author: trouze + Issue: "11607" diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index cee54939489..966daf9bdd6 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -22,7 +22,6 @@ ManifestNode, Metric, ModelNode, - ResultNode, SavedQuery, SemanticModel, SingularTestNode, From c4d8f3c695db235969d0d9e25ce4682ecdbce052 Mon Sep 17 00:00:00 2001 From: trouze Date: Wed, 20 Aug 2025 09:55:21 -0500 Subject: [PATCH 4/6] chore: add functional and unit tests for ConfigSelectorMethod --- .../graph_selection/test_config_selection.py | 281 ++++++++++++++++++ tests/unit/graph/test_selector_methods.py | 122 ++++++++ 2 files changed, 403 insertions(+) create mode 100644 tests/functional/graph_selection/test_config_selection.py diff --git a/tests/functional/graph_selection/test_config_selection.py b/tests/functional/graph_selection/test_config_selection.py new file mode 100644 index 00000000000..63496aa0a50 --- /dev/null +++ b/tests/functional/graph_selection/test_config_selection.py @@ -0,0 +1,281 @@ +import pytest + +from dbt.tests.util import run_dbt + +# Test fixtures - dbt project files +models__model_enabled_sql = """ +{{ config(materialized="table", enabled=true, meta={"env": "prod"}) }} +select 1 as id +""" + +models__model_disabled_sql = """ +{{ config(materialized="view", enabled=false, meta={"env": "dev"}) }} +select 1 as id +""" + +models__base_model_sql = """ +select 1 as id, 'test' as name +""" + +schema_yml = """ +version: 2 + +models: + - name: base_model + description: "Base model for semantic layer" + columns: + - name: id + description: "ID column" + - name: name + description: "Name column" + +metrics: + - name: enabled_metric + description: "Metric that is enabled" + type: simple + type_params: + measure: + name: base_measure + config: + enabled: true + meta: + env: "prod" + + - name: disabled_metric + description: "Metric that is disabled" + type: simple + type_params: + measure: + name: base_measure + config: + enabled: false + meta: + env: "dev" + +semantic_models: + - name: enabled_semantic_model + description: "Semantic model that is enabled" + model: ref('base_model') + config: + enabled: true + meta: + env: "prod" + dimensions: + - name: id + type: categorical + type_params: + values: [1, 2, 3] + measures: + - name: base_measure + agg: count + + - name: disabled_semantic_model + description: "Semantic model that is disabled" + model: ref('base_model') + config: + enabled: false + meta: + env: "dev" + dimensions: + - name: id + type: categorical + type_params: + values: [1, 2, 3] + measures: + - name: base_measure + agg: count + +saved_queries: + - name: enabled_saved_query + description: "Saved query that is enabled" + config: + enabled: true + meta: + env: "prod" + query_params: + metrics: + - enabled_metric + dimensions: + - id + + - name: disabled_saved_query + description: "Saved query that is disabled" + config: + enabled: false + meta: + env: "dev" + query_params: + metrics: + - enabled_metric + dimensions: + - id +""" + + +class TestConfigSelection: + """Test config selectors work on all node types including newly supported ones.""" + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "model_enabled.sql": models__model_enabled_sql, + "model_disabled.sql": models__model_disabled_sql, + "base_model.sql": models__base_model_sql, + "schema.yml": schema_yml, + } + + def test_config_enabled_true_selects_all_enabled_nodes(self, project): + """Test that config.enabled:true selects all enabled nodes including new types.""" + # First run dbt to ensure everything is parsed + run_dbt(["parse"]) + + # Test that config.enabled:true finds all enabled nodes + results = run_dbt(["list", "--select", "config.enabled:true"]) + + # Should include enabled models, metrics, semantic models, and saved queries + selected_nodes = [result.split(".")[-1] for result in results] + + # Models + assert "model_enabled" in selected_nodes + assert "base_model" in selected_nodes # enabled by default + + # New node types that should now be selectable + assert "enabled_metric" in selected_nodes + assert "enabled_semantic_model" in selected_nodes + assert "enabled_saved_query" in selected_nodes + + # Should NOT include disabled nodes + assert "model_disabled" not in selected_nodes + assert "disabled_metric" not in selected_nodes + assert "disabled_semantic_model" not in selected_nodes + assert "disabled_saved_query" not in selected_nodes + + def test_config_enabled_false_selects_disabled_nodes(self, project): + """Test that config.enabled:false selects disabled nodes including new types.""" + run_dbt(["parse"]) + + results = run_dbt(["list", "--select", "config.enabled:false"]) + selected_nodes = [result.split(".")[-1] for result in results] + + # Should include disabled nodes + assert "model_disabled" in selected_nodes + assert "disabled_metric" in selected_nodes + assert "disabled_semantic_model" in selected_nodes + assert "disabled_saved_query" in selected_nodes + + # Should NOT include enabled nodes + assert "model_enabled" not in selected_nodes + assert "enabled_metric" not in selected_nodes + assert "enabled_semantic_model" not in selected_nodes + assert "enabled_saved_query" not in selected_nodes + + def test_config_meta_env_selects_nodes_by_meta(self, project): + """Test that config.meta.env selects nodes by meta config including new types.""" + run_dbt(["parse"]) + + # Test production environment nodes + results = run_dbt(["list", "--select", "config.meta.env:prod"]) + selected_nodes = [result.split(".")[-1] for result in results] + + assert "model_enabled" in selected_nodes + assert "enabled_metric" in selected_nodes + assert "enabled_semantic_model" in selected_nodes + assert "enabled_saved_query" in selected_nodes + + # Test development environment nodes + results = run_dbt(["list", "--select", "config.meta.env:dev"]) + selected_nodes = [result.split(".")[-1] for result in results] + + assert "model_disabled" in selected_nodes + assert "disabled_metric" in selected_nodes + assert "disabled_semantic_model" in selected_nodes + assert "disabled_saved_query" in selected_nodes + + def test_config_materialized_selects_models_only(self, project): + """Test that config.materialized only selects models (since other node types don't have this config).""" + run_dbt(["parse"]) + + # Test table materialization + results = run_dbt(["list", "--select", "config.materialized:table"]) + selected_nodes = [result.split(".")[-1] for result in results] + + assert "model_enabled" in selected_nodes + # Should not include metrics, semantic models, or saved queries + assert "enabled_metric" not in selected_nodes + assert "enabled_semantic_model" not in selected_nodes + assert "enabled_saved_query" not in selected_nodes + + # Test view materialization + results = run_dbt(["list", "--select", "config.materialized:view"]) + selected_nodes = [result.split(".")[-1] for result in results] + + assert "model_disabled" in selected_nodes + assert "base_model" in selected_nodes # default materialization + + def test_config_selector_with_resource_type_filter(self, project): + """Test combining config selectors with resource type filters.""" + run_dbt(["parse"]) + + # Test enabled metrics only + results = run_dbt(["list", "--select", "config.enabled:true", "--resource-type", "metric"]) + selected_nodes = [result.split(".")[-1] for result in results] + + assert "enabled_metric" in selected_nodes + assert "disabled_metric" not in selected_nodes + # Should not include models or other types + assert "model_enabled" not in selected_nodes + assert "enabled_semantic_model" not in selected_nodes + + # Test enabled semantic models only + results = run_dbt( + ["list", "--select", "config.enabled:true", "--resource-type", "semantic_model"] + ) + selected_nodes = [result.split(".")[-1] for result in results] + + assert "enabled_semantic_model" in selected_nodes + assert "disabled_semantic_model" not in selected_nodes + # Should not include other types + assert "enabled_metric" not in selected_nodes + assert "model_enabled" not in selected_nodes + + def test_config_selector_demonstrates_expansion_from_configurable_to_all_nodes(self, project): + """Test that demonstrates the key change: config selectors now work on all node types. + + This test specifically validates that the change from configurable_nodes() to all_nodes() + in ConfigSelectorMethod allows selection of node types that were previously not selectable. + """ + run_dbt(["parse"]) + + # Before the change, this would only find models and sources + # After the change, this should also find metrics, semantic models, and saved queries + results = run_dbt(["list", "--select", "config.enabled:true"]) + + # Count different node types in results + models = [r for r in results if r.startswith("model.")] + metrics = [r for r in results if r.startswith("metric.")] + semantic_models = [r for r in results if r.startswith("semantic_model.")] + saved_queries = [r for r in results if r.startswith("saved_query.")] + + # Verify we found nodes of all types (demonstrating all_nodes() expansion) + assert len(models) > 0, "Should find model nodes" + assert len(metrics) > 0, "Should find metric nodes (new with all_nodes())" + assert len(semantic_models) > 0, "Should find semantic model nodes (new with all_nodes())" + assert len(saved_queries) > 0, "Should find saved query nodes (new with all_nodes())" + + # The total should include nodes from all types + total_nodes = len(results) + traditional_nodes = len(models) # + sources would be here if we had any + new_nodes = len(metrics) + len(semantic_models) + len(saved_queries) + + assert ( + new_nodes > 0 + ), "Should find additional node types beyond traditional configurable nodes" + assert ( + total_nodes >= traditional_nodes + new_nodes + ), "Total should include both traditional and new node types" diff --git a/tests/unit/graph/test_selector_methods.py b/tests/unit/graph/test_selector_methods.py index d500c631a1b..e833048bd1c 100644 --- a/tests/unit/graph/test_selector_methods.py +++ b/tests/unit/graph/test_selector_methods.py @@ -427,6 +427,128 @@ def test_select_config_meta(manifest): assert not search_manifest_using_method(manifest, list_method, "other") == {"table_model"} +def test_select_config_enabled_on_all_nodes(manifest): + """Test that ConfigSelectorMethod can find enabled property on all node types. + + This test verifies that the change from configurable_nodes() to all_nodes() + allows the ConfigSelectorMethod to search through all node types, not just + the previous subset of parsed_nodes and source_nodes. + """ + methods = MethodManager(manifest, None) + method = methods.get_method("config", ["enabled"]) + assert isinstance(method, ConfigSelectorMethod) + assert method.arguments == ["enabled"] + + # All nodes should have enabled: true by default + enabled_nodes = search_manifest_using_method(manifest, method, "true") + + # Should include traditional configurable nodes (models, sources) + assert "table_model" in enabled_nodes + assert "view_model" in enabled_nodes + assert "ext_raw.ext_source" in enabled_nodes + + # Should now also include the new node types that were added via all_nodes() + # Metrics, semantic models, saved queries should all have enabled: true by default + assert "my_metric" in enabled_nodes + assert "test_semantic_model" in enabled_nodes + assert "test_saved_query" in enabled_nodes + + +def test_select_config_searches_more_nodes_than_before(manifest): + """Test that ConfigSelectorMethod now searches more node types than before. + + This test verifies that the change from configurable_nodes() to all_nodes() + actually expands the set of searchable nodes beyond just models and sources. + """ + methods = MethodManager(manifest, None) + + # Test that we can search for config properties that exist on various node types + enabled_method = methods.get_method("config", ["enabled"]) + all_enabled_nodes = search_manifest_using_method(manifest, enabled_method, "true") + + # Verify specific new node types are found (these were not in configurable_nodes()) + metrics = [node for node in all_enabled_nodes if "my_metric" in node] + semantic_models = [node for node in all_enabled_nodes if "test_semantic_model" in node] + saved_queries = [node for node in all_enabled_nodes if "test_saved_query" in node] + + # With all_nodes(), we should find these new node types + assert len(metrics) > 0, "Should find metric nodes (new with all_nodes)" + assert len(semantic_models) > 0, "Should find semantic model nodes (new with all_nodes)" + assert len(saved_queries) > 0, "Should find saved query nodes (new with all_nodes)" + + # Specifically check that these new node types are found + assert "my_metric" in all_enabled_nodes, "Metric should be searchable with config selector" + assert ( + "test_semantic_model" in all_enabled_nodes + ), "Semantic model should be searchable with config selector" + assert ( + "test_saved_query" in all_enabled_nodes + ), "Saved query should be searchable with config selector" + + # The total should include many types of nodes due to all_nodes() expansion + total_nodes = len(all_enabled_nodes) + assert ( + total_nodes > 20 + ), f"Should find many nodes with all_nodes() expansion, found {total_nodes}" + + +def test_config_selector_finds_nodes_not_in_old_configurable_nodes(manifest): + """Test that ConfigSelectorMethod now finds nodes that were excluded by configurable_nodes(). + + Before the change, ConfigSelectorMethod.search() used configurable_nodes() which only + included parsed_nodes() and source_nodes(). Now it uses all_nodes() which includes + exposure_nodes(), metric_nodes(), unit_tests(), semantic_model_nodes(), and saved_query_nodes(). + + This test verifies that the new node types are now searchable. + """ + methods = MethodManager(manifest, None) + method = methods.get_method("config", ["enabled"]) + + # Search for enabled nodes + enabled_nodes = search_manifest_using_method(manifest, method, "true") + + # These node types were NOT included in the old configurable_nodes() method + # but should now be found via all_nodes() + + # Metrics should be found (they have MetricConfig with enabled property) + # search_name for metrics is just the metric name, not the full unique_id + assert ( + "my_metric" in enabled_nodes + ), "Specific metric should be found by config selector (new functionality)" + + # Semantic models should be found (they have SemanticModelConfig with enabled property) + assert ( + "test_semantic_model" in enabled_nodes + ), "Specific semantic model should be found by config selector (new functionality)" + + # Saved queries should be found (they have SavedQueryConfig with enabled property) + assert ( + "test_saved_query" in enabled_nodes + ), "Specific saved query should be found by config selector (new functionality)" + + # Unit tests should also be found (they have config.enabled) + unit_tests_found = [node for node in enabled_nodes if "unit_test" in node] + assert ( + len(unit_tests_found) > 0 + ), "Unit tests should be found by config selector (new functionality)" + + # Count the new node types we can find + new_node_types = 0 + if "my_metric" in enabled_nodes: + new_node_types += 1 + if "test_semantic_model" in enabled_nodes: + new_node_types += 1 + if "test_saved_query" in enabled_nodes: + new_node_types += 1 + new_node_types += len(unit_tests_found) + + # This demonstrates the key change: ConfigSelectorMethod can now search through + # many more node types than just the traditional "configurable" nodes + assert ( + new_node_types >= 4 + ), f"Should find at least 4 nodes from newly included types, found {new_node_types}" + + def test_select_test_name(manifest): methods = MethodManager(manifest, None) method = methods.get_method("test_name", []) From 70b169147e76f4ad8e4668aeb4f0e6cead5e44c1 Mon Sep 17 00:00:00 2001 From: trouze Date: Wed, 20 Aug 2025 10:59:27 -0500 Subject: [PATCH 5/6] chore: update functional tests for ConfigSelectorMethod --- .../graph_selection/test_config_selection.py | 339 ++++++------------ 1 file changed, 114 insertions(+), 225 deletions(-) diff --git a/tests/functional/graph_selection/test_config_selection.py b/tests/functional/graph_selection/test_config_selection.py index 63496aa0a50..f503c7a0fe3 100644 --- a/tests/functional/graph_selection/test_config_selection.py +++ b/tests/functional/graph_selection/test_config_selection.py @@ -1,281 +1,170 @@ -import pytest - -from dbt.tests.util import run_dbt - -# Test fixtures - dbt project files -models__model_enabled_sql = """ -{{ config(materialized="table", enabled=true, meta={"env": "prod"}) }} -select 1 as id """ +Functional tests for config selector functionality. -models__model_disabled_sql = """ -{{ config(materialized="view", enabled=false, meta={"env": "dev"}) }} -select 1 as id +These tests validate that config selectors work correctly after the change from +configurable_nodes() to all_nodes() in ConfigSelectorMethod. """ -models__base_model_sql = """ -select 1 as id, 'test' as name -""" +import pytest -schema_yml = """ -version: 2 +from dbt.tests.util import run_dbt -models: - - name: base_model - description: "Base model for semantic layer" - columns: - - name: id - description: "ID column" - - name: name - description: "Name column" -metrics: - - name: enabled_metric - description: "Metric that is enabled" - type: simple - type_params: - measure: - name: base_measure - config: - enabled: true - meta: - env: "prod" +class TestConfigSelection: + """Test config selectors work on all node types including newly supported ones.""" - - name: disabled_metric - description: "Metric that is disabled" - type: simple - type_params: - measure: - name: base_measure - config: - enabled: false - meta: - env: "dev" + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "base_model.sql": "select 1 as id, 'test' as name, current_timestamp as created_at", + "model_enabled.sql": "select * from {{ ref('base_model') }}", + "model_view.sql": "select * from {{ ref('base_model') }} where id = 1", + "metricflow_time_spine.sql": "SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day", + "semantic_models.yml": """ +version: 2 semantic_models: - - name: enabled_semantic_model - description: "Semantic model that is enabled" + - name: test_semantic_model + label: "Test Semantic Model" model: ref('base_model') config: enabled: true meta: - env: "prod" + semantic_layer: true dimensions: - - name: id - type: categorical + - name: created_at + type: time type_params: - values: [1, 2, 3] + time_granularity: day measures: - - name: base_measure + - name: total_count agg: count - - - name: disabled_semantic_model - description: "Semantic model that is disabled" - model: ref('base_model') - config: - enabled: false - meta: - env: "dev" - dimensions: + expr: 1 + entities: - name: id - type: categorical - type_params: - values: [1, 2, 3] - measures: - - name: base_measure - agg: count + type: primary + defaults: + agg_time_dimension: created_at -saved_queries: - - name: enabled_saved_query - description: "Saved query that is enabled" +metrics: + - name: test_metric + label: "Test Metric" + type: simple config: enabled: true meta: - env: "prod" - query_params: - metrics: - - enabled_metric - dimensions: - - id + metric_type: simple + type_params: + measure: total_count - - name: disabled_saved_query - description: "Saved query that is disabled" +saved_queries: + - name: test_saved_query + label: "Test Saved Query" config: - enabled: false + enabled: true meta: - env: "dev" + query_type: basic query_params: metrics: - - enabled_metric - dimensions: - - id -""" - - -class TestConfigSelection: - """Test config selectors work on all node types including newly supported ones.""" - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "config-version": 2, + - test_metric + group_by: + - "Dimension('test_semantic_model__created_at')" + exports: + - name: test_export + config: + alias: test_export_alias + export_as: table +""", } - @pytest.fixture(scope="class") - def models(self): - return { - "model_enabled.sql": models__model_enabled_sql, - "model_disabled.sql": models__model_disabled_sql, - "base_model.sql": models__base_model_sql, - "schema.yml": schema_yml, - } + def test_basic_semantic_layer_parsing(self, project): + """Test basic parsing of semantic layer components.""" + try: + result = run_dbt(["parse"]) + print(f"Parse result: {result}") - def test_config_enabled_true_selects_all_enabled_nodes(self, project): - """Test that config.enabled:true selects all enabled nodes including new types.""" - # First run dbt to ensure everything is parsed - run_dbt(["parse"]) - - # Test that config.enabled:true finds all enabled nodes - results = run_dbt(["list", "--select", "config.enabled:true"]) + # List all resources to see what was parsed + results = run_dbt(["list"]) + print(f"All resources: {sorted(results)}") - # Should include enabled models, metrics, semantic models, and saved queries - selected_nodes = [result.split(".")[-1] for result in results] + # Look for semantic layer components + semantic_models = [r for r in results if "semantic_model" in r] + metrics = [r for r in results if "metric" in r] + saved_queries = [r for r in results if "saved_query" in r] - # Models - assert "model_enabled" in selected_nodes - assert "base_model" in selected_nodes # enabled by default + print(f"Semantic models: {semantic_models}") + print(f"Metrics: {metrics}") + print(f"Saved queries: {saved_queries}") - # New node types that should now be selectable - assert "enabled_metric" in selected_nodes - assert "enabled_semantic_model" in selected_nodes - assert "enabled_saved_query" in selected_nodes + # At minimum should have models + models = [r for r in results if "model" in r and "semantic" not in r] + assert len(models) >= 3, f"Should have at least 3 models, found: {models}" - # Should NOT include disabled nodes - assert "model_disabled" not in selected_nodes - assert "disabled_metric" not in selected_nodes - assert "disabled_semantic_model" not in selected_nodes - assert "disabled_saved_query" not in selected_nodes + except Exception as e: + print(f"Error during parsing: {e}") + # Let's try a simpler approach - just test models without semantic layer + pass - def test_config_enabled_false_selects_disabled_nodes(self, project): - """Test that config.enabled:false selects disabled nodes including new types.""" - run_dbt(["parse"]) - - results = run_dbt(["list", "--select", "config.enabled:false"]) - selected_nodes = [result.split(".")[-1] for result in results] - - # Should include disabled nodes - assert "model_disabled" in selected_nodes - assert "disabled_metric" in selected_nodes - assert "disabled_semantic_model" in selected_nodes - assert "disabled_saved_query" in selected_nodes - - # Should NOT include enabled nodes - assert "model_enabled" not in selected_nodes - assert "enabled_metric" not in selected_nodes - assert "enabled_semantic_model" not in selected_nodes - assert "enabled_saved_query" not in selected_nodes - - def test_config_meta_env_selects_nodes_by_meta(self, project): - """Test that config.meta.env selects nodes by meta config including new types.""" - run_dbt(["parse"]) - - # Test production environment nodes - results = run_dbt(["list", "--select", "config.meta.env:prod"]) - selected_nodes = [result.split(".")[-1] for result in results] - - assert "model_enabled" in selected_nodes - assert "enabled_metric" in selected_nodes - assert "enabled_semantic_model" in selected_nodes - assert "enabled_saved_query" in selected_nodes - - # Test development environment nodes - results = run_dbt(["list", "--select", "config.meta.env:dev"]) - selected_nodes = [result.split(".")[-1] for result in results] - - assert "model_disabled" in selected_nodes - assert "disabled_metric" in selected_nodes - assert "disabled_semantic_model" in selected_nodes - assert "disabled_saved_query" in selected_nodes - - def test_config_materialized_selects_models_only(self, project): - """Test that config.materialized only selects models (since other node types don't have this config).""" + def test_config_enabled_true_selects_all_enabled_nodes(self, project): + """Test that config.enabled:true selects all enabled nodes.""" run_dbt(["parse"]) - # Test table materialization - results = run_dbt(["list", "--select", "config.materialized:table"]) - selected_nodes = [result.split(".")[-1] for result in results] - - assert "model_enabled" in selected_nodes - # Should not include metrics, semantic models, or saved queries - assert "enabled_metric" not in selected_nodes - assert "enabled_semantic_model" not in selected_nodes - assert "enabled_saved_query" not in selected_nodes - - # Test view materialization - results = run_dbt(["list", "--select", "config.materialized:view"]) - selected_nodes = [result.split(".")[-1] for result in results] + results = run_dbt(["list", "--select", "config.enabled:true"]) + selected_nodes = set(results) - assert "model_disabled" in selected_nodes - assert "base_model" in selected_nodes # default materialization + # Should include all enabled models + assert "test.base_model" in selected_nodes + assert "test.model_enabled" in selected_nodes + assert "test.model_view" in selected_nodes def test_config_selector_with_resource_type_filter(self, project): - """Test combining config selectors with resource type filters.""" + """Test config selectors with resource type filters.""" run_dbt(["parse"]) - # Test enabled metrics only - results = run_dbt(["list", "--select", "config.enabled:true", "--resource-type", "metric"]) - selected_nodes = [result.split(".")[-1] for result in results] - - assert "enabled_metric" in selected_nodes - assert "disabled_metric" not in selected_nodes - # Should not include models or other types - assert "model_enabled" not in selected_nodes - assert "enabled_semantic_model" not in selected_nodes + # Test that config selectors work with resource type filters + results = run_dbt(["list", "--resource-type", "model", "--select", "config.enabled:true"]) + selected_nodes = set(results) - # Test enabled semantic models only - results = run_dbt( - ["list", "--select", "config.enabled:true", "--resource-type", "semantic_model"] - ) - selected_nodes = [result.split(".")[-1] for result in results] - - assert "enabled_semantic_model" in selected_nodes - assert "disabled_semantic_model" not in selected_nodes - # Should not include other types - assert "enabled_metric" not in selected_nodes - assert "model_enabled" not in selected_nodes + # Should only include models + assert "test.base_model" in selected_nodes + assert "test.model_enabled" in selected_nodes + assert "test.model_view" in selected_nodes def test_config_selector_demonstrates_expansion_from_configurable_to_all_nodes(self, project): """Test that demonstrates the key change: config selectors now work on all node types. This test specifically validates that the change from configurable_nodes() to all_nodes() in ConfigSelectorMethod allows selection of node types that were previously not selectable. + + Before the change, ConfigSelectorMethod.configurable_nodes() only returned models and sources. + After the change, ConfigSelectorMethod.all_nodes() returns ALL node types in the graph, + including metrics, semantic models, saved queries, and any other node type that might + have config properties. """ run_dbt(["parse"]) # Before the change, this would only find models and sources - # After the change, this should also find metrics, semantic models, and saved queries + # After the change, this should also find any additional node types results = run_dbt(["list", "--select", "config.enabled:true"]) - # Count different node types in results - models = [r for r in results if r.startswith("model.")] - metrics = [r for r in results if r.startswith("metric.")] - semantic_models = [r for r in results if r.startswith("semantic_model.")] - saved_queries = [r for r in results if r.startswith("saved_query.")] - - # Verify we found nodes of all types (demonstrating all_nodes() expansion) - assert len(models) > 0, "Should find model nodes" - assert len(metrics) > 0, "Should find metric nodes (new with all_nodes())" - assert len(semantic_models) > 0, "Should find semantic model nodes (new with all_nodes())" - assert len(saved_queries) > 0, "Should find saved query nodes (new with all_nodes())" - - # The total should include nodes from all types - total_nodes = len(results) - traditional_nodes = len(models) # + sources would be here if we had any - new_nodes = len(metrics) + len(semantic_models) + len(saved_queries) - - assert ( - new_nodes > 0 - ), "Should find additional node types beyond traditional configurable nodes" - assert ( - total_nodes >= traditional_nodes + new_nodes - ), "Total should include both traditional and new node types" + # Verify we found all our enabled model nodes + selected_nodes = set(results) + assert "test.base_model" in selected_nodes + assert "test.model_enabled" in selected_nodes + assert "test.model_view" in selected_nodes + + # The key demonstration: these config selectors now work on all node types + # This test validates the core functionality works as expected + assert len(results) >= 3, "Should find all enabled nodes" + + # Additional validation: verify config selectors can work with any node type + # that has config properties (this is the expansion from configurable_nodes to all_nodes) + print(f"Selected nodes: {sorted(selected_nodes)}") + for node in selected_nodes: + assert "test." in node, f"All nodes should be from test project: {node}" From 3b12a3667aad4ab3bbe3d211bc3c434e639b86ba Mon Sep 17 00:00:00 2001 From: trouze Date: Wed, 20 Aug 2025 22:22:54 -0500 Subject: [PATCH 6/6] chore: cleanup config selection functional tests --- tests/functional/graph_selection/fixtures.py | 3 +- .../graph_selection/test_config_selection.py | 130 +++++++----------- 2 files changed, 52 insertions(+), 81 deletions(-) diff --git a/tests/functional/graph_selection/fixtures.py b/tests/functional/graph_selection/fixtures.py index b4d4a677d25..7c58039e1c0 100644 --- a/tests/functional/graph_selection/fixtures.py +++ b/tests/functional/graph_selection/fixtures.py @@ -109,7 +109,8 @@ {{ config( materialized = 'table', - tags=['bi', 'users'] + tags=['bi', 'users'], + meta={'contains_pii':true} ) }} diff --git a/tests/functional/graph_selection/test_config_selection.py b/tests/functional/graph_selection/test_config_selection.py index f503c7a0fe3..efd7829aa3c 100644 --- a/tests/functional/graph_selection/test_config_selection.py +++ b/tests/functional/graph_selection/test_config_selection.py @@ -8,6 +8,19 @@ import pytest from dbt.tests.util import run_dbt +from tests.functional.graph_selection.fixtures import ( + alternative_users_sql, + base_users_sql, + emails_alt_sql, + emails_sql, + nested_users_sql, + never_selected_sql, + schema_yml, + subdir_sql, + users_rollup_dependency_sql, + users_rollup_sql, + users_sql, +) class TestConfigSelection: @@ -22,9 +35,20 @@ def project_config_update(self): @pytest.fixture(scope="class") def models(self): return { - "base_model.sql": "select 1 as id, 'test' as name, current_timestamp as created_at", - "model_enabled.sql": "select * from {{ ref('base_model') }}", - "model_view.sql": "select * from {{ ref('base_model') }} where id = 1", + "schema.yml": schema_yml, + "base_users.sql": base_users_sql, + "users.sql": users_sql, + "users_rollup.sql": users_rollup_sql, + "versioned_v3.sql": base_users_sql, + "users_rollup_dependency.sql": users_rollup_dependency_sql, + "emails.sql": emails_sql, + "emails_alt.sql": emails_alt_sql, + "alternative.users.sql": alternative_users_sql, + "never_selected.sql": never_selected_sql, + "test": { + "subdir.sql": subdir_sql, + "subdir": {"nested_users.sql": nested_users_sql}, + }, "metricflow_time_spine.sql": "SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day", "semantic_models.yml": """ version: 2 @@ -32,11 +56,7 @@ def models(self): semantic_models: - name: test_semantic_model label: "Test Semantic Model" - model: ref('base_model') - config: - enabled: true - meta: - semantic_layer: true + model: ref('users') dimensions: - name: created_at type: time @@ -70,6 +90,7 @@ def models(self): enabled: true meta: query_type: basic + contains_pii: true query_params: metrics: - test_metric @@ -83,88 +104,37 @@ def models(self): """, } - def test_basic_semantic_layer_parsing(self, project): - """Test basic parsing of semantic layer components.""" - try: - result = run_dbt(["parse"]) - print(f"Parse result: {result}") - - # List all resources to see what was parsed - results = run_dbt(["list"]) - print(f"All resources: {sorted(results)}") - - # Look for semantic layer components - semantic_models = [r for r in results if "semantic_model" in r] - metrics = [r for r in results if "metric" in r] - saved_queries = [r for r in results if "saved_query" in r] - - print(f"Semantic models: {semantic_models}") - print(f"Metrics: {metrics}") - print(f"Saved queries: {saved_queries}") - - # At minimum should have models - models = [r for r in results if "model" in r and "semantic" not in r] - assert len(models) >= 3, f"Should have at least 3 models, found: {models}" - - except Exception as e: - print(f"Error during parsing: {e}") - # Let's try a simpler approach - just test models without semantic layer - pass - - def test_config_enabled_true_selects_all_enabled_nodes(self, project): - """Test that config.enabled:true selects all enabled nodes.""" - run_dbt(["parse"]) - - results = run_dbt(["list", "--select", "config.enabled:true"]) - selected_nodes = set(results) - - # Should include all enabled models - assert "test.base_model" in selected_nodes - assert "test.model_enabled" in selected_nodes - assert "test.model_view" in selected_nodes - def test_config_selector_with_resource_type_filter(self, project): """Test config selectors with resource type filters.""" - run_dbt(["parse"]) - # Test that config selectors work with resource type filters results = run_dbt(["list", "--resource-type", "model", "--select", "config.enabled:true"]) selected_nodes = set(results) - # Should only include models - assert "test.base_model" in selected_nodes - assert "test.model_enabled" in selected_nodes - assert "test.model_view" in selected_nodes - - def test_config_selector_demonstrates_expansion_from_configurable_to_all_nodes(self, project): - """Test that demonstrates the key change: config selectors now work on all node types. + assert "saved_query:test.test_saved_query" not in selected_nodes + assert "metric:test.test_metric" not in selected_nodes + assert "semantic_model:test.test_semantic_model" not in selected_nodes - This test specifically validates that the change from configurable_nodes() to all_nodes() - in ConfigSelectorMethod allows selection of node types that were previously not selectable. + def test_config_enabled_true_selects_extended_nodes(self, project): + """Test that dbt ls -s config.enabled:true returns the test_saved_query. - Before the change, ConfigSelectorMethod.configurable_nodes() only returned models and sources. - After the change, ConfigSelectorMethod.all_nodes() returns ALL node types in the graph, - including metrics, semantic models, saved queries, and any other node type that might - have config properties. + This specific test validates that the saved query (which has config.enabled:true) + is properly selected by the config selector. This demonstrates that the change from + configurable_nodes() to all_nodes() allows config selectors to work on saved queries. """ - run_dbt(["parse"]) - # Before the change, this would only find models and sources - # After the change, this should also find any additional node types results = run_dbt(["list", "--select", "config.enabled:true"]) + selected_nodes = set(results) + + assert "saved_query:test.test_saved_query" in selected_nodes + assert "metric:test.test_metric" in selected_nodes + assert "semantic_model:test.test_semantic_model" in selected_nodes - # Verify we found all our enabled model nodes + def test_config_meta_selection(self, project): + """ """ + + results = run_dbt(["list", "--select", "config.meta.contains_pii:true"]) selected_nodes = set(results) - assert "test.base_model" in selected_nodes - assert "test.model_enabled" in selected_nodes - assert "test.model_view" in selected_nodes - - # The key demonstration: these config selectors now work on all node types - # This test validates the core functionality works as expected - assert len(results) >= 3, "Should find all enabled nodes" - - # Additional validation: verify config selectors can work with any node type - # that has config properties (this is the expansion from configurable_nodes to all_nodes) - print(f"Selected nodes: {sorted(selected_nodes)}") - for node in selected_nodes: - assert "test." in node, f"All nodes should be from test project: {node}" + + assert "test.users" in selected_nodes + assert "saved_query:test.test_saved_query" in selected_nodes + assert "test.unique_users_id" in selected_nodes