diff --git a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml index 724f5a0b87ce3..69472b80af075 100644 --- a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml @@ -152,6 +152,7 @@ body: - dingding - discord - docker + - dq - edge3 - elasticsearch - exasol diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 2848323162932..a6cd6f810651e 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -141,6 +141,9 @@ labelPRBasedOnFilePath: provider:docker: - providers/docker/** + provider:dq: + - providers/dq/** + provider:edge: - providers/edge3/** diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f0337bb052b80..2f633aadb65da 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -146,6 +146,7 @@ updates: cooldown: default-days: 4 directories: + - /providers/dq/src/airflow/providers/dq/plugins/www - /providers/edge3/src/airflow/providers/edge3/plugins/www schedule: interval: "weekly" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a1a0de273842c..2efa503d5406a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -784,6 +784,7 @@ repos: ^airflow-core/src/airflow/ui/openapi-gen/| ^airflow-core/src/airflow/ui/pnpm-lock\.yaml$| ^providers/common/ai/src/airflow/providers/common/ai/plugins/www/pnpm-lock\.yaml$| + ^providers/dq/src/airflow/providers/dq/plugins/www/pnpm-lock\.yaml$| ^airflow-core/src/airflow/ui/public/i18n/locales/de/README\.md$| ^airflow-core/src/airflow/ui/src/i18n/config\.ts$| ^\.agents/skills/airflow-translations/| diff --git a/airflow-core/docs/extra-packages-ref.rst b/airflow-core/docs/extra-packages-ref.rst index c2bb02ba55dbd..eada3fe487dc1 100644 --- a/airflow-core/docs/extra-packages-ref.rst +++ b/airflow-core/docs/extra-packages-ref.rst @@ -267,6 +267,8 @@ These are extras that add dependencies needed for integration with external serv +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | discord | ``pip install 'apache-airflow[discord]'`` | Discord hooks and sensors | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ +| dq | ``pip install 'apache-airflow[dq]'`` | Data Quality rules and operators | ++---------------------+-----------------------------------------------------+-----------------------------------------------------+ | facebook | ``pip install 'apache-airflow[facebook]'`` | Facebook Social | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | github | ``pip install 'apache-airflow[github]'`` | GitHub operators and hook | diff --git a/airflow-core/tests/unit/always/test_project_structure.py b/airflow-core/tests/unit/always/test_project_structure.py index 750966a320221..392ab726648d8 100644 --- a/airflow-core/tests/unit/always/test_project_structure.py +++ b/airflow-core/tests/unit/always/test_project_structure.py @@ -107,6 +107,7 @@ def test_providers_modules_should_have_tests(self): "providers/common/compat/tests/unit/common/compat/standard/test_utils.py", "providers/common/messaging/tests/unit/common/messaging/providers/test_base_provider.py", "providers/common/messaging/tests/unit/common/messaging/providers/test_sqs.py", + "providers/dq/tests/unit/dq/test_exceptions.py", "providers/edge3/tests/unit/edge3/cli/test_example_extended_sysinfo.py", "providers/edge3/tests/unit/edge3/models/test_edge_job.py", "providers/edge3/tests/unit/edge3/models/test_edge_logs.py", diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_plugins.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_plugins.py index 6bca49d8052ea..f6b1665a6687c 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_plugins.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_plugins.py @@ -34,7 +34,7 @@ class TestGetPlugins: # Filters ( {}, - 18, + 19, [ "InformaticaProviderPlugin", "MetadataCollectionPlugin", @@ -42,6 +42,7 @@ class TestGetPlugins: "business_day_window_plugin", "databricks_workflow", "decreasing_priority_weight_strategy_plugin", + "dq", "edge_executor", "hitl_review", "hive", @@ -58,14 +59,14 @@ class TestGetPlugins: ), ( {"limit": 3, "offset": 3}, - 18, + 19, [ "business_day_window_plugin", "databricks_workflow", "decreasing_priority_weight_strategy_plugin", ], ), - ({"limit": 1}, 18, ["InformaticaProviderPlugin"]), + ({"limit": 1}, 19, ["InformaticaProviderPlugin"]), ], ) def test_should_respond_200( @@ -168,7 +169,7 @@ def test_invalid_external_view_destination_should_log_warning_and_continue(self, assert len(plugins_page) == 7 assert "test_plugin_invalid" not in [p["name"] for p in plugins_page] - assert body["total_entries"] == 18 + assert body["total_entries"] == 19 @skip_if_force_lowest_dependencies_marker diff --git a/airflow-core/tests/unit/plugins/test_plugins_manager.py b/airflow-core/tests/unit/plugins/test_plugins_manager.py index ce55db1e269ea..0dcac96fe0a32 100644 --- a/airflow-core/tests/unit/plugins/test_plugins_manager.py +++ b/airflow-core/tests/unit/plugins/test_plugins_manager.py @@ -408,7 +408,7 @@ def test_does_not_double_import_entrypoint_provider_plugins(self): # Mock/skip loading from plugin dir with mock.patch("airflow.plugins_manager._load_plugins_from_plugin_directory", return_value=([], [])): plugins = plugins_manager._get_plugins()[0] - assert len(plugins) == 6 + assert len(plugins) == 7 class TestWindowPluginRegistration: diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index ca5cb1775bcc0..47d8d7f665a8a 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -245,13 +245,13 @@ apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +dingding | discord | docker | docker-stack | dq | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github +| google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | jdbc | jenkins | keycloak | +microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         +openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     +presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    +sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb +| zendesk]...                                                                                                          Build documents. diff --git a/dev/breeze/doc/images/output_build-docs.txt b/dev/breeze/doc/images/output_build-docs.txt index 7e86964622680..a77b769c19561 100644 --- a/dev/breeze/doc/images/output_build-docs.txt +++ b/dev/breeze/doc/images/output_build-docs.txt @@ -1 +1 @@ -22cd66d36e80473f4fac0fa311c5e846 +1a48397be50e3de3856de21b7d7dce01 diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.svg b/dev/breeze/doc/images/output_release-management_add-back-references.svg index 09e78ba88111a..9cf44bc91c0b0 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.svg +++ b/dev/breeze/doc/images/output_release-management_add-back-references.svg @@ -154,13 +154,13 @@ apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +dingding | discord | docker | docker-stack | dq | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github +| google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | jdbc | jenkins | keycloak | +microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         +openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     +presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    +sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb +| zendesk]...                                                                                                          Command to add back references for documentation to make it backward compatible. diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.txt b/dev/breeze/doc/images/output_release-management_add-back-references.txt index 672e05eb6b26f..d1aabe7ffc715 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.txt +++ b/dev/breeze/doc/images/output_release-management_add-back-references.txt @@ -1 +1 @@ -2700b8a52b9a04bb3cf5e365e70c2f40 +b22bcdaeeec5f5a324f9bab27dcd812d diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg index cef6e4d3b1d14..fe6774f68dae7 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg @@ -166,11 +166,11 @@ apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | dq | edge3 | elasticsearch | exasol |    +fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc |        +jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone +| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    ydb | zendesk]...                                                                                                      diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt index 6e02c700afdf5..dc63186a0cf8d 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt @@ -1 +1 @@ -4cac13b21eee8b732a46c5a15aec7a4b +20d79ab20c9c8a3642541dae56cc07ca diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg index 2f9692be54fa0..075d604ec1ed0 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg @@ -155,11 +155,11 @@ apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | dq | edge3 | elasticsearch | exasol |    +fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc |        +jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone +| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    ydb | zendesk]...                                                                                                      diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index 6afb289b5854f..5c011d40cc77d 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -bd80bf2dd63a27111b8c5745df5dfb86 +195d4f4d2001bd0b681fa2610264778d diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg index 64efb0af6b65b..c0e373a4c4ced 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg @@ -174,7 +174,7 @@ │apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | │ │apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes│ │| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | │ -│datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | │ +│datadog | dbt.cloud | dingding | discord | docker | dq | edge3 | elasticsearch | exasol | fab | │ │facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica│ │| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | │ │microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | │ diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt index 7c14e04e63151..49cc734ebbe27 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt @@ -1 +1 @@ -5865810f5a4b12c9199267b7dd6459b3 +dacac5a22a7ab84d7fdc7e9709375340 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg index 27a1d92de9a4b..f713a828c564c 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg @@ -193,11 +193,11 @@ apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | dq | edge3 | elasticsearch | exasol |    +fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc |        +jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone +| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    ydb | zendesk]...                                                                                                      diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt index f054c03806e05..2b47f1755ef80 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt @@ -1 +1 @@ -a27c1726f5902e5fdb501ecdee226476 +b6edff16004d612e687e5f16d01a5b74 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg index b9c71f9c81aec..74c5c78f5cd46 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg @@ -214,11 +214,11 @@ apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | dq | edge3 | elasticsearch | exasol |    +fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc |        +jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j |    +odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone +| postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |   snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    ydb | zendesk]...                                                                                                      diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index a54080b6bfbb5..77cd5d480f70b 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -c33a2f6d00a3a8dbec8b56c1c2d88d54 +f38ff7c5d51344a214aefe63194a0f71 diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.svg b/dev/breeze/doc/images/output_release-management_publish-docs.svg index 9416d6183c582..210f67fcc9e5f 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.svg +++ b/dev/breeze/doc/images/output_release-management_publish-docs.svg @@ -193,13 +193,13 @@ apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +dingding | discord | docker | docker-stack | dq | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github +| google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | jdbc | jenkins | keycloak | +microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         +openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     +presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    +sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb +| zendesk]...                                                                                                          Command to publish generated documentation to airflow-site diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.txt b/dev/breeze/doc/images/output_release-management_publish-docs.txt index b6433c39bb9be..e472795130dde 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.txt +++ b/dev/breeze/doc/images/output_release-management_publish-docs.txt @@ -1 +1 @@ -cc4ad6886b5ff0da1d4f819d2c5dc80d +ef5f666c153e5c3ec72a2cf8250ee5a5 diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg index a9d2e173ef2b8..4b243694bf80c 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg @@ -188,7 +188,7 @@ │apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | │ │atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | │ │common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud | │ -│dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | │ +│dingding | discord | docker | dq | edge3 | elasticsearch | exasol | fab | facebook | ftp | git |│ │github | google | grpc | hashicorp | http | ibm.mq | imap | influxdb | informatica | jdbc | │ │jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | │ │mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle │ diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt index bb3262922cec4..8aed44f7ddb9e 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt @@ -1 +1 @@ -84f322f2ffa9ded046c2d3678135a8f7 +c07440cc580f3635e601b100ea1aebe6 diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg index dfb0027b3d7ef..5e1893d0d0070 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg @@ -208,13 +208,13 @@ apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +dingding | discord | docker | docker-stack | dq | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github +| google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | jdbc | jenkins | keycloak | +microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         +openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     +presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    +sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb +| zendesk]...                                                                                                          Trigger publish docs to S3 workflow diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt index 4c6ec79dba124..bb40fd9b1f773 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt @@ -1 +1 @@ -e784b5234a05d958f84df96cd5d8d562 +83ee7d480d222cf939bab166bfc30af8 diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index abd6237606f91..8529598384b03 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -843,7 +843,7 @@ def get_airflow_extras(): { "python-version": "3.10", "airflow-version": "2.11.1", - "remove-providers": "anthropic common.messaging edge3 fab git keycloak informatica common.ai opensearch", + "remove-providers": "anthropic common.messaging edge3 fab git keycloak informatica common.ai opensearch dq", "run-unit-tests": "true", }, { diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index cbecbdd94429d..0852fe3bbdbcd 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -2748,7 +2748,7 @@ def test_upgrade_to_newer_dependencies( ("providers/common/sql/src/airflow/providers/common/sql/common_sql_python.py",), { "docs-list-as-string": "amazon apache.drill apache.druid apache.hive apache.iceberg " - "apache.impala apache.pinot clickhousedb common.ai common.compat common.sql databricks elasticsearch " + "apache.impala apache.pinot clickhousedb common.ai common.compat common.sql databricks dq elasticsearch " "exasol google informatica jdbc microsoft.mssql mysql odbc openlineage " "oracle pgvector postgres presto slack snowflake sqlite teradata trino vertica ydb", }, diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 2b7c13ee66036..00650dad7301a 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -292,6 +292,7 @@ conda conf Config config +ConfigDict configfile configMap configmap diff --git a/providers/dq/.gitignore b/providers/dq/.gitignore new file mode 100644 index 0000000000000..f454b41e73c0a --- /dev/null +++ b/providers/dq/.gitignore @@ -0,0 +1,3 @@ +# Do not commit hash files, this is just to speed-up local builds +www-hash.txt +*.iml diff --git a/providers/dq/.pre-commit-config.yaml b/providers/dq/.pre-commit-config.yaml new file mode 100644 index 0000000000000..7defb148dc634 --- /dev/null +++ b/providers/dq/.pre-commit-config.yaml @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 + node: 22.19.0 + golang: 1.24.0 +repos: + - repo: local + hooks: + - id: generate-dq-ruleset-schema + name: Generate Data Quality RuleSet schema + language: python + entry: ../../scripts/ci/prek/generate_dq_ruleset_schema.py + pass_filenames: false + always_run: true + files: > + (?x) + ^src/airflow/providers/dq/rules/.*\.py$| + ^src/airflow/providers/dq/skills/dq-rule-authoring/references/ruleset\.schema\.json$ + - id: ts-compile-lint-dq-ui + name: Compile Data Quality provider UI + description: Type-check changed TypeScript files in the Data Quality provider UI + language: node + files: | + (?x) + ^src/airflow/providers/dq/plugins/www/.*\.(js|ts|tsx|yaml|css|json)$ + exclude: | + (?x) + ^src/airflow/providers/dq/plugins/www/node-modules/.*| + ^src/airflow/providers/dq/plugins/www/.pnpm-store + entry: ../../scripts/ci/prek/ts_compile_lint_dq.py + additional_dependencies: ['pnpm@10.25.0'] + pass_filenames: true + require_serial: true + - id: compile-dq-assets + name: Compile Data Quality provider assets + language: node + stages: ['pre-commit', 'manual'] + files: ^src/airflow/providers/dq/plugins/www/ + entry: ../../scripts/ci/prek/compile_provider_assets.py dq + pass_filenames: false + additional_dependencies: ['pnpm@10.25.0'] + - id: check-http-exception-import-from-fastapi + name: Check HTTPException is imported from fastapi + entry: ../../scripts/ci/prek/check_http_exception_import_from_fastapi.py + language: python + pass_filenames: true + files: > + (?x) + ^src/airflow/providers/dq/(api|plugins)/.*\.py$| + ^tests/unit/dq/(api|plugins)/.*\.py$ diff --git a/providers/dq/LICENSE b/providers/dq/LICENSE new file mode 100644 index 0000000000000..11069edd79019 --- /dev/null +++ b/providers/dq/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/dq/NOTICE b/providers/dq/NOTICE new file mode 100644 index 0000000000000..a51bd9390d030 --- /dev/null +++ b/providers/dq/NOTICE @@ -0,0 +1,5 @@ +Apache Airflow +Copyright 2016-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/providers/dq/README.rst b/providers/dq/README.rst new file mode 100644 index 0000000000000..a6f6ba496bdbc --- /dev/null +++ b/providers/dq/README.rst @@ -0,0 +1,52 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Package ``apache-airflow-providers-dq`` + +Release: ``0.1.0`` + +``Data Quality Provider`` + +Declarative data quality rules with durable, per-rule execution history. Checks run through +``common.sql`` DB-API hooks; results are persisted to a configurable results store (object +storage or local files) so task, run, and rule-level quality can be inspected over time. + +Provider package +---------------- + +This package is for the ``dq`` provider. +All classes for this package are included in the ``airflow.providers.dq`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-dq``. For the minimum Airflow version supported, +see ``Requirements`` below. + +Requirements +------------ + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.15.0`` +``apache-airflow-providers-common-sql`` ``>=2.0.0`` +``pydantic`` ``>=2.11.0`` +``pyyaml`` ``>=6.0.2`` +========================================== ================== diff --git a/providers/dq/docs/agents.rst b/providers/dq/docs/agents.rst new file mode 100644 index 0000000000000..cc820cf88b416 --- /dev/null +++ b/providers/dq/docs/agents.rst @@ -0,0 +1,55 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _dq:agents: + +Generating rules with an LLM +============================== + +Writing a :class:`~airflow.providers.dq.rules.RuleSet` by hand for every table doesn't scale. +An LLM can propose one from a table's column definitions instead, given the check catalog as +context. + +The ``dq-rule-authoring`` skill +--------------------------------- + +This provider ships an `Agent Skill `__ at +``airflow/providers/dq/skills/dq-rule-authoring/``: a ``SKILL.md`` documenting the +``RuleSet``/``DQRule`` fields and check catalog, plus a generated JSON Schema +(``references/ruleset.schema.json``) for validation. + +Point ``common.ai``'s :doc:`AgentSkillsToolset ` at +it, and give the model ``output_type=RuleSet`` so pydantic-ai validates -- and self-corrects -- +its output before the task completes: + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_llm_generated_ruleset.py + :language: python + :start-after: [START howto_task_dq_generate_ruleset_with_llm] + :end-before: [END howto_task_dq_generate_ruleset_with_llm] + +Requires ``apache-airflow-providers-common-ai[skills]`` and a configured ``llm_conn_id``. + +Wiring the result into a check +--------------------------------- + +``@task.dq_check`` can leave ``ruleset=`` unset and return the LLM task's result at execution +time instead: + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_llm_generated_ruleset.py + :language: python + :start-after: [START howto_decorator_dq_check_llm_runtime_ruleset] + :end-before: [END howto_decorator_dq_check_llm_runtime_ruleset] diff --git a/providers/dq/docs/assets.rst b/providers/dq/docs/assets.rst new file mode 100644 index 0000000000000..538952f50a085 --- /dev/null +++ b/providers/dq/docs/assets.rst @@ -0,0 +1,66 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _dq:assets: + +Assets and quality gating +============================ + +Quality rules can travel with the :class:`~airflow.sdk.Asset` they describe instead of being +scattered across every Dag that checks it, and a downstream consumer Dag can refuse to run when +the data it was triggered by did not meet a minimum quality bar. + +Attaching a ruleset to an asset +---------------------------------- + +:func:`~airflow.providers.dq.assets.asset_quality` stores ruleset, connection, and table +configuration inside ``Asset.extra`` under the ``airflow.dq`` key, so it is serialized with the +Dag and needs no Airflow core changes: + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_require_quality.py + :language: python + :start-after: [START howto_asset_quality] + :end-before: [END howto_asset_quality] + +Pass the resulting asset to :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` +via ``asset=`` (see :doc:`operators`) instead of ``table``/``ruleset``/``conn_id``: the operator +resolves all three from the asset's config, adds the asset to its own outlets, and attaches its +summary -- including the quality ``score`` used below -- to the asset event. + +Gating a consumer Dag on quality +------------------------------------ + +:func:`~airflow.providers.dq.assets.require_quality` builds a ``@task.short_circuit`` task that +reads the ``score`` off the asset event that triggered the current run, and skips every +downstream task when that event has no quality summary at all, or its score is below +``min_score``: + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_require_quality.py + :language: python + :start-after: [START howto_require_quality] + :end-before: [END howto_require_quality] + +Put the gate first in a Dag scheduled by the asset, and chain everything else after it: + +.. code-block:: python + + with DAG("orders_consumer", schedule=orders_asset) as consumer: + gate = require_quality(orders_asset, min_score=0.95) + gate >> process_orders() + +``min_score`` must be between ``0`` and ``1``; the check considers only the *most recent* +triggering event for the asset when a run was triggered by several. diff --git a/providers/dq/docs/changelog.rst b/providers/dq/docs/changelog.rst new file mode 100644 index 0000000000000..3441e711a8345 --- /dev/null +++ b/providers/dq/docs/changelog.rst @@ -0,0 +1,33 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE TO CONTRIBUTORS: + Please, only add notes to the Changelog just below the "Changelog" header when there + are some breaking changes and you want to add an explanation to the users on how they + are supposed to deal with them. The changelog is updated and maintained semi-automatically + by release manager. + +``apache-airflow-providers-dq`` + + +Changelog +--------- + +0.1.0 +..... + +Initial version of the provider. diff --git a/providers/dq/docs/commits.rst b/providers/dq/docs/commits.rst new file mode 100644 index 0000000000000..6f999c6f0f339 --- /dev/null +++ b/providers/dq/docs/commits.rst @@ -0,0 +1,35 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + + .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN! + +Package apache-airflow-providers-dq +------------------------------------------------------ + +``Data Quality Provider`` + + +This is detailed commit list of changes for versions provider package: ``dq``. +For high-level changelog, see :doc:`package information including changelog `. + +.. airflow-providers-commits:: diff --git a/providers/dq/docs/conf.py b/providers/dq/docs/conf.py new file mode 100644 index 0000000000000..f133bf86729a7 --- /dev/null +++ b/providers/dq/docs/conf.py @@ -0,0 +1,27 @@ +# Disable Flake8 because of all the sphinx imports +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Configuration of Providers docs building.""" + +from __future__ import annotations + +import os + +os.environ["AIRFLOW_PACKAGE_NAME"] = "apache-airflow-providers-dq" + +from docs.provider_conf import * # noqa: F403 diff --git a/providers/dq/docs/configurations-ref.rst b/providers/dq/docs/configurations-ref.rst new file mode 100644 index 0000000000000..a52b21b2e5679 --- /dev/null +++ b/providers/dq/docs/configurations-ref.rst @@ -0,0 +1,19 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../devel-common/src/sphinx_exts/includes/providers-configurations-ref.rst +.. include:: /../../../devel-common/src/sphinx_exts/includes/sections-and-options.rst diff --git a/providers/dq/docs/decorators.rst b/providers/dq/docs/decorators.rst new file mode 100644 index 0000000000000..257e0de8bb697 --- /dev/null +++ b/providers/dq/docs/decorators.rst @@ -0,0 +1,52 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/decorator:dq_check: + +``@task.dq_check`` +===================== + +``@task.dq_check`` wraps :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` in +the TaskFlow API. ``ruleset`` may be declared as a decorator argument when it exists at +Dag-parse time, or returned by the decorated function as a runtime ruleset. ``table`` or +``asset`` are declared as decorator arguments exactly like the plain operator. The decorated +function is optional plumbing on top: return ``None`` to run the check exactly as declared. + +.. exampleinclude:: /../tests/system/dq/example_dq_check.py + :language: python + :dedent: 4 + :start-after: [START howto_decorator_dq_check] + :end-before: [END howto_decorator_dq_check] + +Runtime rule sets +-------------------- + +Return a :class:`~airflow.providers.dq.rules.RuleSet`, its dict form, or a YAML path to use a +ruleset that is only known at task-execution time -- for example one produced by an upstream +task, loaded from a Variable, or generated by an LLM. Return ``None`` to use the ruleset declared +on the decorator. + +Swapping in a different ruleset at execution time: + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_check_decorator_dynamic.py + :language: python + :start-after: [START howto_decorator_dq_check_runtime_ruleset] + :end-before: [END howto_decorator_dq_check_runtime_ruleset] + +Everything documented for the plain operator in :doc:`operators` -- ``fail_on``, persistence, +``asset``, ``partition_clause``, ``custom_sql`` -- applies unchanged; the decorator only adds the +optional runtime ruleset step before the check runs. diff --git a/providers/dq/docs/index.rst b/providers/dq/docs/index.rst new file mode 100644 index 0000000000000..0559b586e2f16 --- /dev/null +++ b/providers/dq/docs/index.rst @@ -0,0 +1,175 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-dq`` +=============================== + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Basics + + Home + Changelog + Security + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Guides + + Rules, rule sets, and checks + Operators + Decorators + Assets and quality gating + Generating rules with an LLM + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: References + + Configuration + Python API <_api/airflow/providers/dq/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Resources + + PyPI Repository + Installing from sources + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/dq/index> + + +apache-airflow-providers-dq package +----------------------------------- + +``Data Quality Provider`` + +Declarative data quality rules with durable, per-rule execution history. Checks run through +``common.sql`` DB-API hooks; results are persisted to a configurable results store (object +storage or local files) so task, run, and rule-level quality can be inspected over time. + +See :doc:`rules` for the built-in check catalog (and its cross-database caveats), +:doc:`operators`/:doc:`decorators` for running checks, and :doc:`assets` for attaching rules to +an asset and gating a downstream Dag on its quality score. + +Release: 0.1.0 + +Provider package +---------------- + +This package is for the ``dq`` provider. +All classes for this package are included in the ``airflow.providers.dq`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-dq``. For the minimum Airflow version supported, +see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider distribution is ``3.0.0``. + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.15.0`` +``apache-airflow-providers-common-sql`` ``>=2.0.0`` +``pydantic`` ``>=2.11.0`` +``pyyaml`` ``>=6.0.2`` +========================================== ================== + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +apache-airflow-providers-dq package +------------------------------------------------------ + +``Data Quality Provider`` + +Declarative data quality rules with durable, per-rule execution history. +Checks run through ``common.sql`` DB-API hooks; results are persisted to a +configurable results store (object storage or local files) so task, run, +and rule-level quality can be inspected over time. + + +Release: 0.1.0 + +Provider package +---------------- + +This package is for the ``dq`` provider. +All classes for this package are included in the ``airflow.providers.dq`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-dq``. +For the minimum Airflow version supported, see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider distribution is ``3.0.0``. + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.15.0`` +``apache-airflow-providers-common-sql`` ``>=2.0.0`` +``pydantic`` ``>=2.11.0`` +``pyyaml`` ``>=6.0.2`` +========================================== ================== + +Downloading official packages +----------------------------- + +You can download officially released packages and verify their checksums and signatures from the +`Official Apache Download site `_ + +* `The apache-airflow-providers-dq 0.1.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-dq 0.1.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/dq/docs/installing-providers-from-sources.rst b/providers/dq/docs/installing-providers-from-sources.rst new file mode 100644 index 0000000000000..a72b45ffaa6e8 --- /dev/null +++ b/providers/dq/docs/installing-providers-from-sources.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../devel-common/src/sphinx_exts/includes/installing-providers-from-sources.rst diff --git a/providers/dq/docs/operators.rst b/providers/dq/docs/operators.rst new file mode 100644 index 0000000000000..4043e2177298e --- /dev/null +++ b/providers/dq/docs/operators.rst @@ -0,0 +1,119 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/operator:DQCheckOperator: + +``DQCheckOperator`` +===================== + +Use :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` to run a +:doc:`ruleset ` against a table and persist per-rule results to the configured results +store. Every rule is evaluated and recorded regardless of the task outcome -- a rule failing +doesn't stop the others from running, and the full set of results is always written before the +task decides whether to fail. + +Basic usage +------------ + +Pass a connection, table, and ruleset directly: + +.. exampleinclude:: /../tests/system/dq/example_dq_check.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dq_check] + :end-before: [END howto_operator_dq_check] + +Parameters +----------- + +- ``ruleset`` -- a :class:`~airflow.providers.dq.rules.RuleSet`, its dict form, or a path to a + YAML ruleset file (see :doc:`rules`). Optional when ``asset`` carries one. +- ``table`` -- the table to check. Optional when ``asset`` carries one (falling back to the + asset's name). +- ``asset`` -- an :class:`~airflow.sdk.Asset` decorated with + :func:`~airflow.providers.dq.assets.asset_quality`. Supplies defaults for ``ruleset``, + ``table``, and ``conn_id`` (explicit arguments win) and is automatically added to the task's + outlets so its asset events carry the check summary -- see :doc:`assets`. +- ``partition_clause`` -- predicate ANDed into every built-in check's ``WHERE`` clause, e.g. + ``"ds = '{{ ds }}'"`` (templated). +- ``fail_on`` -- severity that fails the task: + + - ``error`` (default) -- only ``error``-severity rule failures fail the task; ``warn`` + failures are recorded but don't fail it. + - ``warn`` -- any rule failure (``error`` or ``warn`` severity) fails the task. + - ``never`` -- rule failures never fail the task; only an execution error (the check query + itself failing) does. + +- ``conn_id`` -- connection to the database to check, any ``common.sql`` ``DbApiHook``- + compatible type. +- ``database`` -- optional database/schema, overriding the connection's default. + +Regardless of ``fail_on``, an execution error -- the check query itself failing, as opposed to +a rule failing its condition -- always fails the task; there's no way to check data whose query +can't even run. + +Persisting results +-------------------- + +Results are persisted to the backend configured under ``[dq] results_path`` (see +:doc:`configurations-ref`). When that's unset, checks still run and the task still passes or +fails normally -- only the history in the Data Quality UI is unavailable. There is no +per-operator override: every check in a deployment shares one results store, so history stays +available across tasks and Dags without stitching together several stores. + +Checking an asset +-------------------- + +Attach a ruleset to an :class:`~airflow.sdk.Asset` with +:func:`~airflow.providers.dq.assets.asset_quality`, then pass the asset instead of ``table``/ +``ruleset``/``conn_id``: + +.. exampleinclude:: /../tests/system/dq/example_dq_check.py + :language: python + :start-after: [START howto_operator_dq_check_asset] + :end-before: [END howto_operator_dq_check_asset] + +The operator adds the asset to its own outlets automatically, and attaches the check's summary +(including its quality ``score``) to the asset event -- which is what makes +:func:`~airflow.providers.dq.assets.require_quality` (see :doc:`assets`) able to gate a +downstream consumer Dag on it. + +``custom_sql`` checks +------------------------ + +Built-in checks are all single-column. For cross-column comparisons, joins, or anything the +catalog doesn't cover, use ``custom_sql``: + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py + :language: python + :start-after: [START howto_operator_dq_check_custom_sql] + :end-before: [END howto_operator_dq_check_custom_sql] + +See :doc:`rules` for the full built-in check catalog and the ``custom_sql`` grammar. + +Loading a ruleset from YAML +------------------------------ + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py + :language: python + :start-after: [START howto_operator_dq_check_ruleset_from_yaml] + :end-before: [END howto_operator_dq_check_ruleset_from_yaml] + +TaskFlow decorator +-------------------- + +See :doc:`decorators` for the ``@task.dq_check`` equivalent, including runtime rule sets. diff --git a/providers/dq/docs/rules.rst b/providers/dq/docs/rules.rst new file mode 100644 index 0000000000000..38366034783d8 --- /dev/null +++ b/providers/dq/docs/rules.rst @@ -0,0 +1,221 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _dq:rules: + +Rules, rule sets, and checks +============================ + +Rules are data, not code. A :class:`~airflow.providers.dq.rules.RuleSet` is a named tuple of +:class:`~airflow.providers.dq.rules.DQRule` -- plain, serializable objects with no behavior of +their own. They describe *what* to check; :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` +(see :doc:`operators`) is what actually runs them. Rules are usually written by hand, but they +can also be proposed by an LLM -- see :doc:`agents`. + +.. code-block:: python + + from airflow.providers.dq.rules import DQRule, RuleSet + + orders_ruleset = RuleSet( + name="orders_quality", + rules=( + DQRule( + name="order_id_not_null", + check="null_count", + column="order_id", + condition={"equal_to": 0}, + ), + DQRule( + name="row_count_present", + check="row_count", + condition={"greater_than": 0}, + ), + ), + ) + +``DQRule`` fields +------------------ + +- ``name`` -- unique within its ruleset. Shows up in the Data Quality UI and logs. +- ``check`` -- one of the built-in checks below, or ``custom_sql``. +- ``condition`` -- the pass/fail condition for the observed value; see `Conditions`_. +- ``column`` -- target column. Required for column-level built-in checks, unused for + ``row_count`` and ``custom_sql``. +- ``sql`` -- a SQL statement returning a single scalar. Required for, and only valid with, + ``check="custom_sql"``. May reference the table being checked as ``{table}``. +- ``severity`` -- ``error`` (default) or ``warn``. Controls whether a failing rule fails the + task, subject to the operator's ``fail_on`` (see :doc:`operators`). +- ``partition_clause`` -- extra SQL predicate ANDed into this rule's ``WHERE`` clause, e.g. + ``"region = 'EU'"``. Combines with the operator-level ``partition_clause``, if any. +- ``previous_name`` -- set when renaming a rule so its execution history stays continuous + (see `Identity and history`_). +- ``description`` -- optional human-readable text shown in results and the Data Quality UI. + When omitted, the provider generates a short default description from the rule and condition. +- ``dimension`` -- one of ``completeness``, ``uniqueness``, ``validity``, ``freshness``, + ``volume``, ``consistency``. Defaults to the check's catalog dimension (see the table below; + ``validity`` for ``custom_sql``) when left unset. Only set this explicitly for a ``custom_sql`` + rule that measures something the default dimension doesn't capture. + +Built-in checks +---------------- + +Each built-in check is backed by a plain SQL expression, rendered with ``{column}`` for +column-level checks: + +.. list-table:: + :header-rows: 1 + + * - Check + - SQL expression + - Column required + - Default dimension + * - ``null_count`` + - ``SUM(CASE WHEN {column} IS NULL THEN 1 ELSE 0 END)`` + - yes + - ``completeness`` + * - ``null_ratio`` + - ``SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*)`` + - yes + - ``completeness`` + * - ``distinct_count`` + - ``COUNT(DISTINCT {column})`` + - yes + - ``uniqueness`` + * - ``unique_violations`` + - ``COUNT({column}) - COUNT(DISTINCT {column})`` + - yes + - ``uniqueness`` + * - ``min`` + - ``MIN({column})`` + - yes + - ``validity`` + * - ``max`` + - ``MAX({column})`` + - yes + - ``validity`` + * - ``mean`` + - ``AVG({column})`` + - yes + - ``validity`` + * - ``row_count`` + - ``COUNT(*)`` + - no + - ``volume`` + +.. note:: + + ``null_ratio`` divides by ``COUNT(*)`` with no guard against an empty table (or an empty + partition, when combined with ``partition_clause``); running it against zero rows is a + division-by-zero at the database level, not a clean "not applicable" result. Guard with a + ``row_count`` rule upstream, or a ``partition_clause`` that guarantees a non-empty set. A + portable guard would wrap the denominator in ``NULLIF(COUNT(*), 0)``, but ``NULLIF`` isn't + supported by every ``DbApiHook`` (see `Supported checks and databases`_) -- this is one + instance of the general rule below: if a built-in check's SQL doesn't work against your + database, express it as ``custom_sql`` instead. + +Conditions +----------- + +A :class:`~airflow.providers.dq.rules.Condition` is the pass/fail rule applied to the observed +value, using the same grammar as the ``common.sql`` check operators: + +- ``equal_to`` -- exact match. Cannot be combined with any other comparison. +- ``greater_than`` / ``geq_to`` -- lower bound, exclusive/inclusive. +- ``less_than`` / ``leq_to`` -- upper bound, exclusive/inclusive. +- ``tolerance`` -- a percentage that widens ``equal_to`` into a range; only valid together + with ``equal_to``. + +``greater_than``/``less_than``/``geq_to``/``leq_to`` may be combined to express a range +(e.g. ``{"geq_to": 0, "leq_to": 10}``). + +``custom_sql``: the escape hatch +---------------------------------- + +Built-in checks are all single-column. The moment a rule needs to compare two columns, join +another table, or use a function the catalog doesn't cover, use ``custom_sql`` -- any SQL +statement that resolves to a single scalar, evaluated exactly like a built-in check: + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py + :language: python + :start-after: [START howto_operator_dq_check_custom_sql] + :end-before: [END howto_operator_dq_check_custom_sql] + +Supported checks and databases +-------------------------------- + +Built-in checks are plain SQL expressions executed through whichever ``common.sql`` +:class:`~airflow.providers.common.sql.hooks.sql.DbApiHook` your connection resolves to. +Airflow does not validate those expressions per database dialect. + +The catalog intentionally uses simple expressions that work on common relational databases +and many distributed SQL engines, but support is not guaranteed for every ``DbApiHook``. +Some hooks expose non-standard SQL layers and may reject or evaluate an expression differently. + +For example, ``null_ratio`` is currently rendered as: + +.. code-block:: text + + SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*) + +This is simple and portable, but it is not guarded against an empty table or an empty +partition. A database-specific version could use a different expression, for example: + +.. code-block:: text + + CASE WHEN COUNT(*) = 0 THEN NULL + ELSE SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*) END + +If a built-in check does not work for your database or you need different empty-table +semantics, use ``custom_sql`` and write the expression for your dialect. + +Loading rules from YAML +------------------------- + +Anywhere a ``RuleSet`` is accepted -- ``DQCheckOperator(ruleset=...)``, +``@task.dq_check(ruleset=...)``, :func:`~airflow.providers.dq.assets.asset_quality` -- a path +string is accepted too, and resolved via :meth:`~airflow.providers.dq.rules.RuleSet.from_file` +at Dag-parse time. This keeps rules editable by people who don't write Python. + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/orders_ruleset.yaml + :language: yaml + +.. exampleinclude:: /../src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py + :language: python + :start-after: [START howto_operator_dq_check_ruleset_from_yaml] + :end-before: [END howto_operator_dq_check_ruleset_from_yaml] + +Identity and history +---------------------- + +Each rule has a stable ``rule_uid``, a hash of the fields that define *what is being measured*: +``name`` (or ``previous_name``, if set), ``check``, ``column``, ``sql``, and ``condition``. +Everything else -- including ``severity``, ``partition_clause``, ``description``, and +``dimension`` -- can change between Dag runs without breaking the rule's execution history, +because it isn't part of the identity hash. + +Renaming a rule outright would normally start a new history under the new name; set +``previous_name`` to the old name for one deploy to carry the old identity forward instead: + +.. code-block:: python + + DQRule( + name="order_id_is_unique", # renamed from order_id_unique + previous_name="order_id_unique", + check="unique_violations", + column="order_id", + condition={"equal_to": 0}, + ) diff --git a/providers/dq/docs/security.rst b/providers/dq/docs/security.rst new file mode 100644 index 0000000000000..15a0ebbb2d054 --- /dev/null +++ b/providers/dq/docs/security.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../devel-common/src/sphinx_exts/includes/security.rst diff --git a/providers/dq/hatch_build.py b/providers/dq/hatch_build.py new file mode 100644 index 0000000000000..ec5ccf72c83cf --- /dev/null +++ b/providers/dq/hatch_build.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +import os +import shutil +from collections.abc import Callable, Iterable +from pathlib import Path +from subprocess import run +from typing import Any + +from hatchling.builders.config import BuilderConfig +from hatchling.builders.plugin.interface import BuilderInterface +from hatchling.plugin.manager import PluginManager + +log = logging.getLogger(__name__) +log_level = logging.getLevelName(os.getenv("CUSTOM_AIRFLOW_BUILD_LOG_LEVEL", "INFO")) +log.setLevel(log_level) + + +class CustomBuild(BuilderInterface[BuilderConfig, PluginManager]): + """Custom build class for Data Quality provider assets.""" + + # Note that this name of the plugin MUST be `custom` - as long as we use it from custom + # hatch_build.py file and not from external plugin. See note in the: + # https://hatch.pypa.io/latest/plugins/build-hook/custom/#example + PLUGIN_NAME = "custom" + + @staticmethod + def clean_dir(path: Path) -> None: + log.warning("Cleaning directory: %s", path) + shutil.rmtree(path, ignore_errors=True) + + def clean(self, directory: str, versions: Iterable[str]) -> None: + work_dir = Path(self.root) + log.warning("Cleaning generated files in directory: %s", work_dir) + dq_package_src = work_dir / "src" / "airflow" / "providers" / "dq" + dq_ui_path = dq_package_src / "plugins" / "www" + self.clean_dir(dq_ui_path / ".pnpm-store") + self.clean_dir(dq_ui_path / "dist") + self.clean_dir(dq_ui_path / "node_modules") + (work_dir / "www-hash.txt").unlink(missing_ok=True) + + def get_version_api(self) -> dict[str, Callable[..., str]]: + """Get custom build target for standard package preparation.""" + return {"standard": self.build_standard} + + def build_standard(self, directory: str, artifacts: Any, **build_data: Any) -> str: + # run this in the airflow repo root + work_dir = Path(self.root).parents[1].resolve() + cmd = ["prek", "run", "compile-dq-assets", "--all-files"] + log.warning("Running command: %s", " ".join(cmd)) + run(cmd, cwd=work_dir.as_posix(), check=True) + dist_path = Path(self.root) / "src" / "airflow" / "providers" / "dq" / "plugins" / "www" / "dist" + return dist_path.resolve().as_posix() diff --git a/providers/dq/provider.yaml b/providers/dq/provider.yaml new file mode 100644 index 0000000000000..363bc0baf6e9f --- /dev/null +++ b/providers/dq/provider.yaml @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +package-name: apache-airflow-providers-dq +name: Data Quality +description: | + ``Data Quality Provider`` + + Declarative data quality rules with durable, per-rule execution history. + Checks run through ``common.sql`` DB-API hooks; results are persisted to a + configurable results store (object storage or local files) so task, run, + and rule-level quality can be inspected over time. + +state: ready +lifecycle: incubation +source-date-epoch: 1751587200 +build-system: hatchling + +# Note that those versions are maintained by release manager - do not update them manually +# with the exception of case where other provider in sources has >= new provider version. +# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have +# to be done in the same PR +versions: + - 0.1.0 + +integrations: + - integration-name: Data Quality + external-doc-url: https://airflow.apache.org/docs/apache-airflow-providers-dq/ + how-to-guide: + - /docs/apache-airflow-providers-dq/operators.rst + tags: [software] + +operators: + - integration-name: Data Quality + python-modules: + - airflow.providers.dq.operators.dq_check + +task-decorators: + - class-name: airflow.providers.dq.decorators.dq_check.dq_check_task + name: dq_check + +plugins: + - name: dq + plugin-class: airflow.providers.dq.plugins.dq.DQPlugin + +config: + dq: + description: | + Configuration for the Data Quality provider results store. + options: + results_path: + description: | + Any fsspec-compatible URL understood by ``ObjectStoragePath`` where data quality + results are persisted, e.g. ``s3://bucket/airflow-dq`` or ``file:///opt/airflow/dq``. + When unset, checks still run but no history is persisted. + version_added: 0.1.0 + type: string + example: s3://data-platform/airflow-dq + default: ~ + results_conn_id: + description: | + Optional Airflow connection used to access ``results_path``. + version_added: 0.1.0 + type: string + example: aws_default + default: ~ diff --git a/providers/dq/pyproject.toml b/providers/dq/pyproject.toml new file mode 100644 index 0000000000000..1a264da2cf6ca --- /dev/null +++ b/providers/dq/pyproject.toml @@ -0,0 +1,161 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE +# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +[build-system] +requires = [ + "hatchling==1.30.1", + "packaging==26.2", + "pathspec==1.1.1", + "pluggy==1.6.0", + "tomli==2.4.1; python_version < '3.11'", + "trove-classifiers==2026.6.1.19", +] +build-backend = "hatchling.build" + +[project] +name = "apache-airflow-providers-dq" +version = "0.1.0" +description = "Provider package apache-airflow-providers-dq for Apache Airflow" +readme = "README.rst" +license = "Apache-2.0" +license-files = ['LICENSE', 'NOTICE'] +authors = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +maintainers = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +keywords = [ "airflow-provider", "dq", "airflow", "integration" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Framework :: Apache Airflow :: Provider", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: System :: Monitoring", +] +requires-python = ">=3.10" + +# The dependencies should be modified in place in the generated file. +# Any change in the dependencies is preserved when the file is regenerated +# Make sure to run ``prek update-providers-dependencies --all-files`` +# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` +dependencies = [ + "apache-airflow>=3.0.0", + "apache-airflow-providers-common-compat>=1.15.0", + "apache-airflow-providers-common-sql>=2.0.0", + "pydantic>=2.11.0", + "pyyaml>=6.0.2", +] + +[dependency-groups] +dev = [ + "apache-airflow", + "apache-airflow-task-sdk", + "apache-airflow-devel-common", + "apache-airflow-providers-common-compat", + "apache-airflow-providers-common-sql", + # Additional devel dependencies (do not remove this line and add extra development dependencies) + "apache-airflow-providers-common-sql", +] + +# To build docs: +# +# uv run --group docs build-docs +# +# To enable auto-refreshing build with server: +# +# uv run --group docs build-docs --autobuild +# +# To see more options: +# +# uv run --group docs build-docs --help +# +docs = [ + "apache-airflow-devel-common[docs]" +] + +[tool.uv.sources] +# These names must match the names as defined in the pyproject.toml of the workspace items, +# *not* the workspace folder paths +apache-airflow = {workspace = true} +apache-airflow-devel-common = {workspace = true} +apache-airflow-task-sdk = {workspace = true} +apache-airflow-providers-common-sql = {workspace = true} +apache-airflow-providers-standard = {workspace = true} + +[project.urls] +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-dq/0.1.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-dq/0.1.0/changelog.html" +"Bug Tracker" = "https://github.com/apache/airflow/issues" +"Source Code" = "https://github.com/apache/airflow" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Mastodon" = "https://fosstodon.org/@airflow" +"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[project.entry-points."apache_airflow_provider"] +provider_info = "airflow.providers.dq.get_provider_info:get_provider_info" + +[project.entry-points."airflow.plugins"] +dq = "airflow.providers.dq.plugins.dq:DQPlugin" + +[tool.hatch.version] +path = "src/airflow/providers/dq/__init__.py" + +[tool.hatch.build.targets.sdist] +include = [ + "docs", + "src/airflow/providers/dq", + "tests", + "NOTICE" +] +exclude = [ + "src/airflow/__init__.py", + "src/airflow/providers/__init__.py", + "src/airflow/providers/dq/plugins/www/.pnpm-store", + "src/airflow/providers/dq/plugins/www/node_modules", +] + +[tool.hatch.build.targets.custom] +path = "./hatch_build.py" + +artifacts = [ + "src/airflow/providers/dq/plugins/www/dist", +] + +[tool.hatch.build.targets.wheel] +packages = ['src/airflow'] +artifacts = [ + "src/airflow/providers/dq/plugins/www/dist", +] +exclude = [ + "src/airflow/__init__.py", + "src/airflow/providers/__init__.py", + "src/airflow/providers/dq/plugins/www/.pnpm-store", + "src/airflow/providers/dq/plugins/www/node_modules", +] diff --git a/providers/dq/src/airflow/__init__.py b/providers/dq/src/airflow/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/dq/src/airflow/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/dq/src/airflow/providers/__init__.py b/providers/dq/src/airflow/providers/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/dq/src/airflow/providers/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/dq/src/airflow/providers/dq/__init__.py b/providers/dq/src/airflow/providers/dq/__init__.py new file mode 100644 index 0000000000000..4fd31dec8bc1a --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/__init__.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE +# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES. +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +# +from __future__ import annotations + +import packaging.version + +from airflow import __version__ as airflow_version + +__all__ = ["__version__"] + +__version__ = "0.1.0" + +if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( + "3.0.0" +): + raise RuntimeError(f"The package `apache-airflow-providers-dq:{__version__}` needs Apache Airflow 3.0.0+") diff --git a/providers/dq/src/airflow/providers/dq/api/__init__.py b/providers/dq/src/airflow/providers/dq/api/__init__.py new file mode 100644 index 0000000000000..21d298ede6ed3 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/api/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations diff --git a/providers/dq/src/airflow/providers/dq/api/app.py b/providers/dq/src/airflow/providers/dq/api/app.py new file mode 100644 index 0000000000000..c431fbe066a1e --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/api/app.py @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Read-only query API for data quality results.""" + +from __future__ import annotations + +from typing import Annotated + +from fastapi import Depends, FastAPI, HTTPException, Query + +from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity +from airflow.api_fastapi.core_api.security import requires_access_dag +from airflow.providers.dq.api.models import PaginatedResponse, RuleHistoryRecordModel, TaskDQRunModel +from airflow.providers.dq.backends import get_backend_from_config +from airflow.providers.dq.backends.object_storage import ObjectStorageResultsBackend + + +def _get_backend() -> ObjectStorageResultsBackend: + backend = get_backend_from_config() + if backend is None: + raise HTTPException( + status_code=503, + detail="No data quality results backend configured; set '[dq] results_path'.", + ) + return backend + + +BackendDep = Annotated[ObjectStorageResultsBackend, Depends(_get_backend)] + +dq_app = FastAPI( + title="Data Quality", + description="Read-only query API over data quality check results.", +) + + +@dq_app.get("/health") +async def health() -> dict[str, str]: + """Liveness check.""" + return {"status": "ok"} + + +@dq_app.get( + "/v1/dags/{dag_id}/tasks/{task_id}/runs", + dependencies=[Depends(requires_access_dag(method="GET", access_entity=DagAccessEntity.TASK_INSTANCE))], +) +async def task_runs( + dag_id: str, + task_id: str, + backend: BackendDep, + limit: Annotated[int, Query(gt=0, le=200)] = 50, + before: Annotated[str | None, Query()] = None, +) -> PaginatedResponse[TaskDQRunModel]: + """ + Recent data quality runs for one task, newest first. + + ``before`` is the opaque ``next_cursor`` from the previous page; omit it for the first page. + """ + return PaginatedResponse[TaskDQRunModel](**backend.read_task_runs(dag_id, task_id, limit, before)) + + +@dq_app.get( + "/v1/dags/{dag_id}/tasks/{task_id}/rules/{rule_uid}/history", + dependencies=[Depends(requires_access_dag(method="GET", access_entity=DagAccessEntity.TASK_INSTANCE))], +) +async def task_rule_history( + dag_id: str, + task_id: str, + rule_uid: str, + backend: BackendDep, + limit: Annotated[int, Query(gt=0, le=1000)] = 100, + before: Annotated[str | None, Query()] = None, +) -> PaginatedResponse[RuleHistoryRecordModel]: + """ + Recent results for one rule produced by one task, newest first. + + ``before`` is the opaque ``next_cursor`` from the previous page; omit it for the first page. + """ + return PaginatedResponse[RuleHistoryRecordModel]( + **backend.read_task_rule_history(dag_id, task_id, rule_uid, limit, before) + ) + + +@dq_app.get( + "/v1/dags/{dag_id}/tasks/{task_id}/runs/by_run/{run_id}", + dependencies=[Depends(requires_access_dag(method="GET", access_entity=DagAccessEntity.TASK_INSTANCE))], +) +async def run_detail_by_task_instance( + dag_id: str, + task_id: str, + run_id: str, + backend: BackendDep, + map_index: Annotated[int, Query()] = -1, +) -> TaskDQRunModel: + """Look up a run by the route params available to the task-instance UI plugin.""" + try: + payload = backend.read_by_task_instance(dag_id, task_id, run_id, map_index) + except FileNotFoundError: + raise HTTPException( + status_code=404, detail="No data quality run found for this task instance." + ) from None + return TaskDQRunModel(**payload) diff --git a/providers/dq/src/airflow/providers/dq/api/models.py b/providers/dq/src/airflow/providers/dq/api/models.py new file mode 100644 index 0000000000000..f874f57819790 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/api/models.py @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from typing import Any, Generic, TypeVar + +from pydantic import BaseModel + +T = TypeVar("T") + + +class RuleResultModel(BaseModel): + """Mirrors ``airflow.providers.dq.results.RuleResult.to_dict()``.""" + + rule_uid: str + rule_name: str + status: str + observed_value: float | str | None = None + condition: dict[str, Any] = {} + dimension: str = "validity" + severity: str = "error" + duration_ms: float | None = None + error_message: str | None = None + description: str | None = None + sql: str | None = None + + +class DQRunModel(BaseModel): + """Mirrors ``airflow.providers.dq.results.DQRun.to_dict()`` -- the full run header.""" + + dag_id: str + task_id: str + run_id: str + try_number: int = 1 + map_index: int = -1 + run_uid: str + ruleset_name: str | None = None + table_ref: str | None = None + asset_names: list[str] = [] + started_at: str | None = None + finished_at: str | None = None + + +class RunContextModel(BaseModel): + """Mirrors ``ObjectStorageResultsBackend._build_run_context()``.""" + + run_uid: str + dag_id: str + task_id: str + run_id: str + map_index: int = -1 + started_at: str | None = None + table_ref: str | None = None + + +class DQSummaryModel(BaseModel): + """Compact summary fields returned by the Data Quality UI API.""" + + errored: int + failed: int + passed: int + score: float | None + warned: int + + +class TaskDQRunModel(BaseModel): + """One run, as returned by ``task_runs`` items and ``run_detail_by_task_instance``.""" + + run: DQRunModel + results: list[RuleResultModel] + summary: DQSummaryModel + + +class RuleHistoryRecordModel(RuleResultModel): + """A rule result plus the run context that produced it, as returned by ``task_rule_history``.""" + + run: RunContextModel + + +class PaginatedResponse(BaseModel, Generic[T]): + """``{"items": [...], "next_cursor": ...}``, shared by every cursor-paginated route.""" + + items: list[T] + next_cursor: str | None = None diff --git a/providers/dq/src/airflow/providers/dq/assets.py b/providers/dq/src/airflow/providers/dq/assets.py new file mode 100644 index 0000000000000..513587a68d9b4 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/assets.py @@ -0,0 +1,172 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Asset-level data quality declarations. + +Quality configuration lives inside ``Asset.extra`` under the ``airflow.dq`` key, so it is +serialized with the Dag and needs no Airflow core changes. The rules travel with the asset +definition instead of being scattered across the Dags that check it. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from airflow.providers.dq.exceptions import DQRuleValidationError +from airflow.providers.dq.rules import RuleSet +from airflow.sdk import task + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + from airflow.sdk import Asset + from airflow.sdk.execution_time.comms import AssetEventDagRunReferenceResult + +log = logging.getLogger(__name__) + +DQ_EXTRA_KEY = "airflow.dq" +DQ_RESULT_EXTRA_KEY = "airflow.dq.result" + + +def asset_quality( + asset: Asset, + *, + ruleset: RuleSet | dict[str, Any] | str, + conn_id: str | None = None, + table: str | None = None, +) -> Asset: + """ + Attach data quality configuration to an asset, returning the same asset. + + :param asset: The asset the rules describe. + :param ruleset: A :class:`~airflow.providers.dq.rules.RuleSet`, its dict form, or a path + to a YAML ruleset file (resolved eagerly, at Dag-parse time). + :param conn_id: Default connection a check operator should use for this asset. + :param table: Default table to check; falls back to the asset name when unset. + + Usage:: + + orders = asset_quality( + Asset("orders", uri="postgres://warehouse/analytics/orders"), + ruleset=rules, + conn_id="warehouse", + table="analytics.orders", + ) + check = DQCheckOperator(task_id="dq", asset=orders) + """ + if isinstance(ruleset, str): + ruleset = RuleSet.from_file(ruleset) + elif isinstance(ruleset, dict): + ruleset = RuleSet.from_dict(ruleset) + config: dict[str, Any] = {"ruleset": ruleset.to_dict()} + if conn_id: + config["conn_id"] = conn_id + if table: + config["table"] = table + asset.extra[DQ_EXTRA_KEY] = config + return asset + + +def get_asset_quality_config(asset: Asset) -> dict[str, Any] | None: + """Return the raw ``airflow.dq`` config attached to an asset, if any.""" + config = asset.extra.get(DQ_EXTRA_KEY) + if not isinstance(config, dict): + return None + return config + + +def get_asset_ruleset(asset: Asset) -> RuleSet: + """Return the ruleset attached to an asset, raising when none is attached.""" + config = get_asset_quality_config(asset) + if not config or "ruleset" not in config: + raise DQRuleValidationError( + f"Asset {asset.name!r} has no data quality config; attach one with asset_quality()" + ) + return RuleSet.from_dict(config["ruleset"]) + + +def _quality_score_passes( + asset: Asset, + min_score: float, + triggering_asset_events: Mapping[Asset, Sequence[AssetEventDagRunReferenceResult]], +) -> bool: + """Pure decision logic behind :func:`require_quality`, kept separate so it is testable without a Dag.""" + events = triggering_asset_events.get(asset, []) + if not events: + log.warning( + "require_quality(%s): no triggering event for this run; skipping downstream tasks", + asset.name, + ) + return False + + summary = events[-1].extra.get(DQ_RESULT_EXTRA_KEY) + score = summary.get("score") if isinstance(summary, dict) else None + if not isinstance(score, (int, float)) or isinstance(score, bool): + log.warning( + "require_quality(%s): triggering event has no data quality summary; skipping downstream tasks", + asset.name, + ) + return False + + if score < min_score: + log.warning( + "require_quality(%s): score %s below required minimum %s; skipping downstream tasks", + asset.name, + score, + min_score, + ) + return False + + return True + + +def require_quality( + asset: Asset, + *, + min_score: float, + task_id: str | None = None, +) -> Any: + """ + Gate a Dag run on the data quality score attached to one of its triggering asset events. + + Reads the summary a :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` + attaches to ``asset``'s outlet event, under ``asset_event.extra["airflow.dq.result"]`` + (see :func:`asset_quality`), and short-circuits the run — skipping every downstream task + — when that summary is missing or its score is below ``min_score``. Call it inside a Dag + scheduled by ``asset``:: + + with DAG("orders_consumer", schedule=orders) as consumer: + start = require_quality(orders, min_score=0.95) + start >> process_orders() + + :param asset: The asset whose triggering event carries the quality summary. + :param min_score: Minimum required score in ``[0, 1]``; the run proceeds only when the + triggering event's score is at least this value. + :param task_id: Task id for the generated gate task. Defaults to + ``f"require_quality_{asset.name}"`` so gating on several assets in one Dag doesn't + collide on task id. + """ + if not 0 <= min_score <= 1: + raise ValueError(f"min_score must be between 0 and 1, got {min_score!r}") + gate_task_id = task_id or f"require_quality_{asset.name}" + + @task.short_circuit(task_id=gate_task_id) + def _require_quality(**context: Any) -> bool: + return _quality_score_passes(asset, min_score, context["triggering_asset_events"]) + + return _require_quality() diff --git a/providers/dq/src/airflow/providers/dq/backends/__init__.py b/providers/dq/src/airflow/providers/dq/backends/__init__.py new file mode 100644 index 0000000000000..7fe7bade843a5 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/backends/__init__.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.dq.backends.object_storage import ObjectStorageResultsBackend + +__all__ = ["ObjectStorageResultsBackend", "get_backend_from_config"] + + +def get_backend_from_config() -> ObjectStorageResultsBackend | None: + """Build the results backend configured under ``[dq]``, or ``None`` when not configured.""" + from airflow.providers.common.compat.sdk import conf + + results_path = conf.get("dq", "results_path", fallback=None) + if not results_path: + return None + conn_id = conf.get("dq", "results_conn_id", fallback=None) + return ObjectStorageResultsBackend(results_path=results_path, conn_id=conn_id or None) diff --git a/providers/dq/src/airflow/providers/dq/backends/object_storage.py b/providers/dq/src/airflow/providers/dq/backends/object_storage.py new file mode 100644 index 0000000000000..c43978a03dbeb --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/backends/object_storage.py @@ -0,0 +1,259 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Object-storage results backend. + +Each DQ check writes a keyed JSON document plus read indexes optimized for the UI: + + runs/by_task/dag_id=/task_id=/date=<2026-07-04>/.json + Canonical run record: ``{"run": ..., "results": [...], "summary": ...}``. + + runs/by_task_instance/dag_id=/task_id=/__.json + Latest result for a task-instance page. Last write wins across retries. + + rules/by_rule/rule_uid=/__.json + One rule result plus run context: ``{"run": ..., "result": ...}``. + + rules/by_task_rule/dag_id=/task_id=/rule_uid=/__.json + Same payload, scoped for task-level rule history views. + +The duplicate files are intentional read indexes: DQ tasks write once, while the UI reads +many times. Keeping these indexes avoids scanning all task runs for common UI views. +""" + +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone +from typing import Any + +from airflow.providers.dq.results import DQRun, RuleResult, build_summary +from airflow.sdk import ObjectStoragePath + +log = logging.getLogger(__name__) + + +class ObjectStorageResultsBackend: + """Persist DQ results as JSON files via ``ObjectStoragePath``.""" + + def __init__(self, results_path: str, conn_id: str | None = None) -> None: + self.root = ObjectStoragePath(results_path, conn_id=conn_id) + + def write_run(self, run: DQRun, results: list[RuleResult]) -> None: + timestamp = run.started_at or datetime.now(tz=timezone.utc).isoformat() + payload = self._build_run_payload(run, results) + + self._write_run_file(run, timestamp[:10], payload) + self._write_task_instance_index(run, payload) + self._write_rule_indexes(run, results, timestamp) + + def read_task_rule_history( + self, dag_id: str, task_id: str, rule_uid: str, limit: int = 100, before: str | None = None + ) -> dict[str, Any]: + """Return recent results for one rule produced by one task, newest first.""" + rule_dir = ( + self.root + / "rules" + / "by_task_rule" + / f"dag_id={dag_id}" + / f"task_id={task_id}" + / f"rule_uid={rule_uid}" + ) + return self._read_rule_history_dir(rule_dir, limit, before) + + def read_task_runs( + self, dag_id: str, task_id: str, limit: int = 50, before: str | None = None + ) -> dict[str, Any]: + """ + Return recent data quality runs for one task, newest first. + + Returns ``{"items": [...], "next_cursor": ...}``. ``before`` is the opaque + ``next_cursor`` from the previous page, so "load more" only reads runs it hasn't + shown yet. ``next_cursor`` is set by reading one extra entry past ``limit``: cheap + on every page except the last one, where there's no way to confirm history is + exhausted without walking to the end. + + ``date=`` partition names sort correctly as plain strings, so directories are walked + newest-first and scanning stops as soon as ``limit + 1`` matching runs have been + collected — a task with years of history doesn't pay for a full scan on every page. + """ + task_dir = self.root / "runs" / "by_task" / f"dag_id={dag_id}" / f"task_id={task_id}" + if not task_dir.exists(): + return {"items": [], "next_cursor": None} + + date_dirs = sorted((path for path in task_dir.iterdir() if path.is_dir()), reverse=True) + runs = [] + for date_dir in date_dirs: + for path in date_dir.iterdir(): + if not path.name.endswith(".json"): + continue + payload = self._read_json(path) + if payload is None: + continue + cursor = self._get_run_payload_cursor(payload) + if before is not None and cursor >= before: + continue + runs.append(payload) + if len(runs) > limit: + break + + ordered = sorted(runs, key=self._get_run_payload_cursor, reverse=True) + page = ordered[:limit] + next_cursor = self._get_run_payload_cursor(page[-1]) if len(ordered) > limit and page else None + return {"items": page, "next_cursor": next_cursor} + + def read_by_task_instance( + self, dag_id: str, task_id: str, run_id: str, map_index: int = -1 + ) -> dict[str, Any]: + """Read the latest run for one task instance as ``{"run": ..., "results": ..., "summary": ...}``.""" + path = ( + self.root + / "runs" + / "by_task_instance" + / f"dag_id={dag_id}" + / f"task_id={task_id}" + / f"{self._get_safe_key(run_id)}__{map_index}.json" + ) + return self._read_json_or_raise(path) + + def _write_run_file(self, run: DQRun, date_part: str, payload: dict[str, Any]) -> None: + run_dir = ( + self.root + / "runs" + / "by_task" + / f"dag_id={run.dag_id}" + / f"task_id={run.task_id}" + / f"date={date_part}" + ) + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / f"{run.run_uid}.json").write_text(json.dumps(payload, default=str)) + + def _write_task_instance_index(self, run: DQRun, payload: dict[str, Any]) -> None: + ti_dir = self.root / "runs" / "by_task_instance" / f"dag_id={run.dag_id}" / f"task_id={run.task_id}" + ti_dir.mkdir(parents=True, exist_ok=True) + (ti_dir / f"{self._get_safe_key(run.run_id)}__{run.map_index}.json").write_text( + json.dumps(payload, default=str) + ) + + def _write_rule_indexes(self, run: DQRun, results: list[RuleResult], timestamp: str) -> None: + run_context = self._build_run_context(run) + compact_ts = self._get_safe_key(timestamp) + for result in results: + payload = {"run": run_context, "result": result.to_dict()} + self._write_rule_index( + self.root / "rules" / "by_rule" / f"rule_uid={result.rule_uid}", + compact_ts, + run.run_uid, + payload, + ) + self._write_rule_index( + self.root + / "rules" + / "by_task_rule" + / f"dag_id={run.dag_id}" + / f"task_id={run.task_id}" + / f"rule_uid={result.rule_uid}", + compact_ts, + run.run_uid, + payload, + ) + + def _write_rule_index( + self, rule_dir: ObjectStoragePath, compact_ts: str, run_uid: str, payload: dict[str, Any] + ) -> None: + rule_dir.mkdir(parents=True, exist_ok=True) + (rule_dir / f"{compact_ts}__{run_uid}.json").write_text(json.dumps(payload, default=str)) + + def _read_rule_history_dir( + self, rule_dir: ObjectStoragePath, limit: int, before: str | None = None + ) -> dict[str, Any]: + """ + Read rule-result records newest-first, as ``{"items": [...], "next_cursor": ...}``. + + Reads one entry past ``limit`` to determine ``next_cursor``: cheap on every page + except the last one, where confirming there's nothing older means reading to the end. + """ + if not rule_dir.exists(): + return {"items": [], "next_cursor": None} + + history: list[dict[str, Any]] = [] + for path in sorted(rule_dir.iterdir(), key=lambda p: p.name, reverse=True): + payload = self._read_json(path) + if payload is None: + continue + record = {**payload["result"], "run": payload["run"]} + cursor = self._get_rule_history_cursor(record) + if before is not None and cursor >= before: + continue + history.append(record) + if len(history) > limit: + break + + page = history[:limit] + next_cursor = self._get_rule_history_cursor(page[-1]) if len(history) > limit and page else None + return {"items": page, "next_cursor": next_cursor} + + @staticmethod + def _get_safe_key(value: str) -> str: + """Sanitize a value (for example an Airflow ``run_id``) for an object key segment.""" + return value.replace("/", "_").replace(":", "_").replace("+", "_") + + @staticmethod + def _build_run_payload(run: DQRun, results: list[RuleResult]) -> dict[str, Any]: + result_records = [result.to_dict() for result in results] + return { + "run": run.to_dict(), + "results": result_records, + "summary": build_summary(run, results), + } + + @staticmethod + def _build_run_context(run: DQRun) -> dict[str, Any]: + return { + "run_uid": run.run_uid, + "dag_id": run.dag_id, + "task_id": run.task_id, + "run_id": run.run_id, + "map_index": run.map_index, + "started_at": run.started_at, + "table_ref": run.table_ref, + } + + @staticmethod + def _get_run_payload_cursor(payload: dict[str, Any]) -> str: + run = payload["run"] + return ObjectStorageResultsBackend._build_cursor(run.get("started_at"), run.get("run_uid")) + + @staticmethod + def _get_rule_history_cursor(record: dict[str, Any]) -> str: + run = record["run"] + return ObjectStorageResultsBackend._build_cursor(run.get("started_at"), run.get("run_uid")) + + @staticmethod + def _build_cursor(started_at: str | None, run_uid: str | None) -> str: + return f"{started_at or ''}|{run_uid or ''}" + + def _read_json_or_raise(self, path: ObjectStoragePath) -> dict[str, Any]: + return json.loads(path.read_text()) + + def _read_json(self, path: ObjectStoragePath) -> dict[str, Any] | None: + try: + return self._read_json_or_raise(path) + except (OSError, json.JSONDecodeError): + log.warning("Skipping unreadable DQ result file %s", path) + return None diff --git a/providers/dq/src/airflow/providers/dq/decorators/__init__.py b/providers/dq/src/airflow/providers/dq/decorators/__init__.py new file mode 100644 index 0000000000000..21d298ede6ed3 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/decorators/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations diff --git a/providers/dq/src/airflow/providers/dq/decorators/dq_check.py b/providers/dq/src/airflow/providers/dq/decorators/dq_check.py new file mode 100644 index 0000000000000..0385187b1d39d --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/decorators/dq_check.py @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""TaskFlow decorator for :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator`.""" + +from __future__ import annotations + +from collections.abc import Callable, Collection, Mapping, Sequence +from typing import TYPE_CHECKING, Any, ClassVar + +from airflow.providers.common.compat.sdk import ( + DecoratedOperator, + context_merge, + determine_kwargs, + task_decorator_factory, +) +from airflow.providers.dq.operators.dq_check import DQCheckOperator +from airflow.providers.dq.rules import RuleSet +from airflow.sdk.definitions._internal.types import SET_DURING_EXECUTION + +if TYPE_CHECKING: + from airflow.sdk import Context + from airflow.sdk.bases.decorator import TaskDecorator + + +class _DQCheckDecoratedOperator(DecoratedOperator, DQCheckOperator): + """ + Wraps a callable that optionally returns a runtime ruleset for a data quality check. + + :param python_callable: A reference to a callable returning ``None`` or a ruleset for this run. + :param op_args: Positional arguments for the callable. + :param op_kwargs: Keyword arguments for the callable. + """ + + template_fields: Sequence[str] = ( + *DecoratedOperator.template_fields, + *DQCheckOperator.template_fields, + ) + template_fields_renderers: ClassVar[dict[str, str]] = { + **DecoratedOperator.template_fields_renderers, + } + + custom_operator_name = "@task.dq_check" + + def __init__( + self, + *, + python_callable: Callable, + op_args: Collection[Any] | None = None, + op_kwargs: Mapping[str, Any] | None = None, + **kwargs, + ) -> None: + kwargs.setdefault("ruleset", SET_DURING_EXECUTION) + super().__init__(python_callable=python_callable, op_args=op_args, op_kwargs=op_kwargs, **kwargs) + + def execute(self, context: Context) -> Any: + context_merge(context, self.op_kwargs) + kwargs = determine_kwargs(self.python_callable, self.op_args, context) + ruleset = self.python_callable(*self.op_args, **kwargs) + if ruleset is not None: + if not isinstance(ruleset, (RuleSet, dict, str)): + raise TypeError( + f"{self.custom_operator_name} function must return a RuleSet, dict, path, or None, " + f"got {type(ruleset).__name__}" + ) + self.ruleset = ruleset + return DQCheckOperator.execute(self, context) + + +def dq_check_task( + python_callable: Callable | None = None, + **kwargs, +) -> TaskDecorator: + """ + Turn a function into a data quality check task. + + ``ruleset`` may be passed as a decorator argument when known at Dag-parse time, or returned + by the decorated function at runtime. ``table`` or ``asset`` are passed through ``kwargs`` + exactly as on :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator`. The + function itself is optional plumbing: return ``None`` to run the check as declared, or a + ruleset to use for this run. + + Usage:: + + @task.dq_check(conn_id="warehouse", ruleset=rules) + def orders_quality(ds=None): + return None + + :param python_callable: Function to decorate. + """ + return task_decorator_factory( + python_callable=python_callable, + decorated_operator_class=_DQCheckDecoratedOperator, + **kwargs, + ) diff --git a/providers/dq/src/airflow/providers/dq/engines/__init__.py b/providers/dq/src/airflow/providers/dq/engines/__init__.py new file mode 100644 index 0000000000000..98a6f6fe9ba80 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/engines/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.dq.engines.sql import Observation, SQLDQEngine + +__all__ = ["Observation", "SQLDQEngine"] diff --git a/providers/dq/src/airflow/providers/dq/engines/sql.py b/providers/dq/src/airflow/providers/dq/engines/sql.py new file mode 100644 index 0000000000000..43c2f9579f8df --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/engines/sql.py @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""SQL execution engine: compiles a ruleset into check queries run through a DB-API hook.""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from airflow.providers.dq.rules import CUSTOM_SQL_CHECK +from airflow.providers.dq.rules.checks import CHECK_SPECS + +if TYPE_CHECKING: + from airflow.providers.common.sql.hooks.sql import DbApiHook + from airflow.providers.dq.rules import DQRule, RuleSet + +log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class Observation: + """Raw value a rule observed, before its condition is evaluated.""" + + rule: DQRule + observed_value: Any = None + duration_ms: float | None = None + error_message: str | None = None + sql: str | None = None + + +class SQLDQEngine: + """ + Runs built-in checks as a single UNION ALL query and custom SQL rules individually. + + Table, column, and partition-clause values come from the Dag author and are interpolated + into SQL the same way the ``common.sql`` check operators do — they are trusted input. + """ + + check_sql_template = "SELECT '{rule_uid}' AS rule_uid, {expression} AS observed FROM {table}{where}" + + def __init__(self, hook: DbApiHook) -> None: + self.hook = hook + + def measure(self, ruleset: RuleSet, table: str, partition_clause: str | None = None) -> list[Observation]: + builtin_rules = [rule for rule in ruleset.rules if rule.check != CUSTOM_SQL_CHECK] + custom_rules = [rule for rule in ruleset.rules if rule.check == CUSTOM_SQL_CHECK] + + observations = [] + if builtin_rules: + observations.extend(self._measure_builtin(builtin_rules, table, partition_clause)) + for rule in custom_rules: + observations.append(self._measure_custom(rule, table)) + return observations + + def build_batch_sql(self, rules: list[DQRule], table: str, partition_clause: str | None) -> str: + return " UNION ALL ".join(self.build_rule_sql(rule, table, partition_clause) for rule in rules) + + def build_rule_sql(self, rule: DQRule, table: str, partition_clause: str | None = None) -> str: + """Build the SQL used to measure one built-in rule.""" + expression = CHECK_SPECS[rule.check].expression.format(column=rule.column) + predicates = [p for p in (partition_clause, rule.partition_clause) if p] + where = f" WHERE {' AND '.join(predicates)}" if predicates else "" + return self.check_sql_template.format( + rule_uid=rule.rule_uid, expression=expression, table=table, where=where + ) + + def _measure_builtin( + self, rules: list[DQRule], table: str, partition_clause: str | None + ) -> list[Observation]: + sql = self.build_batch_sql(rules, table, partition_clause) + log.info("Running %d built-in checks against %s", len(rules), table) + started = time.monotonic() + try: + records = self.hook.get_records(sql) + except Exception as e: + elapsed_ms = (time.monotonic() - started) * 1000 + log.exception("Check query failed for table %s", table) + return [ + Observation( + rule=rule, + duration_ms=elapsed_ms, + error_message=str(e), + sql=self.build_rule_sql(rule, table, partition_clause), + ) + for rule in rules + ] + elapsed_ms = (time.monotonic() - started) * 1000 + observed_by_uid = {str(row[0]): row[1] for row in records or []} + return [ + Observation( + rule=rule, + observed_value=observed_by_uid.get(rule.rule_uid), + duration_ms=elapsed_ms, + error_message=None if rule.rule_uid in observed_by_uid else "No result returned for rule", + sql=self.build_rule_sql(rule, table, partition_clause), + ) + for rule in rules + ] + + def _measure_custom(self, rule: DQRule, table: str) -> Observation: + if rule.sql is None: + raise ValueError(f"Rule {rule.name!r} has no SQL to execute") + sql = rule.sql.replace("{table}", table) + log.info("Running custom SQL check %s", rule.name) + started = time.monotonic() + try: + row = self.hook.get_first(sql) + except Exception as e: + log.exception("Custom SQL check %s failed", rule.name) + return Observation( + rule=rule, + duration_ms=(time.monotonic() - started) * 1000, + error_message=str(e), + sql=sql, + ) + elapsed_ms = (time.monotonic() - started) * 1000 + if row is None: + return Observation( + rule=rule, duration_ms=elapsed_ms, error_message="Query returned no rows", sql=sql + ) + return Observation(rule=rule, observed_value=row[0], duration_ms=elapsed_ms, sql=sql) diff --git a/providers/dq/src/airflow/providers/dq/example_dags/__init__.py b/providers/dq/src/airflow/providers/dq/example_dags/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/example_dags/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py new file mode 100644 index 0000000000000..f07f746be9849 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_custom_sql.py @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: ``custom_sql`` for checks the built-in catalog can't express. + +Built-in checks (``null_count``, ``min``, ``max``, ...) are all single-column. The moment a +rule needs to compare two columns, join against another table, or use a SQL function the +built-in catalog doesn't cover (or that your database's ``DbApiHook`` doesn't support -- see +"Supported checks and databases" in the provider docs), ``custom_sql`` is the escape hatch: +any statement that resolves to a single scalar, evaluated the same way as a built-in check. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.providers.dq.operators.dq_check import DQCheckOperator +from airflow.providers.dq.rules import DQRule, RuleSet, Severity +from airflow.sdk import DAG + +DAG_ID = "example_dq_check_custom_sql" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_custom_sql_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +# [START howto_operator_dq_check_custom_sql] +custom_sql_ruleset = RuleSet( + name="orders_custom_sql", + rules=( + DQRule( + name="shipped_after_ordered", + check="custom_sql", + # {table} is substituted by the SQL engine at check time, not an f-string + # placeholder -- a cross-column comparison no single-column built-in can express. + sql="SELECT COUNT(*) FROM {table} WHERE shipped_at < ordered_at", + condition={"equal_to": 0}, + ), + DQRule( + name="high_value_order_ratio", + check="custom_sql", + sql=( + "SELECT CAST(SUM(CASE WHEN amount > 100 THEN 1 ELSE 0 END) AS REAL) / COUNT(*) FROM {table}" + ), + condition={"leq_to": 0.5}, + severity=Severity.WARN, + ), + ), +) +# [END howto_operator_dq_check_custom_sql] + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f""" + CREATE TABLE {TABLE_NAME} ( + order_id INTEGER, + amount REAL, + ordered_at TEXT, + shipped_at TEXT + ); + """, + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount, ordered_at, shipped_at) VALUES + (1, 42.0, '2026-07-01', '2026-07-02'), + (2, 150.0, '2026-07-01', '2026-07-03'), + (3, 15.5, '2026-07-02', '2026-07-02'); + """, + ) + + check_orders = DQCheckOperator( + task_id="check_orders", + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=custom_sql_ruleset, + fail_on="never", + ) + + create_table >> insert_orders >> check_orders diff --git a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_decorator_dynamic.py b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_decorator_dynamic.py new file mode 100644 index 0000000000000..2a4b782539a11 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_check_decorator_dynamic.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: ``@task.dq_check`` with a runtime ruleset. + +``table`` (or ``asset``) is declared as a decorator argument, exactly like the plain operator. +``ruleset`` may be declared at Dag-parse time, or returned by the decorated function at +execution time. This is useful when an upstream task, variable, or generated value decides which +ruleset to run. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.providers.dq.rules import DQRule, RuleSet +from airflow.sdk import DAG, task + +DAG_ID = "example_dq_check_decorator_dynamic" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_dynamic_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +orders_ruleset = RuleSet( + name="orders_dynamic", + rules=( + DQRule(name="order_id_not_null", check="null_count", column="order_id", condition={"equal_to": 0}), + DQRule(name="row_count_present", check="row_count", condition={"greater_than": 0}), + ), +) + +strict_ruleset = RuleSet( + name="orders_dynamic_strict", + rules=( + *orders_ruleset.rules, + DQRule(name="amount_min_ge_zero", check="min", column="amount", condition={"geq_to": 0}), + ), +) + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f"CREATE TABLE {TABLE_NAME} (order_id INTEGER, amount REAL, ds TEXT);", + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount, ds) VALUES + (1, 10.0, '{{{{ ds }}}}'), + (2, 25.5, '{{{{ ds }}}}'), + (3, 7.25, '2020-01-01'); + """, + ) + + # [START howto_decorator_dq_check_runtime_ruleset] + @task.dq_check(conn_id=CONN_ID, table=TABLE_NAME, ruleset=orders_ruleset, fail_on="never") + def check_with_ruleset_from_upstream(strict: bool = True): + """Swap in a stricter ruleset based on a signal only known at task-execution time.""" + return strict_ruleset if strict else None + + # [END howto_decorator_dq_check_runtime_ruleset] + + create_table >> insert_orders >> check_with_ruleset_from_upstream() diff --git a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_llm_generated_ruleset.py b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_llm_generated_ruleset.py new file mode 100644 index 0000000000000..7ecfd743ceb91 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_llm_generated_ruleset.py @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: generate a ``RuleSet`` with an LLM, guided by the ``dq-rule-authoring`` skill. + +Requires the optional ``apache-airflow-providers-common-ai[skills]`` package and a configured +LLM connection (``llm_conn_id``); this Dag is not registered when common.ai isn't installed. + +An LLM is asked to propose data quality rules for a table's columns. Its ``system_prompt`` +points it at the ``dq-rule-authoring`` skill shipped with this provider (via +``AgentSkillsToolset``), so it knows the exact ``RuleSet``/``DQRule`` schema, the built-in check +catalog, and the ``Condition`` grammar, instead of guessing at field or check names. +``output_type=RuleSet`` makes pydantic-ai validate -- and self-correct -- the model's output +against the real ``RuleSet`` model before the task completes, so a malformed rule never reaches +``DQCheckOperator`` in the first place. + +The generated ``RuleSet`` is then returned by ``@task.dq_check`` at runtime, since the real +ruleset doesn't exist until the LLM task runs. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.providers.dq.rules import RuleSet +from airflow.sdk import DAG, task + +try: + from airflow.providers.common.ai.toolsets.skills import AgentSkillsToolset +except ImportError: + AgentSkillsToolset = None # type: ignore[assignment,misc] + +DAG_ID = "example_dq_llm_generated_ruleset" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_llm_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") +# The skill ships next to this Dag's package, not next to this file -- resolve relative to +# __file__ so the path holds regardless of the Dag processor's working directory. +SKILLS_DIR = Path(__file__).parent.parent / "skills" / "dq-rule-authoring" + +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +if AgentSkillsToolset is not None: + with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq", "ai"], + ) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f""" + CREATE TABLE {TABLE_NAME} ( + order_id INTEGER, + customer_id INTEGER, + amount REAL, + region TEXT, + created_at TEXT + ); + """, + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, region, created_at) VALUES + (1, 101, 10.0, 'US', '2026-07-01'), + (2, 102, 25.5, 'EU', '2026-07-02'), + (3, NULL, 7.25, 'US', '2026-07-03'); + """, + ) + + # [START howto_task_dq_generate_ruleset_with_llm] + @task.llm( + llm_conn_id="pydanticai_default", + system_prompt=( + "You are a data quality engineer. Before answering, consult the " + "dq-rule-authoring skill for the exact RuleSet/DQRule schema, the built-in " + "check catalog, and the Condition grammar -- do not invent field or check names." + ), + output_type=RuleSet, + agent_params={"toolsets": [AgentSkillsToolset(sources=[str(SKILLS_DIR)])]}, + ) + def generate_ruleset(): + return ( + "Generate a RuleSet named 'orders_llm_quality' for a table with these columns: " + "order_id (integer, primary key), customer_id (integer, nullable foreign key), " + "amount (float, must be non-negative), region (short string, low-cardinality), " + "created_at (date). Include at least a not-null check on order_id, a uniqueness " + "check on order_id, and a row-count check." + ) + + # [END howto_task_dq_generate_ruleset_with_llm] + + # [START howto_decorator_dq_check_llm_runtime_ruleset] + @task.dq_check(conn_id=CONN_ID, table=TABLE_NAME, fail_on="never") + def check_with_llm_ruleset(ruleset: RuleSet): + return ruleset + + # [END howto_decorator_dq_check_llm_runtime_ruleset] + + generated_ruleset = generate_ruleset() + checked = check_with_llm_ruleset(generated_ruleset) + + create_table >> insert_orders >> generated_ruleset >> checked diff --git a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_require_quality.py b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_require_quality.py new file mode 100644 index 0000000000000..124156604a11e --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_require_quality.py @@ -0,0 +1,120 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAGs: gate a consumer Dag on the data quality score of the asset that triggers it. + +Two Dags, wired together only through the ``dq_gated_orders`` asset: + +- ``example_dq_require_quality_producer`` checks a table and produces the asset. + :class:`~airflow.providers.dq.operators.dq_check.DQCheckOperator` attaches its summary + (including the quality ``score``) to the asset event automatically because the asset carries + quality config attached with :func:`~airflow.providers.dq.assets.asset_quality`. +- ``example_dq_require_quality_consumer`` is scheduled by that asset. Its first task, built by + :func:`~airflow.providers.dq.assets.require_quality`, reads the score off the triggering + asset event and short-circuits -- skipping every downstream task -- when it's missing or + below ``min_score``. Downstream tasks never see a run triggered by a bad batch of data. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.providers.dq.assets import asset_quality, require_quality +from airflow.providers.dq.operators.dq_check import DQCheckOperator +from airflow.providers.dq.rules import DQRule, RuleSet +from airflow.sdk import DAG, Asset, task + +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_gated_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +# [START howto_asset_quality] +gated_orders_ruleset = RuleSet( + name="gated_orders_quality", + rules=( + DQRule(name="order_id_not_null", check="null_count", column="order_id", condition={"equal_to": 0}), + DQRule(name="amount_min_ge_zero", check="min", column="amount", condition={"geq_to": 0}), + DQRule(name="row_count_present", check="row_count", condition={"greater_than": 0}), + ), +) + +gated_orders_asset = asset_quality( + Asset("dq_gated_orders", uri="file:///tmp/airflow_dq_example/gated_orders"), + ruleset=gated_orders_ruleset, + conn_id=CONN_ID, + table=TABLE_NAME, +) +# [END howto_asset_quality] + +with DAG( + dag_id="example_dq_require_quality_producer", + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as producer_dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f"CREATE TABLE {TABLE_NAME} (order_id INTEGER, amount REAL);", + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount) VALUES + (1, 10.0), + (2, 25.5), + (3, 7.25); + """, + ) + + # Producing check: no explicit outlets needed, DQCheckOperator adds the asset itself + # because it was passed via asset=. + check_orders = DQCheckOperator( + task_id="check_orders", + asset=gated_orders_asset, + fail_on="never", + ) + + create_table >> insert_orders >> check_orders + +# [START howto_require_quality] +with DAG( + dag_id="example_dq_require_quality_consumer", + schedule=gated_orders_asset, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as consumer_dag: + gate = require_quality(gated_orders_asset, min_score=0.95) + + @task + def process_orders(): + print("Processing orders -- quality gate passed.") + + gate >> process_orders() +# [END howto_require_quality] diff --git a/providers/dq/src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py new file mode 100644 index 0000000000000..25b9a6cf76abd --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/example_dags/example_dq_ruleset_from_yaml.py @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: load a ruleset from a YAML file instead of declaring it in Python. + +A path string is accepted anywhere a :class:`~airflow.providers.dq.rules.RuleSet` is -- +``DQCheckOperator(ruleset=...)``, ``@task.dq_check(ruleset=...)``, and +:func:`~airflow.providers.dq.assets.asset_quality` all resolve it via +:meth:`~airflow.providers.dq.rules.RuleSet.from_file` at Dag-parse time. This keeps rules +editable by people who don't write Python -- a data steward can change ``orders_ruleset.yaml`` +without touching the Dag file. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.providers.dq.operators.dq_check import DQCheckOperator +from airflow.sdk import DAG + +DAG_ID = "example_dq_ruleset_from_yaml" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_yaml_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") +RULESET_FILE = Path(__file__).parent / "orders_ruleset.yaml" + +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f"CREATE TABLE {TABLE_NAME} (order_id INTEGER, amount REAL);", + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount) VALUES + (1, 10.0), + (2, 25.5), + (3, 7.25); + """, + ) + + # [START howto_operator_dq_check_ruleset_from_yaml] + check_orders = DQCheckOperator( + task_id="check_orders", + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=str(RULESET_FILE), + fail_on="never", + ) + # [END howto_operator_dq_check_ruleset_from_yaml] + + create_table >> insert_orders >> check_orders diff --git a/providers/dq/src/airflow/providers/dq/example_dags/orders_ruleset.yaml b/providers/dq/src/airflow/providers/dq/example_dags/orders_ruleset.yaml new file mode 100644 index 0000000000000..049e69543bc32 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/example_dags/orders_ruleset.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- + +name: orders_from_yaml +rules: + - name: order_id_not_null + check: null_count + column: order_id + condition: + equal_to: 0 + dimension: completeness + - name: amount_min_ge_zero + check: min + column: amount + condition: + geq_to: 0 + - name: row_count_present + check: row_count + condition: + greater_than: 0 + severity: warn diff --git a/providers/dq/src/airflow/providers/dq/exceptions.py b/providers/dq/src/airflow/providers/dq/exceptions.py new file mode 100644 index 0000000000000..9350308d46501 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/exceptions.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + + +class DQRuleValidationError(ValueError): + """Raised when a rule or ruleset definition is invalid.""" + + +class DQCheckFailedError(RuntimeError): + """Raised when a data quality check fails at or above the operator's ``fail_on`` severity.""" diff --git a/providers/dq/src/airflow/providers/dq/get_provider_info.py b/providers/dq/src/airflow/providers/dq/get_provider_info.py new file mode 100644 index 0000000000000..ae5b6cc9b782c --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/get_provider_info.py @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +def get_provider_info(): + return { + "package-name": "apache-airflow-providers-dq", + "name": "Data Quality", + "description": "``Data Quality Provider``\n\nDeclarative data quality rules with durable, per-rule execution history.\nChecks run through ``common.sql`` DB-API hooks; results are persisted to a\nconfigurable results store (object storage or local files) so task, run,\nand rule-level quality can be inspected over time.\n", + "integrations": [ + { + "integration-name": "Data Quality", + "external-doc-url": "https://airflow.apache.org/docs/apache-airflow-providers-dq/", + "how-to-guide": ["/docs/apache-airflow-providers-dq/operators.rst"], + "tags": ["software"], + } + ], + "operators": [ + { + "integration-name": "Data Quality", + "python-modules": ["airflow.providers.dq.operators.dq_check"], + } + ], + "task-decorators": [ + {"class-name": "airflow.providers.dq.decorators.dq_check.dq_check_task", "name": "dq_check"} + ], + "plugins": [{"name": "dq", "plugin-class": "airflow.providers.dq.plugins.dq.DQPlugin"}], + "config": { + "dq": { + "description": "Configuration for the Data Quality provider results store.\n", + "options": { + "results_path": { + "description": "Any fsspec-compatible URL understood by ``ObjectStoragePath`` where data quality\nresults are persisted, e.g. ``s3://bucket/airflow-dq`` or ``file:///opt/airflow/dq``.\nWhen unset, checks still run but no history is persisted.\n", + "version_added": "0.1.0", + "type": "string", + "example": "s3://data-platform/airflow-dq", + "default": None, + }, + "results_conn_id": { + "description": "Optional Airflow connection used to access ``results_path``.\n", + "version_added": "0.1.0", + "type": "string", + "example": "aws_default", + "default": None, + }, + }, + } + }, + } diff --git a/providers/dq/src/airflow/providers/dq/operators/__init__.py b/providers/dq/src/airflow/providers/dq/operators/__init__.py new file mode 100644 index 0000000000000..f1734cd8846f9 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/operators/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.dq.operators.dq_check import DQCheckOperator + +__all__ = ["DQCheckOperator"] diff --git a/providers/dq/src/airflow/providers/dq/operators/dq_check.py b/providers/dq/src/airflow/providers/dq/operators/dq_check.py new file mode 100644 index 0000000000000..f006a54686010 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/operators/dq_check.py @@ -0,0 +1,247 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +from collections.abc import Sequence +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, cast + +from airflow.providers.common.sql.operators.sql import BaseSQLOperator +from airflow.providers.dq.assets import DQ_RESULT_EXTRA_KEY, get_asset_quality_config +from airflow.providers.dq.backends import get_backend_from_config +from airflow.providers.dq.engines.sql import SQLDQEngine +from airflow.providers.dq.exceptions import DQCheckFailedError +from airflow.providers.dq.results import ERROR, FAIL, PASS, WARN, DQRun, RuleResult, build_summary +from airflow.providers.dq.rules import RuleSet, describe_rule +from airflow.sdk.definitions._internal.types import SET_DURING_EXECUTION + +if TYPE_CHECKING: + from airflow.providers.common.sql.hooks.sql import DbApiHook + from airflow.providers.dq.engines.sql import Observation + from airflow.providers.dq.rules.rule import Condition, Dimension + from airflow.sdk import Asset, Context + +log = logging.getLogger(__name__) + +FAIL_ON_CHOICES = ("error", "warn", "never") + + +class DQCheckOperator(BaseSQLOperator): + """ + Run a ruleset against a table and persist per-rule results to the configured results store. + + Every rule is evaluated and recorded regardless of the task outcome. Execution errors + (a check query failing) always fail the task; rule failures fail the task according to + ``fail_on``. + + :param ruleset: A :class:`~airflow.providers.dq.rules.RuleSet`, its dict form, or a path + to a YAML ruleset file. Optional when ``asset`` carries one. + :param table: The table to run checks against. Optional when ``asset`` carries one + (falling back to the asset name). + :param asset: An asset decorated with :func:`~airflow.providers.dq.assets.asset_quality`. + Supplies defaults for ``ruleset``, ``table``, and ``conn_id`` (explicit arguments + win) and is automatically added to the task's outlets so its asset events carry + the check summary. + :param partition_clause: Predicate ANDed into every built-in check's WHERE clause, + e.g. ``"ds = '{{ ds }}'"``. + :param fail_on: Severity that fails the task: ``error`` (default — warn-severity rule + failures are recorded but don't fail), ``warn`` (any rule failure fails), or + ``never`` (only execution errors fail). + :param conn_id: Connection to the database to check (any ``DbApiHook``-compatible type). + :param database: Optional database/schema overriding the connection's default. + + Results are persisted to the backend configured under ``[dq] results_path``; when that's + unset, checks still run but no history is persisted. There is no per-operator override -- + all checks in a deployment share one results store. + """ + + template_fields: Sequence[str] = ("table", "partition_clause", *BaseSQLOperator.template_fields) + ui_color: str = "#87ceeb" + + def __init__( + self, + *, + ruleset: RuleSet | dict[str, Any] | str | None = None, + table: str | None = None, + asset: Asset | None = None, + partition_clause: str | None = None, + fail_on: str = "error", + **kwargs, + ) -> None: + if asset is not None: + config = get_asset_quality_config(asset) or {} + ruleset = ruleset if ruleset is not None else config.get("ruleset") + table = table or config.get("table") or asset.name + kwargs.setdefault("conn_id", config.get("conn_id")) + outlets = list(kwargs.get("outlets") or []) + if asset not in outlets: + outlets.append(asset) + kwargs["outlets"] = outlets + super().__init__(**kwargs) + if fail_on not in FAIL_ON_CHOICES: + raise ValueError(f"fail_on must be one of {FAIL_ON_CHOICES}, got {fail_on!r}") + if ruleset is None: + raise ValueError("ruleset is required, either directly or via an asset_quality() asset") + if not table: + raise ValueError("table is required, either directly or via an asset_quality() asset") + self.ruleset = ruleset + self.table = table + self.asset = asset + self.partition_clause = partition_clause + self.fail_on = fail_on + + def execute(self, context: Context) -> dict[str, Any]: + ruleset = self._resolve_ruleset() + started_at = datetime.now(tz=timezone.utc).isoformat() + engine = self._get_engine(self.get_db_hook()) + observations = engine.measure(ruleset, self.table, self.partition_clause) + results = [self._evaluate(observation) for observation in observations] + finished_at = datetime.now(tz=timezone.utc).isoformat() + + run = self._build_run(context, ruleset, started_at, finished_at) + summary = build_summary(run, results) + self._log_results(results, summary) + self._persist(run, results) + self._attach_to_outlet_events(context, summary) + self._raise_for_failures(results, summary) + return summary + + def _get_engine(self, hook: DbApiHook) -> SQLDQEngine: + return SQLDQEngine(hook) + + def _resolve_ruleset(self) -> RuleSet: + if self.ruleset is SET_DURING_EXECUTION: + raise ValueError("ruleset is required, either directly or returned by @task.dq_check") + if isinstance(self.ruleset, RuleSet): + return self.ruleset + if isinstance(self.ruleset, dict): + return RuleSet.from_dict(self.ruleset) + return RuleSet.from_file(self.ruleset) + + @staticmethod + def _evaluate(observation: Observation) -> RuleResult: + rule = observation.rule + condition = cast("Condition", rule.condition) + dimension = cast("Dimension", rule.dimension) + error_message = observation.error_message + if error_message is None: + try: + passed = condition.evaluate(observation.observed_value) + except (TypeError, ValueError) as e: + passed = False + error_message = f"Could not evaluate observed value {observation.observed_value!r}: {e}" + else: + passed = False + + if error_message is not None: + status = ERROR + elif passed: + status = PASS + else: + status = WARN if rule.severity == "warn" else FAIL + observed = observation.observed_value + if observed is not None and not isinstance(observed, int | float | str): + observed = str(observed) + return RuleResult( + rule_uid=rule.rule_uid, + rule_name=rule.name, + status=status, + observed_value=observed, + condition=condition.to_dict(), + dimension=dimension.value, + severity=rule.severity.value, + duration_ms=observation.duration_ms, + error_message=error_message, + description=rule.description or describe_rule(rule), + sql=observation.sql, + ) + + def _build_run(self, context: Context, ruleset: RuleSet, started_at: str, finished_at: str) -> DQRun: + ti = context["ti"] + return DQRun( + dag_id=ti.dag_id, + task_id=ti.task_id, + run_id=ti.run_id, + try_number=ti.try_number, + map_index=ti.map_index if ti.map_index is not None else -1, + ruleset_name=ruleset.name, + table_ref=self.table, + asset_names=tuple(self._outlet_asset_names()), + started_at=started_at, + finished_at=finished_at, + ) + + def _outlet_asset_names(self) -> list[str]: + names = [] + for outlet in self.outlets or []: + name = getattr(outlet, "name", None) + if name: + names.append(name) + return names + + def _persist(self, run: DQRun, results: list[RuleResult]) -> None: + backend = get_backend_from_config() + if backend is None: + self.log.info("No [dq] results_path configured; skipping results persistence") + return + # Persistence is best-effort: an unreachable results store leaves a gap in + # history but must not change the outcome of the check itself. + try: + backend.write_run(run, results) + except Exception: + self.log.exception("Failed to persist data quality results; continuing") + + def _attach_to_outlet_events(self, context: Context, summary: dict[str, Any]) -> None: + try: + outlet_events = context["outlet_events"] + except KeyError: + return + for outlet in self.outlets or []: + if getattr(outlet, "name", None): + try: + outlet_events[outlet].extra[DQ_RESULT_EXTRA_KEY] = summary + except Exception: + self.log.warning("Could not attach dq summary to outlet event for %s", outlet) + + def _log_results(self, results: list[RuleResult], summary: dict[str, Any]) -> None: + for result in results: + self.log.info( + "Rule %s [%s/%s]: %s (observed=%s, condition=%s)", + result.rule_name, + result.dimension, + result.severity, + result.status.upper(), + result.observed_value, + result.condition, + ) + self.log.info("Data quality score: %s", summary["score"]) + + def _raise_for_failures(self, results: list[RuleResult], summary: dict[str, Any]) -> None: + errored = [r.rule_name for r in results if r.status == ERROR] + if errored: + raise DQCheckFailedError(f"Check execution errored for rules: {errored}") + failed = [r.rule_name for r in results if r.status == FAIL] + warned = [r.rule_name for r in results if r.status == WARN] + if self.fail_on == "error" and failed: + raise DQCheckFailedError(f"Data quality rules failed: {failed} (score={summary['score']})") + if self.fail_on == "warn" and (failed or warned): + raise DQCheckFailedError( + f"Data quality rules failed: {failed + warned} (score={summary['score']})" + ) + if failed or warned: + self.log.warning("Rule failures below fail_on=%s threshold: %s", self.fail_on, failed + warned) diff --git a/providers/dq/src/airflow/providers/dq/plugins/__init__.py b/providers/dq/src/airflow/providers/dq/plugins/__init__.py new file mode 100644 index 0000000000000..21d298ede6ed3 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations diff --git a/providers/dq/src/airflow/providers/dq/plugins/dq.py b/providers/dq/src/airflow/providers/dq/plugins/dq.py new file mode 100644 index 0000000000000..31221d6925117 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/dq.py @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import mimetypes +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from fastapi.staticfiles import StaticFiles + +from airflow.plugins_manager import AirflowPlugin +from airflow.providers.common.compat.sdk import conf +from airflow.providers.dq.api.app import dq_app +from airflow.providers.dq.version_compat import AIRFLOW_V_3_1_PLUS + +_PLUGIN_PREFIX = "/dq" + + +def _get_base_url_path(path: str) -> str: + """Construct URL path with webserver base_url prefix for non-root deployments.""" + base_url = conf.get("api", "base_url", fallback="/") + if base_url.startswith(("http://", "https://")): + base_path = urlparse(base_url).path + else: + base_path = base_url + base_path = base_path.rstrip("/") + return base_path + path + + +def _get_bundle_url() -> str: + """ + Return the bundle URL for the React plugin. + + Uses an absolute URL when ``api.base_url`` is a full URL so the bundle loads correctly in + Vite dev mode, where ``import()`` resolves relative to the script origin (5175) rather + than the document origin (28080). + """ + path = _get_base_url_path(f"{_PLUGIN_PREFIX}/static/main.umd.cjs") + base_url = conf.get("api", "base_url", fallback="/") + if base_url.startswith(("http://", "https://")): + parsed = urlparse(base_url) + return f"{parsed.scheme}://{parsed.netloc}" + path + return path + + +# Ensure proper MIME type for the plugin bundle (FastAPI serves .cjs as text/plain by default). +mimetypes.add_type("application/javascript", ".cjs") + + +def _react_apps(dist_dir: Path) -> list[dict[str, str]]: + """ + React app entries, gated on the bundle actually being built. + + ``dist_dir`` isn't populated by a plain source checkout — it's a build artifact produced + by ``pnpm build`` (wired into the wheel build via ``hatch_build.py``). Advertising the + entry before that would point the UI at a 404. + """ + if not dist_dir.is_dir(): + return [] + return [ + { + "name": "Data Quality", + "bundle_url": _get_bundle_url(), + "destination": "task", + "url_route": "dq-task", + }, + { + "name": "Data Quality", + "bundle_url": _get_bundle_url(), + "destination": "task_instance", + "url_route": "dq-run", + }, + ] + + +_WWW_DIR = Path(__file__).parent / "www" +_dist_dir = _WWW_DIR / "dist" +if AIRFLOW_V_3_1_PLUS and _dist_dir.is_dir(): + dq_app.mount("/static", StaticFiles(directory=str(_dist_dir.absolute())), name="dq_static") + + +class DQPlugin(AirflowPlugin): + """Register the data quality read-only query API and task-instance React view.""" + + name = "dq" + fastapi_apps: list[dict[str, Any]] = ( + [{"name": "dq", "app": dq_app, "url_prefix": _PLUGIN_PREFIX}] if AIRFLOW_V_3_1_PLUS else [] + ) + react_apps: list[dict[str, str]] = _react_apps(_dist_dir) if AIRFLOW_V_3_1_PLUS else [] diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/.gitignore b/providers/dq/src/airflow/providers/dq/plugins/www/.gitignore new file mode 100644 index 0000000000000..10f8e3ae86fbe --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/.gitignore @@ -0,0 +1,30 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +.pnpm-store +# Stray atomic-write temp files pnpm can leave behind after an interrupted install +_tmp_* +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? + +# Coverage +coverage diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/index.html b/providers/dq/src/airflow/providers/dq/plugins/www/index.html new file mode 100644 index 0000000000000..504c624990137 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/index.html @@ -0,0 +1,12 @@ + + + + + + Data Quality + + +
+ + + diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/package.json b/providers/dq/src/airflow/providers/dq/plugins/www/package.json new file mode 100644 index 0000000000000..0f0d1bbdbe99d --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/package.json @@ -0,0 +1,62 @@ +{ + "name": "dq-plugin", + "packageManager": "pnpm@9.14.2", + "private": true, + "version": "0.0.0", + "engines": { + "node": ">=22" + }, + "type": "module", + "main": "./dist/main.js", + "module": "./dist/main.js", + "types": "./dist/main.d.ts", + "exports": { + ".": { + "import": "./dist/main.js", + "types": "./dist/main.d.ts" + } + }, + "files": [ + "dist" + ], + "scripts": { + "dev": "vite --port 5175 --strictPort", + "build": "vite build", + "build:types": "tsc --p tsconfig.lib.json", + "build:lib": "vite build", + "lint": "tsc --p tsconfig.app.json --noEmit", + "preview": "vite preview" + }, + "dependencies": { + "@chakra-ui/react": "^3.34.0", + "@emotion/react": "^11.14.0", + "react": "^19.2.4", + "react-dom": "^19.2.4" + }, + "devDependencies": { + "@types/react": "^19.2.14", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react-swc": "^4.2.3", + "typescript": "~5.9.3", + "vite": "^8.0.16", + "vite-plugin-css-injected-by-js": "^3.5.2", + "vite-plugin-dts": "^4.5.4" + }, + "pnpm": { + "minimumReleaseAge": 5760, + "overrides": { + "minimatch@>=10.0.0 <10.2.3": ">=10.2.3", + "brace-expansion@>=2.0.0 <2.0.3": ">=2.0.3", + "brace-expansion@>=4.0.0 <5.0.5": ">=5.0.5", + "picomatch@>=4.0.0 <4.0.4": ">=4.0.4", + "yaml@>=1.0.0 <1.10.3": ">=1.10.3", + "lodash@>=4.0.0 <=4.17.23": ">=4.18.0", + "lodash@<=4.17.23": ">=4.18.0", + "vite@>=7.0.0 <=7.3.1": ">=7.3.2", + "vite@>=7.1.0 <=7.3.1": ">=7.3.2", + "postcss@<8.5.10": ">=8.5.10", + "fast-uri@<=3.1.0": ">=3.1.1", + "fast-uri@<=3.1.1": ">=3.1.2" + } + } +} diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/pnpm-lock.yaml b/providers/dq/src/airflow/providers/dq/plugins/www/pnpm-lock.yaml new file mode 100644 index 0000000000000..051dffff5b113 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/pnpm-lock.yaml @@ -0,0 +1,2728 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +overrides: + minimatch@>=10.0.0 <10.2.3: '>=10.2.3' + brace-expansion@>=2.0.0 <2.0.3: '>=2.0.3' + brace-expansion@>=4.0.0 <5.0.5: '>=5.0.5' + picomatch@>=4.0.0 <4.0.4: '>=4.0.4' + yaml@>=1.0.0 <1.10.3: '>=1.10.3' + lodash@>=4.0.0 <=4.17.23: '>=4.18.0' + lodash@<=4.17.23: '>=4.18.0' + vite@>=7.0.0 <=7.3.1: '>=7.3.2' + vite@>=7.1.0 <=7.3.1: '>=7.3.2' + postcss@<8.5.10: '>=8.5.10' + fast-uri@<=3.1.0: '>=3.1.1' + fast-uri@<=3.1.1: '>=3.1.2' + +importers: + + .: + dependencies: + '@chakra-ui/react': + specifier: ^3.34.0 + version: 3.36.0(@emotion/react@11.14.0(@types/react@19.2.17)(react@19.2.7))(react-dom@19.2.7(react@19.2.7))(react@19.2.7) + '@emotion/react': + specifier: ^11.14.0 + version: 11.14.0(@types/react@19.2.17)(react@19.2.7) + react: + specifier: ^19.2.4 + version: 19.2.7 + react-dom: + specifier: ^19.2.4 + version: 19.2.7(react@19.2.7) + devDependencies: + '@types/react': + specifier: ^19.2.14 + version: 19.2.17 + '@types/react-dom': + specifier: ^19.2.3 + version: 19.2.3(@types/react@19.2.17) + '@vitejs/plugin-react-swc': + specifier: ^4.2.3 + version: 4.3.1(@swc/helpers@0.5.23)(vite@8.1.3(yaml@2.9.0)) + typescript: + specifier: ~5.9.3 + version: 5.9.3 + vite: + specifier: ^8.0.16 + version: 8.1.3(yaml@2.9.0) + vite-plugin-css-injected-by-js: + specifier: ^3.5.2 + version: 3.5.2(vite@8.1.3(yaml@2.9.0)) + vite-plugin-dts: + specifier: ^4.5.4 + version: 4.5.4(typescript@5.9.3)(vite@8.1.3(yaml@2.9.0)) + +packages: + + '@ark-ui/react@5.37.2': + resolution: {integrity: sha512-Q0R2Ah50kUhup0Ljxg65zGJq5yBV52BLm1coRkjHHid40d1yclaDGfhPL48kcF/xtjAFlGLkL6SiENkGvfh+mw==} + peerDependencies: + react: '>=18.0.0' + react-dom: '>=18.0.0' + + '@babel/code-frame@7.29.7': + resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} + engines: {node: '>=6.9.0'} + + '@babel/generator@7.29.7': + resolution: {integrity: sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==} + engines: {node: '>=6.9.0'} + + '@babel/helper-globals@7.29.7': + resolution: {integrity: sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==} + engines: {node: '>=6.9.0'} + + '@babel/helper-module-imports@7.29.7': + resolution: {integrity: sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==} + engines: {node: '>=6.9.0'} + + '@babel/helper-string-parser@7.29.7': + resolution: {integrity: sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.29.7': + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} + engines: {node: '>=6.9.0'} + + '@babel/parser@7.29.7': + resolution: {integrity: sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==} + engines: {node: '>=6.0.0'} + hasBin: true + + '@babel/runtime@7.29.7': + resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} + engines: {node: '>=6.9.0'} + + '@babel/template@7.29.7': + resolution: {integrity: sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==} + engines: {node: '>=6.9.0'} + + '@babel/traverse@7.29.7': + resolution: {integrity: sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==} + engines: {node: '>=6.9.0'} + + '@babel/types@7.29.7': + resolution: {integrity: sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==} + engines: {node: '>=6.9.0'} + + '@chakra-ui/react@3.36.0': + resolution: {integrity: sha512-6AxUbJsC6yyTzPeYL8sxyAL07lflT0NA+S6tcPzEuwdMux+benRMFOpPktnkifWKl/Vq/JD7fhxDyMuDQ4M0gA==} + peerDependencies: + '@emotion/react': '>=11' + react: '>=18' + react-dom: '>=18' + + '@emnapi/core@1.11.1': + resolution: {integrity: sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==} + + '@emnapi/runtime@1.11.1': + resolution: {integrity: sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==} + + '@emnapi/wasi-threads@1.2.2': + resolution: {integrity: sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==} + + '@emotion/babel-plugin@11.13.5': + resolution: {integrity: sha512-pxHCpT2ex+0q+HH91/zsdHkw/lXd468DIN2zvfvLtPKLLMo6gQj7oLObq8PhkrxOZb/gGCq03S3Z7PDhS8pduQ==} + + '@emotion/cache@11.14.0': + resolution: {integrity: sha512-L/B1lc/TViYk4DcpGxtAVbx0ZyiKM5ktoIyafGkH6zg/tj+mA+NE//aPYKG0k8kCHSHVJrpLpcAlOBEXQ3SavA==} + + '@emotion/hash@0.9.2': + resolution: {integrity: sha512-MyqliTZGuOm3+5ZRSaaBGP3USLw6+EGykkwZns2EPC5g8jJ4z9OrdZY9apkl3+UP9+sdz76YYkwCKP5gh8iY3g==} + + '@emotion/is-prop-valid@1.4.0': + resolution: {integrity: sha512-QgD4fyscGcbbKwJmqNvUMSE02OsHUa+lAWKdEUIJKgqe5IwRSKd7+KhibEWdaKwgjLj0DRSHA9biAIqGBk05lw==} + + '@emotion/memoize@0.9.0': + resolution: {integrity: sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==} + + '@emotion/react@11.14.0': + resolution: {integrity: sha512-O000MLDBDdk/EohJPFUqvnp4qnHeYkVP5B0xEG0D/L7cOKP9kefu2DXn8dj74cQfsEzUqh+sr1RzFqiL1o+PpA==} + peerDependencies: + '@types/react': '*' + react: '>=16.8.0' + peerDependenciesMeta: + '@types/react': + optional: true + + '@emotion/serialize@1.3.3': + resolution: {integrity: sha512-EISGqt7sSNWHGI76hC7x1CksiXPahbxEOrC5RjmFRJTqLyEK9/9hZvBbiYn70dw4wuwMKiEMCUlR6ZXTSWQqxA==} + + '@emotion/sheet@1.4.0': + resolution: {integrity: sha512-fTBW9/8r2w3dXWYM4HCB1Rdp8NLibOw2+XELH5m5+AkWiL/KqYX6dc0kKYlaYyKjrQ6ds33MCdMPEwgs2z1rqg==} + + '@emotion/unitless@0.10.0': + resolution: {integrity: sha512-dFoMUuQA20zvtVTuxZww6OHoJYgrzfKM1t52mVySDJnMSEa08ruEvdYQbhvyu6soU+NeLVd3yKfTfT0NeV6qGg==} + + '@emotion/use-insertion-effect-with-fallbacks@1.2.0': + resolution: {integrity: sha512-yJMtVdH59sxi/aVJBpk9FQq+OR8ll5GT8oWd57UpeaKEVGab41JWaCFA7FRLoMLloOZF/c/wsPoe+bfGmRKgDg==} + peerDependencies: + react: '>=16.8.0' + + '@emotion/utils@1.4.2': + resolution: {integrity: sha512-3vLclRofFziIa3J2wDh9jjbkUz9qk5Vi3IZ/FSTKViB0k+ef0fPV7dYrUIugbgupYDx7v9ud/SjrtEP8Y4xLoA==} + + '@emotion/weak-memoize@0.4.0': + resolution: {integrity: sha512-snKqtPW01tN0ui7yu9rGv69aJXr/a/Ywvl11sUjNtEcRc+ng/mQriFL0wLXMef74iHa/EkftbDzU9F8iFbH+zg==} + + '@floating-ui/core@1.7.5': + resolution: {integrity: sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ==} + + '@floating-ui/dom@1.7.6': + resolution: {integrity: sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ==} + + '@floating-ui/utils@0.2.11': + resolution: {integrity: sha512-RiB/yIh78pcIxl6lLMG0CgBXAZ2Y0eVHqMPYugu+9U0AeT6YBeiJpf7lbdJNIugFP5SIjwNRgo4DhR1Qxi26Gg==} + + '@internationalized/date@3.12.2': + resolution: {integrity: sha512-FY1Y+H64NDs+HAF6omlnWxm3mEpfgaCSWtL5l551ZZfImA+kGjPFgrnJrGjH6lfmLL0g8Z/mBu1R3kufeCp6Jw==} + + '@internationalized/number@3.6.6': + resolution: {integrity: sha512-iFgmQaXHE0vytNfpLZWOC2mEJCBRzcUxt53Xf/yCXG93lRvqas237i3r7X4RKMwO3txiyZD4mQjKAByFv6UGSQ==} + + '@jridgewell/gen-mapping@0.3.13': + resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + + '@microsoft/api-extractor-model@7.33.8': + resolution: {integrity: sha512-aIcoQggPyer3B6Ze3usz0YWC/oBwUHfRH5ETUsr+oT2BRA6SfTJl7IKPcPZkX4UR+PohowzW4uMxsvjrn8vm+w==} + + '@microsoft/api-extractor@7.58.9': + resolution: {integrity: sha512-S2UF4yza5GoxCmf7hJQNxJNZN9ltOVuOQv8Dy+Z21aol5ERoBNMdWcQHm4MJMPPItW4H/4rZD906iaf4mUojJA==} + hasBin: true + + '@microsoft/tsdoc-config@0.18.1': + resolution: {integrity: sha512-9brPoVdfN9k9g0dcWkFeA7IH9bbcttzDJlXvkf8b2OBzd5MueR1V2wkKBL0abn0otvmkHJC6aapBOTJDDeMCZg==} + + '@microsoft/tsdoc@0.16.0': + resolution: {integrity: sha512-xgAyonlVVS+q7Vc7qLW0UrJU7rSFcETRWsqdXZtjzRU8dF+6CkozTK4V4y1LwOX7j8r/vHphjDeMeGI4tNGeGA==} + + '@napi-rs/wasm-runtime@1.1.6': + resolution: {integrity: sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==} + peerDependencies: + '@emnapi/core': ^1.7.1 + '@emnapi/runtime': ^1.7.1 + + '@oxc-project/types@0.138.0': + resolution: {integrity: sha512-1a7ZKmrRTCoN1XMZ4L0PyyqrMnrNlLyPuOkdSX2MZg7IiIGRUyurNhAm73ptDOraoBcIordsIGKNPKUzy3ZmfA==} + + '@pandacss/is-valid-prop@1.11.4': + resolution: {integrity: sha512-RWxInlS+lGgKiF0fB0HO76vsJFgRvbavm5Z25/GqqN8MPHXYA6n5rZnfdp4itEXy5DJkQ9vt3yrwa2IKiuhtrA==} + engines: {node: '>=20'} + + '@rolldown/binding-android-arm64@1.1.4': + resolution: {integrity: sha512-EZLpf/8y7GXkkra90ML47kzik/GMP3EMcE9bPyHmRfxLC6z9+aW5A8poCsoxjrT5GfEcNAAvWwUHjvP1pUQkfw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + + '@rolldown/binding-darwin-arm64@1.1.4': + resolution: {integrity: sha512-aUi+HBvmYb7j8krl1+qJgkG8C17fO79gk3c+jPw4S8glRFc1DTija9S3EyaTSQUm5GJXYKDAsugBEhFHH2vYiQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + + '@rolldown/binding-darwin-x64@1.1.4': + resolution: {integrity: sha512-F7hHC3gwY11+vByKPRWqwGbeXWVgKmL+pTGCinaEhdihzBV2aQ0fvZOch9cXYUOKuKKq429HeYXOqQLc7wFCEg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + + '@rolldown/binding-freebsd-x64@1.1.4': + resolution: {integrity: sha512-sI5yw+7s92SK6odiEhD5lKCBlWcpjHS5qyqpVQbZAJ0fIzEUXrmbl3DH2ybR3PZogulNJF+COLtmA8hUfvkCCQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + + '@rolldown/binding-linux-arm-gnueabihf@1.1.4': + resolution: {integrity: sha512-mCi0OKgEieFircrtVYmQAFGszRtMnZ6fpZAXrxanXAu7lqZcsK1E1RAaZNG0uKAnxox3B1f4EyQNnoyMfN1vAA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@rolldown/binding-linux-arm64-gnu@1.1.4': + resolution: {integrity: sha512-B9Ial3Kv5sh0SHnB1g/QWcUQCEvCF6QKGAl4zXypYj65mVI+B4AhFBwPtSN7pDrJeIx8Z7zdy4ntx+wQABom7w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + + '@rolldown/binding-linux-arm64-musl@1.1.4': + resolution: {integrity: sha512-lZVym0PuHE1KZ22gmFTC15lAkrg9iTszR617oYRB/iPY1A56ywoJzVKOJBKaot5RiikCObmur6pogpse3gRcng==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + + '@rolldown/binding-linux-ppc64-gnu@1.1.4': + resolution: {integrity: sha512-t2DNiLJWNTbnEHyUzTumldML6ET4/g16467LZoDDJ3tSxGvguL5/NyC2lCsNKuyRycg9XeDQF5SSv+TNOhQEXg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + + '@rolldown/binding-linux-s390x-gnu@1.1.4': + resolution: {integrity: sha512-0WIRnL1Uw4BvTZRLQt+PVgo6ZKTJadlC2btP+/EOXv2f/DWbY0rEgl+y834mIVwP1FkTlWVTrGGJXf12lru7EQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + + '@rolldown/binding-linux-x64-gnu@1.1.4': + resolution: {integrity: sha512-JWtGshGfX+oENAKonoNkqEJX+7hC8yfhi9GUyPX1VX4mdh1y5r+ZiJLR5XzAB0aoP6s/PcILsGjKq8O0mm24bw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + + '@rolldown/binding-linux-x64-musl@1.1.4': + resolution: {integrity: sha512-rT6yQcxUuXs4CnbofqwHRRV0iem349rLMYpTjkgQGLjrY4ado/eDzwPZPTCgTOlF6Nkp8NEv70yLMTn6qkWxsQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + + '@rolldown/binding-openharmony-arm64@1.1.4': + resolution: {integrity: sha512-KXMGoboq5cyaCQjDA4GLuRiOwBQ0EyFnJoVViLeZ45/3rFItRODEr+NdsBcVpll40hhNArlm/speWGRvj08LzA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + + '@rolldown/binding-wasm32-wasi@1.1.4': + resolution: {integrity: sha512-5K83rb36oJiY7BCyE9zLZtGcPV4g5wvq+xwdO0XPIwDVZI8cyB/AUjkNXGb92/rnmezEkjMOpgY61rtwjQtFwg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [wasm32] + + '@rolldown/binding-win32-arm64-msvc@1.1.4': + resolution: {integrity: sha512-PnWBtw3TV5KOg69HQQDR0mnQuyCmSGR2pAB4DC1rPF808fgKeTUMj2EOEyKATpgiuxuR5APQmiDO7PDgEjTFSA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + + '@rolldown/binding-win32-x64-msvc@1.1.4': + resolution: {integrity: sha512-M1lpniBePobTfsa7Ks9a199e1akxsXn+GYBUKsEzv3YFzOm1HJAMNwKI3qr0Zq+mxwx9gOZoTdP1yXRYsZUocQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + + '@rolldown/pluginutils@1.0.1': + resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} + + '@rollup/pluginutils@5.4.0': + resolution: {integrity: sha512-MfPp06CjRLfXQ3wY0R8vJDYBy/MvVcc9OulEfR0B8Iv9ko+GCNaRZ+EpJYFl27LhKsZK0o420sYCRHCjfCgeUg==} + engines: {node: '>=14.0.0'} + peerDependencies: + rollup: ^1.20.0||^2.0.0||^3.0.0||^4.0.0 + peerDependenciesMeta: + rollup: + optional: true + + '@rushstack/node-core-library@5.23.1': + resolution: {integrity: sha512-wlKmIKIYCKuCASbITvOxLZXepPbwXvrv7S6ig6XNWFchSyhL/E2txmVXspHY49Wu2dzf7nI27a2k/yV5BA3EiA==} + peerDependencies: + '@types/node': '*' + peerDependenciesMeta: + '@types/node': + optional: true + + '@rushstack/problem-matcher@0.2.1': + resolution: {integrity: sha512-gulfhBs6n+I5b7DvjKRfhMGyUejtSgOHTclF/eONr8hcgF1APEDjhxIsfdUYYMzC3rvLwGluqLjbwCFZ8nxrog==} + peerDependencies: + '@types/node': '*' + peerDependenciesMeta: + '@types/node': + optional: true + + '@rushstack/rig-package@0.7.3': + resolution: {integrity: sha512-aAA518n6wxxjCfnTAOjQnm7ngNE0FVHxHAw2pxKlIhxrMn0XQjGcXKF0oKWpjBgJOmsaJpVob/v+zr3zxgPWuA==} + + '@rushstack/terminal@0.24.0': + resolution: {integrity: sha512-8ZQS4MMaGsv27EXCBiH7WMPkRZrffeDoIevs6z9TM5dzqiY6+Hn4evfK/G+gvgBTjfvfkHIZPQQmalmI2sM4TQ==} + peerDependencies: + '@types/node': '*' + peerDependenciesMeta: + '@types/node': + optional: true + + '@rushstack/ts-command-line@5.3.10': + resolution: {integrity: sha512-fwI076HYknC0IrMXdY6UmjDv+PH7NHhNJX3/pY2UblSE5XrXgndXZPiOe/6ZtuFpn6DvVDVNhtkIzQ+Qu/MhVQ==} + + '@swc/core-darwin-arm64@1.15.43': + resolution: {integrity: sha512-v1aVuvXdo/BHxJzco9V2xpHrvwWmhfS8t6gziY5wJxd+Z2h8AeJRnAwPD8itCDaGXVBwJ/CaKfxEzTkG0Va0OA==} + engines: {node: '>=10'} + cpu: [arm64] + os: [darwin] + + '@swc/core-darwin-x64@1.15.43': + resolution: {integrity: sha512-lp3d4Lamc8dt5huYdGLSR+9hLxmfr1jb0l+4XXG2zPqZwYWRN9R0U2qYoTrggiU2RWW0oV9VbWM3kBnqIc2kdQ==} + engines: {node: '>=10'} + cpu: [x64] + os: [darwin] + + '@swc/core-linux-arm-gnueabihf@1.15.43': + resolution: {integrity: sha512-JWTQQELtsG5GgphDrr/XqqmM2pDN3cZqbMS0Mrg+iTiXL3F74sn/S2IyYE/5u4h2KLkTf9qQ7dXyxsbx7YzkeA==} + engines: {node: '>=10'} + cpu: [arm] + os: [linux] + + '@swc/core-linux-arm64-gnu@1.15.43': + resolution: {integrity: sha512-B4otJRdPWIsmiSBf0uG7Z/+vMWmkufjz5MmYxubwKuZazDW14Zd3symga1N62QR4RT+kEFeHEgsXfZGyn/w0hw==} + engines: {node: '>=10'} + cpu: [arm64] + os: [linux] + + '@swc/core-linux-arm64-musl@1.15.43': + resolution: {integrity: sha512-6zB6OnpViBxYy4tgY3v2i6AZY9fwkcHZ032UOwtwUuW1d19sdT07qF0kZe6/3UR1tUaK6jjg2rmVcUIBCEYVjQ==} + engines: {node: '>=10'} + cpu: [arm64] + os: [linux] + + '@swc/core-linux-ppc64-gnu@1.15.43': + resolution: {integrity: sha512-coxE1ZWdB3uSDVNoEtYNrRi/1epvckZx9cTJ8ICUxTMTxGk+yvQ/Twacp3ruZSaMPGCriUjP86C37VhaT6nyRg==} + engines: {node: '>=10'} + cpu: [ppc64] + os: [linux] + + '@swc/core-linux-s390x-gnu@1.15.43': + resolution: {integrity: sha512-lXfLhs+LpBsD5inuYx+YDH5WsPPBQ95KPUiy8P5wq9ob9xKDZFqwNfU2QW6bGO8NqRO/H9JQomTSt5Yyh+FGfA==} + engines: {node: '>=10'} + cpu: [s390x] + os: [linux] + + '@swc/core-linux-x64-gnu@1.15.43': + resolution: {integrity: sha512-07XnKwTmKy8TGOZG3D9fRnLWGynxPjwQnZLVmBFbo6F+7vHYzBIOuwXEhemrChBWb6yDNZsVCcMWCPX6FDD2xg==} + engines: {node: '>=10'} + cpu: [x64] + os: [linux] + + '@swc/core-linux-x64-musl@1.15.43': + resolution: {integrity: sha512-TJc+bsSIaBh+hZvZ5GRtW/K1bw66TJ9vsUwvVIsZdiWxU5ObLwZvfcnZ3UpgVfMnFibRes9uriJrQNBHEEogRQ==} + engines: {node: '>=10'} + cpu: [x64] + os: [linux] + + '@swc/core-win32-arm64-msvc@1.15.43': + resolution: {integrity: sha512-jfd7s2/bUQYkOHLs+LWQNKZdmDa8+sufKLllhpWAhVQ2GDCwsHe3vR/j+OSiItZNtkzFuaawa3+SAKz9y5gYfw==} + engines: {node: '>=10'} + cpu: [arm64] + os: [win32] + + '@swc/core-win32-ia32-msvc@1.15.43': + resolution: {integrity: sha512-rLAE8JvucqEW1ZGohxPQrQWPBQeJG4+ypKbWfdlU/qmKScvCkxf9/Jxnzki1dkUQCQ7P5Enp13RlvqOlvx/32g==} + engines: {node: '>=10'} + cpu: [ia32] + os: [win32] + + '@swc/core-win32-x64-msvc@1.15.43': + resolution: {integrity: sha512-h8MLDHZcfIukwQWj03rIJZx1I0E81AYj2X7J/nGErG4nz+QAv6G1Z+peotvinL3lqpbo32tLYSMFo32/ySzxKg==} + engines: {node: '>=10'} + cpu: [x64] + os: [win32] + + '@swc/core@1.15.43': + resolution: {integrity: sha512-1CuKjFkPxIgGdeHVuNbkxmBxkcbdc08u0aiI43pFq6yY1tTVKmXT9hFEooyyKs/sJ3xf1GPHyEwTtk9Xl8dvQw==} + engines: {node: '>=10'} + peerDependencies: + '@swc/helpers': '>=0.5.17' + peerDependenciesMeta: + '@swc/helpers': + optional: true + + '@swc/counter@0.1.3': + resolution: {integrity: sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==} + + '@swc/helpers@0.5.23': + resolution: {integrity: sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==} + + '@swc/types@0.1.27': + resolution: {integrity: sha512-K6h3iUlqeM946U4sXFYeahefR1YBbXJvko+hv8WS8/0BNJ4OHiHRywMnQUJCqkR7Y9+hqQ1TvEpiKqUhz7NEFg==} + + '@tybys/wasm-util@0.10.3': + resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==} + + '@types/argparse@1.0.38': + resolution: {integrity: sha512-ebDJ9b0e702Yr7pWgB0jzm+CX4Srzz8RcXtLJDJB+BSccqMa36uyH/zUsSYao5+BD1ytv3k3rPYCq4mAE1hsXA==} + + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + + '@types/parse-json@4.0.2': + resolution: {integrity: sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==} + + '@types/react-dom@19.2.3': + resolution: {integrity: sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==} + peerDependencies: + '@types/react': ^19.2.0 + + '@types/react@19.2.17': + resolution: {integrity: sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==} + + '@vitejs/plugin-react-swc@4.3.1': + resolution: {integrity: sha512-PaeokKjAGraNN+s5SIApgsktnJprIyt3zgEIu7awnEdfn29QiB2crTcCzyi2XGpX9rUnTc0cKU07Wm0N0g7H2w==} + engines: {node: ^20.19.0 || >=22.12.0} + peerDependencies: + vite: '>=7.3.2' + + '@volar/language-core@2.4.28': + resolution: {integrity: sha512-w4qhIJ8ZSitgLAkVay6AbcnC7gP3glYM3fYwKV3srj8m494E3xtrCv6E+bWviiK/8hs6e6t1ij1s2Endql7vzQ==} + + '@volar/source-map@2.4.28': + resolution: {integrity: sha512-yX2BDBqJkRXfKw8my8VarTyjv48QwxdJtvRgUpNE5erCsgEUdI2DsLbpa+rOQVAJYshY99szEcRDmyHbF10ggQ==} + + '@volar/typescript@2.4.28': + resolution: {integrity: sha512-Ja6yvWrbis2QtN4ClAKreeUZPVYMARDYZl9LMEv1iQ1QdepB6wn0jTRxA9MftYmYa4DQ4k/DaSZpFPUfxl8giw==} + + '@vue/compiler-core@3.5.39': + resolution: {integrity: sha512-16KBTEXAJCpDr0mwlw+AZyhu8iyC7R3S2vBwsI7QnWJU6X3WKc9VKeNEZpiMdZ569qWhz9574L3vV55qRL0Vtw==} + + '@vue/compiler-dom@3.5.39': + resolution: {integrity: sha512-oQPigALqYbNxTNPvNgSOe+czwVExfbVF02lz8jP0S3AXJiu3jxYDygNUiqSep4ezzW8XgnubqH63My2A7JR/vg==} + + '@vue/compiler-vue2@2.7.16': + resolution: {integrity: sha512-qYC3Psj9S/mfu9uVi5WvNZIzq+xnXMhOwbTFKKDD7b1lhpnn71jXSFdTQ+WsIEk0ONCd7VV2IMm7ONl6tbQ86A==} + + '@vue/language-core@2.2.0': + resolution: {integrity: sha512-O1ZZFaaBGkKbsRfnVH1ifOK1/1BUkyK+3SQsfnh6PmMmD4qJcTU8godCeA96jjDRTL6zgnK7YzCHfaUlH2r0Mw==} + peerDependencies: + typescript: '*' + peerDependenciesMeta: + typescript: + optional: true + + '@vue/shared@3.5.39': + resolution: {integrity: sha512-l1rrBtBfTnmxvtsvdQDXltUUy8S1Y+ZaqdfUzmAnJkTd8Z8rv5v/ytW+TKiqEOWyHPoqtPlNFSs0lhRmYVSHVA==} + + '@zag-js/accordion@1.41.2': + resolution: {integrity: sha512-7G//V7svGGT8k5avw7bbQvbRC0Q/9QtX51b4iyAB1alR9E5mFd6Ch8q4njwcXClMQ7xePS3jUfVnzVGiRInEiQ==} + + '@zag-js/anatomy@1.41.2': + resolution: {integrity: sha512-Fm9hqdrvaCzCsdcf19G8WZxYtHElKltkGHdhqMEt4XU+ULTr1DK7KbOtDDv9J27CuzqSLALUz5QfRjPftoKHwg==} + + '@zag-js/angle-slider@1.41.2': + resolution: {integrity: sha512-+7bZHAZx0MEbjTMr2tD+meFJ0EJwFfUEcTqmdLzFGr/ySAMCWlcadDBz+ZmrSn03aKLps8FxliVLzsFJNgUqIQ==} + + '@zag-js/aria-hidden@1.41.2': + resolution: {integrity: sha512-qEcYmwlQr3qjA0T/IZ5a/o7fRUxfQ14tXjAFhR3GXCtxBKaqS+wnq/LN09Xw4bin3QWTINU+Z0oXFs9spWtNwA==} + + '@zag-js/async-list@1.41.2': + resolution: {integrity: sha512-NZZEIGFdeDp2uHjsLVegLAGJOYGwI9HPJI1V2c/P1TQmfmrfWyWELAvnnW4kWYVUKYD9TxKQkm6LvqpHrQzgfQ==} + + '@zag-js/auto-resize@1.41.2': + resolution: {integrity: sha512-CYq+JQ1TTkEiK7OcNcMTS0f4wFvtmManvUltije5o50gDQ7vFYA81oQh4A9pAB1keUi9Zv06CNefFARaTcne5w==} + + '@zag-js/avatar@1.41.2': + resolution: {integrity: sha512-+4K0tRtIQysMGuERh5JRmn46uq7gJ6IjJd5DKj74VBtVVY8T6HGk0D0DZxmOIgADHP1qSvowLDoOX8kUyBHarQ==} + + '@zag-js/carousel@1.41.2': + resolution: {integrity: sha512-H6sDxyWIzkvtHN4M/BYvFy7pi8j+kLEtfPZvgNl2+gRnfAHVK9/XqpoUsjw1DAo5Fh25pPuaTnh52Kml9OK0iA==} + + '@zag-js/cascade-select@1.41.2': + resolution: {integrity: sha512-dBByZmIAJU/B+YIAzczng05lCJXEwEm4df6GmUZtioKHZdeN+WEKUP4qzFDFZdytXympGfJCIBvtgf87gACYVQ==} + + '@zag-js/checkbox@1.41.2': + resolution: {integrity: sha512-aQ5dZWHUHRIw4cbLtzrR+dc8M0N6wSiiCml/zbU3ciHOTBXSrs2rKnp/L99xhi97Lj2uCEhpIZ704Ou/MjGuCg==} + + '@zag-js/clipboard@1.41.2': + resolution: {integrity: sha512-FM+PNGEeY+YpF1dVr9d8kg3DQM66LRnkR4Mva1oprqnrCXTYWj+k7uig893ubSG2xw1GO5b/WJd27kSmNoKnmw==} + + '@zag-js/collapsible@1.41.2': + resolution: {integrity: sha512-uMIp4rqd3iI6VFPMAOcbu9dh9WV4l85nNPYQiBtMKRHgbkfaZdfzF+E+NX3KquIeHW6JiBFUiIyCU0Sf2Cscdw==} + + '@zag-js/collection@1.41.2': + resolution: {integrity: sha512-ZZWuvfPZI8ccWd4aLpuU47k1jSc9eO+F3FM3iBJuvgegCH6g3+HDEwGN6wdePHnYfv7zyIKCGKr816zXcIWQbw==} + + '@zag-js/color-picker@1.41.2': + resolution: {integrity: sha512-UynyJ/bTBeSlq6ziHr9GVrFycDjVxfLFIb8Aivu/XvihrAAvkhXUxwy+1ax+hj7peGlTY590xoQAvQixV+McBA==} + + '@zag-js/color-utils@1.41.2': + resolution: {integrity: sha512-Lsi5c2ztGZqud0oeDtAn3xFhgrYMaeaz1zi4p+mr0zXOuQNuZzfsrJ0stQ45s8L/xT8UJtGhYyEVy1+xjE4ARA==} + + '@zag-js/combobox@1.41.2': + resolution: {integrity: sha512-V9jQteyQHs8ZNQx/FcA98P1NVZas/yjiW1V7s4L+h8MnRS3AK97+FAHAT83KCiZ34/vkpLF6ZEHI2zkcQpNXcg==} + + '@zag-js/core@1.41.2': + resolution: {integrity: sha512-xXTN3zKwOtMI4+5dG2cG+T1B4WR3X9alXiYaPJKaGd4N2eYRj9JEPte3Hv9gtFm+RM0b9VIwksHEg25rqE4apw==} + + '@zag-js/date-input@1.41.2': + resolution: {integrity: sha512-J8PdpEpM9TBxZBEE8/Nxi39LNJOZY7lilTbgcDABR5uiviZUk5++5GSdUTspcjFUIXx5SvqmdCk2RF7hsUEHVQ==} + peerDependencies: + '@internationalized/date': '>=3.0.0' + + '@zag-js/date-picker@1.41.2': + resolution: {integrity: sha512-j9LaznCV4QCfrq/y/mNAN9XgIRFFg+414TxGT4TK8mfWouIaFvxEoKdGP0l/LL4DFv1NRDaRdSBBcw3eXtMVnA==} + peerDependencies: + '@internationalized/date': '>=3.0.0' + + '@zag-js/date-utils@1.41.2': + resolution: {integrity: sha512-jdcWLa5fYLTsvGWJkx4v/9Hzqf6UHdvJDeP6NxHpwoEmzYy7+AghYKBNSnRrJIBCQJgl1vM9OkpMvYrLNHr9jw==} + peerDependencies: + '@internationalized/date': '>=3.0.0' + + '@zag-js/dialog@1.41.2': + resolution: {integrity: sha512-t1N72snpFGiKj7cg2PivPdsy+X1YdgXPv7bzovpqjMLy0kN+gYTPoV1Iy/hQA+g021dz9XGgXzkDuU45sJqsFA==} + + '@zag-js/dismissable@1.41.2': + resolution: {integrity: sha512-hO/tFhRZ7S+LOOljGOQJIubbc3MXg41+iWR1yUXKl76cAenbxaCit1LZmUCwQPvRN0GndK6bDQo5ETjHZz/k7A==} + + '@zag-js/dom-query@1.41.2': + resolution: {integrity: sha512-+eBk1nlJA312mNmY/GSThLRwcCRqMIL+A1pLsWvTlQLQjmH1/UxoAuv6l2yvRCT33XmC8FBlBIKnXhOCpDvIZA==} + + '@zag-js/drawer@1.41.2': + resolution: {integrity: sha512-aJql6L0cfHd1wXbemfLcNnjqTA/CbwVgQdWEVn5Qci6zhy4UJXPQeBBfxfgPgEOAbW60gL/gaaXG3d9Vx6+6oA==} + + '@zag-js/editable@1.41.2': + resolution: {integrity: sha512-loTM1lrHBBqYfR8SJZrYayVdWHMpXCLVir+aDTQ8/d4bb/Gfl/L8CJNs3BRj3yt9zvLoWTLO+4LOY3hLyQSR2w==} + + '@zag-js/file-upload@1.41.2': + resolution: {integrity: sha512-WFwIaKvHpCUjyYMvp42VoydT1WIP5DhDlpmG/nrF4i0ro7pLGb1A4dvyBrAfGcozCB88yR1y1iZKPDY1I9/uUw==} + + '@zag-js/file-utils@1.41.2': + resolution: {integrity: sha512-Ih+8ULbId0M+CFR4IsqG5y/0VLCk2l+1rgPH+21L40dlSB6z6qKSP2tG7W69Cj2/3vryZsn67ibn26iCPG/vOw==} + + '@zag-js/floating-panel@1.41.2': + resolution: {integrity: sha512-nJP3oZ4YrJh+7H5YdwNccSzPGXFqjQN1ujZ/xGDhegjz4XtL48QMFnAasxlRI8VGjse9Tj8VXlQZxXPrASGzlA==} + + '@zag-js/focus-trap@1.41.2': + resolution: {integrity: sha512-3QTtGUjFU2OLbyrDlyoYWvKZecCmtn/+bsfsHW159jJHiEGHVYK6CY8AI1ePsMk9gVay48bXh008j+lVli7gAw==} + + '@zag-js/focus-visible@1.41.2': + resolution: {integrity: sha512-oRjwtgafUdGVwLJUN6mKsnBQbez/CHYAPkPg1FxOnr5GFpEpr8oMTOZJ3wdPM2U1ynS9QnUUu/sXc3KQv/jX0Q==} + + '@zag-js/highlight-word@1.41.2': + resolution: {integrity: sha512-95zcZKqNrL7JlqAckfzHa+LRbnfoz1lj6skUhq/uHSnni1vH6+8fNWT8ruo9G7vpGopbyRmmcie5p41SbAwQjQ==} + + '@zag-js/hover-card@1.41.2': + resolution: {integrity: sha512-Xn9RVzgTkKaVzyJTDdBJiXmCleNi1/hmW8z73tC0vOiQvSSvPejg/JkzqTOLFODvQHK3NOw54QHZvmjYp9Mubw==} + + '@zag-js/i18n-utils@1.41.2': + resolution: {integrity: sha512-f1xqaEY79awBxgUyjFso0UEpIoEHZq+zRvB0nUVFHJRW7Ds/QhaFHKSRnf2QBTlP/ObvCT225R+piNAAc17Aaw==} + + '@zag-js/image-cropper@1.41.2': + resolution: {integrity: sha512-750aT4U+J/TJw/Z1QVoLU9JG0luCtf9CyvShtJFIxeS+i25aUoBI9pOKgSFABsOutIqNJTPq4gYpqtuxFjSxRw==} + + '@zag-js/interact-outside@1.41.2': + resolution: {integrity: sha512-dM4Fn9iyqQeqkCMRYZP+bAgWEPKRVQRqMmcPsN0OXBhhFKC31Em15LTIZXaOtVKAjH+iwx+UvSYFRiWwEjkOEA==} + + '@zag-js/json-tree-utils@1.41.2': + resolution: {integrity: sha512-gNaOzsbCwmTd2HM3/u0xQdWX5UDBfl8tCXFavzbamkFH0iYQOXJb7cqUXBVuI4KScIbHPCKwrzZjqA5Sg9qzAQ==} + + '@zag-js/listbox@1.41.2': + resolution: {integrity: sha512-iDGrZleP3ui2Q6Jgmr9RYlbd7njdJHs8Qb3IJrSIZBIeYyYmFvVUfAFjk0g/z/amjTx6uYxRASWSPy/RETd4ug==} + + '@zag-js/live-region@1.41.2': + resolution: {integrity: sha512-7ubIW5AQt1wx9S/gFN+rU4TyvuFWJrL/DhnDWPNlH5g3luDVHSNeYGeeqf4d4tkcibtpYZa0pg9CbXxRxrfwVA==} + + '@zag-js/marquee@1.41.2': + resolution: {integrity: sha512-cT77aMhrtAK3oe2O6+X3TLtQs8wupdPUtZgHMWVxCcQOwagrk7RUBEQwU3Cx7cTswpBdTKSuDYgUggfgCs96wg==} + + '@zag-js/menu@1.41.2': + resolution: {integrity: sha512-wwix8hcAUSi0scpWXCiDppfdZV01Za8nN0gqLt9GdhCiVSlr0rs9pK1ROgPKJTyc43UZfyFPqtTWVHvEHMM70w==} + + '@zag-js/navigation-menu@1.41.2': + resolution: {integrity: sha512-aROB9CHzskZpnoFuGFkp7dbkZdsXvp2nxQgsgld02I1sDqiwcQd+YMdB0/6Ik0oz6X8I70RKdUuQM9WQFQfDnA==} + + '@zag-js/number-input@1.41.2': + resolution: {integrity: sha512-QoQWCiVHO+ciUbq8uL8Kxhtk4o3UpwzYpJkfsOfitzsZoYpPc7V/A6+n5yABV2SOwpqBODwNASZdxiEa60kfow==} + + '@zag-js/pagination@1.41.2': + resolution: {integrity: sha512-vZTxz4DrIDfedMcTjDeEkOc1iXD9wkP8eKuEcDieHOodnjSnNNwtmoFw5DCWv+yEa/TByIamXBZ8ZxHY144JgA==} + + '@zag-js/password-input@1.41.2': + resolution: {integrity: sha512-OWKFl0S12Qnlf4R4WbMCJ/YU0kGfezm5tP0UiyubMO/Fixv6H0twDFaJSPg2F6POv5uomCcGubc1H7gO+fIhsQ==} + + '@zag-js/pin-input@1.41.2': + resolution: {integrity: sha512-Wrn3YDbmWL3qvUIzN4QyLO7PzEhunX7z8DwBpmrK04p7HxR9pniTLVIPk27xk2MzyAPYcl4mwd9/Mc88tBxHsg==} + + '@zag-js/popover@1.41.2': + resolution: {integrity: sha512-h/LlVMIERM+NWzYV0ZHURlJuaqkT8XxwyEV96XfVzjknDRFNoPSl5IttVYtoaPoUqC0p/Y4oTSiee4mUZKOLHw==} + + '@zag-js/popper@1.41.2': + resolution: {integrity: sha512-Iz4D5YAIiIPn4IHGjhX3QatR/RyGaDt43lBSZv0RYcPQYtFg9sUuek3wizjW9qXgdvItevvNMqRdpl7f3es09g==} + + '@zag-js/presence@1.41.2': + resolution: {integrity: sha512-OhOLPAf/DYPmgoEntrlrf3LOrkwA+Y0J3K03NXHXPnkRB7h8jyIbHqzHS0jRTb1pvsO2P/yowRyoYtptY2Zmog==} + + '@zag-js/progress@1.41.2': + resolution: {integrity: sha512-SnzrqN+Z568NoO4rrgjrIc/S4EXMEne5CDgWwt2kQ8yq9VysdH8TtHAjyciFRIio7cdgEZNHKw9jSccpMWgRwA==} + + '@zag-js/qr-code@1.41.2': + resolution: {integrity: sha512-+JLswCNnzf58aQTaX0SMUA9wRC8Hb/a/ZveJIXTz6853Siemay2HqOqB2WQIeF33HNldUFD/a4+4Q8ugg93ubA==} + + '@zag-js/radio-group@1.41.2': + resolution: {integrity: sha512-EZjos61jKHZlNw8ez80vG2T9gUwpooxcVpYOKS2hyhuEXn3wEoefS5v1WFbmpoA/8TUpUQnYxisAeNuQfEQCuQ==} + + '@zag-js/rating-group@1.41.2': + resolution: {integrity: sha512-SbJP4HiK5XRy/oC3xRowjZOCThq1n/QA2Z/XAkvKLJArVQCYjrH3MoWwWvMVNfNC5+ZJluborR2AGF7tlVLzxA==} + + '@zag-js/react@1.41.2': + resolution: {integrity: sha512-5Bx7mQAron4LFWI8Hhs/uw5kwQ19s2Tn30HhctozLqmCu4nnJSTSh7GRvX+uwRZnztGXBXoOrgBWIepU4RXFGg==} + peerDependencies: + react: '>=18.0.0' + react-dom: '>=18.0.0' + + '@zag-js/rect-utils@1.41.2': + resolution: {integrity: sha512-GWBTamaMLMG1p7Fe6V0dsXeTEmk+tqG3ciovzmjxURCJ3Yq2EkAMRhS0v5DG0oo+PyrPEIzEWukGBQkh/XRgYQ==} + + '@zag-js/remove-scroll@1.41.2': + resolution: {integrity: sha512-ieIrOgPKlCikAGEBIboQJoU7oxrL5BEY670tDOu7Eg7rNOdAwGXLEKPX2A/q+lREhOiVYjx0D3//vHTvdte80Q==} + + '@zag-js/scroll-area@1.41.2': + resolution: {integrity: sha512-hJFAwfIFuS7XmNsS3nB5rPm9OWXfB4d98sID7fjurcaZtDH5LqFriqfXhceNvs7rz7K4f/u8rufXrb6tcvATIA==} + + '@zag-js/scroll-snap@1.41.2': + resolution: {integrity: sha512-+70Al6LSASyEZtFyyffUJlDbE88KgYJDud05z8oZTqyEOLlTqnlSNkXq/P3siO7r3sNykg9H+TmAKn+/dVSKuw==} + + '@zag-js/select@1.41.2': + resolution: {integrity: sha512-3wGaKABILexoNBJ1bJiHqLLTctR/VMZaNA4cyKiqZeBEWtAkdMhgyY2xKobrP6KtUTqAeUFNVSTu4yCDrAQnUw==} + + '@zag-js/signature-pad@1.41.2': + resolution: {integrity: sha512-iNrOxY4gtqhsZdXYvlhF7s+LiOvwV7/kBpNnq8tJ1oYhgTs8wvKtJHrP86/CR05irEXQTaLfmeAJZuBEys9iRA==} + + '@zag-js/slider@1.41.2': + resolution: {integrity: sha512-mKK2BwoDbIGxAdkdKkPZJA1SHtEQt3lS9hJ6WghefYU2vyd0BXoIKvcDV3xJOzly5LXYhH5cJITn6JtGK8353A==} + + '@zag-js/splitter@1.41.2': + resolution: {integrity: sha512-Ubp4hkmzvVysU31jCINYbBXVqruu7rEPGqugWMzeXC5Hwda648aHpbg4Jix/wRtaGLjsyh6KOVEwAmoJU9NwhQ==} + + '@zag-js/steps@1.41.2': + resolution: {integrity: sha512-m0t2r8+FWwa2b2aU5JiNrHVdYHyNZYHK0G3Tq9lCOSQoDeoJIkyta+sIVehLVSY+0Ba9kOlkRUmYLbsnfXaW+Q==} + + '@zag-js/store@1.41.2': + resolution: {integrity: sha512-dVZF7E1ezXzynrKhMH3rfSr2rBbCfvTjvXbXz7//1PNULuq58UU5dG93V+9l834npCZxI2+PrpY45wZLJPTsIA==} + + '@zag-js/switch@1.41.2': + resolution: {integrity: sha512-qHbQK95UUHN0tj+bf9LLphLcMo/uTg2Pvs0c1Gs03Zh4g3NtHf0uYIhMZKYlCH0hNVlKrmdzLKWgeoDxv9gySw==} + + '@zag-js/tabs@1.41.2': + resolution: {integrity: sha512-7YVj2mCcxRbn1wMXP9anaTOVf+J0fa5uaPScr8e4+e2xc+/1WKzqN6V8IDeKS5wV/xzi1r3Ny307sX7Xz4ZJVQ==} + + '@zag-js/tags-input@1.41.2': + resolution: {integrity: sha512-aIPEndSO+9LHxyoXLUr0Uttxw2cYMyDuii5w50wn/N7lFJD49U3Sj+6XaB/oJSbDAwn10WrsRDtb7Gs2kmU+Qw==} + + '@zag-js/timer@1.41.2': + resolution: {integrity: sha512-PRYLWaip0+1FeVGEMNk5wMGAAIYgBIWwulZ4U5I+2Ayjohzp2NUAfwJ3sqoYvRrfjNYUNAPdU4eGu1zetC+oVQ==} + + '@zag-js/toast@1.41.2': + resolution: {integrity: sha512-+F3PsAo6EIz4rh73IOMCb/+FOUp7e3VjUY2q5sdU4IbfOzJBIbVSJxn0PQmHkuxkzWdCom0Lv0qSPFg/UTplnQ==} + + '@zag-js/toggle-group@1.41.2': + resolution: {integrity: sha512-C6wn3A89h24hTs0BN9ryEuKatfR493u7QqxS06TeK9oI/KZBvm5Rwm8FPHSUJvsUTkAkow4PsXC0Ra19duzEdQ==} + + '@zag-js/toggle@1.41.2': + resolution: {integrity: sha512-EFB9pb3pEtwXt7RSivVLWXV64dKUF8gsn75tt8TbYBgfy+zW85MsRLu8U48TXZN5teQWAKKNujr9bxQsW/CWvQ==} + + '@zag-js/tooltip@1.41.2': + resolution: {integrity: sha512-68okWJCFXfW8r0h97kEcU2yKPkq6e0S6QkiYh09ifMXoYjrQw/shPol8PgrS1poqKJigUWtsKm+bw73abhMn3w==} + + '@zag-js/tour@1.41.2': + resolution: {integrity: sha512-Q7UsvuHYYBo1Cs4b4OS0e/D8lxd2GpSRII93s5BQPi5HTcBjaWhVysAykmCFbat6Z56z0NBmFHNdhZ8oVPls1A==} + + '@zag-js/tree-view@1.41.2': + resolution: {integrity: sha512-QNi0VpV+RyzF4NP72+kSleUpauF9SMAzOAe59nxvs8jAUHqV5diDCInnbQos4+cyFDXFzGq3lElot26A1yI+7Q==} + + '@zag-js/types@1.41.2': + resolution: {integrity: sha512-L6CNvK06lIVpy0X8eG3kbDIx8Uuv+3KHElxXYSzRXSJ7/OLCv1sTRgEvnxNtdIWOrksGgxF4JtT7PXtoClGqNQ==} + + '@zag-js/utils@1.41.2': + resolution: {integrity: sha512-Yj8FSrR7vGA6ahUhjrThfHAF+PM2Y1Yv2lkXkqZZd60mPBhixcot1+SHOfEMV63JimQcWrmQ8QbeYYMmF+ZpLQ==} + + acorn@8.17.0: + resolution: {integrity: sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==} + engines: {node: '>=0.4.0'} + hasBin: true + + ajv-draft-04@1.0.0: + resolution: {integrity: sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==} + peerDependencies: + ajv: ^8.5.0 + peerDependenciesMeta: + ajv: + optional: true + + ajv-formats@3.0.1: + resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + + ajv@8.18.0: + resolution: {integrity: sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==} + + alien-signals@0.4.14: + resolution: {integrity: sha512-itUAVzhczTmP2U5yX67xVpsbbOiquusbWVyA9N+sy6+r6YVbFkahXvNCeEPWEOMhwDYwbVbGHFkVL03N9I5g+Q==} + + argparse@1.0.10: + resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} + + babel-plugin-macros@3.1.0: + resolution: {integrity: sha512-Cg7TFGpIr01vOQNODXOOaGz2NpCU5gl8x1qJFbb6hbZxR7XrcE2vtbAsTAbJ7/xwJtUuJEw8K8Zr/AE0LHlesg==} + engines: {node: '>=10', npm: '>=6'} + + balanced-match@4.0.4: + resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} + engines: {node: 18 || 20 || >=22} + + brace-expansion@5.0.7: + resolution: {integrity: sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==} + engines: {node: 18 || 20 || >=22} + + callsites@3.1.0: + resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} + engines: {node: '>=6'} + + compare-versions@6.1.1: + resolution: {integrity: sha512-4hm4VPpIecmlg59CHXnRDnqGplJFrbLG4aFEl5vl6cK1u76ws3LLvX7ikFnTDl5vo39sjWD6AaDPYodJp/NNHg==} + + confbox@0.1.8: + resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} + + confbox@0.2.4: + resolution: {integrity: sha512-ysOGlgTFbN2/Y6Cg3Iye8YKulHw+R2fNXHrgSmXISQdMnomY6eNDprVdW9R5xBguEqI954+S6709UyiO7B+6OQ==} + + convert-source-map@1.9.0: + resolution: {integrity: sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==} + + cosmiconfig@7.1.0: + resolution: {integrity: sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==} + engines: {node: '>=10'} + + csstype@3.2.3: + resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + + de-indent@1.0.2: + resolution: {integrity: sha512-e/1zu3xH5MQryN2zdVaF0OrdNLUbvWxzMbi+iNA6Bky7l1RoP8a2fIbRocyHclXt/arDrrR6lL3TqFD9pMQTsg==} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + diff@8.0.4: + resolution: {integrity: sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==} + engines: {node: '>=0.3.1'} + + entities@7.0.1: + resolution: {integrity: sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==} + engines: {node: '>=0.12'} + + error-ex@1.3.4: + resolution: {integrity: sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + + estree-walker@2.0.2: + resolution: {integrity: sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==} + + exsolve@1.1.0: + resolution: {integrity: sha512-D+42+T12DdIlJM3uepa55qGiL3sYdLBOxIl2ifQCzCHz4c7eiolaHsi3BIqEr7JxBzxv2pYZQX9kw16ziMcEmw==} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-uri@4.1.0: + resolution: {integrity: sha512-ZodJ2cRiLVWGi9IgPb3mbgSqM4CD3LexCHkuv0FfBXHJI1ADfucTD06m6clO2Cy5RZYsw/SiCVl/dyrFI/SYWA==} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: '>=4.0.4' + peerDependenciesMeta: + picomatch: + optional: true + + find-root@1.1.0: + resolution: {integrity: sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==} + + fs-extra@11.3.6: + resolution: {integrity: sha512-w8ZNZr2mKIc7qeNaQ9AVPT1+iFaI+Avd4xudVOvdDJ8VytREi1Ft5Ih7hd9jjehod8vAM5GMsfQ/TpPf4EyoEA==} + engines: {node: '>=14.14'} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + graceful-fs@4.2.11: + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + + hasown@2.0.4: + resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} + engines: {node: '>= 0.4'} + + he@1.2.0: + resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} + hasBin: true + + hoist-non-react-statics@3.3.2: + resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==} + + import-fresh@3.3.1: + resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} + engines: {node: '>=6'} + + import-lazy@4.0.0: + resolution: {integrity: sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==} + engines: {node: '>=8'} + + is-arrayish@0.2.1: + resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} + + is-core-module@2.16.2: + resolution: {integrity: sha512-evOr8xfXKxE6qSR0hSXL2r3sd7ALj8+7jQEUvPYcm5sgZFdJ+AYzT6yNmJenvIYQBgIGwfwz08sL8zoL7yq2BA==} + engines: {node: '>= 0.4'} + + jju@1.4.0: + resolution: {integrity: sha512-8wb9Yw966OSxApiCt0K3yNJL8pnNeIv+OEq2YMidz4FKP6nonSRoOXc80iXY4JaN2FC11B9qsNmDsm+ZOfMROA==} + + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + + jsesc@3.1.0: + resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==} + engines: {node: '>=6'} + hasBin: true + + json-parse-even-better-errors@2.3.1: + resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} + + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + jsonfile@6.2.1: + resolution: {integrity: sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==} + + kolorist@1.8.0: + resolution: {integrity: sha512-Y+60/zizpJ3HRH8DCss+q95yr6145JXZo46OTpFvDZWLfRCE4qChOyk1b26nMaNpfHHgxagk9dXT5OP0Tfe+dQ==} + + lightningcss-android-arm64@1.32.0: + resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + + lightningcss-darwin-arm64@1.32.0: + resolution: {integrity: sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + + lightningcss-darwin-x64@1.32.0: + resolution: {integrity: sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + + lightningcss-freebsd-x64@1.32.0: + resolution: {integrity: sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + + lightningcss-linux-arm-gnueabihf@1.32.0: + resolution: {integrity: sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + + lightningcss-linux-arm64-gnu@1.32.0: + resolution: {integrity: sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + + lightningcss-linux-arm64-musl@1.32.0: + resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + + lightningcss-linux-x64-gnu@1.32.0: + resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + + lightningcss-linux-x64-musl@1.32.0: + resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + + lightningcss-win32-arm64-msvc@1.32.0: + resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + + lightningcss-win32-x64-msvc@1.32.0: + resolution: {integrity: sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + + lightningcss@1.32.0: + resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==} + engines: {node: '>= 12.0.0'} + + lines-and-columns@1.2.4: + resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + + local-pkg@1.2.1: + resolution: {integrity: sha512-++gUqRDEvcnN6Zhqrr+y/CkVEHhlrR96vZn3nZZPYzMcBUyBtTKzB9NadClFIsIVSsu+3i9tfk/erqy9kAmt7Q==} + engines: {node: '>=14'} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + minimatch@10.2.3: + resolution: {integrity: sha512-Rwi3pnapEqirPSbWbrZaa6N3nmqq4Xer/2XooiOKyV3q12ML06f7MOuc5DVH8ONZIFhwIYQ3yzPH4nt7iWHaTg==} + engines: {node: 18 || 20 || >=22} + + minimatch@9.0.9: + resolution: {integrity: sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==} + engines: {node: '>=16 || 14 >=14.17'} + + mlly@1.8.2: + resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + muggle-string@0.4.1: + resolution: {integrity: sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==} + + nanoid@3.3.15: + resolution: {integrity: sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + parent-module@1.0.1: + resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} + engines: {node: '>=6'} + + parse-json@5.2.0: + resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} + engines: {node: '>=8'} + + path-browserify@1.0.1: + resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==} + + path-parse@1.0.7: + resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + + path-type@4.0.0: + resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} + engines: {node: '>=8'} + + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + + perfect-freehand@1.2.3: + resolution: {integrity: sha512-bHZSfqDHGNlPpgH2yxXgPHlQSPpEbo+qg7li0M78J9vNAi2yjwLeA4x79BEQhX44lEWpCLSFCeRZwpw0niiXPA==} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.5: + resolution: {integrity: sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==} + engines: {node: '>=12'} + + pkg-types@1.3.1: + resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} + + pkg-types@2.3.1: + resolution: {integrity: sha512-y+ichcgc2LrADuhLNAx8DFjVfgz91pRxfZdI3UDhxHvcVEZsenLO+7XaU5vOp0u/7V/wZ+plyuQxtrDlZJ+yeg==} + + postcss@8.5.16: + resolution: {integrity: sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==} + engines: {node: ^10 || ^12 || >=14} + + proxy-compare@3.0.1: + resolution: {integrity: sha512-V9plBAt3qjMlS1+nC8771KNf6oJ12gExvaxnNzN/9yVRLdTv/lc+oJlnSzrdYDAvBfTStPCoiaCOTmTs0adv7Q==} + + proxy-memoize@3.0.1: + resolution: {integrity: sha512-VDdG/VYtOgdGkWJx7y0o7p+zArSf2383Isci8C+BP3YXgMYDoPd3cCBjw0JdWb6YBb9sFiOPbAADDVTPJnh+9g==} + + quansync@0.2.11: + resolution: {integrity: sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==} + + react-dom@19.2.7: + resolution: {integrity: sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ==} + peerDependencies: + react: ^19.2.7 + + react-is@16.13.1: + resolution: {integrity: sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==} + + react@19.2.7: + resolution: {integrity: sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ==} + engines: {node: '>=0.10.0'} + + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + + resolve-from@4.0.0: + resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} + engines: {node: '>=4'} + + resolve@1.22.12: + resolution: {integrity: sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==} + engines: {node: '>= 0.4'} + hasBin: true + + rolldown@1.1.4: + resolution: {integrity: sha512-IjZYiLxZwpnhwhdBH2ugdTGVSdhCQUmLxLoqyjiL0JxYjyRst+5a0P3xfrTxJ5F638j4Mvvw5FAX5XE6eHpXbA==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + + scheduler@0.27.0: + resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==} + + semver@7.7.4: + resolution: {integrity: sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==} + engines: {node: '>=10'} + hasBin: true + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + source-map@0.5.7: + resolution: {integrity: sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==} + engines: {node: '>=0.10.0'} + + source-map@0.6.1: + resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} + engines: {node: '>=0.10.0'} + + sprintf-js@1.0.3: + resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + + string-argv@0.3.2: + resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==} + engines: {node: '>=0.6.19'} + + stylis@4.2.0: + resolution: {integrity: sha512-Orov6g6BB1sDfYgzWfTHDOxamtX1bE/zo104Dh9e6fqJ3PooipYyfJ0pUmrZO2wAvO8YbEyeFrkV91XTsGMSrw==} + + supports-color@8.1.1: + resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} + engines: {node: '>=10'} + + supports-preserve-symlinks-flag@1.0.0: + resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} + engines: {node: '>= 0.4'} + + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + ufo@1.6.4: + resolution: {integrity: sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==} + + universalify@2.0.1: + resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} + engines: {node: '>= 10.0.0'} + + uqr@0.1.3: + resolution: {integrity: sha512-0rjE8iEJe4YmT9TOhwsZtqCMRLc5DXZUI2UEYUUg63ikBkqqE5EYWaI0etFe/5KUcmcYwLih2RND1kq+hrUJXA==} + + vite-plugin-css-injected-by-js@3.5.2: + resolution: {integrity: sha512-2MpU/Y+SCZyWUB6ua3HbJCrgnF0KACAsmzOQt1UvRVJCGF6S8xdA3ZUhWcWdM9ivG4I5az8PnQmwwrkC2CAQrQ==} + peerDependencies: + vite: '>=7.3.2' + + vite-plugin-dts@4.5.4: + resolution: {integrity: sha512-d4sOM8M/8z7vRXHHq/ebbblfaxENjogAAekcfcDCCwAyvGqnPrc7f4NZbvItS+g4WTgerW0xDwSz5qz11JT3vg==} + peerDependencies: + typescript: '*' + vite: '>=7.3.2' + peerDependenciesMeta: + vite: + optional: true + + vite@8.1.3: + resolution: {integrity: sha512-Ds+gBRbj0lwRO2Y5hwnUBdxSwlAve9LeRyU4sNnAr0ewW0gWF0n5bgXgUzbgZ49MV9BVUAQUFYVcDUcilUExMA==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + '@vitejs/devtools': ^0.3.0 + esbuild: ^0.27.0 || ^0.28.0 + jiti: '>=1.21.0' + less: ^4.0.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + '@vitejs/devtools': + optional: true + esbuild: + optional: true + jiti: + optional: true + less: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vscode-uri@3.1.0: + resolution: {integrity: sha512-/BpdSx+yCQGnCvecbyXdxHDkuk55/G3xwnC0GqY4gmQ3j+A+g8kzzgB4Nk/SINjqn6+waqw3EgbVF2QKExkRxQ==} + + yaml@2.9.0: + resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} + engines: {node: '>= 14.6'} + hasBin: true + +snapshots: + + '@ark-ui/react@5.37.2(react-dom@19.2.7(react@19.2.7))(react@19.2.7)': + dependencies: + '@internationalized/date': 3.12.2 + '@zag-js/accordion': 1.41.2 + '@zag-js/anatomy': 1.41.2 + '@zag-js/angle-slider': 1.41.2 + '@zag-js/async-list': 1.41.2 + '@zag-js/auto-resize': 1.41.2 + '@zag-js/avatar': 1.41.2 + '@zag-js/carousel': 1.41.2 + '@zag-js/cascade-select': 1.41.2 + '@zag-js/checkbox': 1.41.2 + '@zag-js/clipboard': 1.41.2 + '@zag-js/collapsible': 1.41.2 + '@zag-js/collection': 1.41.2 + '@zag-js/color-picker': 1.41.2 + '@zag-js/color-utils': 1.41.2 + '@zag-js/combobox': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/date-input': 1.41.2(@internationalized/date@3.12.2) + '@zag-js/date-picker': 1.41.2(@internationalized/date@3.12.2) + '@zag-js/date-utils': 1.41.2(@internationalized/date@3.12.2) + '@zag-js/dialog': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/drawer': 1.41.2 + '@zag-js/editable': 1.41.2 + '@zag-js/file-upload': 1.41.2 + '@zag-js/file-utils': 1.41.2 + '@zag-js/floating-panel': 1.41.2 + '@zag-js/focus-trap': 1.41.2 + '@zag-js/focus-visible': 1.41.2 + '@zag-js/highlight-word': 1.41.2 + '@zag-js/hover-card': 1.41.2 + '@zag-js/i18n-utils': 1.41.2 + '@zag-js/image-cropper': 1.41.2 + '@zag-js/json-tree-utils': 1.41.2 + '@zag-js/listbox': 1.41.2 + '@zag-js/marquee': 1.41.2 + '@zag-js/menu': 1.41.2 + '@zag-js/navigation-menu': 1.41.2 + '@zag-js/number-input': 1.41.2 + '@zag-js/pagination': 1.41.2 + '@zag-js/password-input': 1.41.2 + '@zag-js/pin-input': 1.41.2 + '@zag-js/popover': 1.41.2 + '@zag-js/presence': 1.41.2 + '@zag-js/progress': 1.41.2 + '@zag-js/qr-code': 1.41.2 + '@zag-js/radio-group': 1.41.2 + '@zag-js/rating-group': 1.41.2 + '@zag-js/react': 1.41.2(react-dom@19.2.7(react@19.2.7))(react@19.2.7) + '@zag-js/scroll-area': 1.41.2 + '@zag-js/select': 1.41.2 + '@zag-js/signature-pad': 1.41.2 + '@zag-js/slider': 1.41.2 + '@zag-js/splitter': 1.41.2 + '@zag-js/steps': 1.41.2 + '@zag-js/switch': 1.41.2 + '@zag-js/tabs': 1.41.2 + '@zag-js/tags-input': 1.41.2 + '@zag-js/timer': 1.41.2 + '@zag-js/toast': 1.41.2 + '@zag-js/toggle': 1.41.2 + '@zag-js/toggle-group': 1.41.2 + '@zag-js/tooltip': 1.41.2 + '@zag-js/tour': 1.41.2 + '@zag-js/tree-view': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + react: 19.2.7 + react-dom: 19.2.7(react@19.2.7) + + '@babel/code-frame@7.29.7': + dependencies: + '@babel/helper-validator-identifier': 7.29.7 + js-tokens: 4.0.0 + picocolors: 1.1.1 + + '@babel/generator@7.29.7': + dependencies: + '@babel/parser': 7.29.7 + '@babel/types': 7.29.7 + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + jsesc: 3.1.0 + + '@babel/helper-globals@7.29.7': {} + + '@babel/helper-module-imports@7.29.7': + dependencies: + '@babel/traverse': 7.29.7 + '@babel/types': 7.29.7 + transitivePeerDependencies: + - supports-color + + '@babel/helper-string-parser@7.29.7': {} + + '@babel/helper-validator-identifier@7.29.7': {} + + '@babel/parser@7.29.7': + dependencies: + '@babel/types': 7.29.7 + + '@babel/runtime@7.29.7': {} + + '@babel/template@7.29.7': + dependencies: + '@babel/code-frame': 7.29.7 + '@babel/parser': 7.29.7 + '@babel/types': 7.29.7 + + '@babel/traverse@7.29.7': + dependencies: + '@babel/code-frame': 7.29.7 + '@babel/generator': 7.29.7 + '@babel/helper-globals': 7.29.7 + '@babel/parser': 7.29.7 + '@babel/template': 7.29.7 + '@babel/types': 7.29.7 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + '@babel/types@7.29.7': + dependencies: + '@babel/helper-string-parser': 7.29.7 + '@babel/helper-validator-identifier': 7.29.7 + + '@chakra-ui/react@3.36.0(@emotion/react@11.14.0(@types/react@19.2.17)(react@19.2.7))(react-dom@19.2.7(react@19.2.7))(react@19.2.7)': + dependencies: + '@ark-ui/react': 5.37.2(react-dom@19.2.7(react@19.2.7))(react@19.2.7) + '@emotion/is-prop-valid': 1.4.0 + '@emotion/react': 11.14.0(@types/react@19.2.17)(react@19.2.7) + '@emotion/serialize': 1.3.3 + '@emotion/use-insertion-effect-with-fallbacks': 1.2.0(react@19.2.7) + '@emotion/utils': 1.4.2 + '@pandacss/is-valid-prop': 1.11.4 + csstype: 3.2.3 + react: 19.2.7 + react-dom: 19.2.7(react@19.2.7) + + '@emnapi/core@1.11.1': + dependencies: + '@emnapi/wasi-threads': 1.2.2 + tslib: 2.8.1 + optional: true + + '@emnapi/runtime@1.11.1': + dependencies: + tslib: 2.8.1 + optional: true + + '@emnapi/wasi-threads@1.2.2': + dependencies: + tslib: 2.8.1 + optional: true + + '@emotion/babel-plugin@11.13.5': + dependencies: + '@babel/helper-module-imports': 7.29.7 + '@babel/runtime': 7.29.7 + '@emotion/hash': 0.9.2 + '@emotion/memoize': 0.9.0 + '@emotion/serialize': 1.3.3 + babel-plugin-macros: 3.1.0 + convert-source-map: 1.9.0 + escape-string-regexp: 4.0.0 + find-root: 1.1.0 + source-map: 0.5.7 + stylis: 4.2.0 + transitivePeerDependencies: + - supports-color + + '@emotion/cache@11.14.0': + dependencies: + '@emotion/memoize': 0.9.0 + '@emotion/sheet': 1.4.0 + '@emotion/utils': 1.4.2 + '@emotion/weak-memoize': 0.4.0 + stylis: 4.2.0 + + '@emotion/hash@0.9.2': {} + + '@emotion/is-prop-valid@1.4.0': + dependencies: + '@emotion/memoize': 0.9.0 + + '@emotion/memoize@0.9.0': {} + + '@emotion/react@11.14.0(@types/react@19.2.17)(react@19.2.7)': + dependencies: + '@babel/runtime': 7.29.7 + '@emotion/babel-plugin': 11.13.5 + '@emotion/cache': 11.14.0 + '@emotion/serialize': 1.3.3 + '@emotion/use-insertion-effect-with-fallbacks': 1.2.0(react@19.2.7) + '@emotion/utils': 1.4.2 + '@emotion/weak-memoize': 0.4.0 + hoist-non-react-statics: 3.3.2 + react: 19.2.7 + optionalDependencies: + '@types/react': 19.2.17 + transitivePeerDependencies: + - supports-color + + '@emotion/serialize@1.3.3': + dependencies: + '@emotion/hash': 0.9.2 + '@emotion/memoize': 0.9.0 + '@emotion/unitless': 0.10.0 + '@emotion/utils': 1.4.2 + csstype: 3.2.3 + + '@emotion/sheet@1.4.0': {} + + '@emotion/unitless@0.10.0': {} + + '@emotion/use-insertion-effect-with-fallbacks@1.2.0(react@19.2.7)': + dependencies: + react: 19.2.7 + + '@emotion/utils@1.4.2': {} + + '@emotion/weak-memoize@0.4.0': {} + + '@floating-ui/core@1.7.5': + dependencies: + '@floating-ui/utils': 0.2.11 + + '@floating-ui/dom@1.7.6': + dependencies: + '@floating-ui/core': 1.7.5 + '@floating-ui/utils': 0.2.11 + + '@floating-ui/utils@0.2.11': {} + + '@internationalized/date@3.12.2': + dependencies: + '@swc/helpers': 0.5.23 + + '@internationalized/number@3.6.6': + dependencies: + '@swc/helpers': 0.5.23 + + '@jridgewell/gen-mapping@0.3.13': + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@jridgewell/trace-mapping@0.3.31': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + + '@microsoft/api-extractor-model@7.33.8': + dependencies: + '@microsoft/tsdoc': 0.16.0 + '@microsoft/tsdoc-config': 0.18.1 + '@rushstack/node-core-library': 5.23.1 + transitivePeerDependencies: + - '@types/node' + + '@microsoft/api-extractor@7.58.9': + dependencies: + '@microsoft/api-extractor-model': 7.33.8 + '@microsoft/tsdoc': 0.16.0 + '@microsoft/tsdoc-config': 0.18.1 + '@rushstack/node-core-library': 5.23.1 + '@rushstack/rig-package': 0.7.3 + '@rushstack/terminal': 0.24.0 + '@rushstack/ts-command-line': 5.3.10 + diff: 8.0.4 + minimatch: 10.2.3 + resolve: 1.22.12 + semver: 7.7.4 + source-map: 0.6.1 + typescript: 5.9.3 + transitivePeerDependencies: + - '@types/node' + + '@microsoft/tsdoc-config@0.18.1': + dependencies: + '@microsoft/tsdoc': 0.16.0 + ajv: 8.18.0 + jju: 1.4.0 + resolve: 1.22.12 + + '@microsoft/tsdoc@0.16.0': {} + + '@napi-rs/wasm-runtime@1.1.6(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)': + dependencies: + '@emnapi/core': 1.11.1 + '@emnapi/runtime': 1.11.1 + '@tybys/wasm-util': 0.10.3 + optional: true + + '@oxc-project/types@0.138.0': {} + + '@pandacss/is-valid-prop@1.11.4': {} + + '@rolldown/binding-android-arm64@1.1.4': + optional: true + + '@rolldown/binding-darwin-arm64@1.1.4': + optional: true + + '@rolldown/binding-darwin-x64@1.1.4': + optional: true + + '@rolldown/binding-freebsd-x64@1.1.4': + optional: true + + '@rolldown/binding-linux-arm-gnueabihf@1.1.4': + optional: true + + '@rolldown/binding-linux-arm64-gnu@1.1.4': + optional: true + + '@rolldown/binding-linux-arm64-musl@1.1.4': + optional: true + + '@rolldown/binding-linux-ppc64-gnu@1.1.4': + optional: true + + '@rolldown/binding-linux-s390x-gnu@1.1.4': + optional: true + + '@rolldown/binding-linux-x64-gnu@1.1.4': + optional: true + + '@rolldown/binding-linux-x64-musl@1.1.4': + optional: true + + '@rolldown/binding-openharmony-arm64@1.1.4': + optional: true + + '@rolldown/binding-wasm32-wasi@1.1.4': + dependencies: + '@emnapi/core': 1.11.1 + '@emnapi/runtime': 1.11.1 + '@napi-rs/wasm-runtime': 1.1.6(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1) + optional: true + + '@rolldown/binding-win32-arm64-msvc@1.1.4': + optional: true + + '@rolldown/binding-win32-x64-msvc@1.1.4': + optional: true + + '@rolldown/pluginutils@1.0.1': {} + + '@rollup/pluginutils@5.4.0': + dependencies: + '@types/estree': 1.0.9 + estree-walker: 2.0.2 + picomatch: 4.0.5 + + '@rushstack/node-core-library@5.23.1': + dependencies: + ajv: 8.18.0 + ajv-draft-04: 1.0.0(ajv@8.18.0) + ajv-formats: 3.0.1(ajv@8.18.0) + fs-extra: 11.3.6 + import-lazy: 4.0.0 + jju: 1.4.0 + resolve: 1.22.12 + semver: 7.7.4 + + '@rushstack/problem-matcher@0.2.1': {} + + '@rushstack/rig-package@0.7.3': + dependencies: + jju: 1.4.0 + resolve: 1.22.12 + + '@rushstack/terminal@0.24.0': + dependencies: + '@rushstack/node-core-library': 5.23.1 + '@rushstack/problem-matcher': 0.2.1 + supports-color: 8.1.1 + + '@rushstack/ts-command-line@5.3.10': + dependencies: + '@rushstack/terminal': 0.24.0 + '@types/argparse': 1.0.38 + argparse: 1.0.10 + string-argv: 0.3.2 + transitivePeerDependencies: + - '@types/node' + + '@swc/core-darwin-arm64@1.15.43': + optional: true + + '@swc/core-darwin-x64@1.15.43': + optional: true + + '@swc/core-linux-arm-gnueabihf@1.15.43': + optional: true + + '@swc/core-linux-arm64-gnu@1.15.43': + optional: true + + '@swc/core-linux-arm64-musl@1.15.43': + optional: true + + '@swc/core-linux-ppc64-gnu@1.15.43': + optional: true + + '@swc/core-linux-s390x-gnu@1.15.43': + optional: true + + '@swc/core-linux-x64-gnu@1.15.43': + optional: true + + '@swc/core-linux-x64-musl@1.15.43': + optional: true + + '@swc/core-win32-arm64-msvc@1.15.43': + optional: true + + '@swc/core-win32-ia32-msvc@1.15.43': + optional: true + + '@swc/core-win32-x64-msvc@1.15.43': + optional: true + + '@swc/core@1.15.43(@swc/helpers@0.5.23)': + dependencies: + '@swc/counter': 0.1.3 + '@swc/types': 0.1.27 + optionalDependencies: + '@swc/core-darwin-arm64': 1.15.43 + '@swc/core-darwin-x64': 1.15.43 + '@swc/core-linux-arm-gnueabihf': 1.15.43 + '@swc/core-linux-arm64-gnu': 1.15.43 + '@swc/core-linux-arm64-musl': 1.15.43 + '@swc/core-linux-ppc64-gnu': 1.15.43 + '@swc/core-linux-s390x-gnu': 1.15.43 + '@swc/core-linux-x64-gnu': 1.15.43 + '@swc/core-linux-x64-musl': 1.15.43 + '@swc/core-win32-arm64-msvc': 1.15.43 + '@swc/core-win32-ia32-msvc': 1.15.43 + '@swc/core-win32-x64-msvc': 1.15.43 + '@swc/helpers': 0.5.23 + + '@swc/counter@0.1.3': {} + + '@swc/helpers@0.5.23': + dependencies: + tslib: 2.8.1 + + '@swc/types@0.1.27': + dependencies: + '@swc/counter': 0.1.3 + + '@tybys/wasm-util@0.10.3': + dependencies: + tslib: 2.8.1 + optional: true + + '@types/argparse@1.0.38': {} + + '@types/estree@1.0.9': {} + + '@types/parse-json@4.0.2': {} + + '@types/react-dom@19.2.3(@types/react@19.2.17)': + dependencies: + '@types/react': 19.2.17 + + '@types/react@19.2.17': + dependencies: + csstype: 3.2.3 + + '@vitejs/plugin-react-swc@4.3.1(@swc/helpers@0.5.23)(vite@8.1.3(yaml@2.9.0))': + dependencies: + '@rolldown/pluginutils': 1.0.1 + '@swc/core': 1.15.43(@swc/helpers@0.5.23) + vite: 8.1.3(yaml@2.9.0) + transitivePeerDependencies: + - '@swc/helpers' + + '@volar/language-core@2.4.28': + dependencies: + '@volar/source-map': 2.4.28 + + '@volar/source-map@2.4.28': {} + + '@volar/typescript@2.4.28': + dependencies: + '@volar/language-core': 2.4.28 + path-browserify: 1.0.1 + vscode-uri: 3.1.0 + + '@vue/compiler-core@3.5.39': + dependencies: + '@babel/parser': 7.29.7 + '@vue/shared': 3.5.39 + entities: 7.0.1 + estree-walker: 2.0.2 + source-map-js: 1.2.1 + + '@vue/compiler-dom@3.5.39': + dependencies: + '@vue/compiler-core': 3.5.39 + '@vue/shared': 3.5.39 + + '@vue/compiler-vue2@2.7.16': + dependencies: + de-indent: 1.0.2 + he: 1.2.0 + + '@vue/language-core@2.2.0(typescript@5.9.3)': + dependencies: + '@volar/language-core': 2.4.28 + '@vue/compiler-dom': 3.5.39 + '@vue/compiler-vue2': 2.7.16 + '@vue/shared': 3.5.39 + alien-signals: 0.4.14 + minimatch: 9.0.9 + muggle-string: 0.4.1 + path-browserify: 1.0.1 + optionalDependencies: + typescript: 5.9.3 + + '@vue/shared@3.5.39': {} + + '@zag-js/accordion@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/anatomy@1.41.2': {} + + '@zag-js/angle-slider@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/rect-utils': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/aria-hidden@1.41.2': + dependencies: + '@zag-js/dom-query': 1.41.2 + + '@zag-js/async-list@1.41.2': + dependencies: + '@zag-js/core': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/auto-resize@1.41.2': + dependencies: + '@zag-js/dom-query': 1.41.2 + + '@zag-js/avatar@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/carousel@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/scroll-snap': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/cascade-select@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/collection': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dismissable': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/focus-visible': 1.41.2 + '@zag-js/popper': 1.41.2 + '@zag-js/rect-utils': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/checkbox@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/focus-visible': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/clipboard@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/collapsible@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/collection@1.41.2': + dependencies: + '@zag-js/utils': 1.41.2 + + '@zag-js/color-picker@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/color-utils': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dismissable': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/popper': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/color-utils@1.41.2': + dependencies: + '@zag-js/utils': 1.41.2 + + '@zag-js/combobox@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/collection': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dismissable': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/focus-visible': 1.41.2 + '@zag-js/live-region': 1.41.2 + '@zag-js/popper': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/core@1.41.2': + dependencies: + '@zag-js/dom-query': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/date-input@1.41.2(@internationalized/date@3.12.2)': + dependencies: + '@internationalized/date': 3.12.2 + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/date-utils': 1.41.2(@internationalized/date@3.12.2) + '@zag-js/dom-query': 1.41.2 + '@zag-js/live-region': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/date-picker@1.41.2(@internationalized/date@3.12.2)': + dependencies: + '@internationalized/date': 3.12.2 + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/date-utils': 1.41.2(@internationalized/date@3.12.2) + '@zag-js/dismissable': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/live-region': 1.41.2 + '@zag-js/popper': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/date-utils@1.41.2(@internationalized/date@3.12.2)': + dependencies: + '@internationalized/date': 3.12.2 + + '@zag-js/dialog@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/aria-hidden': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dismissable': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/focus-trap': 1.41.2 + '@zag-js/remove-scroll': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/dismissable@1.41.2': + dependencies: + '@zag-js/dom-query': 1.41.2 + '@zag-js/interact-outside': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/dom-query@1.41.2': + dependencies: + '@zag-js/types': 1.41.2 + + '@zag-js/drawer@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/aria-hidden': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dismissable': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/focus-trap': 1.41.2 + '@zag-js/remove-scroll': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/editable@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/interact-outside': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/file-upload@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/file-utils': 1.41.2 + '@zag-js/i18n-utils': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/file-utils@1.41.2': + dependencies: + '@zag-js/i18n-utils': 1.41.2 + + '@zag-js/floating-panel@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/popper': 1.41.2 + '@zag-js/rect-utils': 1.41.2 + '@zag-js/store': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/focus-trap@1.41.2': + dependencies: + '@zag-js/dom-query': 1.41.2 + + '@zag-js/focus-visible@1.41.2': + dependencies: + '@zag-js/dom-query': 1.41.2 + + '@zag-js/highlight-word@1.41.2': {} + + '@zag-js/hover-card@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dismissable': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/popper': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/i18n-utils@1.41.2': + dependencies: + '@zag-js/dom-query': 1.41.2 + + '@zag-js/image-cropper@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/interact-outside@1.41.2': + dependencies: + '@zag-js/dom-query': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/json-tree-utils@1.41.2': {} + + '@zag-js/listbox@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/collection': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/focus-visible': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/live-region@1.41.2': {} + + '@zag-js/marquee@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/menu@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dismissable': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/focus-visible': 1.41.2 + '@zag-js/popper': 1.41.2 + '@zag-js/rect-utils': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/navigation-menu@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dismissable': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/number-input@1.41.2': + dependencies: + '@internationalized/number': 3.6.6 + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/pagination@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/password-input@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/pin-input@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/popover@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/aria-hidden': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dismissable': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/focus-trap': 1.41.2 + '@zag-js/popper': 1.41.2 + '@zag-js/remove-scroll': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/popper@1.41.2': + dependencies: + '@floating-ui/dom': 1.7.6 + '@zag-js/dom-query': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/presence@1.41.2': + dependencies: + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + + '@zag-js/progress@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/qr-code@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + proxy-memoize: 3.0.1 + uqr: 0.1.3 + + '@zag-js/radio-group@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/focus-visible': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/rating-group@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/react@1.41.2(react-dom@19.2.7(react@19.2.7))(react@19.2.7)': + dependencies: + '@zag-js/core': 1.41.2 + '@zag-js/store': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + react: 19.2.7 + react-dom: 19.2.7(react@19.2.7) + + '@zag-js/rect-utils@1.41.2': {} + + '@zag-js/remove-scroll@1.41.2': + dependencies: + '@zag-js/dom-query': 1.41.2 + + '@zag-js/scroll-area@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/scroll-snap@1.41.2': + dependencies: + '@zag-js/dom-query': 1.41.2 + + '@zag-js/select@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/collection': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dismissable': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/focus-visible': 1.41.2 + '@zag-js/popper': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/signature-pad@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + perfect-freehand: 1.2.3 + + '@zag-js/slider@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/splitter@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/steps@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/store@1.41.2': + dependencies: + proxy-compare: 3.0.1 + + '@zag-js/switch@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/focus-visible': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/tabs@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/tags-input@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/auto-resize': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/interact-outside': 1.41.2 + '@zag-js/live-region': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/timer@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/toast@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dismissable': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/toggle-group@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/toggle@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/tooltip@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/focus-visible': 1.41.2 + '@zag-js/popper': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/tour@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dismissable': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/focus-trap': 1.41.2 + '@zag-js/interact-outside': 1.41.2 + '@zag-js/popper': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/tree-view@1.41.2': + dependencies: + '@zag-js/anatomy': 1.41.2 + '@zag-js/collection': 1.41.2 + '@zag-js/core': 1.41.2 + '@zag-js/dom-query': 1.41.2 + '@zag-js/types': 1.41.2 + '@zag-js/utils': 1.41.2 + + '@zag-js/types@1.41.2': + dependencies: + csstype: 3.2.3 + + '@zag-js/utils@1.41.2': {} + + acorn@8.17.0: {} + + ajv-draft-04@1.0.0(ajv@8.18.0): + optionalDependencies: + ajv: 8.18.0 + + ajv-formats@3.0.1(ajv@8.18.0): + optionalDependencies: + ajv: 8.18.0 + + ajv@8.18.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 4.1.0 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + + alien-signals@0.4.14: {} + + argparse@1.0.10: + dependencies: + sprintf-js: 1.0.3 + + babel-plugin-macros@3.1.0: + dependencies: + '@babel/runtime': 7.29.7 + cosmiconfig: 7.1.0 + resolve: 1.22.12 + + balanced-match@4.0.4: {} + + brace-expansion@5.0.7: + dependencies: + balanced-match: 4.0.4 + + callsites@3.1.0: {} + + compare-versions@6.1.1: {} + + confbox@0.1.8: {} + + confbox@0.2.4: {} + + convert-source-map@1.9.0: {} + + cosmiconfig@7.1.0: + dependencies: + '@types/parse-json': 4.0.2 + import-fresh: 3.3.1 + parse-json: 5.2.0 + path-type: 4.0.0 + yaml: 2.9.0 + + csstype@3.2.3: {} + + de-indent@1.0.2: {} + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + detect-libc@2.1.2: {} + + diff@8.0.4: {} + + entities@7.0.1: {} + + error-ex@1.3.4: + dependencies: + is-arrayish: 0.2.1 + + es-errors@1.3.0: {} + + escape-string-regexp@4.0.0: {} + + estree-walker@2.0.2: {} + + exsolve@1.1.0: {} + + fast-deep-equal@3.1.3: {} + + fast-uri@4.1.0: {} + + fdir@6.5.0(picomatch@4.0.5): + optionalDependencies: + picomatch: 4.0.5 + + find-root@1.1.0: {} + + fs-extra@11.3.6: + dependencies: + graceful-fs: 4.2.11 + jsonfile: 6.2.1 + universalify: 2.0.1 + + fsevents@2.3.3: + optional: true + + function-bind@1.1.2: {} + + graceful-fs@4.2.11: {} + + has-flag@4.0.0: {} + + hasown@2.0.4: + dependencies: + function-bind: 1.1.2 + + he@1.2.0: {} + + hoist-non-react-statics@3.3.2: + dependencies: + react-is: 16.13.1 + + import-fresh@3.3.1: + dependencies: + parent-module: 1.0.1 + resolve-from: 4.0.0 + + import-lazy@4.0.0: {} + + is-arrayish@0.2.1: {} + + is-core-module@2.16.2: + dependencies: + hasown: 2.0.4 + + jju@1.4.0: {} + + js-tokens@4.0.0: {} + + jsesc@3.1.0: {} + + json-parse-even-better-errors@2.3.1: {} + + json-schema-traverse@1.0.0: {} + + jsonfile@6.2.1: + dependencies: + universalify: 2.0.1 + optionalDependencies: + graceful-fs: 4.2.11 + + kolorist@1.8.0: {} + + lightningcss-android-arm64@1.32.0: + optional: true + + lightningcss-darwin-arm64@1.32.0: + optional: true + + lightningcss-darwin-x64@1.32.0: + optional: true + + lightningcss-freebsd-x64@1.32.0: + optional: true + + lightningcss-linux-arm-gnueabihf@1.32.0: + optional: true + + lightningcss-linux-arm64-gnu@1.32.0: + optional: true + + lightningcss-linux-arm64-musl@1.32.0: + optional: true + + lightningcss-linux-x64-gnu@1.32.0: + optional: true + + lightningcss-linux-x64-musl@1.32.0: + optional: true + + lightningcss-win32-arm64-msvc@1.32.0: + optional: true + + lightningcss-win32-x64-msvc@1.32.0: + optional: true + + lightningcss@1.32.0: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.32.0 + lightningcss-darwin-arm64: 1.32.0 + lightningcss-darwin-x64: 1.32.0 + lightningcss-freebsd-x64: 1.32.0 + lightningcss-linux-arm-gnueabihf: 1.32.0 + lightningcss-linux-arm64-gnu: 1.32.0 + lightningcss-linux-arm64-musl: 1.32.0 + lightningcss-linux-x64-gnu: 1.32.0 + lightningcss-linux-x64-musl: 1.32.0 + lightningcss-win32-arm64-msvc: 1.32.0 + lightningcss-win32-x64-msvc: 1.32.0 + + lines-and-columns@1.2.4: {} + + local-pkg@1.2.1: + dependencies: + mlly: 1.8.2 + pkg-types: 2.3.1 + quansync: 0.2.11 + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + + minimatch@10.2.3: + dependencies: + brace-expansion: 5.0.7 + + minimatch@9.0.9: + dependencies: + brace-expansion: 5.0.7 + + mlly@1.8.2: + dependencies: + acorn: 8.17.0 + pathe: 2.0.3 + pkg-types: 1.3.1 + ufo: 1.6.4 + + ms@2.1.3: {} + + muggle-string@0.4.1: {} + + nanoid@3.3.15: {} + + parent-module@1.0.1: + dependencies: + callsites: 3.1.0 + + parse-json@5.2.0: + dependencies: + '@babel/code-frame': 7.29.7 + error-ex: 1.3.4 + json-parse-even-better-errors: 2.3.1 + lines-and-columns: 1.2.4 + + path-browserify@1.0.1: {} + + path-parse@1.0.7: {} + + path-type@4.0.0: {} + + pathe@2.0.3: {} + + perfect-freehand@1.2.3: {} + + picocolors@1.1.1: {} + + picomatch@4.0.5: {} + + pkg-types@1.3.1: + dependencies: + confbox: 0.1.8 + mlly: 1.8.2 + pathe: 2.0.3 + + pkg-types@2.3.1: + dependencies: + confbox: 0.2.4 + exsolve: 1.1.0 + pathe: 2.0.3 + + postcss@8.5.16: + dependencies: + nanoid: 3.3.15 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + proxy-compare@3.0.1: {} + + proxy-memoize@3.0.1: + dependencies: + proxy-compare: 3.0.1 + + quansync@0.2.11: {} + + react-dom@19.2.7(react@19.2.7): + dependencies: + react: 19.2.7 + scheduler: 0.27.0 + + react-is@16.13.1: {} + + react@19.2.7: {} + + require-from-string@2.0.2: {} + + resolve-from@4.0.0: {} + + resolve@1.22.12: + dependencies: + es-errors: 1.3.0 + is-core-module: 2.16.2 + path-parse: 1.0.7 + supports-preserve-symlinks-flag: 1.0.0 + + rolldown@1.1.4: + dependencies: + '@oxc-project/types': 0.138.0 + '@rolldown/pluginutils': 1.0.1 + optionalDependencies: + '@rolldown/binding-android-arm64': 1.1.4 + '@rolldown/binding-darwin-arm64': 1.1.4 + '@rolldown/binding-darwin-x64': 1.1.4 + '@rolldown/binding-freebsd-x64': 1.1.4 + '@rolldown/binding-linux-arm-gnueabihf': 1.1.4 + '@rolldown/binding-linux-arm64-gnu': 1.1.4 + '@rolldown/binding-linux-arm64-musl': 1.1.4 + '@rolldown/binding-linux-ppc64-gnu': 1.1.4 + '@rolldown/binding-linux-s390x-gnu': 1.1.4 + '@rolldown/binding-linux-x64-gnu': 1.1.4 + '@rolldown/binding-linux-x64-musl': 1.1.4 + '@rolldown/binding-openharmony-arm64': 1.1.4 + '@rolldown/binding-wasm32-wasi': 1.1.4 + '@rolldown/binding-win32-arm64-msvc': 1.1.4 + '@rolldown/binding-win32-x64-msvc': 1.1.4 + + scheduler@0.27.0: {} + + semver@7.7.4: {} + + source-map-js@1.2.1: {} + + source-map@0.5.7: {} + + source-map@0.6.1: {} + + sprintf-js@1.0.3: {} + + string-argv@0.3.2: {} + + stylis@4.2.0: {} + + supports-color@8.1.1: + dependencies: + has-flag: 4.0.0 + + supports-preserve-symlinks-flag@1.0.0: {} + + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.5) + picomatch: 4.0.5 + + tslib@2.8.1: {} + + typescript@5.9.3: {} + + ufo@1.6.4: {} + + universalify@2.0.1: {} + + uqr@0.1.3: {} + + vite-plugin-css-injected-by-js@3.5.2(vite@8.1.3(yaml@2.9.0)): + dependencies: + vite: 8.1.3(yaml@2.9.0) + + vite-plugin-dts@4.5.4(typescript@5.9.3)(vite@8.1.3(yaml@2.9.0)): + dependencies: + '@microsoft/api-extractor': 7.58.9 + '@rollup/pluginutils': 5.4.0 + '@volar/typescript': 2.4.28 + '@vue/language-core': 2.2.0(typescript@5.9.3) + compare-versions: 6.1.1 + debug: 4.4.3 + kolorist: 1.8.0 + local-pkg: 1.2.1 + magic-string: 0.30.21 + typescript: 5.9.3 + optionalDependencies: + vite: 8.1.3(yaml@2.9.0) + transitivePeerDependencies: + - '@types/node' + - rollup + - supports-color + + vite@8.1.3(yaml@2.9.0): + dependencies: + lightningcss: 1.32.0 + picomatch: 4.0.5 + postcss: 8.5.16 + rolldown: 1.1.4 + tinyglobby: 0.2.17 + optionalDependencies: + fsevents: 2.3.3 + yaml: 2.9.0 + + vscode-uri@3.1.0: {} + + yaml@2.9.0: {} diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/api.ts b/providers/dq/src/airflow/providers/dq/plugins/www/src/api.ts new file mode 100644 index 0000000000000..d88fc3f88c515 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/api.ts @@ -0,0 +1,82 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { PaginatedResult, RuleHistoryRecord, TaskDQRunRecord } from "src/types/dq"; + +function getBase(): string { + if (typeof document === "undefined") return "/dq"; + const baseHref = document.querySelector("head > base")?.getAttribute("href") ?? ""; + const baseUrl = new URL(baseHref, globalThis.location.origin); + const basePath = baseUrl.pathname.replace(/\/$/, "") || ""; + return basePath ? `${basePath}/dq` : "/dq"; +} + +const BASE = getBase(); + +export class ApiError extends Error { + status: number; + + constructor(message: string, status: number) { + super(message); + this.status = status; + } +} + +async function apiFetch(path: string): Promise { + const res = await fetch(`${BASE}${path}`); + if (!res.ok) { + const body: unknown = await res.json().catch(() => ({})); + const detail = (body as { detail?: string })?.detail; + throw new ApiError(detail ?? res.statusText, res.status); + } + return res.json() as Promise; +} + +interface PageOptions { + before?: string | null; + limit?: number; +} + +function buildPageQuery({ before, limit }: PageOptions): string { + const params = new URLSearchParams(); + if (limit !== undefined) params.set("limit", String(limit)); + if (before) params.set("before", before); + const query = params.toString(); + return query ? `?${query}` : ""; +} + +export function createApi(dagId: string, taskId: string, runId: string, mapIndex: number) { + return { + fetchRuleHistory: (ruleUid: string, options: PageOptions = {}) => + apiFetch>( + `/v1/dags/${encodeURIComponent(dagId)}/tasks/${encodeURIComponent(taskId)}` + + `/rules/${encodeURIComponent(ruleUid)}/history${buildPageQuery({ limit: 100, ...options })}`, + ), + fetchTaskRuns: (options: PageOptions = {}) => + apiFetch>( + `/v1/dags/${encodeURIComponent(dagId)}/tasks/${encodeURIComponent(taskId)}/runs` + + buildPageQuery({ limit: 50, ...options }), + ), + fetchTaskInstanceRun: () => + apiFetch( + `/v1/dags/${encodeURIComponent(dagId)}/tasks/${encodeURIComponent(taskId)}` + + `/runs/by_run/${encodeURIComponent(runId)}?map_index=${mapIndex}`, + ), + }; +} diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/components/DQErrorState.tsx b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/DQErrorState.tsx new file mode 100644 index 0000000000000..411bc368b7e23 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/DQErrorState.tsx @@ -0,0 +1,35 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Alert } from "@chakra-ui/react"; +import type { FC } from "react"; + +interface DQErrorStateProps { + message: string; +} + +export const DQErrorState: FC = ({ message }) => ( + + + + Could not load data quality results + {message} + + +); diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/components/DQSummaryCards.tsx b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/DQSummaryCards.tsx new file mode 100644 index 0000000000000..42e239e9ae307 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/DQSummaryCards.tsx @@ -0,0 +1,55 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Box, Grid, Text } from "@chakra-ui/react"; +import type { FC } from "react"; + +import type { DQSummaryRecord } from "src/types/dq"; + +interface DQSummaryCardsProps { + summary: DQSummaryRecord; +} + +const METRICS: ReadonlyArray<{ colorPalette: string; key: keyof DQSummaryRecord; label: string }> = [ + { colorPalette: "blue", key: "score", label: "Score" }, + { colorPalette: "green", key: "passed", label: "Passed" }, + { colorPalette: "yellow", key: "warned", label: "Warned" }, + { colorPalette: "orange", key: "failed", label: "Failed" }, + { colorPalette: "red", key: "errored", label: "Errored" }, +]; + +function formatMetricValue(key: keyof DQSummaryRecord, value: number | null): string { + if (value === null) return "—"; + return key === "score" ? value.toFixed(2) : String(value); +} + +export const DQSummaryCards: FC = ({ summary }) => ( + + {METRICS.map(({ colorPalette, key, label }) => ( + + + {label} + + + {formatMetricValue(key, summary[key])} + + + ))} + +); diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/components/NoDQResult.tsx b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/NoDQResult.tsx new file mode 100644 index 0000000000000..cd54cb769d258 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/NoDQResult.tsx @@ -0,0 +1,39 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Box, Text } from "@chakra-ui/react"; +import type { FC } from "react"; + +export const NoDQResult: FC = () => ( + + + + 📋 + + + No data quality results for this task instance + + + Either this task doesn't run a DQCheckOperator, or it + hasn't executed yet. Once it runs, results are persisted to the configured{" "} + [dq] results_path and appear here. + + + +); diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/components/RuleHistoryPage.tsx b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/RuleHistoryPage.tsx new file mode 100644 index 0000000000000..04ed9126e8196 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/RuleHistoryPage.tsx @@ -0,0 +1,222 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Box, Button, Code, Flex, Grid, Link, Spinner, Table, Text } from "@chakra-ui/react"; +import { type FC, type ReactNode, useState } from "react"; + +import { DQErrorState } from "src/components/DQErrorState"; +import { RuleStatusBadge } from "src/components/RuleStatusBadge"; +import type { RuleHistoryRecord, RuleResultRecord } from "src/types/dq"; +import { buildTaskInstanceHref } from "src/utils/taskInstanceLink"; + +interface RuleHistoryPageProps { + history: Array; + loading: boolean; + loadingMore: boolean; + nextCursor: string | null; + onBack: () => void; + onLoadMore: () => void; + onRefresh: () => void; + rule: RuleResultRecord; + ruleHistoryError: string | null; +} + +const SummaryItem: FC<{ label: string; value: ReactNode }> = ({ label, value }) => ( + + + {label} + + + {value} + + +); + +function formatDate(value: string | null): string { + if (!value) return "-"; + const date = new Date(value); + if (Number.isNaN(date.valueOf())) return value; + return date.toLocaleString(); +} + +function formatDuration(value: number | null): string { + return value === null ? "-" : `${value.toFixed(0)} ms`; +} + +function formatCondition(condition: Record): string { + if ("equal_to" in condition) return `= ${condition.equal_to}`; + if ("geq_to" in condition) return `>= ${condition.geq_to}`; + if ("greater_than" in condition) return `> ${condition.greater_than}`; + if ("leq_to" in condition) return `<= ${condition.leq_to}`; + if ("less_than" in condition) return `< ${condition.less_than}`; + return Object.entries(condition) + .map(([key, value]) => `${key}=${value}`) + .join(", "); +} + +function getTriggeredBy(runId: string): string { + if (runId.startsWith("manual__")) return "manual"; + if (runId.startsWith("scheduled__")) return "scheduler"; + return "-"; +} + +function getRuleDescription(rule: RuleResultRecord): string { + return rule.description ?? rule.rule_name.replaceAll("_", " "); +} + +export const RuleHistoryPage: FC = ({ + history, + loading, + loadingMore, + nextCursor, + onBack, + onLoadMore, + onRefresh, + rule, + ruleHistoryError, +}) => { + const [sqlExpanded, setSqlExpanded] = useState(false); + const latestHistory = history[0]; + const latest = latestHistory ?? rule; + const sql = latest.sql ?? "SQL was not captured for this run."; + + return ( + + + + + + + + + Rule History + + + {rule.rule_name} + + + {getRuleDescription(latest)} + + + + + + Rule Details + + + + } /> + + + + + + + + + Execution History + + {ruleHistoryError ? : null} + {loading ? ( + + + + ) : history.length === 0 ? ( + + No execution history available. + + Run the task to collect history. + + + ) : ( + + + + Started + Status + Observed Value + Condition + Duration + Triggered By + Run + + {history.map((record) => ( + + {formatDate(record.run.started_at)} + + + + {record.observed_value ?? "-"} + {formatCondition(record.condition)} + {formatDuration(record.duration_ms)} + {getTriggeredBy(record.run.run_id)} + + + {record.run.run_id} + + + + ))} + + + )} + {!loading && history.length > 0 ? ( + + {nextCursor === null ? ( + + No more history. + + ) : ( + + )} + + ) : null} + + + + + + SQL Executed + + + + {sqlExpanded ? ( + + + + + + {sql} + + + ) : null} + + + ); +}; diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/components/RuleResultsTable.tsx b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/RuleResultsTable.tsx new file mode 100644 index 0000000000000..3bf14ea325108 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/RuleResultsTable.tsx @@ -0,0 +1,70 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Table, Text } from "@chakra-ui/react"; +import type { FC } from "react"; + +import { RuleStatusBadge } from "src/components/RuleStatusBadge"; +import type { RuleResultRecord } from "src/types/dq"; + +function formatCondition(condition: Record): string { + return Object.entries(condition) + .map(([key, value]) => `${key}=${value}`) + .join(", "); +} + +interface RuleResultsTableProps { + results: Array; +} + +export const RuleResultsTable: FC = ({ results }) => ( + + + + Rule + Status + Dimension + Severity + Observed Value + Condition + Duration + + {results.map((result) => ( + + + {result.rule_name} + {result.error_message ? ( + + {result.error_message} + + ) : null} + + + + + {result.dimension} + {result.severity} + {result.observed_value ?? "—"} + {formatCondition(result.condition)} + {result.duration_ms === null ? "—" : `${result.duration_ms.toFixed(0)} ms`} + + ))} + + +); diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/components/RuleStatusBadge.tsx b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/RuleStatusBadge.tsx new file mode 100644 index 0000000000000..e6a95071d2e80 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/RuleStatusBadge.tsx @@ -0,0 +1,40 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Badge } from "@chakra-ui/react"; +import type { FC } from "react"; + +import type { RuleStatus } from "src/types/dq"; + +const COLOR_BY_STATUS: Record = { + error: "gray", + fail: "red", + pass: "green", + warn: "yellow", +}; + +interface RuleStatusBadgeProps { + status: RuleStatus; +} + +export const RuleStatusBadge: FC = ({ status }) => ( + + {status} + +); diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/components/TaskDQOverview.tsx b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/TaskDQOverview.tsx new file mode 100644 index 0000000000000..aaafef1b7ca03 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/TaskDQOverview.tsx @@ -0,0 +1,311 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Box, Button, Flex, Input, Link, Spinner, Table, Text } from "@chakra-ui/react"; +import { type FC, useMemo, useState } from "react"; + +import { DQErrorState } from "src/components/DQErrorState"; +import { DQSummaryCards } from "src/components/DQSummaryCards"; +import { NoDQResult } from "src/components/NoDQResult"; +import { RuleHistoryPage } from "src/components/RuleHistoryPage"; +import { RuleStatusBadge } from "src/components/RuleStatusBadge"; +import { useTaskDQ } from "src/hooks/useTaskDQ"; +import type { RuleResultRecord, TaskDQRunRecord } from "src/types/dq"; +import { buildTaskInstanceHref } from "src/utils/taskInstanceLink"; + +interface TaskDQOverviewProps { + dagId: string; + taskId: string; +} + +interface RuleOverview { + errored: number; + failures: number; + latest: RuleResultRecord; + latestStartedAt: string | null; + passes: number; + runs: number; + warnings: number; +} + +type RuleFilter = "all" | "error" | "fail" | "issues" | "pass" | "warn"; + +const RULE_FILTERS: Array<{ label: string; value: RuleFilter }> = [ + { label: "All", value: "all" }, + { label: "Issues", value: "issues" }, + { label: "Pass", value: "pass" }, + { label: "Warn", value: "warn" }, + { label: "Fail", value: "fail" }, + { label: "Error", value: "error" }, +]; + +function formatDate(value: string | null): string { + if (!value) return "-"; + const date = new Date(value); + if (Number.isNaN(date.valueOf())) return value; + return date.toLocaleString(); +} + +function getRuleOverviews(runs: Array): Array { + const rules = new Map(); + + for (const run of runs) { + for (const result of run.results) { + const current = rules.get(result.rule_uid); + const errored = result.status === "error" ? 1 : 0; + const failures = result.status === "fail" || result.status === "error" ? 1 : 0; + const passes = result.status === "pass" ? 1 : 0; + const warnings = result.status === "warn" ? 1 : 0; + + if (current === undefined) { + rules.set(result.rule_uid, { + errored, + failures, + latest: result, + latestStartedAt: run.run.started_at, + passes, + runs: 1, + warnings, + }); + } else { + rules.set(result.rule_uid, { + errored: current.errored + errored, + failures: current.failures + failures, + latest: current.latest, + latestStartedAt: current.latestStartedAt, + passes: current.passes + passes, + runs: current.runs + 1, + warnings: current.warnings + warnings, + }); + } + } + } + + return [...rules.values()].sort((left, right) => left.latest.rule_name.localeCompare(right.latest.rule_name)); +} + +function getLatestRun(runs: Array): TaskDQRunRecord | undefined { + return runs[0]; +} + +export const TaskDQOverview: FC = ({ dagId, taskId }) => { + const [ruleFilter, setRuleFilter] = useState("all"); + const [refreshKey, setRefreshKey] = useState(0); + const [ruleQuery, setRuleQuery] = useState(""); + const [historyRuleUid, setHistoryRuleUid] = useState(null); + const { + error, + loading, + loadMoreRuleHistory, + loadMoreRuns, + notFound, + ruleHistory, + ruleHistoryError, + ruleHistoryLoading, + ruleHistoryLoadingMore, + ruleHistoryNextCursor, + runs, + runsLoadingMore, + runsNextCursor, + } = useTaskDQ(dagId, taskId, historyRuleUid, refreshKey); + const ruleOverviews = useMemo(() => getRuleOverviews(runs), [runs]); + const filteredRuleOverviews = useMemo( + () => + ruleOverviews.filter((rule) => { + const matchesQuery = rule.latest.rule_name.toLowerCase().includes(ruleQuery.trim().toLowerCase()); + const matchesFilter = + ruleFilter === "all" || + (ruleFilter === "issues" && (rule.failures > 0 || rule.warnings > 0)) || + (ruleFilter === "pass" && rule.passes > 0) || + (ruleFilter === "warn" && rule.warnings > 0) || + (ruleFilter === "fail" && rule.failures > rule.errored) || + (ruleFilter === "error" && rule.errored > 0); + + return matchesQuery && matchesFilter; + }), + [ruleFilter, ruleOverviews, ruleQuery], + ); + const latestRun = getLatestRun(runs); + const historyRule = ruleOverviews.find((rule) => rule.latest.rule_uid === historyRuleUid); + + if (loading) { + return ( + + + + ); + } + + if (error) { + return ; + } + + if (notFound || runs.length === 0 || latestRun === undefined) { + return ; + } + + if (historyRule) { + return ( + setHistoryRuleUid(null)} + onLoadMore={loadMoreRuleHistory} + onRefresh={() => setRefreshKey((key) => key + 1)} + rule={historyRule.latest} + ruleHistoryError={ruleHistoryError} + /> + ); + } + + return ( + + + + + Data Quality Results + + + + + + + + setRuleQuery(event.target.value)} + placeholder="Filter rules" + size="sm" + value={ruleQuery} + /> + + {RULE_FILTERS.map((filter) => ( + + ))} + + + + + + + Rule + Latest + Runs + Failures + Warnings + Last Value + Last Run + History + + {filteredRuleOverviews.map((rule) => ( + + + {rule.latest.rule_name} + {rule.latest.description ? ( + + {rule.latest.description} + + ) : null} + + + + + {rule.runs} + {rule.failures} + {rule.warnings} + {rule.latest.observed_value ?? "-"} + {formatDate(rule.latestStartedAt)} + + + + + ))} + {filteredRuleOverviews.length === 0 ? ( + + + + No rules match the current filters. + + + + ) : null} + + + + + + + Recent Runs + + + + + Started + Score + Passed + Warned + Failed + Errored + + + {runs.map((run) => ( + + {formatDate(run.run.started_at)} + {run.summary.score === null ? "-" : run.summary.score.toFixed(2)} + {run.summary.passed} + {run.summary.warned} + {run.summary.failed} + {run.summary.errored} + + + View run + + + + ))} + + + + {runsNextCursor === null ? ( + + No more runs. + + ) : ( + + )} + + + + ); +}; diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/components/TaskDQPanel.tsx b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/TaskDQPanel.tsx new file mode 100644 index 0000000000000..68e6ad6897a58 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/components/TaskDQPanel.tsx @@ -0,0 +1,81 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Box, Flex, Spinner, Text } from "@chakra-ui/react"; +import type { FC } from "react"; + +import { DQErrorState } from "src/components/DQErrorState"; +import { DQSummaryCards } from "src/components/DQSummaryCards"; +import { NoDQResult } from "src/components/NoDQResult"; +import { RuleResultsTable } from "src/components/RuleResultsTable"; +import { useTaskInstanceDQ } from "src/hooks/useTaskInstanceDQ"; + +interface TaskDQPanelProps { + dagId: string; + mapIndex: number; + runId: string; + taskId: string; +} + +const MetadataItem: FC<{ label: string; value: string | null }> = ({ label, value }) => + value ? ( + + + {label}:{" "} + + {value} + + ) : null; + +export const TaskDQPanel: FC = ({ dagId, mapIndex, runId, taskId }) => { + const { error, loading, notFound, result } = useTaskInstanceDQ(dagId, taskId, runId, mapIndex); + + if (loading) { + return ( + + + + ); + } + + if (error) { + return ; + } + + if (notFound || !result) { + return ; + } + + return ( + + + + Data Quality Results + + + + + + + + + + + ); +}; diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/dev.tsx b/providers/dq/src/airflow/providers/dq/plugins/www/src/dev.tsx new file mode 100644 index 0000000000000..67b54eebb937e --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/dev.tsx @@ -0,0 +1,32 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { StrictMode } from "react"; +import { createRoot } from "react-dom/client"; + +import PluginComponent from "./main"; + +// Development entry point - mount with mock task instance params for local testing. +// `vite dev` proxies /dq to a running Airflow API server (see vite.config.ts), so point +// dagId/runId/taskId/mapIndex at a real task instance that has a persisted DQ run. +createRoot(document.querySelector("#root") as HTMLDivElement).render( + + + , +); diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/hooks/useTaskDQ.ts b/providers/dq/src/airflow/providers/dq/plugins/www/src/hooks/useTaskDQ.ts new file mode 100644 index 0000000000000..a185444e1c22c --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/hooks/useTaskDQ.ts @@ -0,0 +1,170 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { useCallback, useEffect, useState } from "react"; + +import { ApiError, createApi } from "src/api"; +import type { RuleHistoryRecord, TaskDQRunRecord } from "src/types/dq"; + +interface UseTaskDQReturn { + error: string | null; + loading: boolean; + loadMoreRuleHistory: () => void; + loadMoreRuns: () => void; + notFound: boolean; + ruleHistory: Array; + ruleHistoryError: string | null; + ruleHistoryLoading: boolean; + ruleHistoryLoadingMore: boolean; + ruleHistoryNextCursor: string | null; + runs: Array; + runsLoadingMore: boolean; + runsNextCursor: string | null; +} + +export function useTaskDQ( + dagId: string, + taskId: string, + selectedRuleUid: string | null, + refreshKey = 0, +): UseTaskDQReturn { + const [runs, setRuns] = useState>([]); + const [runsNextCursor, setRunsNextCursor] = useState(null); + const [runsLoadingMore, setRunsLoadingMore] = useState(false); + const [error, setError] = useState(null); + const [notFound, setNotFound] = useState(false); + const [loading, setLoading] = useState(true); + const [ruleHistory, setRuleHistory] = useState>([]); + const [ruleHistoryNextCursor, setRuleHistoryNextCursor] = useState(null); + const [ruleHistoryLoadingMore, setRuleHistoryLoadingMore] = useState(false); + const [ruleHistoryError, setRuleHistoryError] = useState(null); + const [ruleHistoryLoading, setRuleHistoryLoading] = useState(false); + + useEffect(() => { + let cancelled = false; + + async function loadRuns() { + setLoading(true); + try { + const page = await createApi(dagId, taskId, "", -1).fetchTaskRuns(); + if (cancelled) return; + setRuns(page.items); + setRunsNextCursor(page.next_cursor); + setNotFound(page.items.length === 0); + setError(null); + } catch (err) { + if (cancelled) return; + if (err instanceof ApiError && err.status === 404) { + setNotFound(true); + setError(null); + } else { + setError(err instanceof Error ? err.message : String(err)); + } + } finally { + if (!cancelled) setLoading(false); + } + } + + void loadRuns(); + return () => { + cancelled = true; + }; + }, [dagId, refreshKey, taskId]); + + useEffect(() => { + let cancelled = false; + + async function loadRuleHistory(ruleUid: string) { + setRuleHistoryLoading(true); + try { + const page = await createApi(dagId, taskId, "", -1).fetchRuleHistory(ruleUid); + if (cancelled) return; + setRuleHistory(page.items); + setRuleHistoryNextCursor(page.next_cursor); + setRuleHistoryError(null); + } catch (err) { + if (cancelled) return; + setRuleHistoryError(err instanceof Error ? err.message : String(err)); + } finally { + if (!cancelled) setRuleHistoryLoading(false); + } + } + + if (selectedRuleUid === null) { + setRuleHistory([]); + setRuleHistoryNextCursor(null); + setRuleHistoryError(null); + setRuleHistoryLoading(false); + return undefined; + } + + void loadRuleHistory(selectedRuleUid); + return () => { + cancelled = true; + }; + }, [dagId, refreshKey, selectedRuleUid, taskId]); + + const loadMoreRuns = useCallback(() => { + if (runsNextCursor === null || runsLoadingMore) return; + + setRunsLoadingMore(true); + createApi(dagId, taskId, "", -1) + .fetchTaskRuns({ before: runsNextCursor }) + .then((page) => { + setRuns((current) => [...current, ...page.items]); + setRunsNextCursor(page.next_cursor); + }) + .catch((err: unknown) => { + setError(err instanceof Error ? err.message : String(err)); + }) + .finally(() => setRunsLoadingMore(false)); + }, [dagId, runsLoadingMore, runsNextCursor, taskId]); + + const loadMoreRuleHistory = useCallback(() => { + if (selectedRuleUid === null || ruleHistoryNextCursor === null || ruleHistoryLoadingMore) return; + + setRuleHistoryLoadingMore(true); + createApi(dagId, taskId, "", -1) + .fetchRuleHistory(selectedRuleUid, { before: ruleHistoryNextCursor }) + .then((page) => { + setRuleHistory((current) => [...current, ...page.items]); + setRuleHistoryNextCursor(page.next_cursor); + }) + .catch((err: unknown) => { + setRuleHistoryError(err instanceof Error ? err.message : String(err)); + }) + .finally(() => setRuleHistoryLoadingMore(false)); + }, [dagId, ruleHistoryLoadingMore, ruleHistoryNextCursor, selectedRuleUid, taskId]); + + return { + error, + loading, + loadMoreRuleHistory, + loadMoreRuns, + notFound, + ruleHistory, + ruleHistoryError, + ruleHistoryLoading, + ruleHistoryLoadingMore, + ruleHistoryNextCursor, + runs, + runsLoadingMore, + runsNextCursor, + }; +} diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/hooks/useTaskInstanceDQ.ts b/providers/dq/src/airflow/providers/dq/plugins/www/src/hooks/useTaskInstanceDQ.ts new file mode 100644 index 0000000000000..0c3ff0ba2f759 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/hooks/useTaskInstanceDQ.ts @@ -0,0 +1,76 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { useEffect, useState } from "react"; + +import { ApiError, createApi } from "src/api"; +import type { TaskDQRunRecord } from "src/types/dq"; + +interface UseTaskInstanceDQReturn { + error: string | null; + loading: boolean; + /** True when no data quality check has run yet for this task instance (HTTP 404). */ + notFound: boolean; + result: TaskDQRunRecord | null; +} + +export function useTaskInstanceDQ( + dagId: string, + taskId: string, + runId: string, + mapIndex: number, +): UseTaskInstanceDQReturn { + const [result, setResult] = useState(null); + const [error, setError] = useState(null); + const [notFound, setNotFound] = useState(false); + const [loading, setLoading] = useState(true); + + useEffect(() => { + let cancelled = false; + const api = createApi(dagId, taskId, runId, mapIndex); + + async function load() { + setLoading(true); + try { + const record = await api.fetchTaskInstanceRun(); + if (cancelled) return; + setResult(record); + setNotFound(false); + setError(null); + } catch (err) { + if (cancelled) return; + if (err instanceof ApiError && err.status === 404) { + setNotFound(true); + setError(null); + } else { + setError(err instanceof Error ? err.message : String(err)); + } + } finally { + if (!cancelled) setLoading(false); + } + } + + void load(); + return () => { + cancelled = true; + }; + }, [dagId, mapIndex, runId, taskId]); + + return { error, loading, notFound, result }; +} diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/main.tsx b/providers/dq/src/airflow/providers/dq/plugins/www/src/main.tsx new file mode 100644 index 0000000000000..8b2ea285412d2 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/main.tsx @@ -0,0 +1,73 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Box, ChakraProvider } from "@chakra-ui/react"; +import type { FC } from "react"; + +import { TaskDQOverview } from "src/components/TaskDQOverview"; +import { TaskDQPanel } from "src/components/TaskDQPanel"; + +import { localSystem } from "./theme"; + +export interface PluginComponentProps { + dagId?: string; + mapIndex?: string; + runId?: string; + taskId?: string; +} + +/** + * Main plugin component for the Data Quality task and task-instance views. + * + * Receives dagId, runId, taskId, mapIndex as props from the Airflow React plugin host (from + * route params), matching the react_apps contract. + */ +const PluginComponent: FC = ({ + dagId = "", + runId = "", + taskId = "", + mapIndex: mapIndexProp = "-1", +}) => { + if (!runId) { + return ; + } + + const mapIndex = /^-?\d+$/.test(String(mapIndexProp)) ? parseInt(String(mapIndexProp), 10) : -1; + + return ; +}; + +/** + * Plugin component wrapped with ChakraProvider for consistent theming with the host. + * Chakra semantic tokens handle light/dark mode automatically. + */ +const WrappedPluginComponent: FC = (props) => { + const system = + (globalThis as { ChakraUISystem?: typeof localSystem }).ChakraUISystem ?? localSystem; + + return ( + + + + + + ); +}; + +export default WrappedPluginComponent; diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/theme.ts b/providers/dq/src/airflow/providers/dq/plugins/www/src/theme.ts new file mode 100644 index 0000000000000..dfb4866cedb58 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/theme.ts @@ -0,0 +1,22 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createSystem, defaultConfig } from "@chakra-ui/react"; + +export const localSystem = createSystem(defaultConfig); diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/types/dq.ts b/providers/dq/src/airflow/providers/dq/plugins/www/src/types/dq.ts new file mode 100644 index 0000000000000..879643e0c8986 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/types/dq.ts @@ -0,0 +1,84 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** Mirrors ``airflow.providers.dq.results.DQRun.to_dict()``. */ +export interface DQRunRecord { + asset_names: Array; + dag_id: string; + finished_at: string | null; + map_index: number; + run_id: string; + run_uid: string; + ruleset_name: string | null; + started_at: string | null; + table_ref: string | null; + task_id: string; + try_number: number; +} + +export type RuleStatus = "error" | "fail" | "pass" | "warn"; + +export type RuleSeverity = "error" | "warn"; + +/** Mirrors ``airflow.providers.dq.results.RuleResult.to_dict()``. */ +export interface RuleResultRecord { + condition: Record; + description?: string | null; + dimension: string; + duration_ms: number | null; + error_message: string | null; + observed_value: number | string | null; + rule_name: string; + rule_uid: string; + severity: RuleSeverity; + sql?: string | null; + status: RuleStatus; +} + +export interface DQSummaryRecord { + errored: number; + failed: number; + passed: number; + score: number | null; + warned: number; +} + +export interface TaskDQRunRecord { + results: Array; + run: DQRunRecord; + summary: DQSummaryRecord; +} + +export interface RuleHistoryRecord extends RuleResultRecord { + run: { + dag_id: string; + map_index?: number; + run_id: string; + run_uid: string; + started_at: string | null; + table_ref: string | null; + task_id: string; + }; +} + +/** Mirrors the ``{"items": [...], "next_cursor": ...}`` shape returned by paginated routes. */ +export interface PaginatedResult { + items: Array; + next_cursor: string | null; +} diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/utils/taskInstanceLink.ts b/providers/dq/src/airflow/providers/dq/plugins/www/src/utils/taskInstanceLink.ts new file mode 100644 index 0000000000000..5c4db71bdbcb4 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/utils/taskInstanceLink.ts @@ -0,0 +1,34 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +interface TaskInstanceRef { + dag_id: string; + map_index?: number; + run_id: string; + task_id: string; +} + +/** Link back to a run's own task-instance page, matching the host UI's route shape. */ +export function buildTaskInstanceHref(run: TaskInstanceRef): string { + const base = `/dags/${encodeURIComponent(run.dag_id)}/runs/${encodeURIComponent( + run.run_id, + )}/tasks/${encodeURIComponent(run.task_id)}`; + + return run.map_index !== undefined && run.map_index >= 0 ? `${base}/mapped/${run.map_index}` : base; +} diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/src/vite-env.d.ts b/providers/dq/src/airflow/providers/dq/plugins/www/src/vite-env.d.ts new file mode 100644 index 0000000000000..a1fdcdd1e6fc5 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/src/vite-env.d.ts @@ -0,0 +1,20 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/// diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/tsconfig.app.json b/providers/dq/src/airflow/providers/dq/plugins/www/tsconfig.app.json new file mode 100644 index 0000000000000..4705c6983131f --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/tsconfig.app.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedIndexedAccess": true, + "baseUrl": ".", + "paths": { + "src/*": ["./src/*"] + } + }, + "include": ["src"] +} diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/tsconfig.json b/providers/dq/src/airflow/providers/dq/plugins/www/tsconfig.json new file mode 100644 index 0000000000000..b6a01bf3b76ac --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/tsconfig.json @@ -0,0 +1,8 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.app.json" }, + { "path": "./tsconfig.node.json" }, + { "path": "./tsconfig.lib.json" } + ] +} diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/tsconfig.lib.json b/providers/dq/src/airflow/providers/dq/plugins/www/tsconfig.lib.json new file mode 100644 index 0000000000000..d423df60f8313 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/tsconfig.lib.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "./tsconfig.app.json", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": true, + "outDir": "./dist", + "rootDir": "./src", + "noEmit": false, + "allowImportingTsExtensions": false + }, + "include": ["src/main.tsx"], + "exclude": ["**/*.test.*", "**/*.spec.*"] +} diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/tsconfig.node.json b/providers/dq/src/airflow/providers/dq/plugins/www/tsconfig.node.json new file mode 100644 index 0000000000000..a865d28180fa7 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/tsconfig.node.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2023"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedIndexedAccess": true, + "baseUrl": ".", + "paths": { + "src/*": ["./src/*"] + } + }, + "include": ["vite.config.ts"] +} diff --git a/providers/dq/src/airflow/providers/dq/plugins/www/vite.config.ts b/providers/dq/src/airflow/providers/dq/plugins/www/vite.config.ts new file mode 100644 index 0000000000000..bb5b8e2e60f92 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/plugins/www/vite.config.ts @@ -0,0 +1,91 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import react from "@vitejs/plugin-react-swc"; +import { resolve } from "node:path"; +import cssInjectedByJsPlugin from "vite-plugin-css-injected-by-js"; +import dts from "vite-plugin-dts"; +import { defineConfig } from "vite"; + +export default defineConfig(({ command }) => { + const isLibraryBuild = command === "build"; + + return { + base: "./", + build: isLibraryBuild + ? { + chunkSizeWarningLimit: 1600, + lib: { + entry: resolve("src", "main.tsx"), + fileName: "main", + formats: ["umd"], + name: "AirflowPlugin", + }, + rollupOptions: { + external: [ + "react", + "react-dom", + "react/jsx-runtime", + "@chakra-ui/react", + "@emotion/react", + ], + output: { + entryFileNames: "[name].umd.cjs", + globals: { + react: "React", + "react-dom": "ReactDOM", + "react/jsx-runtime": "ReactJSXRuntime", + "@chakra-ui/react": "ChakraUI", + "@emotion/react": "EmotionReact", + }, + }, + }, + } + : { + chunkSizeWarningLimit: 1600, + }, + define: { + global: "globalThis", + "process.env": "{}", + "process.env.NODE_ENV": JSON.stringify("production"), + }, + plugins: [ + react(), + cssInjectedByJsPlugin(), + ...(isLibraryBuild + ? [ + dts({ + include: ["src/main.tsx"], + insertTypesEntry: true, + outDir: "dist", + }), + ] + : []), + ], + resolve: { alias: { src: "/src" } }, + server: { + cors: true, + proxy: { + "/dq": { + changeOrigin: true, + target: "http://localhost:28080", + }, + }, + }, + }; +}); diff --git a/providers/dq/src/airflow/providers/dq/results.py b/providers/dq/src/airflow/providers/dq/results.py new file mode 100644 index 0000000000000..5fbcf04573cc7 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/results.py @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Result records produced by a data quality check run — immutable facts, JSON-serializable.""" + +from __future__ import annotations + +import uuid +from dataclasses import asdict, dataclass, field +from typing import Any + +PASS = "pass" +WARN = "warn" +FAIL = "fail" +ERROR = "error" + +# Weight of a warn-severity failure in the run score, relative to an error-severity failure. +WARN_SCORE_WEIGHT = 0.25 + + +@dataclass(frozen=True) +class RuleResult: + """Outcome of evaluating one rule in one run.""" + + rule_uid: str + rule_name: str + status: str # pass | warn | fail | error + observed_value: float | str | None = None + condition: dict[str, Any] = field(default_factory=dict) + dimension: str = "validity" + severity: str = "error" + duration_ms: float | None = None + error_message: str | None = None + description: str | None = None + sql: str | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> RuleResult: + return cls(**data) + + +@dataclass(frozen=True) +class DQRun: + """One execution of a ruleset by one task instance.""" + + dag_id: str + task_id: str + run_id: str + try_number: int = 1 + map_index: int = -1 + run_uid: str = field(default_factory=lambda: uuid.uuid4().hex) + ruleset_name: str | None = None + table_ref: str | None = None + asset_names: tuple[str, ...] = () + started_at: str | None = None + finished_at: str | None = None + + def to_dict(self) -> dict[str, Any]: + data = asdict(self) + data["asset_names"] = list(self.asset_names) + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> DQRun: + data = {**data, "asset_names": tuple(data.get("asset_names", ()))} + return cls(**data) + + +def compute_score(results: list[RuleResult]) -> float | None: + """ + Weighted pass rate for a run in [0, 1]. + + Error-severity failures (and execution errors) count fully against the score; + warn-severity failures count at ``WARN_SCORE_WEIGHT``. + """ + if not results: + return None + penalty = 0.0 + for result in results: + if result.status in (FAIL, ERROR): + penalty += 1.0 + elif result.status == WARN: + penalty += WARN_SCORE_WEIGHT + return round(1.0 - penalty / len(results), 4) + + +def build_summary(run: DQRun, results: list[RuleResult]) -> dict[str, Any]: + """Compact run summary attached to XCom and outlet asset events.""" + return { + "run_uid": run.run_uid, + "ruleset": run.ruleset_name, + "table": run.table_ref, + "score": compute_score(results), + "passed": sum(1 for r in results if r.status == PASS), + "warned": sum(1 for r in results if r.status == WARN), + "failed": sum(1 for r in results if r.status == FAIL), + "errored": sum(1 for r in results if r.status == ERROR), + "failed_rules": sorted(r.rule_name for r in results if r.status in (FAIL, ERROR)), + "warned_rules": sorted(r.rule_name for r in results if r.status == WARN), + } diff --git a/providers/dq/src/airflow/providers/dq/rules/__init__.py b/providers/dq/src/airflow/providers/dq/rules/__init__.py new file mode 100644 index 0000000000000..1536086480383 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/rules/__init__.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.dq.rules.checks import CHECK_SPECS, CheckSpec, Dimension +from airflow.providers.dq.rules.rule import ( + CUSTOM_SQL_CHECK, + Condition, + DQRule, + RuleSet, + Severity, + describe_rule, +) + +__all__ = [ + "CHECK_SPECS", + "CUSTOM_SQL_CHECK", + "CheckSpec", + "Condition", + "DQRule", + "Dimension", + "RuleSet", + "Severity", + "describe_rule", +] diff --git a/providers/dq/src/airflow/providers/dq/rules/checks.py b/providers/dq/src/airflow/providers/dq/rules/checks.py new file mode 100644 index 0000000000000..11e73a5208f30 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/rules/checks.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Built-in check catalog: one place for each check's SQL expression and metadata.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any + + +class Dimension(str, Enum): + """Data quality dimension assigned to a rule result.""" + + COMPLETENESS = "completeness" + UNIQUENESS = "uniqueness" + VALIDITY = "validity" + FRESHNESS = "freshness" + VOLUME = "volume" + CONSISTENCY = "consistency" + + +@dataclass(frozen=True) +class CheckSpec: + """ + Metadata for one built-in check. + + :param expression: SQL expression template. Column-level checks are rendered with + ``{column}``; table-level checks (``requires_column=False``) ignore it. + :param dimension: Default value for ``DQRule.dimension`` when a rule doesn't set it + explicitly. ``DQRule.dimension`` is a settable field; this is only the fallback. + :param requires_column: Whether a rule using this check must set ``column``. + :param default_condition: Default ``DQRule.condition`` (as a dict) for a rule that doesn't + set one explicitly; ``None`` means the rule must always specify a condition. + """ + + expression: str + dimension: Dimension = Dimension.VALIDITY + requires_column: bool = True + default_condition: dict[str, Any] | None = None + + +CHECK_SPECS: dict[str, CheckSpec] = { + "null_count": CheckSpec( + expression="SUM(CASE WHEN {column} IS NULL THEN 1 ELSE 0 END)", + dimension=Dimension.COMPLETENESS, + ), + "null_ratio": CheckSpec( + expression="SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*)", + dimension=Dimension.COMPLETENESS, + ), + "distinct_count": CheckSpec( + expression="COUNT(DISTINCT {column})", + dimension=Dimension.UNIQUENESS, + ), + "unique_violations": CheckSpec( + expression="COUNT({column}) - COUNT(DISTINCT {column})", + dimension=Dimension.UNIQUENESS, + ), + "min": CheckSpec(expression="MIN({column})"), + "max": CheckSpec(expression="MAX({column})"), + "mean": CheckSpec(expression="AVG({column})"), + "row_count": CheckSpec( + expression="COUNT(*)", + dimension=Dimension.VOLUME, + requires_column=False, + ), +} diff --git a/providers/dq/src/airflow/providers/dq/rules/rule.py b/providers/dq/src/airflow/providers/dq/rules/rule.py new file mode 100644 index 0000000000000..8151be8e41143 --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/rules/rule.py @@ -0,0 +1,279 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Declarative data quality rule model: rules are data, not code.""" + +from __future__ import annotations + +import hashlib +import json +from enum import Enum +from typing import Any, cast + +import yaml +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from airflow.providers.dq.exceptions import DQRuleValidationError +from airflow.providers.dq.rules.checks import CHECK_SPECS, Dimension + +CUSTOM_SQL_CHECK = "custom_sql" + + +class Severity(str, Enum): + """Severity used to decide whether a failing rule fails the task.""" + + WARN = "warn" + ERROR = "error" + + +class Condition(BaseModel): + """ + Pass/fail condition evaluated against a rule's observed value. + + Uses the same grammar as the ``common.sql`` check operators: ``equal_to``, + ``greater_than``, ``less_than``, ``geq_to``, ``leq_to``, plus a percentage + ``tolerance`` that widens ``equal_to`` into a range. + + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + equal_to: float | None = None + greater_than: float | None = None + less_than: float | None = None + geq_to: float | None = None + leq_to: float | None = None + tolerance: float | None = None + + @model_validator(mode="after") + def _validate_comparisons(self) -> Condition: + comparisons = { + "equal_to": self.equal_to, + "greater_than": self.greater_than, + "less_than": self.less_than, + "geq_to": self.geq_to, + "leq_to": self.leq_to, + } + set_comparisons = {name for name, value in comparisons.items() if value is not None} + if not set_comparisons: + raise ValueError(f"Condition needs at least one comparison out of: {', '.join(comparisons)}") + if self.equal_to is not None and len(set_comparisons) > 1: + raise ValueError("equal_to cannot be combined with other comparisons") + if self.tolerance is not None and self.equal_to is None: + raise ValueError("tolerance is only supported together with equal_to") + return self + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Condition: + return cls(**data) + + def to_dict(self) -> dict[str, float]: + data = { + "equal_to": self.equal_to, + "greater_than": self.greater_than, + "less_than": self.less_than, + "geq_to": self.geq_to, + "leq_to": self.leq_to, + "tolerance": self.tolerance, + } + return {k: v for k, v in data.items() if v is not None} + + def evaluate(self, observed: Any) -> bool: + if observed is None: + return False + value = float(observed) + if self.equal_to is not None: + if self.tolerance is not None: + low = self.equal_to * (1 - self.tolerance) + high = self.equal_to * (1 + self.tolerance) + return min(low, high) <= value <= max(low, high) + return value == self.equal_to + return ( + (self.greater_than is None or value > self.greater_than) + and (self.less_than is None or value < self.less_than) + and (self.geq_to is None or value >= self.geq_to) + and (self.leq_to is None or value <= self.leq_to) + ) + + +class DQRule(BaseModel): + """ + A single, named data quality rule. + + :param name: Rule name, unique within its ruleset. + :param check: One of the built-in checks (see ``CHECK_SPECS``) or ``custom_sql``. Built-in + checks are plain ANSI SQL and are not guaranteed to work against every ``DbApiHook``; + see "Supported checks and databases" in the provider docs. Use ``custom_sql`` if a + built-in check doesn't fit your database's dialect. + :param condition: Pass condition for the observed value (``Condition`` or its dict form). + Optional only for checks whose ``CheckSpec.default_condition`` is set; every current + built-in check requires one explicitly. + :param column: Target column; required for column-level built-in checks. + :param sql: SQL statement returning a single scalar; required for ``custom_sql``. + May reference the target table as ``{table}``. + :param severity: ``error`` (fails the task by default) or ``warn`` (recorded only). + :param partition_clause: Extra predicate ANDed into the check's WHERE clause. + :param previous_name: Set when renaming a rule, to keep its history continuous. + :param description: Human-readable description shown in results and the UI. When omitted, + the provider generates a short default description from the rule and condition. + + Invalid input raises pydantic's own :class:`~pydantic.ValidationError`. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str + check: str = Field(description="One of the built-in checks, or custom_sql.") + condition: Condition | dict[str, Any] | None = None + column: str | None = None + sql: str | None = None + severity: Severity = Severity.ERROR + partition_clause: str | None = None + previous_name: str | None = None + description: str | None = Field( + default=None, + description=( + "Human-readable description shown in results and the UI. When omitted, the provider " + "generates a short default description from the rule and condition." + ), + ) + dimension: Dimension | None = Field( + default=None, + description=( + "Data quality dimension recorded for this rule's results. Defaults to the check's " + "catalog dimension (validity for custom_sql) when not set explicitly." + ), + ) + + @model_validator(mode="after") + def _validate(self) -> DQRule: + if not self.name: + raise ValueError("Rule name cannot be empty") + catalog_spec = None + if self.check == CUSTOM_SQL_CHECK: + if not self.sql: + raise ValueError(f"Rule {self.name!r}: custom_sql check requires 'sql'") + else: + catalog_spec = CHECK_SPECS.get(self.check) + if catalog_spec is None: + supported = sorted([*CHECK_SPECS, CUSTOM_SQL_CHECK]) + raise ValueError(f"Rule {self.name!r}: unknown check {self.check!r}; supported: {supported}") + if self.sql: + raise ValueError(f"Rule {self.name!r}: 'sql' is only valid with custom_sql") + if catalog_spec.requires_column and not self.column: + raise ValueError(f"Rule {self.name!r}: check {self.check!r} requires 'column'") + if self.condition is None and catalog_spec.default_condition is not None: + object.__setattr__(self, "condition", Condition.from_dict(catalog_spec.default_condition)) + if self.condition is None: + raise ValueError(f"Rule {self.name!r}: condition is required for check {self.check!r}") + if self.dimension is None: + object.__setattr__( + self, "dimension", catalog_spec.dimension if catalog_spec else Dimension.VALIDITY + ) + return self + + @property + def rule_uid(self) -> str: + """Stable identity across runs: survives severity/dimension tweaks and Dag refactors.""" + condition = cast("Condition", self.condition) + identity = { + "name": self.previous_name or self.name, + "check": self.check, + "column": self.column, + "sql": self.sql, + "condition": condition.to_dict(), + } + digest = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest() + return digest[:16] + + def to_dict(self) -> dict[str, Any]: + condition = cast("Condition", self.condition) + data: dict[str, Any] = { + "name": self.name, + "check": self.check, + "condition": condition.to_dict(), + "severity": self.severity.value, + } + for optional in ("column", "sql", "partition_clause", "previous_name", "description"): + value = getattr(self, optional) + if value is not None: + data[optional] = value + # Only emit dimension when it overrides the check's catalog default, so a rule that + # never set one explicitly round-trips through to_dict()/from_dict() unchanged. + catalog_spec = CHECK_SPECS.get(self.check) + default_dimension = catalog_spec.dimension if catalog_spec else Dimension.VALIDITY + if self.dimension != default_dimension: + data["dimension"] = cast("Dimension", self.dimension).value + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> DQRule: + return cls(**data) + + +def describe_rule(rule: DQRule) -> str: + """Return a default human-readable description for a rule.""" + condition = cast("Condition", rule.condition).to_dict() + subject = rule.column or rule.name.replace("_", " ") + if "equal_to" in condition: + return f"{subject} should equal {condition['equal_to']}" + if "geq_to" in condition: + return f"{subject} should be greater than or equal to {condition['geq_to']}" + if "greater_than" in condition: + return f"{subject} should be greater than {condition['greater_than']}" + if "leq_to" in condition: + return f"{subject} should be less than or equal to {condition['leq_to']}" + if "less_than" in condition: + return f"{subject} should be less than {condition['less_than']}" + return subject + + +class RuleSet(BaseModel): + """A named collection of rules, typically attached to one table or asset.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str + rules: tuple[DQRule, ...] = () + + @model_validator(mode="after") + def _validate(self) -> RuleSet: + if not self.name: + raise ValueError("RuleSet name cannot be empty") + names = [rule.name for rule in self.rules] + duplicates = {name for name in names if names.count(name) > 1} + if duplicates: + raise ValueError(f"Duplicate rule names in ruleset {self.name!r}: {sorted(duplicates)}") + return self + + def to_dict(self) -> dict[str, Any]: + return {"name": self.name, "rules": [rule.to_dict() for rule in self.rules]} + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> RuleSet: + rules = [ + rule if isinstance(rule, DQRule) else DQRule.from_dict(rule) for rule in data.get("rules", []) + ] + return cls(name=data.get("name", ""), rules=tuple(rules)) + + @classmethod + def from_file(cls, path: str) -> RuleSet: + """Load a ruleset from a YAML (or JSON, being a YAML subset) file.""" + with open(path) as f: + data = yaml.safe_load(f) + if not isinstance(data, dict): + raise DQRuleValidationError(f"Ruleset file {path!r} must contain a mapping at top level") + return cls.from_dict(data) diff --git a/providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/SKILL.md b/providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/SKILL.md new file mode 100644 index 0000000000000..9fdb6eccc3b1f --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/SKILL.md @@ -0,0 +1,174 @@ +--- +name: dq-rule-authoring +description: "Generate a RuleSet/DQRule JSON payload for Apache Airflow's dq provider from a table's column definitions. Use this whenever asked to write, generate, or suggest data quality rules, checks, or a ruleset for a table or dataset -- especially when the output is structured JSON handed to DQCheckOperator, @task.dq_check, asset_quality(), or RuleSet.from_file()." +--- + + + +# dq rule authoring + +You are producing a **RuleSet**: a JSON object naming a table's data quality checks. This +document is the schema and the full list of valid values -- do not guess field names or check +names, and do not invent checks that aren't in the catalog below. + +## Output shape + +```json +{ + "name": "orders_quality", + "rules": [ + { + "name": "order_id_not_null", + "check": "null_count", + "column": "order_id", + "condition": {"equal_to": 0} + } + ] +} +``` + +`name` (ruleset name) and `rules` (array) are the only top-level keys. Every rule name must be +unique within the ruleset. + +## `DQRule` fields + +| Field | Required | Notes | +|---|---|---| +| `name` | yes | Unique within the ruleset. Shows up in the Data Quality UI and logs. | +| `check` | yes | One of the catalog names below, or `"custom_sql"`. Exact string match -- there is no fuzzy matching. | +| `condition` | yes* | See `Condition` grammar below. *Every catalog check currently requires one explicitly (no defaults) -- always include it. | +| `column` | check-dependent | Required for every catalog check except `row_count`. Not used with `custom_sql`. | +| `sql` | `custom_sql` only | A SQL statement returning a single scalar. May reference the checked table as `{table}`. Invalid together with any catalog check. | +| `severity` | no | `"error"` (default) or `"warn"`. `"error"` fails the task on a failing rule (subject to the operator's `fail_on`); `"warn"` only records it. | +| `partition_clause` | no | Extra SQL predicate ANDed into this rule's `WHERE` clause, e.g. `"region = 'EU'"`. | +| `previous_name` | no | Only set when told a rule is being renamed, to keep its history continuous. | +| `description` | no | Human-readable text shown in results and the Data Quality UI. Use it when a clear business meaning is known; otherwise omit it and let Airflow generate a default description. | +| `dimension` | no | One of `completeness`, `uniqueness`, `validity`, `freshness`, `volume`, `consistency`. Defaults to the check's catalog dimension (`validity` for `custom_sql`) -- only set this explicitly when a `custom_sql` rule measures something the default doesn't capture (e.g. a freshness check written as `custom_sql` should set `"dimension": "freshness"`). Leave it unset for every built-in check. | + +**Do not add any key not listed above.** The schema rejects unrecognized fields. + +## Built-in check catalog + +| `check` | SQL expression | needs `column` | typical use | +|---|---|---|---| +| `null_count` | `SUM(CASE WHEN {column} IS NULL THEN 1 ELSE 0 END)` | yes | count of nulls in a column | +| `null_ratio` | `SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*)` | yes | fraction of nulls, 0.0-1.0 | +| `distinct_count` | `COUNT(DISTINCT {column})` | yes | number of distinct values | +| `unique_violations` | `COUNT({column}) - COUNT(DISTINCT {column})` | yes | 0 means the column is fully unique | +| `min` | `MIN({column})` | yes | minimum value | +| `max` | `MAX({column})` | yes | maximum value | +| `mean` | `AVG({column})` | yes | average value | +| `row_count` | `COUNT(*)` | no | total row count -- omit `column` entirely | + +These are the *only* built-in check names. Anything else -- comparing two columns, joining +another table, checking a format/regex, freshness against `now()`, referential integrity -- must +be written as `custom_sql`. + +## `Condition` grammar + +`condition` is an object with these optional numeric keys; at least one is required: + +- `equal_to` -- exact match. Cannot be combined with any other key below. +- `greater_than` / `geq_to` -- lower bound, exclusive / inclusive. +- `less_than` / `leq_to` -- upper bound, exclusive / inclusive. +- `tolerance` -- a fraction (e.g. `0.1` for 10%) that widens `equal_to` into a range. Only valid + together with `equal_to`. + +Combine `greater_than`/`less_than`/`geq_to`/`leq_to` freely to express a range, e.g. +`{"geq_to": 0, "leq_to": 100}`. Never combine `equal_to` with another comparison key (other than +`tolerance`). + +## `custom_sql`: when a catalog check doesn't fit + +Use `check: "custom_sql"` with a `sql` statement resolving to a single scalar whenever: + +- The check needs more than one column (e.g. `end_date >= start_date`), a join, or a subquery. +- A catalog expression doesn't run correctly against the target database's SQL dialect (for + example, some engines don't support `NULLIF`, or evaluate `CASE`/`COUNT DISTINCT` + differently) -- write the equivalent expression for that dialect instead. +- The check is conceptually one of the checks above but needs different null/empty-table + semantics than the catalog expression provides. + +Reference the table being checked as `{table}` in the SQL: + +```json +{ + "name": "no_future_order_dates", + "check": "custom_sql", + "sql": "SELECT COUNT(*) FROM {table} WHERE order_date > CURRENT_DATE", + "condition": {"equal_to": 0} +} +``` + +## Worked example + +Given these column definitions for an `orders` table -- +`order_id` (integer, primary key), `customer_id` (integer, nullable foreign key), +`amount` (decimal, must be non-negative), `region` (string, low-cardinality), +`created_at` (timestamp) -- a reasonable ruleset: + +```json +{ + "name": "orders_quality", + "rules": [ + { + "name": "order_id_not_null", + "check": "null_count", + "column": "order_id", + "condition": {"equal_to": 0} + }, + { + "name": "order_id_unique", + "check": "unique_violations", + "column": "order_id", + "condition": {"equal_to": 0} + }, + { + "name": "customer_id_null_ratio_low", + "check": "null_ratio", + "column": "customer_id", + "condition": {"leq_to": 0.05}, + "severity": "warn" + }, + { + "name": "amount_non_negative", + "check": "min", + "column": "amount", + "condition": {"geq_to": 0} + }, + { + "name": "region_cardinality_reasonable", + "check": "distinct_count", + "column": "region", + "condition": {"leq_to": 20} + }, + { + "name": "table_not_empty", + "check": "row_count", + "condition": {"greater_than": 0} + }, + { + "name": "no_future_order_dates", + "check": "custom_sql", + "sql": "SELECT COUNT(*) FROM {table} WHERE created_at > CURRENT_TIMESTAMP", + "condition": {"equal_to": 0} + } + ] +} +``` + +## Reference schema + +`references/ruleset.schema.json` in this skill's directory is the pydantic-generated JSON +Schema for this exact shape (`RuleSet.model_json_schema()`), useful for validating output +structurally. It does not enumerate valid `check` values (that field is a plain string) -- +this document is the source of truth for which check names exist. + +## How this is consumed + +The generated JSON is typically produced by an LLM task (e.g. `@task.llm(output_type=RuleSet, ...)` +from `common.ai`) and passed straight to `DQCheckOperator(ruleset=...)`, `@task.dq_check(ruleset=...)`, +or `asset_quality(ruleset=...)` -- all three accept a `RuleSet`, its dict form, or a path to a +YAML file written in this same shape. Invalid output raises a `pydantic.ValidationError` +describing exactly which field or value was wrong. diff --git a/providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/references/ruleset.schema.json b/providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/references/ruleset.schema.json new file mode 100644 index 0000000000000..a8f4387981d6d --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/references/ruleset.schema.json @@ -0,0 +1,241 @@ +{ + "$defs": { + "Condition": { + "additionalProperties": false, + "description": "Pass/fail condition evaluated against a rule's observed value.\n\nUses the same grammar as the ``common.sql`` check operators: ``equal_to``,\n``greater_than``, ``less_than``, ``geq_to``, ``leq_to``, plus a percentage\n``tolerance`` that widens ``equal_to`` into a range.", + "properties": { + "equal_to": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Equal To" + }, + "greater_than": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Greater Than" + }, + "less_than": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Less Than" + }, + "geq_to": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Geq To" + }, + "leq_to": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Leq To" + }, + "tolerance": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tolerance" + } + }, + "title": "Condition", + "type": "object" + }, + "DQRule": { + "additionalProperties": false, + "description": "A single, named data quality rule.\n\n:param name: Rule name, unique within its ruleset.\n:param check: One of the built-in checks (see ``CHECK_SPECS``) or ``custom_sql``. Built-in\n checks are plain ANSI SQL and are not guaranteed to work against every ``DbApiHook``;\n see \"Supported checks and databases\" in the provider docs. Use ``custom_sql`` if a\n built-in check doesn't fit your database's dialect.\n:param condition: Pass condition for the observed value (``Condition`` or its dict form).\n Optional only for checks whose ``CheckSpec.default_condition`` is set; every current\n built-in check requires one explicitly.\n:param column: Target column; required for column-level built-in checks.\n:param sql: SQL statement returning a single scalar; required for ``custom_sql``.\n May reference the target table as ``{table}``.\n:param severity: ``error`` (fails the task by default) or ``warn`` (recorded only).\n:param partition_clause: Extra predicate ANDed into the check's WHERE clause.\n:param previous_name: Set when renaming a rule, to keep its history continuous.\n:param description: Human-readable description shown in results and the UI. When omitted,\n the provider generates a short default description from the rule and condition.\n\nInvalid input raises pydantic's own :class:`~pydantic.ValidationError`.", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "check": { + "description": "One of the built-in checks, or custom_sql.", + "title": "Check", + "type": "string" + }, + "condition": { + "anyOf": [ + { + "$ref": "#/$defs/Condition" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Condition" + }, + "column": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Column" + }, + "sql": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sql" + }, + "severity": { + "$ref": "#/$defs/Severity", + "default": "error" + }, + "partition_clause": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Partition Clause" + }, + "previous_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Previous Name" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Human-readable description shown in results and the UI. When omitted, the provider generates a short default description from the rule and condition.", + "title": "Description" + }, + "dimension": { + "anyOf": [ + { + "$ref": "#/$defs/Dimension" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Data quality dimension recorded for this rule's results. Defaults to the check's catalog dimension (validity for custom_sql) when not set explicitly." + } + }, + "required": [ + "name", + "check" + ], + "title": "DQRule", + "type": "object" + }, + "Dimension": { + "description": "Data quality dimension assigned to a rule result.", + "enum": [ + "completeness", + "uniqueness", + "validity", + "freshness", + "volume", + "consistency" + ], + "title": "Dimension", + "type": "string" + }, + "Severity": { + "description": "Severity used to decide whether a failing rule fails the task.", + "enum": [ + "warn", + "error" + ], + "title": "Severity", + "type": "string" + } + }, + "additionalProperties": false, + "description": "A named collection of rules, typically attached to one table or asset.", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "rules": { + "default": [], + "items": { + "$ref": "#/$defs/DQRule" + }, + "title": "Rules", + "type": "array" + } + }, + "required": [ + "name" + ], + "title": "RuleSet", + "type": "object" +} diff --git a/providers/dq/src/airflow/providers/dq/version_compat.py b/providers/dq/src/airflow/providers/dq/version_compat.py new file mode 100644 index 0000000000000..b47842705282b --- /dev/null +++ b/providers/dq/src/airflow/providers/dq/version_compat.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS COPIED MANUALLY IN OTHER PROVIDERS DELIBERATELY TO AVOID ADDING UNNECESSARY +# DEPENDENCIES BETWEEN PROVIDERS. IF YOU WANT TO ADD CONDITIONAL CODE IN YOUR PROVIDER THAT DEPENDS +# ON AIRFLOW VERSION, PLEASE COPY THIS FILE TO THE ROOT PACKAGE OF YOUR PROVIDER AND IMPORT +# THOSE CONSTANTS FROM IT RATHER THAN IMPORTING THEM FROM ANOTHER PROVIDER OR TEST CODE +# +from __future__ import annotations + + +def get_base_airflow_version_tuple() -> tuple[int, int, int]: + from packaging.version import Version + + from airflow import __version__ + + airflow_version = Version(__version__) + return airflow_version.major, airflow_version.minor, airflow_version.micro + + +AIRFLOW_V_3_1_PLUS: bool = get_base_airflow_version_tuple() >= (3, 1, 0) + +__all__ = ["AIRFLOW_V_3_1_PLUS"] diff --git a/providers/dq/tests/conftest.py b/providers/dq/tests/conftest.py new file mode 100644 index 0000000000000..f56ccce0a3f69 --- /dev/null +++ b/providers/dq/tests/conftest.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +pytest_plugins = "tests_common.pytest_plugin" diff --git a/providers/dq/tests/system/__init__.py b/providers/dq/tests/system/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/dq/tests/system/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/dq/tests/system/dq/__init__.py b/providers/dq/tests/system/dq/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/system/dq/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/system/dq/example_dq_check.py b/providers/dq/tests/system/dq/example_dq_check.py new file mode 100644 index 0000000000000..8cae644ffbf90 --- /dev/null +++ b/providers/dq/tests/system/dq/example_dq_check.py @@ -0,0 +1,354 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG for the Data Quality provider's ``DQCheckOperator``. + +Runs against the ``sqlite_default`` connection (Airflow's default local backend, no extra +infrastructure required). Results are persisted through the ``[dq] results_path`` config +option, seeded here via ``AIRFLOW__DQ__RESULTS_PATH`` so the history can be inspected +afterwards under ``/tmp/airflow_dq_example/results`` -- a real deployment would instead set +``results_path`` in ``airflow.cfg`` (or its env var) once, for every check to share. + +Trigger the Dag repeatedly with different ``dq_scenario`` values to create useful +history data in the Data Quality tab: + +- ``pass`` inserts clean data. +- ``negative`` inserts a negative amount, failing ``amount_non_negative``. +- ``duplicate_order`` inserts a duplicate ``order_id``, failing uniqueness. +- ``nulls`` inserts null ``order_id``/``customer_id`` values and a high discount null ratio. +- ``outlier`` inserts amounts above the allowed maximum. +- ``small_table`` inserts too few rows for volume checks. +- ``zero_quantity`` inserts an invalid quantity minimum. +- ``empty`` leaves the table empty. +- ``mixed`` combines several bad values to show multiple failures in one run. + +Demonstrates +three equivalent ways to run the same ruleset: + +- Plain ``DQCheckOperator`` with an explicit ``table``/``conn_id``. +- ``asset_quality()`` attaching the ruleset to an ``Asset``, then ``DQCheckOperator(asset=...)`` + picking up ``table``/``conn_id``/``ruleset`` from it and adding the asset to the task's + outlets automatically. +- A second asset-producing ``DQCheckOperator`` with stricter rules, so the Browse > Data Quality + page shows multiple producers for the same asset with both pass and fail outcomes. +- The ``@task.dq_check`` TaskFlow decorator. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.providers.dq.assets import asset_quality +from airflow.providers.dq.operators.dq_check import DQCheckOperator +from airflow.providers.dq.rules import DQRule, RuleSet, Severity +from airflow.sdk import DAG, Asset, task + +DAG_ID = "example_dq_check" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_example_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +# DQCheckOperator has no per-operator results-store override; every check in a deployment +# shares the one store configured under [dq] results_path. setdefault() so a real deployment's +# own config is never overridden. +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +# [START howto_operator_dq_check_ruleset] +orders_ruleset = RuleSet( + name="orders_quality", + rules=( + DQRule( + name="order_id_not_null", + check="null_count", + column="order_id", + condition={"equal_to": 0}, + ), + DQRule( + name="order_id_unique", + check="unique_violations", + column="order_id", + condition={"equal_to": 0}, + ), + DQRule( + name="customer_id_not_null", + check="null_count", + column="customer_id", + condition={"equal_to": 0}, + severity=Severity.WARN, + ), + DQRule( + name="discount_null_ratio", + check="null_ratio", + column="discount", + condition={"leq_to": 0.25}, + severity=Severity.WARN, + ), + DQRule( + name="region_distinct_count", + check="distinct_count", + column="region", + condition={"geq_to": 2}, + severity=Severity.WARN, + ), + DQRule( + name="amount_min_ge_zero", + check="min", + column="amount", + condition={"geq_to": 0}, + ), + DQRule( + name="amount_max_le_100", + check="max", + column="amount", + condition={"leq_to": 100}, + ), + DQRule( + name="amount_mean_between_5_and_60", + check="mean", + column="amount", + condition={"geq_to": 5, "leq_to": 60}, + ), + DQRule( + name="quantity_min_ge_one", + check="min", + column="quantity", + condition={"geq_to": 1}, + ), + DQRule( + name="quantity_max_le_ten", + check="max", + column="quantity", + condition={"leq_to": 10}, + ), + DQRule( + name="amount_non_negative", + check="custom_sql", + # {table} is substituted by the SQL engine at check time, not an f-string placeholder. + sql="SELECT COUNT(*) FROM {table} WHERE amount < 0", + condition={"equal_to": 0}, + ), + DQRule( + name="high_value_order_count", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE amount > 100", + condition={"equal_to": 0}, + severity=Severity.WARN, + ), + DQRule( + name="row_count_present", + check="row_count", + condition={"greater_than": 0}, + severity=Severity.WARN, + ), + DQRule( + name="row_count_at_least_three", + check="row_count", + condition={"geq_to": 3}, + ), + ), +) +# [END howto_operator_dq_check_ruleset] + +# [START howto_operator_dq_check_asset] +orders_asset = asset_quality( + Asset("dq_example_orders", uri="file:///tmp/airflow_dq_example/orders"), + ruleset=orders_ruleset, + conn_id=CONN_ID, + table=TABLE_NAME, +) +# [END howto_operator_dq_check_asset] + +strict_orders_ruleset = RuleSet( + name="orders_strict_quality", + rules=( + DQRule( + name="strict_order_id_not_null", + check="null_count", + column="order_id", + condition={"equal_to": 0}, + ), + DQRule( + name="strict_amount_max_le_20", + check="max", + column="amount", + condition={"leq_to": 20}, + ), + DQRule( + name="strict_row_count_equal_two", + check="row_count", + condition={"equal_to": 2}, + ), + DQRule( + name="strict_region_distinct_count", + check="distinct_count", + column="region", + condition={"geq_to": 4}, + severity=Severity.WARN, + ), + ), +) + +strict_orders_asset = asset_quality( + Asset("dq_example_orders", uri="file:///tmp/airflow_dq_example/orders"), + ruleset=strict_orders_ruleset, + conn_id=CONN_ID, + table=TABLE_NAME, +) + + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f""" + CREATE TABLE {TABLE_NAME} ( + order_id INTEGER, + customer_id INTEGER, + amount REAL, + discount REAL, + quantity INTEGER, + region TEXT + ); + """, + ], + ) + + clear_table = SQLExecuteQueryOperator( + task_id="clear_table", + conn_id=CONN_ID, + sql=f"DELETE FROM {TABLE_NAME};", + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + {{% set scenario = dag_run.conf.get("dq_scenario", "pass") if dag_run else "pass" %}} + {{% if scenario == "negative" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (2, 102, -25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "duplicate_order" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (1, 102, 25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "nulls" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, NULL, 1, 'US'), + (NULL, NULL, 25.5, NULL, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "outlier" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (2, 102, 120.0, 0.10, 2, 'EU'), + (3, 103, 150.0, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "small_table" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'); + {{% elif scenario == "zero_quantity" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 0, 'US'), + (2, 102, 25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "mixed" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, NULL, 0, 'US'), + (1, NULL, -25.5, NULL, 2, 'US'), + (NULL, 103, 150.0, NULL, 11, 'US'), + (4, 104, 55.0, 0.20, 4, 'US'); + {{% elif scenario == "empty" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) + SELECT NULL, NULL, NULL, NULL, NULL, NULL WHERE 0; + {{% else %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (2, 102, 25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% endif %}} + """, + ) + + # [START howto_operator_dq_check] + check_orders = DQCheckOperator( + task_id="check_orders", + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=orders_ruleset, + fail_on="never", + ) + # [END howto_operator_dq_check] + + check_orders_via_asset = DQCheckOperator( + task_id="check_orders_via_asset", + asset=orders_asset, + fail_on="never", + ) + + check_orders_strict_via_asset = DQCheckOperator( + task_id="check_orders_strict_via_asset", + asset=strict_orders_asset, + fail_on="never", + ) + + # [START howto_decorator_dq_check] + @task.dq_check( + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=orders_ruleset, + fail_on="never", + ) + def check_orders_decorated(): + return None + + # [END howto_decorator_dq_check] + + ( + create_table + >> clear_table + >> insert_orders + >> [check_orders, check_orders_via_asset, check_orders_strict_via_asset, check_orders_decorated()] + ) + + from tests_common.test_utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) +test_run = get_test_run(dag) diff --git a/providers/dq/tests/unit/__init__.py b/providers/dq/tests/unit/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/dq/tests/unit/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/dq/tests/unit/dq/__init__.py b/providers/dq/tests/unit/dq/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/unit/dq/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/unit/dq/api/__init__.py b/providers/dq/tests/unit/dq/api/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/unit/dq/api/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/unit/dq/api/test_app.py b/providers/dq/tests/unit/dq/api/test_app.py new file mode 100644 index 0000000000000..eaab8611ae1c3 --- /dev/null +++ b/providers/dq/tests/unit/dq/api/test_app.py @@ -0,0 +1,390 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import asyncio +import datetime +from typing import TYPE_CHECKING +from unittest import mock + +import pytest +import time_machine +from fastapi import HTTPException +from fastapi.testclient import TestClient + +from airflow.api_fastapi.app import create_app, purge_cached_app +from airflow.api_fastapi.auth.managers.simple.user import SimpleAuthManagerUser +from airflow.providers.dq.api.app import ( + _get_backend, + dq_app, + health, + run_detail_by_task_instance, + task_rule_history, + task_runs, +) +from airflow.providers.dq.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.dq.results import DQRun, RuleResult + +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.version_compat import AIRFLOW_V_3_1_PLUS + +if TYPE_CHECKING: + from airflow.api_fastapi.auth.managers.simple.simple_auth_manager import SimpleAuthManager + +BASE_URL = "http://testserver" + +pytestmark = pytest.mark.skipif( + not AIRFLOW_V_3_1_PLUS, + reason="Plugin endpoints are not supported before Airflow 3.1", +) + + +def run(coro): + return asyncio.run(coro) + + +class TestGetBackend: + def test_raises_503_when_unconfigured(self): + with mock.patch("airflow.providers.dq.api.app.get_backend_from_config", return_value=None): + with pytest.raises(HTTPException) as exc_info: + _get_backend() + assert exc_info.value.status_code == 503 + + def test_returns_configured_backend(self, tmp_path): + configured = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + with mock.patch("airflow.providers.dq.api.app.get_backend_from_config", return_value=configured): + assert _get_backend() is configured + + +class TestHealth: + def test_health(self): + assert run(health()) == {"status": "ok"} + + +class TestTaskRuns: + def test_reads_task_runs_from_object_storage_backend(self, tmp_path): + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + dq_run = DQRun( + dag_id="orders_pipeline", + task_id="dq", + run_id="manual__2026-07-04", + started_at="2026-07-04T00:00:00+00:00", + ) + backend.write_run(dq_run, [RuleResult(rule_uid="r1", rule_name="nulls", status="pass")]) + + result = run(task_runs(dag_id="orders_pipeline", task_id="dq", backend=backend)) + + assert result.items[0].run.dag_id == "orders_pipeline" + assert result.items[0].summary.passed == 1 + assert result.next_cursor is None + + def test_passes_before_cursor_through_to_backend(self, tmp_path): + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + for day in range(1, 4): + backend.write_run( + DQRun( + dag_id="orders_pipeline", + task_id="dq", + run_id=f"manual__2026-07-0{day}", + run_uid=f"run{day}", + started_at=f"2026-07-0{day}T00:00:00+00:00", + ), + [RuleResult(rule_uid="r1", rule_name="nulls", status="pass")], + ) + + first_page = run(task_runs(dag_id="orders_pipeline", task_id="dq", backend=backend, limit=2)) + second_page = run( + task_runs( + dag_id="orders_pipeline", + task_id="dq", + backend=backend, + limit=2, + before=first_page.next_cursor, + ) + ) + + assert [item.run.run_uid for item in first_page.items] == ["run3", "run2"] + assert [item.run.run_uid for item in second_page.items] == ["run1"] + assert second_page.next_cursor is None + assert first_page.next_cursor == "2026-07-02T00:00:00+00:00|run2" + + +class TestTaskRuleHistory: + def test_reads_task_rule_history_from_object_storage_backend(self, tmp_path): + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + dq_run = DQRun( + dag_id="orders_pipeline", + task_id="dq", + run_id="manual__2026-07-04", + started_at="2026-07-04T00:00:00+00:00", + ) + backend.write_run(dq_run, [RuleResult(rule_uid="r1", rule_name="nulls", status="pass")]) + + result = run( + task_rule_history( + dag_id="orders_pipeline", + task_id="dq", + rule_uid="r1", + backend=backend, + ) + ) + + assert result.items[0].rule_name == "nulls" + assert result.items[0].run.task_id == "dq" + assert result.next_cursor is None + + +class TestRunDetailByTaskInstance: + def test_reads_run_by_task_instance_route_params(self, tmp_path): + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + dq_run = DQRun( + dag_id="orders_pipeline", + task_id="dq", + run_id="scheduled__2026-07-04", + started_at="2026-07-04T00:00:00+00:00", + ) + backend.write_run(dq_run, [RuleResult(rule_uid="r1", rule_name="nulls", status="pass")]) + + result = run( + run_detail_by_task_instance( + dag_id="orders_pipeline", + task_id="dq", + run_id="scheduled__2026-07-04", + backend=backend, + ) + ) + + assert result.run.dag_id == "orders_pipeline" + assert result.results[0].rule_name == "nulls" + assert result.summary.passed == 1 + + def test_missing_task_instance_run_raises_404(self, tmp_path): + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + + with pytest.raises(HTTPException) as exc_info: + run( + run_detail_by_task_instance( + dag_id="orders_pipeline", + task_id="dq", + run_id="does_not_exist", + backend=backend, + ) + ) + + assert exc_info.value.status_code == 404 + + +@pytest.fixture +def dq_backend(tmp_path): + """Override the route-level backend dependency so requests hit an isolated tmp_path backend.""" + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + dq_app.dependency_overrides[_get_backend] = lambda: backend + yield backend + del dq_app.dependency_overrides[_get_backend] + + +def _make_test_client(*, role: str | None): + with ( + conf_vars( + { + ( + "core", + "auth_manager", + ): "airflow.api_fastapi.auth.managers.simple.simple_auth_manager.SimpleAuthManager", + ("core", "lazy_discover_providers"): "false", + } + ), + mock.patch("airflow.settings.LAZY_LOAD_PROVIDERS", False), + ): + purge_cached_app() + app = create_app() + auth_manager: SimpleAuthManager = app.state.auth_manager + time_very_before = datetime.datetime(2014, 1, 1, 0, 0, 0) + time_after = datetime.datetime.now() + datetime.timedelta(days=1) + with time_machine.travel(time_very_before, tick=False): + token = auth_manager._get_token_signer( + expiration_time_in_seconds=(time_after - time_very_before).total_seconds() + ).generate( + auth_manager.serialize_user(SimpleAuthManagerUser(username="test", role=role)), + ) + return TestClient(app, headers={"Authorization": f"Bearer {token}"}, base_url=BASE_URL) + + +@pytest.fixture +def test_client(): + """Authenticated, authorized (admin) test client for the mounted dq sub-app.""" + return _make_test_client(role="admin") + + +@pytest.fixture +def unauthorized_test_client(): + """Authenticated but role-less test client, for 403 coverage.""" + return _make_test_client(role=None) + + +@pytest.fixture +def unauthenticated_test_client(): + """No Authorization header at all, for 401 coverage.""" + with ( + conf_vars( + { + ( + "core", + "auth_manager", + ): "airflow.api_fastapi.auth.managers.simple.simple_auth_manager.SimpleAuthManager", + ("core", "lazy_discover_providers"): "false", + } + ), + mock.patch("airflow.settings.LAZY_LOAD_PROVIDERS", False), + ): + purge_cached_app() + app = create_app() + yield TestClient(app, base_url=BASE_URL) + + +class TestHealthEndpointHTTP: + """``/health`` needs no auth and no backend, so it's tested against the bare sub-app.""" + + def test_health_returns_ok(self): + client = TestClient(dq_app) + response = client.get("/health") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +@pytest.mark.db_test +class TestEndpointAuthorization: + """401/403 coverage for every route gated by ``requires_access_dag``.""" + + ROUTES = ( + ("get", "/dq/v1/dags/orders_pipeline/tasks/dq/runs"), + ("get", "/dq/v1/dags/orders_pipeline/tasks/dq/rules/r1/history"), + ("get", "/dq/v1/dags/orders_pipeline/tasks/dq/runs/by_run/manual__2026-07-04"), + ) + + @pytest.mark.parametrize(("method", "path"), ROUTES) + def test_401_unauthenticated(self, unauthenticated_test_client, method, path): + response = getattr(unauthenticated_test_client, method)(path) + assert response.status_code == 401 + + @pytest.mark.parametrize(("method", "path"), ROUTES) + def test_403_forbidden(self, unauthorized_test_client, method, path): + response = getattr(unauthorized_test_client, method)(path) + assert response.status_code == 403 + + +@pytest.mark.db_test +class TestTaskRunsHTTP: + def test_returns_persisted_runs(self, test_client, dq_backend): + dq_backend.write_run( + DQRun( + dag_id="orders_pipeline", + task_id="dq", + run_id="manual__2026-07-04", + started_at="2026-07-04T00:00:00+00:00", + ), + [RuleResult(rule_uid="r1", rule_name="nulls", status="pass")], + ) + + response = test_client.get("/dq/v1/dags/orders_pipeline/tasks/dq/runs") + + assert response.status_code == 200 + data = response.json() + assert data["items"][0]["run"]["dag_id"] == "orders_pipeline" + assert data["items"][0]["summary"]["passed"] == 1 + assert data["next_cursor"] is None + + def test_before_cursor_pages_further_back(self, test_client, dq_backend): + for day in range(1, 4): + dq_backend.write_run( + DQRun( + dag_id="orders_pipeline", + task_id="dq", + run_id=f"manual__2026-07-0{day}", + run_uid=f"run{day}", + started_at=f"2026-07-0{day}T00:00:00+00:00", + ), + [RuleResult(rule_uid="r1", rule_name="nulls", status="pass")], + ) + + first_page = test_client.get("/dq/v1/dags/orders_pipeline/tasks/dq/runs", params={"limit": 2}).json() + second_page = test_client.get( + "/dq/v1/dags/orders_pipeline/tasks/dq/runs", + params={"limit": 2, "before": first_page["next_cursor"]}, + ).json() + + assert [item["run"]["run_uid"] for item in first_page["items"]] == ["run3", "run2"] + assert [item["run"]["run_uid"] for item in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + assert first_page["next_cursor"] == "2026-07-02T00:00:00+00:00|run2" + + def test_503_when_backend_unconfigured(self, test_client): + with mock.patch("airflow.providers.dq.api.app.get_backend_from_config", return_value=None): + response = test_client.get("/dq/v1/dags/orders_pipeline/tasks/dq/runs") + assert response.status_code == 503 + + def test_limit_out_of_range_is_422(self, test_client, dq_backend): + response = test_client.get("/dq/v1/dags/orders_pipeline/tasks/dq/runs", params={"limit": 0}) + assert response.status_code == 422 + + +@pytest.mark.db_test +class TestTaskRuleHistoryHTTP: + def test_returns_rule_history_for_task(self, test_client, dq_backend): + dq_backend.write_run( + DQRun( + dag_id="orders_pipeline", + task_id="dq", + run_id="manual__2026-07-04", + started_at="2026-07-04T00:00:00+00:00", + ), + [RuleResult(rule_uid="r1", rule_name="nulls", status="fail")], + ) + + response = test_client.get("/dq/v1/dags/orders_pipeline/tasks/dq/rules/r1/history") + + assert response.status_code == 200 + data = response.json() + assert data["items"][0]["rule_name"] == "nulls" + assert data["items"][0]["run"]["task_id"] == "dq" + assert data["next_cursor"] is None + + +@pytest.mark.db_test +class TestRunDetailByTaskInstanceHTTP: + def test_returns_run_for_task_instance(self, test_client, dq_backend): + dq_backend.write_run( + DQRun( + dag_id="orders_pipeline", + task_id="dq", + run_id="scheduled__2026-07-04", + started_at="2026-07-04T00:00:00+00:00", + ), + [RuleResult(rule_uid="r1", rule_name="nulls", status="pass")], + ) + + response = test_client.get("/dq/v1/dags/orders_pipeline/tasks/dq/runs/by_run/scheduled__2026-07-04") + + assert response.status_code == 200 + data = response.json() + assert data["run"]["dag_id"] == "orders_pipeline" + assert data["results"][0]["rule_name"] == "nulls" + assert data["summary"]["passed"] == 1 + + def test_404_when_run_not_found(self, test_client, dq_backend): + response = test_client.get("/dq/v1/dags/orders_pipeline/tasks/dq/runs/by_run/does_not_exist") + assert response.status_code == 404 diff --git a/providers/dq/tests/unit/dq/api/test_models.py b/providers/dq/tests/unit/dq/api/test_models.py new file mode 100644 index 0000000000000..e3fcba3d6c345 --- /dev/null +++ b/providers/dq/tests/unit/dq/api/test_models.py @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.dq.api.models import ( + PaginatedResponse, + RuleHistoryRecordModel, + TaskDQRunModel, +) +from airflow.providers.dq.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.dq.results import DQRun, RuleResult + + +def make_backend(tmp_path) -> ObjectStorageResultsBackend: + return ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + + +class TestTaskDQRunModel: + def test_validates_backend_run_payload(self, tmp_path): + backend = make_backend(tmp_path) + dq_run = DQRun( + dag_id="orders_pipeline", + task_id="dq", + run_id="manual__2026-07-04", + started_at="2026-07-04T00:00:00+00:00", + ) + backend.write_run(dq_run, [RuleResult(rule_uid="r1", rule_name="nulls", status="pass")]) + + payload = backend.read_by_task_instance("orders_pipeline", "dq", "manual__2026-07-04") + model = TaskDQRunModel(**payload) + + assert model.run.dag_id == "orders_pipeline" + assert model.results[0].rule_name == "nulls" + assert model.summary.passed == 1 + + +class TestPaginatedResponseOfTaskDQRunModel: + def test_validates_backend_task_runs_page(self, tmp_path): + backend = make_backend(tmp_path) + for day in range(1, 3): + backend.write_run( + DQRun( + dag_id="orders_pipeline", + task_id="dq", + run_id=f"manual__2026-07-0{day}", + run_uid=f"run{day}", + started_at=f"2026-07-0{day}T00:00:00+00:00", + ), + [RuleResult(rule_uid="r1", rule_name="nulls", status="pass")], + ) + + page = backend.read_task_runs("orders_pipeline", "dq") + model = PaginatedResponse[TaskDQRunModel](**page) + + assert [item.run.run_uid for item in model.items] == ["run2", "run1"] + assert model.next_cursor is None + + +class TestPaginatedResponseOfRuleHistoryRecordModel: + def test_validates_backend_rule_history_page(self, tmp_path): + backend = make_backend(tmp_path) + backend.write_run( + DQRun( + dag_id="orders_pipeline", + task_id="dq", + run_id="manual__2026-07-04", + started_at="2026-07-04T00:00:00+00:00", + ), + [RuleResult(rule_uid="r1", rule_name="nulls", status="fail")], + ) + + page = backend.read_task_rule_history("orders_pipeline", "dq", "r1") + model = PaginatedResponse[RuleHistoryRecordModel](**page) + + assert model.items[0].rule_name == "nulls" + assert model.items[0].run.task_id == "dq" + assert model.next_cursor is None diff --git a/providers/dq/tests/unit/dq/backends/__init__.py b/providers/dq/tests/unit/dq/backends/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/unit/dq/backends/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/unit/dq/backends/test_object_storage.py b/providers/dq/tests/unit/dq/backends/test_object_storage.py new file mode 100644 index 0000000000000..bc6ac2d4b3b28 --- /dev/null +++ b/providers/dq/tests/unit/dq/backends/test_object_storage.py @@ -0,0 +1,357 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json +from typing import Any + +import pytest + +from airflow.providers.dq.backends import get_backend_from_config +from airflow.providers.dq.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.dq.results import DQRun, RuleResult +from airflow.providers.dq.rules import Severity + +from tests_common.test_utils.config import conf_vars + + +def make_run(run_uid: str = "abc123", started_at: str = "2026-07-04T06:00:00+00:00") -> DQRun: + return DQRun( + dag_id="orders_pipeline", + task_id="dq", + run_id="scheduled__2026-07-04", + run_uid=run_uid, + ruleset_name="orders", + table_ref="analytics.orders", + asset_names=("dq_example_orders",), + started_at=started_at, + finished_at="2026-07-04T06:00:03+00:00", + ) + + +def make_result(rule_uid: str = "rule-1", status: str = "pass") -> RuleResult: + return RuleResult( + rule_uid=rule_uid, + rule_name="ids_not_null", + status=status, + observed_value=0, + condition={"equal_to": 0}, + dimension="completeness", + severity=Severity.ERROR, + duration_ms=12.5, + ) + + +class TestObjectStorageResultsBackend: + @pytest.fixture + def backend(self, tmp_path): + return ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + + def test_write_run_creates_run_file_with_run_and_results(self, backend): + backend.write_run(make_run(), [make_result()]) + + record = backend.read_by_task_instance("orders_pipeline", "dq", "scheduled__2026-07-04") + assert record["run"]["run_uid"] == "abc123" + assert record["run"]["dag_id"] == "orders_pipeline" + assert record["results"][0]["rule_uid"] == "rule-1" + assert record["results"][0]["status"] == "pass" + + def test_write_run_stores_keyed_json_payload(self, backend): + backend.write_run(make_run(), [make_result()]) + + path = ( + backend.root + / "runs" + / "by_task" + / "dag_id=orders_pipeline" + / "task_id=dq" + / "date=2026-07-04" + / "abc123.json" + ) + payload = json.loads(path.read_text()) + + assert payload["run"]["run_uid"] == "abc123" + assert payload["results"][0]["rule_uid"] == "rule-1" + assert payload["summary"]["passed"] == 1 + + def test_write_run_stores_task_rule_index(self, backend): + backend.write_run(make_run(), [make_result()]) + + path = ( + backend.root + / "rules" + / "by_task_rule" + / "dag_id=orders_pipeline" + / "task_id=dq" + / "rule_uid=rule-1" + / "2026-07-04T06_00_00_00_00__abc123.json" + ) + payload = json.loads(path.read_text()) + + assert payload["run"]["dag_id"] == "orders_pipeline" + assert payload["run"]["task_id"] == "dq" + assert payload["result"]["rule_uid"] == "rule-1" + + def test_rule_history_is_newest_first(self, backend): + backend.write_run( + make_run(run_uid="run1", started_at="2026-07-01T06:00:00+00:00"), + [make_result(status="pass")], + ) + backend.write_run( + make_run(run_uid="run2", started_at="2026-07-02T06:00:00+00:00"), + [make_result(status="fail")], + ) + + result = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1") + + assert [record["status"] for record in result["items"]] == ["fail", "pass"] + assert result["items"][0]["run"]["run_uid"] == "run2" + assert result["next_cursor"] is None + + def test_rule_history_includes_task_instance_route_context(self, backend): + backend.write_run(make_run(), [make_result()]) + + history = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1")["items"] + + assert history[0]["run"]["dag_id"] == "orders_pipeline" + assert history[0]["run"]["task_id"] == "dq" + assert history[0]["run"]["run_id"] == "scheduled__2026-07-04" + assert history[0]["run"]["map_index"] == -1 + + def test_rule_history_respects_limit(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + result = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + + assert len(result["items"]) == 2 + assert result["next_cursor"] == "2026-07-02T06:00:00+00:00|run2" + + def test_rule_history_before_cursor_pages_further_back(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + first_page = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + second_page = backend.read_task_rule_history( + "orders_pipeline", "dq", "rule-1", limit=2, before=first_page["next_cursor"] + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_concurrent_style_writes_do_not_collide(self, backend): + backend.write_run(make_run(run_uid="run1"), [make_result()]) + backend.write_run(make_run(run_uid="run2"), [make_result()]) + + runs = backend.read_task_runs("orders_pipeline", "dq")["items"] + assert {run["run"]["run_uid"] for run in runs} == {"run1", "run2"} + + def test_read_by_task_instance_matches_run(self, backend): + backend.write_run(make_run(), [make_result()]) + + record = backend.read_by_task_instance("orders_pipeline", "dq", "scheduled__2026-07-04") + + assert record["run"]["run_uid"] == "abc123" + assert record["results"][0]["rule_uid"] == "rule-1" + assert record["summary"]["passed"] == 1 + + def test_read_by_task_instance_unknown_raises(self, backend): + with pytest.raises(FileNotFoundError): + backend.read_by_task_instance("orders_pipeline", "dq", "no_such_run") + + def test_read_by_task_instance_last_write_wins_across_retries(self, backend): + first = make_run(run_uid="try1") + backend.write_run(first, [make_result(status="fail")]) + second = make_run(run_uid="try2") + backend.write_run(second, [make_result(status="pass")]) + + record = backend.read_by_task_instance("orders_pipeline", "dq", "scheduled__2026-07-04") + + assert record["run"]["run_uid"] == "try2" + assert record["results"][0]["status"] == "pass" + assert record["summary"]["passed"] == 1 + + def test_read_by_task_instance_sanitizes_run_id(self, backend): + run_id = "manual__2026-07-04T06:00:00+00:00" + run = DQRun(dag_id="orders_pipeline", task_id="dq", run_id=run_id, run_uid="abc123") + backend.write_run(run, [make_result()]) + + record = backend.read_by_task_instance("orders_pipeline", "dq", run_id) + + assert record["run"]["run_id"] == run_id + + def test_read_task_runs_returns_newest_first_with_summaries(self, backend): + backend.write_run( + make_run(run_uid="run1", started_at="2026-07-01T06:00:00+00:00"), + [make_result(status="pass")], + ) + backend.write_run( + make_run(run_uid="run2", started_at="2026-07-02T06:00:00+00:00"), + [make_result(status="fail")], + ) + + result = backend.read_task_runs("orders_pipeline", "dq") + runs = result["items"] + + assert [record["run"]["run_uid"] for record in runs] == ["run2", "run1"] + assert runs[0]["summary"]["failed"] == 1 + assert runs[1]["summary"]["passed"] == 1 + assert result["next_cursor"] is None + + def test_read_task_runs_respects_limit(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + result = backend.read_task_runs("orders_pipeline", "dq", limit=2) + + assert len(result["items"]) == 2 + assert result["next_cursor"] == "2026-07-02T06:00:00+00:00|run2" + + def test_read_task_runs_before_cursor_pages_further_back(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + first_page = backend.read_task_runs("orders_pipeline", "dq", limit=2) + second_page = backend.read_task_runs( + "orders_pipeline", "dq", limit=2, before=first_page["next_cursor"] + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_read_task_runs_cursor_keeps_same_timestamp_records(self, backend): + for run_uid in ("run1", "run2", "run3"): + backend.write_run( + make_run(run_uid=run_uid, started_at="2026-07-04T06:00:00+00:00"), + [make_result()], + ) + + first_page = backend.read_task_runs("orders_pipeline", "dq", limit=2) + second_page = backend.read_task_runs( + "orders_pipeline", "dq", limit=2, before=first_page["next_cursor"] + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_read_task_runs_stops_scanning_once_limit_is_reached(self, backend, monkeypatch): + for day in range(1, 5): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + read_paths: list[Any] = [] + original_read_json = backend._read_json + monkeypatch.setattr( + backend, "_read_json", lambda path: (read_paths.append(path), original_read_json(path))[1] + ) + + result = backend.read_task_runs("orders_pipeline", "dq", limit=1) + + assert [record["run"]["run_uid"] for record in result["items"]] == ["run4"] + assert result["next_cursor"] == "2026-07-04T06:00:00+00:00|run4" + # Determining next_cursor reads one entry past the limit (date=2026-07-03's run3), but + # no further — date=2026-07-02/01 must not be read. + assert len(read_paths) == 2 + + def test_read_task_rule_history_filters_to_task(self, backend): + backend.write_run(make_run(run_uid="run1"), [make_result()]) + backend.write_run( + DQRun( + dag_id="other_pipeline", + task_id="dq", + run_id="scheduled__2026-07-04", + run_uid="run2", + started_at="2026-07-04T07:00:00+00:00", + ), + [make_result()], + ) + + history = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1")["items"] + + assert len(history) == 1 + assert history[0]["run"]["dag_id"] == "orders_pipeline" + + def test_read_task_rule_history_respects_limit(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + result = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + + assert [record["run"]["run_uid"] for record in result["items"]] == ["run3", "run2"] + assert result["next_cursor"] == "2026-07-02T06:00:00+00:00|run2" + + def test_read_task_rule_history_before_cursor_pages_further_back(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + first_page = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + second_page = backend.read_task_rule_history( + "orders_pipeline", "dq", "rule-1", limit=2, before=first_page["next_cursor"] + ) + + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_read_task_rule_history_cursor_keeps_same_timestamp_records(self, backend): + for run_uid in ("run1", "run2", "run3"): + backend.write_run( + make_run(run_uid=run_uid, started_at="2026-07-04T06:00:00+00:00"), + [make_result()], + ) + + first_page = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + second_page = backend.read_task_rule_history( + "orders_pipeline", "dq", "rule-1", limit=2, before=first_page["next_cursor"] + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + +class TestGetBackendFromConfig: + @conf_vars({("dq", "results_path"): None}) + def test_no_results_path_returns_none(self): + assert get_backend_from_config() is None + + def test_results_path_builds_object_storage_backend(self, tmp_path): + with conf_vars({("dq", "results_path"): f"file://{tmp_path}"}): + backend = get_backend_from_config() + assert isinstance(backend, ObjectStorageResultsBackend) diff --git a/providers/dq/tests/unit/dq/decorators/__init__.py b/providers/dq/tests/unit/dq/decorators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/unit/dq/decorators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/unit/dq/decorators/test_dq_check.py b/providers/dq/tests/unit/dq/decorators/test_dq_check.py new file mode 100644 index 0000000000000..cbe88a1fb69f6 --- /dev/null +++ b/providers/dq/tests/unit/dq/decorators/test_dq_check.py @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.common.sql.hooks.sql import DbApiHook +from airflow.providers.dq.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.dq.decorators.dq_check import _DQCheckDecoratedOperator +from airflow.providers.dq.rules import DQRule, RuleSet +from airflow.sdk.execution_time.context import OutletEventAccessors + +NULLS = DQRule(name="nulls", check="null_count", column="id", condition={"equal_to": 0}) +RULESET = RuleSet(name="orders", rules=(NULLS,)) + + +def make_context(): + # spec'd to the attributes DQCheckOperator actually reads off `ti`, since the real + # RuntimeTaskInstance is a heavyweight pydantic model with many unrelated required fields. + ti = mock.Mock(spec=["dag_id", "task_id", "run_id", "try_number", "map_index"]) + ti.dag_id = "orders_pipeline" + ti.task_id = "dq" + ti.run_id = "manual__2026-07-04" + ti.try_number = 1 + ti.map_index = -1 + return {"ti": ti, "outlet_events": mock.create_autospec(OutletEventAccessors, instance=True)} + + +@pytest.fixture(autouse=True) +def results_backend(tmp_path): + """Patch the module-level config lookup so persisted results land in an isolated tmp_path.""" + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + with mock.patch("airflow.providers.dq.operators.dq_check.get_backend_from_config", return_value=backend): + yield backend + + +def make_decorated_operator(records, python_callable, **kwargs): + kwargs.setdefault("task_id", "dq") + if not kwargs.pop("omit_ruleset", False): + kwargs.setdefault("ruleset", RULESET) + kwargs.setdefault("table", "orders") + operator = _DQCheckDecoratedOperator(conn_id="warehouse", python_callable=python_callable, **kwargs) + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = records + return operator, hook + + +class TestDQCheckDecoratedOperator: + def test_custom_operator_name(self): + assert _DQCheckDecoratedOperator.custom_operator_name == "@task.dq_check" + + @mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook") + def test_none_return_runs_check_as_declared(self, mock_get_db_hook): + operator, hook = make_decorated_operator(records=[(NULLS.rule_uid, 0)], python_callable=lambda: None) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert operator.table == "orders" + assert summary["passed"] == 1 + + @mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook") + def test_ruleset_can_be_provided_only_at_runtime(self, mock_get_db_hook): + other_rule = DQRule(name="row_count_ok", check="row_count", condition={"greater_than": 0}) + other_ruleset = RuleSet(name="dynamic", rules=(other_rule,)) + + operator, hook = make_decorated_operator( + records=[(other_rule.rule_uid, 10)], + python_callable=lambda: other_ruleset, + omit_ruleset=True, + ) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["passed"] == 1 + + def test_missing_runtime_ruleset_raises_value_error(self): + operator, hook = make_decorated_operator(records=[], python_callable=lambda: None, omit_ruleset=True) + with mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook", return_value=hook): + with pytest.raises(ValueError, match="ruleset is required"): + operator.execute(make_context()) + + @mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook") + def test_op_kwargs_are_passed_to_callable(self, mock_get_db_hook): + seen = {} + + def resolve_target(suffix): + seen["suffix"] = suffix + return None + + operator, hook = make_decorated_operator( + records=[(NULLS.rule_uid, 0)], + python_callable=resolve_target, + op_kwargs={"suffix": "eu"}, + ) + mock_get_db_hook.return_value = hook + + operator.execute(make_context()) + + assert seen["suffix"] == "eu" + + def test_invalid_return_raises_type_error(self): + operator, hook = make_decorated_operator(records=[], python_callable=lambda: 42) + with mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook", return_value=hook): + with pytest.raises(TypeError, match="must return a RuleSet"): + operator.execute(make_context()) diff --git a/providers/dq/tests/unit/dq/engines/__init__.py b/providers/dq/tests/unit/dq/engines/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/unit/dq/engines/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/unit/dq/engines/test_sql.py b/providers/dq/tests/unit/dq/engines/test_sql.py new file mode 100644 index 0000000000000..9161275c3849a --- /dev/null +++ b/providers/dq/tests/unit/dq/engines/test_sql.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.common.sql.hooks.sql import DbApiHook +from airflow.providers.dq.engines.sql import SQLDQEngine +from airflow.providers.dq.rules import DQRule, RuleSet + + +@pytest.fixture +def hook(): + return mock.create_autospec(DbApiHook, instance=True) + + +NULLS = DQRule(name="nulls", check="null_count", column="id", condition={"equal_to": 0}) +VOLUME = DQRule(name="volume", check="row_count", condition={"greater_than": 0}) +CUSTOM = DQRule( + name="negatives", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE amount < 0", + condition={"equal_to": 0}, +) + + +class TestBuildBatchSql: + def test_batch_sql_unions_one_select_per_rule(self, hook): + engine = SQLDQEngine(hook) + sql = engine.build_batch_sql([NULLS, VOLUME], "orders", None) + assert sql.count("UNION ALL") == 1 + assert f"'{NULLS.rule_uid}' AS rule_uid" in sql + assert "SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END)" in sql + assert "COUNT(*)" in sql + assert "WHERE" not in sql + + def test_partition_clauses_are_anded(self, hook): + rule = DQRule( + name="nulls", + check="null_count", + column="id", + condition={"equal_to": 0}, + partition_clause="region = 'EU'", + ) + sql = SQLDQEngine(hook).build_batch_sql([rule], "orders", "ds = '2026-07-04'") + assert "WHERE ds = '2026-07-04' AND region = 'EU'" in sql + + +class TestMeasure: + def test_builtin_rules_measured_from_single_query(self, hook): + hook.get_records.return_value = [(NULLS.rule_uid, 0), (VOLUME.rule_uid, 42)] + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + hook.get_records.assert_called_once() + by_name = {obs.rule.name: obs for obs in observations} + assert by_name["nulls"].observed_value == 0 + assert by_name["volume"].observed_value == 42 + assert by_name["nulls"].sql == SQLDQEngine(hook).build_rule_sql(NULLS, "orders") + assert by_name["volume"].sql == SQLDQEngine(hook).build_rule_sql(VOLUME, "orders") + assert all(obs.error_message is None for obs in observations) + + def test_query_failure_yields_error_observations(self, hook): + hook.get_records.side_effect = RuntimeError("connection refused") + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + assert len(observations) == 2 + assert all(obs.error_message == "connection refused" for obs in observations) + assert all(obs.observed_value is None for obs in observations) + assert all(obs.sql for obs in observations) + + def test_missing_rule_in_result_is_an_error(self, hook): + hook.get_records.return_value = [(NULLS.rule_uid, 0)] + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + by_name = {obs.rule.name: obs for obs in observations} + assert by_name["nulls"].error_message is None + assert by_name["volume"].error_message == "No result returned for rule" + + def test_custom_sql_rule_runs_individually_with_table_substituted(self, hook): + hook.get_first.return_value = (3,) + ruleset = RuleSet(name="s", rules=(CUSTOM,)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + hook.get_first.assert_called_once_with("SELECT COUNT(*) FROM orders WHERE amount < 0") + hook.get_records.assert_not_called() + assert observations[0].observed_value == 3 + assert observations[0].sql == "SELECT COUNT(*) FROM orders WHERE amount < 0" + + def test_custom_sql_no_rows_is_an_error(self, hook): + hook.get_first.return_value = None + ruleset = RuleSet(name="s", rules=(CUSTOM,)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + assert observations[0].error_message == "Query returned no rows" diff --git a/providers/dq/tests/unit/dq/operators/__init__.py b/providers/dq/tests/unit/dq/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/unit/dq/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/unit/dq/operators/test_dq_check.py b/providers/dq/tests/unit/dq/operators/test_dq_check.py new file mode 100644 index 0000000000000..9e74a8dc94642 --- /dev/null +++ b/providers/dq/tests/unit/dq/operators/test_dq_check.py @@ -0,0 +1,247 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.common.sql.hooks.sql import DbApiHook +from airflow.providers.dq.assets import asset_quality +from airflow.providers.dq.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.dq.exceptions import DQCheckFailedError +from airflow.providers.dq.operators.dq_check import DQCheckOperator +from airflow.providers.dq.rules import DQRule, RuleSet, Severity +from airflow.sdk import Asset +from airflow.sdk.execution_time.context import OutletEventAccessors + +NULLS = DQRule(name="nulls", check="null_count", column="id", condition={"equal_to": 0}) +VOLUME_WARN = DQRule( + name="volume", check="row_count", condition={"greater_than": 100}, severity=Severity.WARN +) +RULESET = RuleSet(name="orders", rules=(NULLS, VOLUME_WARN)) + + +def make_context(): + # spec'd to the attributes DQCheckOperator actually reads off `ti`, since the real + # RuntimeTaskInstance is a heavyweight pydantic model with many unrelated required fields. + ti = mock.Mock(spec=["dag_id", "task_id", "run_id", "try_number", "map_index"]) + ti.dag_id = "orders_pipeline" + ti.task_id = "dq" + ti.run_id = "manual__2026-07-04" + ti.try_number = 1 + ti.map_index = -1 + return {"ti": ti, "outlet_events": mock.create_autospec(OutletEventAccessors, instance=True)} + + +def make_operator(records, **kwargs): + kwargs.setdefault("task_id", "dq") + kwargs.setdefault("ruleset", RULESET) + kwargs.setdefault("table", "orders") + operator = DQCheckOperator(conn_id="warehouse", **kwargs) + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = records + return operator, hook + + +@pytest.fixture(autouse=True) +def results_backend(tmp_path): + """Patch the module-level config lookup so persisted results land in an isolated tmp_path.""" + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + with mock.patch("airflow.providers.dq.operators.dq_check.get_backend_from_config", return_value=backend): + yield backend + + +class TestDQCheckOperatorConstruction: + def test_requires_ruleset(self): + with pytest.raises(ValueError, match="ruleset is required"): + DQCheckOperator(task_id="dq", table="orders", conn_id="c") + + def test_requires_table(self): + with pytest.raises(ValueError, match="table is required"): + DQCheckOperator(task_id="dq", ruleset=RULESET, conn_id="c") + + def test_rejects_bad_fail_on(self): + with pytest.raises(ValueError, match="fail_on"): + DQCheckOperator(task_id="dq", ruleset=RULESET, table="orders", conn_id="c", fail_on="maybe") + + def test_asset_supplies_defaults_and_outlet(self): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="analytics.orders", + ) + operator = DQCheckOperator(task_id="dq", asset=asset) + assert operator.table == "analytics.orders" + assert operator.conn_id == "warehouse" + assert asset in operator.outlets + + def test_explicit_arguments_beat_asset_config(self): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="analytics.orders", + ) + operator = DQCheckOperator(task_id="dq", asset=asset, table="other_table", conn_id="other_conn") + assert operator.table == "other_table" + assert operator.conn_id == "other_conn" + + +class TestDQCheckOperatorExecute: + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_all_pass_returns_summary_and_persists(self, mock_get_db_hook, results_backend): + operator, hook = make_operator(records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 500)]) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["passed"] == 2 + assert summary["failed"] == 0 + assert summary["score"] == 1.0 + history = results_backend.read_task_rule_history("orders_pipeline", "dq", NULLS.rule_uid)["items"] + assert len(history) == 1 + assert history[0]["status"] == "pass" + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_error_severity_failure_fails_task(self, mock_get_db_hook, results_backend): + operator, hook = make_operator(records=[(NULLS.rule_uid, 3), (VOLUME_WARN.rule_uid, 500)]) + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="nulls"): + operator.execute(make_context()) + + # The failing result is persisted even though the task fails. + assert ( + results_backend.read_task_rule_history("orders_pipeline", "dq", NULLS.rule_uid)["items"][0][ + "status" + ] + == "fail" + ) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_warn_severity_failure_does_not_fail_task_by_default(self, mock_get_db_hook): + operator, hook = make_operator(records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 5)]) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["warned"] == 1 + assert summary["warned_rules"] == ["volume"] + assert summary["score"] == pytest.approx(0.875) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_rule_description_is_persisted(self, mock_get_db_hook, results_backend): + described_rule = DQRule( + name="nulls", + check="null_count", + column="id", + condition={"equal_to": 0}, + description="Order IDs must always be present.", + ) + operator, hook = make_operator( + records=[(described_rule.rule_uid, 0)], + ruleset=RuleSet(name="orders", rules=(described_rule,)), + ) + mock_get_db_hook.return_value = hook + + operator.execute(make_context()) + + history = results_backend.read_task_rule_history("orders_pipeline", "dq", described_rule.rule_uid)[ + "items" + ] + assert history[0]["description"] == "Order IDs must always be present." + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_fail_on_warn_fails_task_on_warn_failure(self, mock_get_db_hook): + operator, hook = make_operator( + records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 5)], fail_on="warn" + ) + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="volume"): + operator.execute(make_context()) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_fail_on_never_records_failures_without_raising(self, mock_get_db_hook): + operator, hook = make_operator( + records=[(NULLS.rule_uid, 3), (VOLUME_WARN.rule_uid, 5)], fail_on="never" + ) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["failed"] == 1 + assert summary["warned"] == 1 + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_execution_error_always_fails_task(self, mock_get_db_hook): + operator, hook = make_operator(records=None, fail_on="never") + hook.get_records.side_effect = RuntimeError("boom") + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="errored"): + operator.execute(make_context()) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_non_numeric_observed_value_is_persisted_as_error(self, mock_get_db_hook, results_backend): + operator, hook = make_operator(records=[(NULLS.rule_uid, "not-a-number")], fail_on="never") + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="errored"): + operator.execute(make_context()) + + history = results_backend.read_task_rule_history("orders_pipeline", "dq", NULLS.rule_uid)["items"] + assert history[0]["status"] == "error" + assert "Could not evaluate observed value" in history[0]["error_message"] + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_backend_failure_does_not_fail_the_check(self, mock_get_db_hook): + backend = mock.create_autospec(ObjectStorageResultsBackend, instance=True) + backend.write_run.side_effect = OSError("bucket unreachable") + operator, hook = make_operator(records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 500)]) + mock_get_db_hook.return_value = hook + + with mock.patch( + "airflow.providers.dq.operators.dq_check.get_backend_from_config", return_value=backend + ): + summary = operator.execute(make_context()) + + assert summary["passed"] == 2 + backend.write_run.assert_called_once() + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_summary_attached_to_outlet_events(self, mock_get_db_hook): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="orders", + ) + operator, hook = make_operator( + records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 500)], asset=asset + ) + mock_get_db_hook.return_value = hook + context = make_context() + + summary = operator.execute(context) + + outlet_events = context["outlet_events"] + outlet_events.__getitem__.assert_called_with(asset) + extra = outlet_events.__getitem__.return_value.extra + extra.__setitem__.assert_called_once_with("airflow.dq.result", summary) diff --git a/providers/dq/tests/unit/dq/plugins/__init__.py b/providers/dq/tests/unit/dq/plugins/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/unit/dq/plugins/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/unit/dq/plugins/test_dq.py b/providers/dq/tests/unit/dq/plugins/test_dq.py new file mode 100644 index 0000000000000..36e35fb12c04e --- /dev/null +++ b/providers/dq/tests/unit/dq/plugins/test_dq.py @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.providers.dq.api.app import dq_app +from airflow.providers.dq.plugins.dq import DQPlugin, _get_base_url_path, _get_bundle_url, _react_apps + +from tests_common.test_utils.config import conf_vars +from tests_common.test_utils.version_compat import AIRFLOW_V_3_1_PLUS + + +class TestDQPlugin: + def test_name(self): + assert DQPlugin.name == "dq" + + @pytest.mark.skipif( + not AIRFLOW_V_3_1_PLUS, reason="Plugin endpoints are not supported before Airflow 3.1" + ) + def test_registers_the_query_api_under_dq_prefix(self): + assert DQPlugin.fastapi_apps == [{"name": "dq", "app": dq_app, "url_prefix": "/dq"}] + + +@pytest.mark.skipif(not AIRFLOW_V_3_1_PLUS, reason="Plugin React apps are not supported before Airflow 3.1") +class TestReactApps: + def test_empty_when_bundle_not_built(self, tmp_path): + # A plain source checkout has no dist/ until `pnpm build` runs (wired into the wheel + # build via hatch_build.py); the entry must not advertise a bundle URL that 404s. + assert _react_apps(tmp_path / "dist") == [] + + def test_registers_task_and_task_instance_views_once_bundle_exists(self, tmp_path): + dist_dir = tmp_path / "dist" + dist_dir.mkdir() + (dist_dir / "main.umd.cjs").write_text("") + + with conf_vars({("api", "base_url"): "/"}): + apps = _react_apps(dist_dir) + + assert apps == [ + { + "name": "Data Quality", + "bundle_url": "/dq/static/main.umd.cjs", + "destination": "task", + "url_route": "dq-task", + }, + { + "name": "Data Quality", + "bundle_url": "/dq/static/main.umd.cjs", + "destination": "task_instance", + "url_route": "dq-run", + }, + ] + + +class TestGetBaseUrlPath: + def test_root_base_url(self): + with conf_vars({("api", "base_url"): "/"}): + assert _get_base_url_path("/dq") == "/dq" + + def test_http_base_url_extracts_path(self): + with conf_vars({("api", "base_url"): "http://example.com/airflow/"}): + assert _get_base_url_path("/dq") == "/airflow/dq" + + +class TestGetBundleUrl: + def test_root_base_url_returns_relative_path(self): + with conf_vars({("api", "base_url"): "/"}): + assert _get_bundle_url() == "/dq/static/main.umd.cjs" + + def test_http_base_url_returns_absolute_url(self): + with conf_vars({("api", "base_url"): "http://example.com:28080/airflow/"}): + assert _get_bundle_url() == "http://example.com:28080/airflow/dq/static/main.umd.cjs" diff --git a/providers/dq/tests/unit/dq/rules/__init__.py b/providers/dq/tests/unit/dq/rules/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/dq/tests/unit/dq/rules/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/dq/tests/unit/dq/rules/test_checks.py b/providers/dq/tests/unit/dq/rules/test_checks.py new file mode 100644 index 0000000000000..8221a9f040da7 --- /dev/null +++ b/providers/dq/tests/unit/dq/rules/test_checks.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.dq.rules import CHECK_SPECS + + +class TestCheckSpecs: + def test_every_column_check_expression_renders(self): + for name, spec in CHECK_SPECS.items(): + if spec.requires_column: + assert "{column}" in spec.expression, name + + def test_row_count_is_the_only_table_level_check(self): + table_level = {name for name, spec in CHECK_SPECS.items() if not spec.requires_column} + assert table_level == {"row_count"} diff --git a/providers/dq/tests/unit/dq/rules/test_rule.py b/providers/dq/tests/unit/dq/rules/test_rule.py new file mode 100644 index 0000000000000..4c5cd46d38784 --- /dev/null +++ b/providers/dq/tests/unit/dq/rules/test_rule.py @@ -0,0 +1,264 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +import airflow.providers.dq +from airflow.providers.dq.rules import Condition, DQRule, RuleSet, Severity, describe_rule + + +def test_skill_reference_schema_matches_live_model(): + """ + The dq-rule-authoring skill ships a generated RuleSet.model_json_schema() snapshot for + agents to consult. Guard against it silently drifting out of sync with the real model -- + regenerate with ``json.dumps(RuleSet.model_json_schema(), indent=2)`` (plus a trailing + newline) if this fails after an intentional schema change. + """ + schema_path = ( + Path(airflow.providers.dq.__file__).parent + / "skills" + / "dq-rule-authoring" + / "references" + / "ruleset.schema.json" + ) + checked_in = json.loads(schema_path.read_text()) + assert checked_in == RuleSet.model_json_schema() + + +class TestCondition: + @pytest.mark.parametrize( + ("condition", "observed", "expected"), + [ + ({"equal_to": 0}, 0, True), + ({"equal_to": 0}, 1, False), + ({"equal_to": 100, "tolerance": 0.1}, 105, True), + ({"equal_to": 100, "tolerance": 0.1}, 111, False), + ({"greater_than": 5}, 6, True), + ({"greater_than": 5}, 5, False), + ({"less_than": 5}, 4, True), + ({"geq_to": 5}, 5, True), + ({"leq_to": 5}, 6, False), + ({"geq_to": 0, "leq_to": 10}, 5, True), + ({"geq_to": 0, "leq_to": 10}, 11, False), + ({"equal_to": 0}, None, False), + ], + ) + def test_evaluate(self, condition, observed, expected): + assert Condition.from_dict(condition).evaluate(observed) is expected + + @pytest.mark.parametrize( + "condition", + [ + {}, + {"tolerance": 0.1}, + {"equal_to": 1, "greater_than": 0}, + {"greater_than": 0, "tolerance": 0.1}, + {"nonsense": 1}, + ], + ) + def test_invalid_conditions_rejected(self, condition): + with pytest.raises(ValidationError): + Condition.from_dict(condition) + + def test_to_dict_round_trip(self): + condition = Condition.from_dict({"geq_to": 1, "leq_to": 2}) + assert Condition.from_dict(condition.to_dict()) == condition + + +class TestDQRule: + def test_condition_dict_is_coerced(self): + rule = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) + assert isinstance(rule.condition, Condition) + + def test_rule_uid_is_stable_across_severity(self): + base = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) + tweaked = DQRule( + name="r", + check="null_count", + column="c", + condition={"equal_to": 0}, + severity=Severity.WARN, + ) + assert base.rule_uid == tweaked.rule_uid + + def test_rule_uid_is_stable_across_description(self): + base = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) + described = DQRule( + name="r", + check="null_count", + column="c", + condition={"equal_to": 0}, + description="Column c must be populated.", + ) + assert base.rule_uid == described.rule_uid + + def test_rule_uid_changes_with_condition(self): + one = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) + two = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 1}) + assert one.rule_uid != two.rule_uid + + def test_previous_name_keeps_uid(self): + old = DQRule(name="old_name", check="null_count", column="c", condition={"equal_to": 0}) + renamed = DQRule( + name="new_name", + check="null_count", + column="c", + condition={"equal_to": 0}, + previous_name="old_name", + ) + assert renamed.rule_uid == old.rule_uid + + @pytest.mark.parametrize( + "kwargs", + [ + {"name": "", "check": "row_count", "condition": {"equal_to": 0}}, + {"name": "r", "check": "no_such_check", "condition": {"equal_to": 0}}, + {"name": "r", "check": "null_count", "condition": {"equal_to": 0}}, # missing column + {"name": "r", "check": "custom_sql", "condition": {"equal_to": 0}}, # missing sql + {"name": "r", "check": "row_count", "condition": {"equal_to": 0}, "sql": "SELECT 1"}, + {"name": "r", "check": "row_count", "condition": {"equal_to": 0}, "severity": "fatal"}, + ], + ) + def test_invalid_rules_rejected(self, kwargs): + with pytest.raises(ValidationError): + DQRule(**kwargs) + + def test_row_count_needs_no_column(self): + rule = DQRule(name="volume", check="row_count", condition={"greater_than": 0}) + assert rule.column is None + + @pytest.mark.parametrize( + ("check", "column", "expected_dimension"), + [ + ("null_count", "c", "completeness"), + ("null_ratio", "c", "completeness"), + ("distinct_count", "c", "uniqueness"), + ("unique_violations", "c", "uniqueness"), + ("min", "c", "validity"), + ("max", "c", "validity"), + ("mean", "c", "validity"), + ("row_count", None, "volume"), + ], + ) + def test_default_dimension_comes_from_check_catalog(self, check, column, expected_dimension): + rule = DQRule(name="r", check=check, column=column, condition={"equal_to": 0}) + assert rule.dimension.value == expected_dimension + + def test_custom_sql_dimension_can_be_overridden(self): + rule = DQRule( + name="freshness_check", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE updated_at < NOW() - INTERVAL '1 day'", + condition={"equal_to": 0}, + dimension="freshness", + ) + assert rule.dimension.value == "freshness" + + def test_invalid_dimension_rejected(self): + with pytest.raises(ValidationError): + DQRule(name="r", check="row_count", condition={"greater_than": 0}, dimension="not_a_dimension") + + def test_explicit_dimension_matching_default_is_omitted_from_to_dict(self): + rule = DQRule( + name="r", check="null_count", column="c", condition={"equal_to": 0}, dimension="completeness" + ) + assert "dimension" not in rule.to_dict() + + def test_dimension_override_round_trips(self): + rule = DQRule( + name="freshness_check", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE updated_at < NOW() - INTERVAL '1 day'", + condition={"equal_to": 0}, + dimension="freshness", + ) + assert rule.to_dict()["dimension"] == "freshness" + assert DQRule.from_dict(rule.to_dict()) == rule + + def test_missing_condition_without_catalog_default_raises(self): + with pytest.raises(ValidationError, match="condition is required"): + DQRule(name="r", check="null_count", column="c") + + def test_to_dict_round_trip(self): + rule = DQRule( + name="r", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE x < 0", + condition={"equal_to": 0}, + severity=Severity.WARN, + partition_clause="ds = '2026-07-04'", + description="No rows should have negative x.", + ) + assert DQRule.from_dict(rule.to_dict()) == rule + + def test_description_is_serialized(self): + rule = DQRule( + name="r", + check="row_count", + condition={"greater_than": 0}, + description="Orders table should not be empty.", + ) + assert rule.to_dict()["description"] == "Orders table should not be empty." + + def test_default_description_uses_column_when_available(self): + rule = DQRule(name="amount_min", check="min", column="amount", condition={"geq_to": 0}) + assert describe_rule(rule) == "amount should be greater than or equal to 0.0" + + +class TestRuleSet: + def test_duplicate_rule_names_rejected(self): + rule = DQRule(name="r", check="row_count", condition={"greater_than": 0}) + with pytest.raises(ValidationError, match="Duplicate rule names"): + RuleSet(name="s", rules=(rule, rule)) + + def test_from_dict_round_trip(self): + ruleset = RuleSet( + name="orders", + rules=( + DQRule(name="volume", check="row_count", condition={"greater_than": 0}), + DQRule(name="ids", check="null_count", column="id", condition={"equal_to": 0}), + ), + ) + assert RuleSet.from_dict(ruleset.to_dict()) == ruleset + + def test_from_file(self, tmp_path): + ruleset_file = tmp_path / "orders.yaml" + ruleset_file.write_text( + """ + name: orders + rules: + - name: ids_not_null + check: null_count + column: id + condition: + equal_to: 0 + - name: volume + check: row_count + condition: + greater_than: 100 + severity: warn + """ + ) + ruleset = RuleSet.from_file(str(ruleset_file)) + assert ruleset.name == "orders" + assert [rule.name for rule in ruleset.rules] == ["ids_not_null", "volume"] + assert ruleset.rules[1].severity.value == "warn" diff --git a/providers/dq/tests/unit/dq/test_assets.py b/providers/dq/tests/unit/dq/test_assets.py new file mode 100644 index 0000000000000..9079b0ce567d9 --- /dev/null +++ b/providers/dq/tests/unit/dq/test_assets.py @@ -0,0 +1,170 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.providers.dq.assets import ( + DQ_EXTRA_KEY, + DQ_RESULT_EXTRA_KEY, + _quality_score_passes, + asset_quality, + get_asset_quality_config, + get_asset_ruleset, + require_quality, +) +from airflow.providers.dq.exceptions import DQRuleValidationError +from airflow.providers.dq.rules import DQRule, RuleSet +from airflow.sdk import Asset +from airflow.sdk.execution_time.comms import AssetEventDagRunReferenceResult +from airflow.sdk.execution_time.context import TriggeringAssetEventsAccessor + +RULESET = RuleSet( + name="orders", + rules=(DQRule(name="volume", check="row_count", condition={"greater_than": 0}),), +) + +ORDERS = Asset("orders") + + +def make_triggering_events(*, extra: dict, asset: Asset = ORDERS) -> TriggeringAssetEventsAccessor: + event = { + "asset": {"name": asset.name, "uri": asset.uri, "extra": {}}, + "extra": extra, + "source_task_id": "dq", + "source_dag_id": "orders_pipeline", + "source_run_id": "manual__2026-07-04", + "source_map_index": -1, + "source_aliases": [], + "timestamp": "2026-07-04T06:00:00Z", + } + return TriggeringAssetEventsAccessor.build([AssetEventDagRunReferenceResult.model_validate(event)]) + + +class TestAssetQuality: + def test_config_stored_under_dq_extra_key(self): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="analytics.orders", + ) + config = asset.extra[DQ_EXTRA_KEY] + assert config["conn_id"] == "warehouse" + assert config["table"] == "analytics.orders" + assert config["ruleset"] == RULESET.to_dict() + + def test_config_is_json_serializable(self): + import json + + asset = asset_quality(Asset("orders"), ruleset=RULESET) + json.dumps(asset.extra) + + def test_ruleset_round_trips_through_asset(self): + asset = asset_quality(Asset("orders"), ruleset=RULESET) + assert get_asset_ruleset(asset) == RULESET + + def test_ruleset_from_yaml_file(self, tmp_path): + ruleset_file = tmp_path / "orders.yaml" + ruleset_file.write_text( + """ + name: orders + rules: + - name: volume + check: row_count + condition: + greater_than: 0 + """ + ) + asset = asset_quality(Asset("orders"), ruleset=str(ruleset_file)) + assert get_asset_ruleset(asset).rules[0].name == "volume" + + def test_get_config_returns_none_without_quality(self): + assert get_asset_quality_config(Asset("plain")) is None + + def test_get_ruleset_raises_without_quality(self): + with pytest.raises(DQRuleValidationError, match="no data quality config"): + get_asset_ruleset(Asset("plain")) + + +class TestQualityScorePasses: + """ + Covers the pure decision logic behind ``require_quality()``. + + Kept independent of ``@task.short_circuit`` / Dag wiring, which is exercised by the + standard provider's own tests — this only needs to prove "given this triggering event, + does the gate pass or fail". + """ + + def test_passes_when_score_at_min_score(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"score": 0.95}}) + assert _quality_score_passes(ORDERS, 0.95, events) is True + + def test_fails_when_score_below_min_score(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"score": 0.8}}) + assert _quality_score_passes(ORDERS, 0.95, events) is False + + def test_fails_when_no_triggering_event_for_asset(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"score": 1.0}}, asset=Asset("other")) + assert _quality_score_passes(ORDERS, 0.5, events) is False + + def test_fails_when_event_has_no_dq_summary(self): + events = make_triggering_events(extra={}) + assert _quality_score_passes(ORDERS, 0.5, events) is False + + def test_fails_when_summary_has_no_score(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"passed": 3}}) + assert _quality_score_passes(ORDERS, 0.5, events) is False + + def test_uses_most_recent_event_when_several_present(self): + first = AssetEventDagRunReferenceResult.model_validate( + { + "asset": {"name": ORDERS.name, "uri": ORDERS.uri, "extra": {}}, + "extra": {DQ_RESULT_EXTRA_KEY: {"score": 0.4}}, + "source_task_id": "dq", + "source_dag_id": "orders_pipeline", + "source_run_id": "run1", + "source_map_index": -1, + "source_aliases": [], + "timestamp": "2026-07-03T06:00:00Z", + } + ) + second = AssetEventDagRunReferenceResult.model_validate( + { + "asset": {"name": ORDERS.name, "uri": ORDERS.uri, "extra": {}}, + "extra": {DQ_RESULT_EXTRA_KEY: {"score": 0.99}}, + "source_task_id": "dq", + "source_dag_id": "orders_pipeline", + "source_run_id": "run2", + "source_map_index": -1, + "source_aliases": [], + "timestamp": "2026-07-04T06:00:00Z", + } + ) + events = TriggeringAssetEventsAccessor.build([first, second]) + + assert _quality_score_passes(ORDERS, 0.9, events) is True + + +class TestRequireQuality: + def test_rejects_min_score_below_zero(self): + with pytest.raises(ValueError, match="min_score must be between 0 and 1"): + require_quality(ORDERS, min_score=-0.1) + + def test_rejects_min_score_above_one(self): + with pytest.raises(ValueError, match="min_score must be between 0 and 1"): + require_quality(ORDERS, min_score=1.1) diff --git a/providers/dq/tests/unit/dq/test_results.py b/providers/dq/tests/unit/dq/test_results.py new file mode 100644 index 0000000000000..76b37ad1b1714 --- /dev/null +++ b/providers/dq/tests/unit/dq/test_results.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.providers.dq.results import DQRun, RuleResult, build_summary, compute_score + + +def make_result(status: str, name: str = "r") -> RuleResult: + return RuleResult(rule_uid=f"uid-{name}", rule_name=name, status=status) + + +class TestComputeScore: + @pytest.mark.parametrize( + ("statuses", "expected"), + [ + ([], None), + (["pass", "pass"], 1.0), + (["pass", "fail"], 0.5), + (["pass", "error"], 0.5), + (["pass", "pass", "pass", "warn"], 0.9375), + (["fail", "fail"], 0.0), + ], + ) + def test_score(self, statuses, expected): + results = [make_result(status, name=f"r{i}") for i, status in enumerate(statuses)] + assert compute_score(results) == expected + + +class TestBuildSummary: + def test_summary_counts_and_rule_names(self): + run = DQRun(dag_id="d", task_id="t", run_id="r", ruleset_name="orders", table_ref="orders") + results = [ + make_result("pass", "a"), + make_result("warn", "b"), + make_result("fail", "c"), + make_result("error", "d"), + ] + + summary = build_summary(run, results) + + assert summary["passed"] == 1 + assert summary["warned"] == 1 + assert summary["failed"] == 1 + assert summary["errored"] == 1 + assert summary["failed_rules"] == ["c", "d"] + assert summary["warned_rules"] == ["b"] + assert summary["run_uid"] == run.run_uid + + +class TestSerialization: + def test_rule_result_round_trip(self): + result = RuleResult( + rule_uid="u", + rule_name="r", + status="fail", + observed_value=3, + condition={"equal_to": 0}, + description="r should equal 0", + duration_ms=1.5, + sql="SELECT 3", + ) + assert RuleResult.from_dict(result.to_dict()) == result + + def test_run_round_trip(self): + run = DQRun(dag_id="d", task_id="t", run_id="r", asset_names=("orders",)) + assert DQRun.from_dict(run.to_dict()) == run diff --git a/pyproject.toml b/pyproject.toml index 633a208e4dcfe..25a432afb7828 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -218,6 +218,9 @@ apache-airflow = "airflow.__main__:main" "docker" = [ "apache-airflow-providers-docker>=3.14.1" ] +"dq" = [ + "apache-airflow-providers-dq>=0.1.0" +] "edge3" = [ "apache-airflow-providers-edge3>=1.0.0" ] @@ -450,6 +453,7 @@ apache-airflow = "airflow.__main__:main" "apache-airflow-providers-dingding>=3.7.0", "apache-airflow-providers-discord>=3.9.0", "apache-airflow-providers-docker>=3.14.1", + "apache-airflow-providers-dq>=0.1.0", "apache-airflow-providers-edge3>=1.0.0", "apache-airflow-providers-elasticsearch>=6.5.0", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py "apache-airflow-providers-exasol>=4.6.1", @@ -1166,6 +1170,8 @@ mypy_path = [ "$MYPY_CONFIG_FILE_DIR/providers/discord/tests", "$MYPY_CONFIG_FILE_DIR/providers/docker/src", "$MYPY_CONFIG_FILE_DIR/providers/docker/tests", + "$MYPY_CONFIG_FILE_DIR/providers/dq/src", + "$MYPY_CONFIG_FILE_DIR/providers/dq/tests", "$MYPY_CONFIG_FILE_DIR/providers/edge3/src", "$MYPY_CONFIG_FILE_DIR/providers/edge3/tests", "$MYPY_CONFIG_FILE_DIR/providers/elasticsearch/src", @@ -1469,6 +1475,7 @@ apache-airflow-providers-dbt-cloud = false apache-airflow-providers-dingding = false apache-airflow-providers-discord = false apache-airflow-providers-docker = false +apache-airflow-providers-dq = false apache-airflow-providers-edge3 = false apache-airflow-providers-elasticsearch = false apache-airflow-providers-exasol = false @@ -1614,6 +1621,7 @@ apache-airflow-providers-dbt-cloud = false apache-airflow-providers-dingding = false apache-airflow-providers-discord = false apache-airflow-providers-docker = false +apache-airflow-providers-dq = false apache-airflow-providers-edge3 = false apache-airflow-providers-elasticsearch = false apache-airflow-providers-exasol = false @@ -1774,6 +1782,7 @@ apache-airflow-providers-dbt-cloud = { workspace = true } apache-airflow-providers-dingding = { workspace = true } apache-airflow-providers-discord = { workspace = true } apache-airflow-providers-docker = { workspace = true } +apache-airflow-providers-dq = { workspace = true } apache-airflow-providers-edge3 = { workspace = true } apache-airflow-providers-elasticsearch = { workspace = true } apache-airflow-providers-exasol = { workspace = true } @@ -1914,6 +1923,7 @@ members = [ "providers/dingding", "providers/discord", "providers/docker", + "providers/dq", "providers/edge3", "providers/elasticsearch", "providers/exasol", diff --git a/scripts/ci/docker-compose/remove-sources.yml b/scripts/ci/docker-compose/remove-sources.yml index ed2e4b1bed789..bf82c9932b468 100644 --- a/scripts/ci/docker-compose/remove-sources.yml +++ b/scripts/ci/docker-compose/remove-sources.yml @@ -67,6 +67,7 @@ services: - ../../../empty:/opt/airflow/providers/dingding/src - ../../../empty:/opt/airflow/providers/discord/src - ../../../empty:/opt/airflow/providers/docker/src + - ../../../empty:/opt/airflow/providers/dq/src - ../../../empty:/opt/airflow/providers/edge3/src - ../../../empty:/opt/airflow/providers/elasticsearch/src - ../../../empty:/opt/airflow/providers/exasol/src diff --git a/scripts/ci/docker-compose/tests-sources.yml b/scripts/ci/docker-compose/tests-sources.yml index 1f6e47ba961c2..1469c7c227fed 100644 --- a/scripts/ci/docker-compose/tests-sources.yml +++ b/scripts/ci/docker-compose/tests-sources.yml @@ -80,6 +80,7 @@ services: - ../../../providers/dingding/tests:/opt/airflow/providers/dingding/tests - ../../../providers/discord/tests:/opt/airflow/providers/discord/tests - ../../../providers/docker/tests:/opt/airflow/providers/docker/tests + - ../../../providers/dq/tests:/opt/airflow/providers/dq/tests - ../../../providers/edge3/tests:/opt/airflow/providers/edge3/tests - ../../../providers/elasticsearch/tests:/opt/airflow/providers/elasticsearch/tests - ../../../providers/exasol/tests:/opt/airflow/providers/exasol/tests diff --git a/scripts/ci/prek/check_http_exception_import_from_fastapi.py b/scripts/ci/prek/check_http_exception_import_from_fastapi.py index 0e74e74775801..a9b820c5ff8c1 100755 --- a/scripts/ci/prek/check_http_exception_import_from_fastapi.py +++ b/scripts/ci/prek/check_http_exception_import_from_fastapi.py @@ -20,8 +20,9 @@ The hook is wired into per-distribution ``.pre-commit-config.yaml`` files (``airflow-core``, ``providers/amazon``, ``providers/common/ai``, -``providers/edge3``, ``providers/fab``, ``providers/keycloak``), each -scoped to the subtree that actually wires a FastAPI app. In +``providers/dq``, ``providers/edge3``, ``providers/fab``, +``providers/keycloak``), each scoped to the subtree that actually wires +a FastAPI app. In ``airflow-core`` that includes ``api_fastapi/`` and ``utils/serve_logs/`` (the worker log-serving FastAPI app). Provider trees that mix client and server code (e.g. edge3's ``cli/`` is a diff --git a/scripts/ci/prek/check_providers_subpackages_all_have_init.py b/scripts/ci/prek/check_providers_subpackages_all_have_init.py index a67f914e9829b..68094e54bafbe 100755 --- a/scripts/ci/prek/check_providers_subpackages_all_have_init.py +++ b/scripts/ci/prek/check_providers_subpackages_all_have_init.py @@ -50,7 +50,11 @@ "skills", ] -IGNORE_DIR_PATTERNS = ["airflow/providers/edge3/plugins", "airflow/providers/common/ai/plugins"] +IGNORE_DIR_PATTERNS = [ + "airflow/providers/edge3/plugins", + "airflow/providers/common/ai/plugins", + "airflow/providers/dq/plugins", +] PATH_EXTENSION_STRING = '__path__ = __import__("pkgutil").extend_path(__path__, __name__)' diff --git a/scripts/ci/prek/compile_provider_assets.py b/scripts/ci/prek/compile_provider_assets.py index a0f16fb594fa1..d973dd09b0ab5 100755 --- a/scripts/ci/prek/compile_provider_assets.py +++ b/scripts/ci/prek/compile_provider_assets.py @@ -80,6 +80,12 @@ / "dist", "hash": PROVIDERS_ROOT / "common" / "ai" / "www-hash.txt", }, + "dq": { + "root": PROVIDERS_ROOT / "dq", + "www": PROVIDERS_ROOT / "dq" / "src" / "airflow" / "providers" / "dq" / "plugins" / "www", + "dist": PROVIDERS_ROOT / "dq" / "src" / "airflow" / "providers" / "dq" / "plugins" / "www" / "dist", + "hash": PROVIDERS_ROOT / "dq" / "www-hash.txt", + }, } diff --git a/scripts/ci/prek/generate_dq_ruleset_schema.py b/scripts/ci/prek/generate_dq_ruleset_schema.py new file mode 100755 index 0000000000000..348c8d015b122 --- /dev/null +++ b/scripts/ci/prek/generate_dq_ruleset_schema.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Regenerate the Data Quality RuleSet JSON schema snapshot at +``providers/dq/src/airflow/providers/dq/skills/dq-rule-authoring/references/ruleset.schema.json``. +""" + +from __future__ import annotations + +import subprocess +import sys + +from common_prek_utils import AIRFLOW_ROOT_PATH, console + +DQ_PROVIDER_PATH = AIRFLOW_ROOT_PATH / "providers" / "dq" +SCHEMA_PATH = DQ_PROVIDER_PATH.joinpath( + "src", + "airflow", + "providers", + "dq", + "skills", + "dq-rule-authoring", + "references", + "ruleset.schema.json", +) + +DUMP_SCHEMA = r""" +import json +import sys + +from airflow.providers.dq.rules import RuleSet + +sys.stdout.write(json.dumps(RuleSet.model_json_schema(), indent=2)) +sys.stdout.write("\n") +""" + + +def dump_schema() -> str: + """Run the schema-dump snippet in the dq provider project and return its stdout.""" + result = subprocess.run( + [ + "uv", + "run", + "--frozen", + "--no-progress", + "--project", + str(DQ_PROVIDER_PATH), + "python", + "-c", + DUMP_SCHEMA, + ], + cwd=DQ_PROVIDER_PATH, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"Schema generation failed: {result.stderr}") + return result.stdout + + +def main() -> int: + try: + new_content = dump_schema() + except Exception as e: + console.print(f"[bold red]ERROR:[/] {e}") + return 1 + + if SCHEMA_PATH.exists(): + old_content = SCHEMA_PATH.read_text() + if old_content == new_content: + return 0 + else: + SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True) + + SCHEMA_PATH.write_text(new_content) + rel = SCHEMA_PATH.relative_to(AIRFLOW_ROOT_PATH) + console.print(f"[yellow]Regenerated[/] [cyan]{rel}[/]. Please review the diff and re-stage the file.") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/prek/ts_compile_lint_dq.py b/scripts/ci/prek/ts_compile_lint_dq.py new file mode 100755 index 0000000000000..b7220edd9761e --- /dev/null +++ b/scripts/ci/prek/ts_compile_lint_dq.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Type-check the dq provider's React UI. + +Unlike edge3 / common.ai, the dq plugin's ``package.json`` has no eslint or +prettier dependencies, so this hook only runs ``tsc --noEmit`` against the +changed TypeScript files. Keep it that way unless the UI grows enough to +justify the extra tooling. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from common_prek_utils import ( + AIRFLOW_PROVIDERS_ROOT_PATH, + AIRFLOW_ROOT_PATH, + run_command, + temporary_tsc_project, +) + +if __name__ not in ("__main__", "__mp_main__"): + raise SystemExit( + "This file is intended to be executed as an executable program. You cannot use it as a module." + f"To run this script, run the ./{__file__} command" + ) + +if __name__ == "__main__": + original_files = sys.argv[1:] + print("Original files:", original_files) + dir = AIRFLOW_PROVIDERS_ROOT_PATH / "dq" / "src" / "airflow" / "providers" / "dq" / "plugins" / "www" + relative_dir = Path(dir).relative_to(AIRFLOW_ROOT_PATH) + files = [ + file[len(relative_dir.as_posix()) + 1 :] + for file in original_files + if Path(file).is_relative_to(relative_dir) + ] + all_ts_files = [file for file in files if file.endswith(".ts") or file.endswith(".tsx")] + print("All TypeScript files:", all_ts_files) + + if not all_ts_files: + sys.exit(0) + + run_command(["pnpm", "config", "set", "store-dir", ".pnpm-store"], cwd=dir) + run_command(["pnpm", "install", "--frozen-lockfile", "--config.confirmModulesPurge=false"], cwd=dir) + with temporary_tsc_project(dir / "tsconfig.app.json", all_ts_files) as tsc_project: + run_command(["pnpm", "tsc", "--p", tsc_project.name], cwd=dir) diff --git a/uv.lock b/uv.lock index 1ff0a8117e71f..0d19031984775 100644 --- a/uv.lock +++ b/uv.lock @@ -80,6 +80,7 @@ apache-airflow-providers-weaviate = false apache-airflow-providers-akeyless = false apache-airflow-providers-salesforce = false apache-airflow-providers-ssh = false +apache-airflow-providers-dq = false apache-airflow-providers-papermill = false apache-airflow-providers-google = false apache-airflow-providers-microsoft-psrp = false @@ -212,6 +213,7 @@ members = [ "apache-airflow-providers-dingding", "apache-airflow-providers-discord", "apache-airflow-providers-docker", + "apache-airflow-providers-dq", "apache-airflow-providers-edge3", "apache-airflow-providers-elasticsearch", "apache-airflow-providers-exasol", @@ -1052,6 +1054,7 @@ all = [ { name = "apache-airflow-providers-dingding" }, { name = "apache-airflow-providers-discord" }, { name = "apache-airflow-providers-docker" }, + { name = "apache-airflow-providers-dq" }, { name = "apache-airflow-providers-edge3" }, { name = "apache-airflow-providers-elasticsearch" }, { name = "apache-airflow-providers-exasol" }, @@ -1256,6 +1259,9 @@ discord = [ docker = [ { name = "apache-airflow-providers-docker" }, ] +dq = [ + { name = "apache-airflow-providers-dq" }, +] edge3 = [ { name = "apache-airflow-providers-edge3" }, ] @@ -1661,6 +1667,8 @@ requires-dist = [ { name = "apache-airflow-providers-discord", marker = "extra == 'discord'", editable = "providers/discord" }, { name = "apache-airflow-providers-docker", marker = "extra == 'all'", editable = "providers/docker" }, { name = "apache-airflow-providers-docker", marker = "extra == 'docker'", editable = "providers/docker" }, + { name = "apache-airflow-providers-dq", marker = "extra == 'all'", editable = "providers/dq" }, + { name = "apache-airflow-providers-dq", marker = "extra == 'dq'", editable = "providers/dq" }, { name = "apache-airflow-providers-edge3", marker = "extra == 'all'", editable = "providers/edge3" }, { name = "apache-airflow-providers-edge3", marker = "extra == 'edge3'", editable = "providers/edge3" }, { name = "apache-airflow-providers-elasticsearch", marker = "extra == 'all'", editable = "providers/elasticsearch" }, @@ -1799,7 +1807,7 @@ requires-dist = [ { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.30.0" }, { name = "uv", marker = "extra == 'uv'", specifier = ">=0.11.25" }, ] -provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] +provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "dq", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] [package.metadata.requires-dev] ci-image = [ @@ -5080,6 +5088,49 @@ dev = [ ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] +[[package]] +name = "apache-airflow-providers-dq" +version = "0.1.0" +source = { editable = "providers/dq" } +dependencies = [ + { name = "apache-airflow" }, + { name = "apache-airflow-providers-common-compat" }, + { name = "apache-airflow-providers-common-sql" }, + { name = "pydantic" }, + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "apache-airflow" }, + { name = "apache-airflow-devel-common" }, + { name = "apache-airflow-providers-common-compat" }, + { name = "apache-airflow-providers-common-sql" }, + { name = "apache-airflow-task-sdk" }, +] +docs = [ + { name = "apache-airflow-devel-common", extra = ["docs"] }, +] + +[package.metadata] +requires-dist = [ + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, + { name = "pydantic", specifier = ">=2.11.0" }, + { name = "pyyaml", specifier = ">=6.0.2" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-devel-common", editable = "devel-common" }, + { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, + { name = "apache-airflow-task-sdk", editable = "task-sdk" }, +] +docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] + [[package]] name = "apache-airflow-providers-edge3" version = "4.1.0"