From a58edbc1230121661c06a65c4611136b6e3e63ca Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Wed, 18 Jun 2025 07:46:53 -0400 Subject: [PATCH 1/6] INTPYTHON-527 Add test --- django_mongodb_backend/__init__.py | 4 ++-- django_mongodb_backend/features.py | 10 ++++++++++ django_mongodb_backend/utils.py | 11 +++++++++++ tests/backend_/utils/test_parse_uri.py | 5 ++++- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/django_mongodb_backend/__init__.py b/django_mongodb_backend/__init__.py index 00700421..1c9f88f3 100644 --- a/django_mongodb_backend/__init__.py +++ b/django_mongodb_backend/__init__.py @@ -2,7 +2,7 @@ # Check Django compatibility before other imports which may fail if the # wrong version of Django is installed. -from .utils import check_django_compatability, parse_uri +from .utils import check_django_compatability, get_auto_encryption_options, parse_uri check_django_compatability() @@ -15,7 +15,7 @@ from .lookups import register_lookups # noqa: E402 from .query import register_nodes # noqa: E402 -__all__ = ["parse_uri"] +__all__ = ["get_auto_encryption_options", "parse_uri"] register_aggregates() register_checks() diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index fa73461d..98cfd0cb 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -577,3 +577,13 @@ def supports_atlas_search(self): return False else: return True + + @cached_property + def supports_queryable_encryption(self): + """ + Queryable Encryption is supported if the server is Atlas or Enterprise. + """ + self.connection.ensure_connection() + client = self.connection.connection.admin + build_info = client.command("buildInfo") + return "enterprise" in build_info.get("modules") diff --git a/django_mongodb_backend/utils.py b/django_mongodb_backend/utils.py index ced60bc8..8ffed30c 100644 --- a/django_mongodb_backend/utils.py +++ b/django_mongodb_backend/utils.py @@ -8,6 +8,7 @@ from django.utils.functional import SimpleLazyObject from django.utils.text import format_lazy from django.utils.version import get_version_tuple +from pymongo.encryption_options import AutoEncryptionOpts from pymongo.uri_parser import parse_uri as pymongo_parse_uri @@ -28,6 +29,16 @@ def check_django_compatability(): ) +def get_auto_encryption_options(crypt_shared_lib_path=None): + key_vault_database_name = "encryption" + key_vault_collection_name = "__keyVault" + key_vault_namespace = f"{key_vault_database_name}.{key_vault_collection_name}" + kms_providers = {} + return AutoEncryptionOpts( + kms_providers, key_vault_namespace, crypt_shared_lib_path=crypt_shared_lib_path + ) + + def parse_uri(uri, *, db_name=None, test=None): """ Convert the given uri into a dictionary suitable for Django's DATABASES diff --git a/tests/backend_/utils/test_parse_uri.py b/tests/backend_/utils/test_parse_uri.py index 3198a463..a730fc8d 100644 --- a/tests/backend_/utils/test_parse_uri.py +++ b/tests/backend_/utils/test_parse_uri.py @@ -4,7 +4,7 @@ from django.core.exceptions import ImproperlyConfigured from django.test import SimpleTestCase -from django_mongodb_backend import parse_uri +from django_mongodb_backend import get_auto_encryption_options, parse_uri class ParseURITests(SimpleTestCase): @@ -94,3 +94,6 @@ def test_invalid_credentials(self): def test_no_scheme(self): with self.assertRaisesMessage(pymongo.errors.InvalidURI, "Invalid URI scheme"): parse_uri("cluster0.example.mongodb.net") + + def test_queryable_encryption_config(self): + get_auto_encryption_options() From 09fd8ec684ac041a70497e95706c46536e38ec08 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Wed, 18 Jun 2025 11:46:53 -0400 Subject: [PATCH 2/6] Update test --- django_mongodb_backend/features.py | 12 ++++++++++-- tests/backend_/utils/test_parse_uri.py | 9 +++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index 98cfd0cb..dd5efec2 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -581,9 +581,17 @@ def supports_atlas_search(self): @cached_property def supports_queryable_encryption(self): """ - Queryable Encryption is supported if the server is Atlas or Enterprise. + Queryable Encryption is supported if the server is Atlas or Enterprise + and if pymongocrypt is installed. """ self.connection.ensure_connection() client = self.connection.connection.admin build_info = client.command("buildInfo") - return "enterprise" in build_info.get("modules") + is_enterprise = "enterprise" in build_info.get("modules") + try: + import pymongocrypt # noqa: F401 + + has_pymongocrypt = True + except ImportError: + has_pymongocrypt = False + return is_enterprise and has_pymongocrypt diff --git a/tests/backend_/utils/test_parse_uri.py b/tests/backend_/utils/test_parse_uri.py index a730fc8d..f8687f8e 100644 --- a/tests/backend_/utils/test_parse_uri.py +++ b/tests/backend_/utils/test_parse_uri.py @@ -2,7 +2,7 @@ import pymongo from django.core.exceptions import ImproperlyConfigured -from django.test import SimpleTestCase +from django.test import SimpleTestCase, TestCase, skipUnlessDBFeature from django_mongodb_backend import get_auto_encryption_options, parse_uri @@ -95,5 +95,10 @@ def test_no_scheme(self): with self.assertRaisesMessage(pymongo.errors.InvalidURI, "Invalid URI scheme"): parse_uri("cluster0.example.mongodb.net") + +# TODO: This can go in `test_features` once transaction support is added. +class ParseUriOptionsTests(TestCase): + @skipUnlessDBFeature("supports_queryable_encryption") def test_queryable_encryption_config(self): - get_auto_encryption_options() + auto_encryption_options = get_auto_encryption_options() + self.assertEqual(auto_encryption_options._key_vault_namespace, "encryption.__keyVault") From ea923dc109e84eb4fc336f0bd374f60738a8a775 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Wed, 18 Jun 2025 12:16:17 -0400 Subject: [PATCH 3/6] Update test --- django_mongodb_backend/utils.py | 16 ++++++++++------ tests/backend_/utils/test_parse_uri.py | 10 +++++++--- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/django_mongodb_backend/utils.py b/django_mongodb_backend/utils.py index 8ffed30c..662a0153 100644 --- a/django_mongodb_backend/utils.py +++ b/django_mongodb_backend/utils.py @@ -8,7 +8,6 @@ from django.utils.functional import SimpleLazyObject from django.utils.text import format_lazy from django.utils.version import get_version_tuple -from pymongo.encryption_options import AutoEncryptionOpts from pymongo.uri_parser import parse_uri as pymongo_parse_uri @@ -34,12 +33,14 @@ def get_auto_encryption_options(crypt_shared_lib_path=None): key_vault_collection_name = "__keyVault" key_vault_namespace = f"{key_vault_database_name}.{key_vault_collection_name}" kms_providers = {} - return AutoEncryptionOpts( - kms_providers, key_vault_namespace, crypt_shared_lib_path=crypt_shared_lib_path - ) + return { + "kms_providers": kms_providers, + "key_vault_namespace": key_vault_namespace, + "crypt_shared_lib_path": crypt_shared_lib_path, + } -def parse_uri(uri, *, db_name=None, test=None): +def parse_uri(uri, *, auto_encryption_options=None, db_name=None, test=None): """ Convert the given uri into a dictionary suitable for Django's DATABASES setting. @@ -59,6 +60,9 @@ def parse_uri(uri, *, db_name=None, test=None): db_name = db_name or uri["database"] if not db_name: raise ImproperlyConfigured("You must provide the db_name parameter.") + options = uri.get("options") + if auto_encryption_options: + options = {**uri.get("options"), **auto_encryption_options} settings_dict = { "ENGINE": "django_mongodb_backend", "NAME": db_name, @@ -66,7 +70,7 @@ def parse_uri(uri, *, db_name=None, test=None): "PORT": port, "USER": uri.get("username"), "PASSWORD": uri.get("password"), - "OPTIONS": uri.get("options"), + "OPTIONS": options, } if "authSource" not in settings_dict["OPTIONS"] and uri["database"]: settings_dict["OPTIONS"]["authSource"] = uri["database"] diff --git a/tests/backend_/utils/test_parse_uri.py b/tests/backend_/utils/test_parse_uri.py index f8687f8e..ab3d9816 100644 --- a/tests/backend_/utils/test_parse_uri.py +++ b/tests/backend_/utils/test_parse_uri.py @@ -96,9 +96,13 @@ def test_no_scheme(self): parse_uri("cluster0.example.mongodb.net") -# TODO: This can go in `test_features` once transaction support is added. +# TODO: This can be moved to `test_features` once transaction support is merged. class ParseUriOptionsTests(TestCase): @skipUnlessDBFeature("supports_queryable_encryption") - def test_queryable_encryption_config(self): + def test_auto_encryption_options(self): auto_encryption_options = get_auto_encryption_options() - self.assertEqual(auto_encryption_options._key_vault_namespace, "encryption.__keyVault") + settings_dict = parse_uri( + "mongodb://cluster0.example.mongodb.net/myDatabase", + auto_encryption_options=auto_encryption_options, + ) + self.assertEqual(settings_dict["OPTIONS"]["key_vault_namespace"], "encryption.__keyVault") From eff8f8b491c894475210736726e7a4f5aea36e94 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Wed, 18 Jun 2025 13:04:56 -0400 Subject: [PATCH 4/6] Update test --- django_mongodb_backend/utils.py | 13 +++++++------ tests/backend_/utils/test_parse_uri.py | 8 ++------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/django_mongodb_backend/utils.py b/django_mongodb_backend/utils.py index 662a0153..b9df63a6 100644 --- a/django_mongodb_backend/utils.py +++ b/django_mongodb_backend/utils.py @@ -8,6 +8,7 @@ from django.utils.functional import SimpleLazyObject from django.utils.text import format_lazy from django.utils.version import get_version_tuple +from pymongo.encryption_options import AutoEncryptionOpts from pymongo.uri_parser import parse_uri as pymongo_parse_uri @@ -33,11 +34,11 @@ def get_auto_encryption_options(crypt_shared_lib_path=None): key_vault_collection_name = "__keyVault" key_vault_namespace = f"{key_vault_database_name}.{key_vault_collection_name}" kms_providers = {} - return { - "kms_providers": kms_providers, - "key_vault_namespace": key_vault_namespace, - "crypt_shared_lib_path": crypt_shared_lib_path, - } + return AutoEncryptionOpts( + key_vault_namespace=key_vault_namespace, + kms_providers=kms_providers, + crypt_shared_lib_path=crypt_shared_lib_path, + ) def parse_uri(uri, *, auto_encryption_options=None, db_name=None, test=None): @@ -62,7 +63,7 @@ def parse_uri(uri, *, auto_encryption_options=None, db_name=None, test=None): raise ImproperlyConfigured("You must provide the db_name parameter.") options = uri.get("options") if auto_encryption_options: - options = {**uri.get("options"), **auto_encryption_options} + options = {**uri.get("options"), "auto_encryption_options": auto_encryption_options} settings_dict = { "ENGINE": "django_mongodb_backend", "NAME": db_name, diff --git a/tests/backend_/utils/test_parse_uri.py b/tests/backend_/utils/test_parse_uri.py index ab3d9816..0c239cfe 100644 --- a/tests/backend_/utils/test_parse_uri.py +++ b/tests/backend_/utils/test_parse_uri.py @@ -100,9 +100,5 @@ def test_no_scheme(self): class ParseUriOptionsTests(TestCase): @skipUnlessDBFeature("supports_queryable_encryption") def test_auto_encryption_options(self): - auto_encryption_options = get_auto_encryption_options() - settings_dict = parse_uri( - "mongodb://cluster0.example.mongodb.net/myDatabase", - auto_encryption_options=auto_encryption_options, - ) - self.assertEqual(settings_dict["OPTIONS"]["key_vault_namespace"], "encryption.__keyVault") + auto_encryption_options = get_auto_encryption_options(crypt_shared_lib_path="/path/to/lib") + parse_uri("mongodb://localhost/db", auto_encryption_options=auto_encryption_options) From 7d5e39b84cd0209c1a3869d6cd0fd2ebf32a6043 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Wed, 18 Jun 2025 13:56:21 -0400 Subject: [PATCH 5/6] "Fix" kms provider --- django_mongodb_backend/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_mongodb_backend/utils.py b/django_mongodb_backend/utils.py index b9df63a6..aa5da735 100644 --- a/django_mongodb_backend/utils.py +++ b/django_mongodb_backend/utils.py @@ -1,4 +1,5 @@ import copy +import os import time import django @@ -33,7 +34,7 @@ def get_auto_encryption_options(crypt_shared_lib_path=None): key_vault_database_name = "encryption" key_vault_collection_name = "__keyVault" key_vault_namespace = f"{key_vault_database_name}.{key_vault_collection_name}" - kms_providers = {} + kms_providers = {"local": {"key": os.urandom(96)}} return AutoEncryptionOpts( key_vault_namespace=key_vault_namespace, kms_providers=kms_providers, From ef493131280d2fc6fb0e9a9719895f89146ed254 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Wed, 18 Jun 2025 21:19:15 -0400 Subject: [PATCH 6/6] Add encrypted model stubs --- django_mongodb_backend/models.py | 5 +++++ tests/model_fields_/models.py | 6 +++++- tests/model_fields_/test_encrypted_model.py | 8 ++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/model_fields_/test_encrypted_model.py diff --git a/django_mongodb_backend/models.py b/django_mongodb_backend/models.py index adeba21e..a34f5191 100644 --- a/django_mongodb_backend/models.py +++ b/django_mongodb_backend/models.py @@ -14,3 +14,8 @@ def delete(self, *args, **kwargs): def save(self, *args, **kwargs): raise NotSupportedError("EmbeddedModels cannot be saved.") + + +class EncryptedModel(models.Model): + class Meta: + abstract = True diff --git a/tests/model_fields_/models.py b/tests/model_fields_/models.py index 3d3a1584..4839f3db 100644 --- a/tests/model_fields_/models.py +++ b/tests/model_fields_/models.py @@ -8,7 +8,7 @@ EmbeddedModelField, ObjectIdField, ) -from django_mongodb_backend.models import EmbeddedModel +from django_mongodb_backend.models import EmbeddedModel, EncryptedModel # ObjectIdField @@ -136,6 +136,10 @@ class Author(EmbeddedModel): skills = ArrayField(models.CharField(max_length=100), null=True, blank=True) +class EncryptedData(EncryptedModel): + pass + + class Book(models.Model): name = models.CharField(max_length=100) author = EmbeddedModelField(Author) diff --git a/tests/model_fields_/test_encrypted_model.py b/tests/model_fields_/test_encrypted_model.py new file mode 100644 index 00000000..f5426f73 --- /dev/null +++ b/tests/model_fields_/test_encrypted_model.py @@ -0,0 +1,8 @@ +from django.test import TestCase + +from .models import EncryptedData + + +class ModelTests(TestCase): + def test_save_load(self): + EncryptedData.objects.create()