From 95fc8ddb8ac73afcf55bce4618d008398a29153d Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 7 Nov 2025 19:36:15 -0500 Subject: [PATCH 01/26] [temp] remove Django tests for faster CI --- .github/workflows/runtests.py | 145 ---------------------------------- 1 file changed, 145 deletions(-) diff --git a/.github/workflows/runtests.py b/.github/workflows/runtests.py index cc258f363..3775c422b 100755 --- a/.github/workflows/runtests.py +++ b/.github/workflows/runtests.py @@ -6,151 +6,6 @@ from django.core.exceptions import ImproperlyConfigured test_apps = [ - "admin_changelist", - "admin_checks", - "admin_custom_urls", - "admin_docs", - "admin_filters", - "admin_inlines", - "admin_ordering", - "admin_scripts", - "admin_utils", - "admin_views", - "admin_widgets", - "aggregation", - "aggregation_regress", - "annotations", - "apps", - "async", - "auth_tests", - "backends", - "basic", - "bulk_create", - "cache", - "check_framework", - "constraints", - "contenttypes_tests", - "context_processors", - "custom_columns", - "custom_lookups", - "custom_managers", - "custom_pk", - "datatypes", - "dates", - "datetimes", - "db_functions", - "defer", - "defer_regress", - "delete", - "delete_regress", - "empty", - "empty_models", - "expressions", - "expressions_case", - "field_defaults", - "file_storage", - "file_uploads", - "fixtures", - "fixtures_model_package", - "fixtures_regress", - "flatpages_tests", - "force_insert_update", - "foreign_object", - "forms_tests", - "from_db_value", - "generic_inline_admin", - "generic_relations", - "generic_relations_regress", - "generic_views", - "get_earliest_or_latest", - "get_object_or_404", - "get_or_create", - "i18n", - "indexes", - "inline_formsets", - "introspection", - "invalid_models_tests", - "known_related_objects", - "lookup", - "m2m_and_m2o", - "m2m_intermediary", - "m2m_multiple", - "m2m_recursive", - "m2m_regress", - "m2m_signals", - "m2m_through", - "m2m_through_regress", - "m2o_recursive", - "managers_regress", - "many_to_many", - "many_to_one", - "many_to_one_null", - "max_lengths", - "messages_tests", - "migrate_signals", - "migration_test_data_persistence", - "migrations", - "model_fields", - "model_forms", - "model_formsets", - "model_formsets_regress", - "model_indexes", - "model_inheritance", - "model_inheritance_regress", - "model_options", - "model_package", - "model_regress", - "model_utils", - "modeladmin", - "multiple_database", - "mutually_referential", - "nested_foreign_keys", - "null_fk", - "null_fk_ordering", - "null_queries", - "one_to_one", - "or_lookups", - "order_with_respect_to", - "ordering", - "pagination", - "prefetch_related", - "proxy_model_inheritance", - "proxy_models", - "queries", - "queryset_pickle", - "redirects_tests", - "reserved_names", - "reverse_lookup", - "save_delete_hooks", - "schema", - "select_for_update", - "select_related", - "select_related_onetoone", - "select_related_regress", - "serializers", - "servers", - "sessions_tests", - "shortcuts", - "signals", - "sitemaps_tests", - "sites_framework", - "sites_tests", - "string_lookup", - "swappable_models", - "syndication_tests", - "test_client", - "test_client_regress", - "test_runner", - "test_utils", - "timezones", - "transactions", - "unmanaged_models", - "update", - "update_only_fields", - "user_commands", - "validation", - "view_tests", - "xor_lookups", # Add directories in django_mongodb_backend/tests *sorted( [ From 3e4e2443fbba8d7fffc68fe3f58695c0064ce78d Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Wed, 25 Jun 2025 15:20:54 -0400 Subject: [PATCH 02/26] INTPYTHON-527 Add Queryable Encryption support Co-authored-by: Tim Graham --- .github/workflows/encrypted_settings.py | 40 ++ .github/workflows/mongodb_settings.py | 3 +- .github/workflows/test-python-atlas.yml | 26 +- django_mongodb_backend/__init__.py | 2 + django_mongodb_backend/base.py | 11 + django_mongodb_backend/creation.py | 16 +- django_mongodb_backend/features.py | 23 + django_mongodb_backend/fields/__init__.py | 52 +++ django_mongodb_backend/fields/encryption.py | 139 ++++++ .../commands/showencryptedfieldsmap.py | 35 ++ django_mongodb_backend/query.py | 2 +- django_mongodb_backend/routers.py | 25 +- django_mongodb_backend/schema.py | 110 ++++- django_mongodb_backend/utils.py | 19 + docs/howto/index.rst | 1 + docs/howto/queryable-encryption.rst | 289 +++++++++++++ docs/index.rst | 2 + docs/ref/django-admin.rst | 23 + docs/ref/index.rst | 2 + docs/ref/models/encrypted-fields.rst | 108 +++++ docs/ref/settings.rst | 45 ++ docs/releases/6.0.x.rst | 2 +- docs/topics/index.rst | 1 + docs/topics/known-issues.rst | 2 + docs/topics/queryable-encryption.rst | 140 ++++++ pyproject.toml | 1 + tests/backend_/test_features.py | 81 ++++ tests/encryption_/__init__.py | 0 tests/encryption_/models.py | 184 ++++++++ tests/encryption_/test_base.py | 21 + tests/encryption_/test_fields.py | 399 ++++++++++++++++++ tests/encryption_/test_management.py | 131 ++++++ tests/encryption_/test_schema.py | 152 +++++++ tests/raw_query_/test_raw_aggregate.py | 2 +- 34 files changed, 2078 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/encrypted_settings.py create mode 100644 django_mongodb_backend/fields/encryption.py create mode 100644 django_mongodb_backend/management/commands/showencryptedfieldsmap.py create mode 100644 docs/howto/queryable-encryption.rst create mode 100644 docs/ref/models/encrypted-fields.rst create mode 100644 docs/ref/settings.rst create mode 100644 docs/topics/queryable-encryption.rst create mode 100644 tests/encryption_/__init__.py create mode 100644 tests/encryption_/models.py create mode 100644 tests/encryption_/test_base.py create mode 100644 tests/encryption_/test_fields.py create mode 100644 tests/encryption_/test_management.py create mode 100644 tests/encryption_/test_schema.py diff --git a/.github/workflows/encrypted_settings.py b/.github/workflows/encrypted_settings.py new file mode 100644 index 000000000..02dc3447d --- /dev/null +++ b/.github/workflows/encrypted_settings.py @@ -0,0 +1,40 @@ +# Settings for django_mongodb_backend/tests when encryption is supported. +import os + +from mongodb_settings import * # noqa: F403 +from pymongo.encryption import AutoEncryptionOpts + +DATABASES["encrypted"] = { # noqa: F405 + "ENGINE": "django_mongodb_backend", + "NAME": "djangotests_encrypted", + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + key_vault_namespace="djangotests_encrypted.__keyVault", + kms_providers={"local": {"key": os.urandom(96)}}, + ), + "directConnection": True, + }, + "KMS_CREDENTIALS": {}, +} + + +class EncryptedRouter: + def db_for_read(self, model, **hints): + if model._meta.app_label == "encryption_": + return "encrypted" + return None + + db_for_write = db_for_read + + def allow_migrate(self, db, app_label, model_name=None, **hints): + # The encryption_ app's models are only created in the encrypted + # database. + if app_label == "encryption_": + return db == "encrypted" + # Don't create other app's models in the encrypted database. + if db == "encrypted": + return False + return None + + +DATABASE_ROUTERS.append(EncryptedRouter()) # noqa: F405 diff --git a/.github/workflows/mongodb_settings.py b/.github/workflows/mongodb_settings.py index 4dce3c0d5..619bdcd95 100644 --- a/.github/workflows/mongodb_settings.py +++ b/.github/workflows/mongodb_settings.py @@ -1,4 +1,5 @@ -# Settings for django_mongodb_backend/tests. +# Settings for django_mongodb_backend/tests when encryption isn't supported. from django_settings import * # noqa: F403 +DATABASES["encrypted"] = {} # noqa: F405 DATABASE_ROUTERS = ["django_mongodb_backend.routers.MongoRouter"] diff --git a/.github/workflows/test-python-atlas.yml b/.github/workflows/test-python-atlas.yml index bbda0f9f4..33f1c1b3f 100644 --- a/.github/workflows/test-python-atlas.yml +++ b/.github/workflows/test-python-atlas.yml @@ -28,7 +28,7 @@ jobs: - name: install django-mongodb-backend run: | pip3 install --upgrade pip - pip3 install -e . + pip3 install -e .[encryption] - name: Checkout Django uses: actions/checkout@v6 with: @@ -51,8 +51,30 @@ jobs: run: cp .github/workflows/runtests.py django_repo/tests/runtests_.py - name: Start local Atlas working-directory: . - run: bash .github/workflows/start_local_atlas.sh mongodb/mongodb-atlas-local:7 + run: bash .github/workflows/start_local_atlas.sh mongodb/mongodb-atlas-local:8.0.15 + - name: Install mongosh + run: | + wget -q https://downloads.mongodb.com/compass/mongosh-2.2.10-linux-x64.tgz + tar -xzf mongosh-*-linux-x64.tgz + sudo cp mongosh-*-linux-x64/bin/mongosh /usr/local/bin/ + mongosh --version + - name: Install mongocryptd from Enterprise tarball + run: | + curl -sSL -o mongodb-enterprise.tgz "https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-ubuntu2204-8.0.15.tgz" + tar -xzf mongodb-enterprise.tgz + sudo cp mongodb-linux-x86_64-enterprise-ubuntu2204-8.0.15/bin/mongocryptd /usr/local/bin/ + - name: Start mongocryptd + run: | + nohup mongocryptd --logpath=/tmp/mongocryptd.log & + - name: Verify MongoDB installation + run: | + mongosh --eval 'db.runCommand({ connectionStatus: 1 })' + - name: Verify mongocryptd is running + run: | + pgrep mongocryptd - name: Run tests run: python3 django_repo/tests/runtests_.py permissions: contents: read + env: + DJANGO_SETTINGS_MODULE: "encrypted_settings" diff --git a/django_mongodb_backend/__init__.py b/django_mongodb_backend/__init__.py index ed0deadef..d1d9cee5d 100644 --- a/django_mongodb_backend/__init__.py +++ b/django_mongodb_backend/__init__.py @@ -13,6 +13,7 @@ from .indexes import register_indexes # noqa: E402 from .lookups import register_lookups # noqa: E402 from .query import register_nodes # noqa: E402 +from .routers import register_routers # noqa: E402 register_aggregates() register_expressions() @@ -21,3 +22,4 @@ register_indexes() register_lookups() register_nodes() +register_routers() diff --git a/django_mongodb_backend/base.py b/django_mongodb_backend/base.py index 88c2a1189..b1afc1b03 100644 --- a/django_mongodb_backend/base.py +++ b/django_mongodb_backend/base.py @@ -11,6 +11,7 @@ from django.utils.functional import cached_property from pymongo.collection import Collection from pymongo.driver_info import DriverInfo +from pymongo.encryption import ClientEncryption from pymongo.mongo_client import MongoClient from pymongo.uri_parser import parse_uri @@ -241,6 +242,16 @@ def get_database(self): return OperationDebugWrapper(self) return self.database + @cached_property + def client_encryption(self): + auto_encryption_opts = self.connection._options.auto_encryption_opts + return ClientEncryption( + auto_encryption_opts._kms_providers, + auto_encryption_opts._key_vault_namespace, + self.connection, + self.connection.codec_options, + ) + @cached_property def database(self): """Connect to the database the first time it's accessed.""" diff --git a/django_mongodb_backend/creation.py b/django_mongodb_backend/creation.py index c8002b2c4..a1d45277e 100644 --- a/django_mongodb_backend/creation.py +++ b/django_mongodb_backend/creation.py @@ -1,5 +1,5 @@ from django.conf import settings -from django.db.backends.base.creation import BaseDatabaseCreation +from django.db.backends.base.creation import TEST_DATABASE_PREFIX, BaseDatabaseCreation class DatabaseCreation(BaseDatabaseCreation): @@ -7,6 +7,14 @@ def _execute_create_test_db(self, cursor, parameters, keepdb=False): # Close the connection (which may point to the non-test database) so # that a new connection to the test database can be established later. self.connection.close_pool() + # Use a test _key_vault_namespace. This assumes the key vault database + # is the same as the encrypted database so that _destroy_test_db() can + # reset the collection by dropping it. + opts = self.connection.settings_dict["OPTIONS"].get("auto_encryption_opts") + if opts: + self.connection.settings_dict["OPTIONS"][ + "auto_encryption_opts" + ]._key_vault_namespace = TEST_DATABASE_PREFIX + opts._key_vault_namespace if not keepdb: self._destroy_test_db(parameters["dbname"], verbosity=0) @@ -24,3 +32,9 @@ def destroy_test_db(self, old_database_name=None, verbosity=1, keepdb=False, suf super().destroy_test_db(old_database_name, verbosity, keepdb, suffix) # Close the connection to the test database. self.connection.close_pool() + # Restore the original _key_vault_namespace. + opts = self.connection.settings_dict["OPTIONS"].get("auto_encryption_opts") + if opts: + self.connection.settings_dict["OPTIONS"][ + "auto_encryption_opts" + ]._key_vault_namespace = opts._key_vault_namespace[len(TEST_DATABASE_PREFIX) :] diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index 10f206f1c..7cc0b9ebb 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -597,6 +597,14 @@ def django_test_skips(self): skips.update(self._django_test_skips) return skips + @cached_property + def mongodb_version(self): + return self.connection.get_database_version() # e.g., (6, 3, 0) + + @cached_property + def is_mongodb_8_0(self): + return self.mongodb_version >= (8, 0) + @cached_property def supports_atlas_search(self): """Does the server support Atlas search queries and search indexes?""" @@ -623,3 +631,18 @@ def _supports_transactions(self): hello = client.command("hello") # a replica set or a sharded cluster return "setName" in hello or hello.get("msg") == "isdbgrid" + + @cached_property + def supports_queryable_encryption(self): + """ + Queryable Encryption requires a MongoDB 8.0 or later replica set or sharded + cluster, as well as MongoDB Atlas or Enterprise. + """ + self.connection.ensure_connection() + build_info = self.connection.connection.admin.command("buildInfo") + is_enterprise = "enterprise" in build_info.get("modules") + return ( + (is_enterprise or self.supports_atlas_search) + and self._supports_transactions + and self.is_mongodb_8_0 + ) diff --git a/django_mongodb_backend/fields/__init__.py b/django_mongodb_backend/fields/__init__.py index 0c95afd69..6cc4bcc18 100644 --- a/django_mongodb_backend/fields/__init__.py +++ b/django_mongodb_backend/fields/__init__.py @@ -3,6 +3,33 @@ from .duration import register_duration_field from .embedded_model import EmbeddedModelField from .embedded_model_array import EmbeddedModelArrayField +from .encryption import ( + EncryptedArrayField, + EncryptedBigIntegerField, + EncryptedBinaryField, + EncryptedBooleanField, + EncryptedCharField, + EncryptedDateField, + EncryptedDateTimeField, + EncryptedDecimalField, + EncryptedDurationField, + EncryptedEmailField, + EncryptedEmbeddedModelArrayField, + EncryptedEmbeddedModelField, + EncryptedFieldMixin, + EncryptedFloatField, + EncryptedGenericIPAddressField, + EncryptedIntegerField, + EncryptedObjectIdField, + EncryptedPositiveBigIntegerField, + EncryptedPositiveIntegerField, + EncryptedPositiveSmallIntegerField, + EncryptedSmallIntegerField, + EncryptedTextField, + EncryptedTimeField, + EncryptedURLField, + EncryptedUUIDField, +) from .json import register_json_field from .objectid import ObjectIdField from .polymorphic_embedded_model import PolymorphicEmbeddedModelField @@ -12,6 +39,31 @@ "ArrayField", "EmbeddedModelArrayField", "EmbeddedModelField", + "EncryptedArrayField", + "EncryptedBigIntegerField", + "EncryptedBinaryField", + "EncryptedBooleanField", + "EncryptedCharField", + "EncryptedDateField", + "EncryptedDateTimeField", + "EncryptedDecimalField", + "EncryptedDurationField", + "EncryptedEmailField", + "EncryptedEmbeddedModelArrayField", + "EncryptedEmbeddedModelField", + "EncryptedFieldMixin", + "EncryptedFloatField", + "EncryptedGenericIPAddressField", + "EncryptedIntegerField", + "EncryptedObjectIdField", + "EncryptedPositiveBigIntegerField", + "EncryptedPositiveIntegerField", + "EncryptedPositiveSmallIntegerField", + "EncryptedSmallIntegerField", + "EncryptedTextField", + "EncryptedTimeField", + "EncryptedURLField", + "EncryptedUUIDField", "ObjectIdAutoField", "ObjectIdField", "PolymorphicEmbeddedModelArrayField", diff --git a/django_mongodb_backend/fields/encryption.py b/django_mongodb_backend/fields/encryption.py new file mode 100644 index 000000000..3ced82769 --- /dev/null +++ b/django_mongodb_backend/fields/encryption.py @@ -0,0 +1,139 @@ +from django.db import models + +from django_mongodb_backend.fields import ArrayField, EmbeddedModelArrayField, EmbeddedModelField +from django_mongodb_backend.fields.objectid import ObjectIdField + + +class EncryptedFieldMixin: + encrypted = True + + def __init__(self, *args, queries=None, db_index=False, null=False, unique=False, **kwargs): + if db_index: + raise ValueError("'db_index=True' is not supported on encrypted fields.") + if null: + raise ValueError("'null=True' is not supported on encrypted fields.") + if unique: + raise ValueError("'unique=True' is not supported on encrypted fields.") + self.queries = queries + super().__init__(*args, **kwargs) + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + + if self.queries is not None: + kwargs["queries"] = self.queries + + if path.startswith("django_mongodb_backend.fields.encryption"): + path = path.replace( + "django_mongodb_backend.fields.encryption", + "django_mongodb_backend.fields", + ) + + return name, path, args, kwargs + + +class NoQueriesMixin: + def __init__(self, *args, **kwargs): + if "queries" in kwargs: + raise ValueError(f"{self.__class__.__name__} does not support the queries argument.") + super().__init__(*args, **kwargs) + + +# Django fields +class EncryptedBinaryField(EncryptedFieldMixin, models.BinaryField): + pass + + +class EncryptedBigIntegerField(EncryptedFieldMixin, models.BigIntegerField): + pass + + +class EncryptedBooleanField(EncryptedFieldMixin, models.BooleanField): + pass + + +class EncryptedCharField(EncryptedFieldMixin, models.CharField): + pass + + +class EncryptedDateField(EncryptedFieldMixin, models.DateField): + pass + + +class EncryptedDateTimeField(EncryptedFieldMixin, models.DateTimeField): + pass + + +class EncryptedDecimalField(EncryptedFieldMixin, models.DecimalField): + pass + + +class EncryptedDurationField(EncryptedFieldMixin, models.DurationField): + pass + + +class EncryptedEmailField(EncryptedFieldMixin, models.EmailField): + pass + + +class EncryptedFloatField(EncryptedFieldMixin, models.FloatField): + pass + + +class EncryptedGenericIPAddressField(EncryptedFieldMixin, models.GenericIPAddressField): + pass + + +class EncryptedIntegerField(EncryptedFieldMixin, models.IntegerField): + pass + + +class EncryptedPositiveBigIntegerField(EncryptedFieldMixin, models.PositiveBigIntegerField): + pass + + +class EncryptedPositiveIntegerField(EncryptedFieldMixin, models.PositiveIntegerField): + pass + + +class EncryptedPositiveSmallIntegerField(EncryptedFieldMixin, models.PositiveSmallIntegerField): + pass + + +class EncryptedSmallIntegerField(EncryptedFieldMixin, models.SmallIntegerField): + pass + + +class EncryptedTextField(EncryptedFieldMixin, models.TextField): + pass + + +class EncryptedTimeField(EncryptedFieldMixin, models.TimeField): + pass + + +class EncryptedURLField(EncryptedFieldMixin, models.URLField): + pass + + +class EncryptedUUIDField(EncryptedFieldMixin, models.UUIDField): + pass + + +# MongoDB fields +class EncryptedArrayField(NoQueriesMixin, EncryptedFieldMixin, ArrayField): + pass + + +class EncryptedEmbeddedModelArrayField( + NoQueriesMixin, EncryptedFieldMixin, EmbeddedModelArrayField +): + pass + + +class EncryptedEmbeddedModelField(NoQueriesMixin, EncryptedFieldMixin, EmbeddedModelField): + pass + + +class EncryptedObjectIdField(EncryptedFieldMixin, ObjectIdField): + pass diff --git a/django_mongodb_backend/management/commands/showencryptedfieldsmap.py b/django_mongodb_backend/management/commands/showencryptedfieldsmap.py new file mode 100644 index 000000000..017fabde5 --- /dev/null +++ b/django_mongodb_backend/management/commands/showencryptedfieldsmap.py @@ -0,0 +1,35 @@ +from bson import json_util +from django.apps import apps +from django.core.management.base import BaseCommand +from django.db import DEFAULT_DB_ALIAS, connections, router + +from django_mongodb_backend.utils import model_has_encrypted_fields + + +class Command(BaseCommand): + help = """ + Shows the mapping of encrypted fields to field attributes, including data + type, data keys and query types. The output can be used to set + ``encrypted_fields_map`` in ``AutoEncryptionOpts``. + """ + + def add_arguments(self, parser): + parser.add_argument( + "--database", + default=DEFAULT_DB_ALIAS, + help=""" + Specifies the database to use. Defaults to ``default``.""", + ) + + def handle(self, *args, **options): + db = options["database"] + connection = connections[db] + connection.ensure_connection() + encrypted_fields_map = {} + with connection.schema_editor() as editor: + for app_config in apps.get_app_configs(): + for model in router.get_migratable_models(app_config, db): + if model_has_encrypted_fields(model): + fields = editor._get_encrypted_fields(model, create_data_keys=False) + encrypted_fields_map[model._meta.db_table] = fields + self.stdout.write(json_util.dumps(encrypted_fields_map, indent=4)) diff --git a/django_mongodb_backend/query.py b/django_mongodb_backend/query.py index 7155d6de2..72a552945 100644 --- a/django_mongodb_backend/query.py +++ b/django_mongodb_backend/query.py @@ -25,7 +25,7 @@ def wrapper(*args, **kwargs): except DuplicateKeyError as e: raise IntegrityError from e except PyMongoError as e: - raise DatabaseError from e + raise DatabaseError(str(e)) from e return wrapper diff --git a/django_mongodb_backend/routers.py b/django_mongodb_backend/routers.py index 60e54bbd8..b17f4b021 100644 --- a/django_mongodb_backend/routers.py +++ b/django_mongodb_backend/routers.py @@ -1,6 +1,6 @@ from django.apps import apps - -from django_mongodb_backend.models import EmbeddedModel +from django.core.exceptions import ImproperlyConfigured +from django.db.utils import ConnectionRouter class MongoRouter: @@ -9,6 +9,8 @@ def allow_migrate(self, db, app_label, model_name=None, **hints): EmbeddedModels don't have their own collection and must be ignored by dumpdata. """ + from django_mongodb_backend.models import EmbeddedModel # noqa: PLC0415 + if not model_name: return None try: @@ -16,3 +18,22 @@ def allow_migrate(self, db, app_label, model_name=None, **hints): except LookupError: return None return False if issubclass(model, EmbeddedModel) else None + + +# This function is intended to be monkey-patched as a method of ConnectionRouter. +def kms_provider(self, model, *args, **kwargs): + """ + Return the Key Management Service (KMS) provider for a given model. + + Call each router's kms_provider() method (if present), and return the + first non-None result. Raise ImproperlyConfigured if no provider is found. + """ + for router in self.routers: + func = getattr(router, "kms_provider", None) + if func and callable(func) and (result := func(model, *args, **kwargs)): + return result + raise ImproperlyConfigured("No kms_provider found in database routers.") + + +def register_routers(): + ConnectionRouter.kms_provider = kms_provider diff --git a/django_mongodb_backend/schema.py b/django_mongodb_backend/schema.py index 9bcaecc63..4ee0009fe 100644 --- a/django_mongodb_backend/schema.py +++ b/django_mongodb_backend/schema.py @@ -1,5 +1,7 @@ from time import monotonic, sleep +from django.core.exceptions import ImproperlyConfigured +from django.db import router from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.models import Index, UniqueConstraint from pymongo.operations import SearchIndexModel @@ -9,7 +11,7 @@ from .fields import EmbeddedModelField from .gis.schema import GISSchemaEditor from .query import wrap_database_errors -from .utils import OperationCollector +from .utils import OperationCollector, model_has_encrypted_fields def ignore_embedded_models(func): @@ -44,7 +46,7 @@ def get_database(self): @wrap_database_errors @ignore_embedded_models def create_model(self, model): - self.get_database().create_collection(model._meta.db_table) + self._create_collection(model) self._create_model_indexes(model) # Make implicit M2M tables. for field in model._meta.local_many_to_many: @@ -452,6 +454,110 @@ def wait_until_index_dropped(collection, index_name, timeout=60, interval=0.5): sleep(interval) raise TimeoutError(f"Index {index_name} not dropped after {timeout} seconds.") + def _create_collection(self, model): + """ + Create a collection for the model. + If the model has encrypted fields, build (or retrieve) the encrypted_fields schema. + """ + db = self.get_database() + db_table = model._meta.db_table + + if model_has_encrypted_fields(model): + # Encrypted path + client = self.connection.connection + auto_encryption_opts = getattr(client._options, "auto_encryption_opts", None) + if not auto_encryption_opts: + raise ImproperlyConfigured( + f"Tried to create model {model._meta.label} in " + f"'{self.connection.alias}' database. The model has " + "encrypted fields but " + f"DATABASES['{self.connection.alias}']['OPTIONS'] is " + 'missing the "auto_encryption_opts" parameter. If the ' + "model should not be created in this database, adjust " + "your database routers." + ) + encrypted_fields = self._get_encrypted_fields(model) + db.create_collection(db_table, encryptedFields=encrypted_fields) + else: + # Unencrypted path + db.create_collection(db_table) + + def _get_encrypted_fields( + self, model, *, key_alt_name_prefix=None, path_prefix=None, create_data_keys=True + ): + """ + Return the encrypted fields map for the given model. The "prefix" + arguments are used when this method is called recursively on embedded + models. + """ + connection = self.connection + client = connection.connection + key_alt_name_prefix = key_alt_name_prefix or model._meta.db_table + path_prefix = path_prefix or "" + auto_encryption_opts = client._options.auto_encryption_opts + _, key_vault_collection = auto_encryption_opts._key_vault_namespace.split(".", 1) + key_vault = self.get_collection(key_vault_collection) + # Create partial unique index on keyAltNames. + # TODO: find a better place for this. It only needs to run once for an + # application's lifetime. + key_vault.create_index( + "keyAltNames", unique=True, partialFilterExpression={"keyAltNames": {"$exists": True}} + ) + # Select the KMS provider. + kms_providers = auto_encryption_opts._kms_providers + if len(kms_providers) == 1: + # If one provider is configured, no need to consult the router. + kms_provider = next(iter(kms_providers.keys())) + else: + # Otherwise, call the user-defined router.kms_provider(). + kms_provider = router.kms_provider(model) + if kms_provider == "local": + master_key = None + else: + master_key = connection.settings_dict["KMS_CREDENTIALS"][kms_provider] + # Generate the encrypted fields map. + field_list = [] + for field in model._meta.fields: + key_alt_name = f"{key_alt_name_prefix}.{field.column}" + path = f"{path_prefix}.{field.column}" if path_prefix else field.column + # Check non-encrypted EmbeddedModelFields for encrypted fields. + if isinstance(field, EmbeddedModelField) and not getattr(field, "encrypted", False): + embedded_result = self._get_encrypted_fields( + field.embedded_model, + key_alt_name_prefix=key_alt_name, + path_prefix=path, + create_data_keys=create_data_keys, + ) + # An EmbeddedModelField may not have any encrypted fields. + if embedded_result: + field_list.extend(embedded_result["fields"]) + # Populate data for encrypted field. + elif getattr(field, "encrypted", False): + if create_data_keys: + data_key = connection.client_encryption.create_data_key( + kms_provider=kms_provider, + key_alt_names=[key_alt_name], + master_key=master_key, + ) + else: + data_key = key_vault.find_one({"keyAltNames": key_alt_name}) + if data_key: + data_key = data_key["_id"] + else: + raise ImproperlyConfigured( + f"Encryption key {key_alt_name} not found. Have " + f"migrated the {model} model?" + ) + field_dict = { + "bsonType": field.db_type(connection), + "path": path, + "keyId": data_key, + } + if queries := getattr(field, "queries", None): + field_dict["queries"] = queries + field_list.append(field_dict) + return {"fields": field_list} + # GISSchemaEditor extends some SchemaEditor methods. class DatabaseSchemaEditor(GISSchemaEditor, BaseSchemaEditor): diff --git a/django_mongodb_backend/utils.py b/django_mongodb_backend/utils.py index 8c68bf442..0dcd2ff44 100644 --- a/django_mongodb_backend/utils.py +++ b/django_mongodb_backend/utils.py @@ -71,6 +71,7 @@ class OperationDebugWrapper: "create_indexes", "create_search_index", "drop", + "find_one", "index_information", "insert_many", "delete_many", @@ -146,3 +147,21 @@ def wrapper(self, *args, **kwargs): self.log(method, args, kwargs) return wrapper + + +def model_has_encrypted_fields(model): + """ + Recursively check if this model or any embedded models contain encrypted fields. + Returns True if encryption is found anywhere in the hierarchy. + """ + from django_mongodb_backend.fields import EmbeddedModelField # noqa: PLC0415 + + # Recursively check embedded models. + return any( + getattr(field, "encrypted", False) + or ( + isinstance(field, EmbeddedModelField) + and model_has_encrypted_fields(field.embedded_model) + ) + for field in model._meta.fields + ) diff --git a/docs/howto/index.rst b/docs/howto/index.rst index 95d7ef632..8451960ef 100644 --- a/docs/howto/index.rst +++ b/docs/howto/index.rst @@ -11,3 +11,4 @@ Project configuration :maxdepth: 1 contrib-apps + queryable-encryption diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst new file mode 100644 index 000000000..4c72f554d --- /dev/null +++ b/docs/howto/queryable-encryption.rst @@ -0,0 +1,289 @@ +================================ +Configuring Queryable Encryption +================================ + +.. versionadded:: 5.2.3 + +:doc:`manual:core/queryable-encryption` is a powerful MongoDB feature that +allows you to encrypt sensitive fields in your database while still supporting +queries on that encrypted data. + +This section will guide you through the process of configuring Queryable +Encryption in your Django project. + +.. admonition:: MongoDB requirements + + Queryable Encryption can be used with MongoDB replica sets or sharded + clusters running version 8.0 or later. Standalone instances are not + supported. The :ref:`manual:qe-compatibility-reference` table summarizes + which MongoDB server products support Queryable Encryption. + +Installation +============ + +In addition to Django MongoDB Backend's regular :doc:`installation +` and :doc:`configuration ` steps, Queryable +Encryption has additional Python dependencies: + +.. code-block:: console + + $ pip install django-mongodb-backend[encryption] + +.. _qe-configuring-databases-setting: + +Configuring the ``DATABASES`` setting +===================================== + +In addition to the :ref:`database settings ` +required to use Django MongoDB Backend, Queryable Encryption requires you to +configure a separate encrypted database connection in your +:setting:`django:DATABASES` setting. + +.. admonition:: Encrypted database + + An encrypted database is a separate database connection in your + :setting:`django:DATABASES` setting that is configured to use PyMongo's + :class:`automatic encryption + `. + +Here's how to configure an encrypted database using a local KMS provider and +encryption keys stored in the ``encryption.__keyVault`` collection:: + + import os + + from pymongo.encryption_options import AutoEncryptionOpts + + DATABASES = { + "default": { + "ENGINE": "django_mongodb_backend", + "HOST": "mongodb+srv://cluster0.example.mongodb.net", + "NAME": "my_database", + # ... + }, + "encrypted": { + "ENGINE": "django_mongodb_backend", + "HOST": "mongodb+srv://cluster0.example.mongodb.net", + "NAME": "my_database_encrypted", + "USER": "my_user", + "PASSWORD": "my_password", + "PORT": 27017, + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + key_vault_namespace="encryption.__keyVault", + kms_providers={"local": {"key": os.urandom(96)}}, + ) + }, + }, + } + +.. admonition:: Local KMS provider key + + In the example above, a random key is generated for the local KMS provider + using ``os.urandom(96)``. In a production environment, you should securely + :ref:`store and manage your encryption keys + `. + +.. _qe-configuring-database-routers-setting: + +Configuring the ``DATABASE_ROUTERS`` setting +============================================ + +Similar to configuring the :ref:`DATABASE_ROUTERS +` setting for +:doc:`embedded models `, Queryable Encryption +requires a :setting:`DATABASE_ROUTERS ` setting to +route database operations to the encrypted database. + +The following example shows how to configure a router for the "myapp" +application that routes database operations to the encrypted database for all +models in that application:: + + # myapp/routers.py + class EncryptedRouter: + def allow_migrate(self, db, app_label, model_name=None, **hints): + if app_label == "myapp": + return db == "encrypted" + # Prevent migrations on the encrypted database for other apps + if db == "encrypted": + return False + return None + + def db_for_read(self, model, **hints): + if model._meta.app_label == "myapp": + return "encrypted" + return None + + db_for_write = db_for_read + +Then in your Django settings, add the custom database router to the +:setting:`django:DATABASE_ROUTERS` setting:: + + # settings.py + DATABASE_ROUTERS = ["myapp.routers.EncryptedRouter"] + +.. _qe-configuring-kms: + +Configuring the Key Management Service (KMS) +============================================ + +To use Queryable Encryption, you must configure a Key Management Service (KMS) +to store and manage your encryption keys. Django MongoDB Backend allows you to +configure multiple KMS providers and select the appropriate provider for each +model using a custom database router. + +The KMS is responsible for managing the encryption keys used to encrypt and +decrypt data. The following table summarizes the available KMS configuration +options followed by an example of how to use them. + ++-------------------------------------------------------------------------+--------------------------------------------------------+ +| :setting:`KMS_CREDENTIALS ` | A dictionary of Key Management Service (KMS) | +| | credentials configured in the | +| | :setting:`django:DATABASES` setting. | ++-------------------------------------------------------------------------+--------------------------------------------------------+ +| :class:`kms_providers ` | A dictionary of KMS provider credentials used to | +| | access the KMS with ``kms_provider``. | ++-------------------------------------------------------------------------+--------------------------------------------------------+ +| :ref:`kms_provider ` | A single KMS provider name | +| | configured in your custom database | +| | router. | ++-------------------------------------------------------------------------+--------------------------------------------------------+ + +Example of KMS configuration with ``aws`` in your :class:`kms_providers +` setting:: + + from pymongo.encryption_options import AutoEncryptionOpts + + DATABASES = { + "encrypted": { + # ... + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + # ... + kms_providers={ + "aws": { + "accessKeyId": "your-access-key-id", + "secretAccessKey": "your-secret-access-key", + }, + }, + ), + }, + "KMS_CREDENTIALS": { + "aws": { + "key": os.getenv("AWS_KEY_ARN", ""), + "region": os.getenv("AWS_KEY_REGION", ""), + }, + }, + }, + } + +(TODO: If there's a use case for multiple providers, motivate with a use case +and add a test.) + +If you've configured multiple KMS providers, you must define logic to determine +the provider for each model in your :ref:`database router +`:: + + class EncryptedRouter: + # ... + def kms_provider(self, model, **hints): + return "aws" + +.. _qe-configuring-encrypted-fields-map: + +Configuring the ``encrypted_fields_map`` option +=============================================== + +When you configure the :ref:`DATABASES ` +setting for Queryable Encryption *without* specifying an +``encrypted_fields_map``, Django MongoDB Backend will create encrypted +collections, including encryption keys, when you :ref:`run migrations for models +that have encrypted fields `. + +Encryption keys for encrypted fields are stored in the key vault specified in +the :ref:`DATABASES ` setting. To see the keys created by +Django MongoDB Backend, along with the entire schema, you can run the +:djadmin:`showencryptedfieldsmap` command:: + + $ python manage.py showencryptedfieldsmap --database encrypted + +Use the output of :djadmin:`showencryptedfieldsmap` to set the +``encrypted_fields_map`` in :class:`AutoEncryptionOpts +` in your Django settings:: + + from bson import json_util + from pymongo.encryption_options import AutoEncryptionOpts + + DATABASES = { + "encrypted": { + # ... + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + # ... + encrypted_fields_map=json_util.loads( + """{ + "encrypt_patient": { + "fields": [ + { + "bsonType": "string", + "path": "patient_record.ssn", + "keyId": { + "$binary": { + "base64": "2MA29LaARIOqymYHGmi2mQ==", + "subType": "04" + } + }, + "queries": { + "queryType": "equality" + } + }, + ] + }}""" + ), + ), + }, + }, + } + +.. admonition:: Security consideration + + Supplying an encrypted fields map provides more security than relying on an + encrypted fields map obtained from the server. It protects against a + malicious server advertising a false encrypted fields map. + +Configuring the Automatic Encryption Shared Library +=================================================== + +The :ref:`manual:qe-reference-shared-library` is a preferred alternative to +:ref:`manual:qe-mongocryptd` and does not require you to start another process +to perform automatic encryption. + +In practice, if you use Atlas or Enterprise MongoDB, ``mongocryptd`` is already +configured for you, however in such cases the shared library is still +recommended for use with Queryable Encryption. + +You can :ref:`download the shared library +` from the +:ref:`manual:enterprise-official-packages` and configure it in your Django +settings using the ``crypt_shared_lib_path`` option in +:class:`AutoEncryptionOpts `. + +The following example shows how to configure the shared library in your Django +settings:: + + from pymongo.encryption_options import AutoEncryptionOpts + + DATABASES = { + "encrypted": { + # ... + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + # ... + crypt_shared_lib_path="/path/to/mongo_crypt_shared_v1.dylib", + ) + }, + # ... + }, + } + +You are now ready to :doc:`start developing applications +` with Queryable Encryption! diff --git a/docs/index.rst b/docs/index.rst index dc2124f85..7e3690249 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -46,10 +46,12 @@ Models - :doc:`ref/database` - :doc:`ref/contrib/gis` - :doc:`ref/django-admin` +- :doc:`ref/models/encrypted-fields` **Topic guides:** - :doc:`topics/embedded-models` +- :doc:`topics/queryable-encryption` - :doc:`topics/transactions` Forms diff --git a/docs/ref/django-admin.rst b/docs/ref/django-admin.rst index a491714cf..1e111eee8 100644 --- a/docs/ref/django-admin.rst +++ b/docs/ref/django-admin.rst @@ -13,3 +13,26 @@ in the :setting:`INSTALLED_APPS` setting. Available commands ================== + +``showencryptedfieldsmap`` +-------------------------- + +.. versionadded:: 5.2.3 + +.. django-admin:: showencryptedfieldsmap + + This command shows the mapping of encrypted fields to attributes including + data type, data keys and query types. Its output can be used to set the + :ref:`encrypted_fields_map ` argument + in :class:`AutoEncryptionOpts + `. + + .. django-admin-option:: --database DATABASE + + Specifies the database to use. Defaults to ``default``. + + To show the encrypted fields map for a database named ``encrypted``, run: + + .. code-block:: console + + $ python manage.py showencryptedfieldsmap --database encrypted diff --git a/docs/ref/index.rst b/docs/ref/index.rst index 94a11a2a8..47b27d466 100644 --- a/docs/ref/index.rst +++ b/docs/ref/index.rst @@ -9,5 +9,7 @@ API reference forms contrib/index database + models/encrypted-fields django-admin utils + settings diff --git a/docs/ref/models/encrypted-fields.rst b/docs/ref/models/encrypted-fields.rst new file mode 100644 index 000000000..f09dfa168 --- /dev/null +++ b/docs/ref/models/encrypted-fields.rst @@ -0,0 +1,108 @@ +================ +Encrypted fields +================ + +.. versionadded:: 5.2.3 + +Django MongoDB Backend supports :doc:`manual:core/queryable-encryption`. + +See :doc:`/howto/queryable-encryption` for more information on how to use +Queryable Encryption with Django MongoDB Backend. + +See the :doc:`/topics/queryable-encryption` topic guide for +more information on developing applications with Queryable Encryption. + +The following tables detailed which fields have encrypted counterparts. In all +cases, the encrypted field names are simply prefixed with ``Encrypted``, e.g. +``EncryptedCharField``. They are importable from +``django_mongodb_backend.fields``. + +.. csv-table:: ``django.db.models`` + :header: "Model Field", "Encrypted version available?" + + :class:`~django.db.models.BigIntegerField`, Yes + :class:`~django.db.models.BinaryField`, Yes + :class:`~django.db.models.BooleanField`, Yes + :class:`~django.db.models.CharField`, Yes + :class:`~django.db.models.DateField`, Yes + :class:`~django.db.models.DateTimeField`, Yes + :class:`~django.db.models.DecimalField`, Yes + :class:`~django.db.models.DurationField`, Yes + :class:`~django.db.models.EmailField`, Yes + :class:`~django.db.models.FileField`, No: the use case for encrypting this field is unclear. + :class:`~django.db.models.FilePathField`, No: the use case for encrypting this field is unclear. + :class:`~django.db.models.GenericIPAddressField`, Yes + :class:`~django.db.models.ImageField`, No: the use case for encrypting this field is unclear. + :class:`~django.db.models.IntegerField`, Yes + :class:`~django.db.models.JSONField`, No: ``JSONField`` isn't recommended. + :class:`~django.db.models.PositiveIntegerField`, Yes + :class:`~django.db.models.PositiveBigIntegerField`, Yes + :class:`~django.db.models.PositiveSmallIntegerField`, Yes + :class:`~django.db.models.SlugField`, No: it requires a unique index which Queryable Encryption doesn't support. + :class:`~django.db.models.SmallIntegerField`, Yes + :class:`~django.db.models.TimeField`, Yes + :class:`~django.db.models.TextField`, Yes + :class:`~django.db.models.URLField`, Yes + :class:`~django.db.models.UUIDField`, Yes + +.. csv-table:: ``django_mongodb_backend.fields`` + :header: "Model Field", "Encrypted version available?" + + :class:`~.fields.ArrayField`, Yes + :class:`~.fields.EmbeddedModelArrayField`, Yes + :class:`~.fields.EmbeddedModelField`, Yes + :class:`~.fields.ObjectIdField`, Yes + :class:`~.fields.PolymorphicEmbeddedModelField`, No: may be implemented in the future. + :class:`~.fields.PolymorphicEmbeddedModelArrayField`, No: may be implemented in the future. + +These fields don't support the ``queries`` argument: + +- ``EncryptedArrayField`` +- ``EncryptedEmbeddedModelArrayField`` +- ``EncryptedEmbeddedModelField`` + +Limitations +=========== + +MongoDB imposes some restrictions on encrypted fields: + +* They cannot be indexed. +* They cannot be part of a unique constraint. +* They cannot be null. + +``EncryptedFieldMixin`` +======================= + +.. class:: EncryptedFieldMixin + + .. versionadded:: 5.2.3 + + A mixin that can be used to create custom encrypted fields with Queryable + Encryption. + + To create an encrypted field, inherit from ``EncryptedFieldMixin`` and + your custom field class: + + .. code-block:: python + + from django.db import models + from django_mongodb_backend.fields import EncryptedFieldMixin + from myapp.fields import MyField + + + class MyEncryptedField(EncryptedFieldMixin, MyField): + pass + + + You can then use your custom encrypted field in a model, specifying the + desired query types: + + .. code-block:: python + + class MyModel(models.Model): + my_encrypted_field = MyEncryptedField( + queries={"queryType": "equality"}, + ) + my_encrypted_field_too = MyEncryptedField( + queries={"queryType": "range"}, + ) diff --git a/docs/ref/settings.rst b/docs/ref/settings.rst new file mode 100644 index 000000000..233515262 --- /dev/null +++ b/docs/ref/settings.rst @@ -0,0 +1,45 @@ +======== +Settings +======== + +.. _queryable-encryption-settings: + +Queryable Encryption +==================== + +The following :setting:`django:DATABASES` inner options support configuration of +Key Management Service (KMS) credentials for Queryable Encryption. + +.. setting:: DATABASE-KMS-CREDENTIALS + +``KMS_CREDENTIALS`` +------------------- + +Default: ``{}`` (empty dictionary) + +A dictionary of Key Management Service (KMS) credential key-value pairs. These +credentials are required to access your KMS provider (such as AWS KMS, Azure Key +Vault, or GCP KMS) for encrypting and decrypting data using Queryable +Encryption. + +For example after :doc:`/howto/queryable-encryption`, to configure AWS KMS, +Azure Key Vault, or GCP KMS credentials, you can set ``KMS_CREDENTIALS`` in +your :setting:`django:DATABASES` settings as follows: + +.. code-block:: python + + DATABASES["encrypted"]["KMS_CREDENTIALS"] = { + "aws": { + "key": os.getenv("AWS_KEY_ARN", ""), + "region": os.getenv("AWS_KEY_REGION", ""), + }, + "azure": { + "key": os.getenv("AZURE_KEY_VAULT_URL", ""), + "client_id": os.getenv("AZURE_CLIENT_ID", ""), + "client_secret": os.getenv("AZURE_CLIENT_SECRET", ""), + }, + "gcp": { + "key": os.getenv("GCP_KEY_NAME", ""), + "project_id": os.getenv("GCP_PROJECT_ID", ""), + }, + } diff --git a/docs/releases/6.0.x.rst b/docs/releases/6.0.x.rst index 0427b6958..a446ac809 100644 --- a/docs/releases/6.0.x.rst +++ b/docs/releases/6.0.x.rst @@ -10,7 +10,7 @@ Django MongoDB Backend 6.0.x New features ------------ -- ... +- Added support for :doc:`Queryable Encryption`. Bug fixes --------- diff --git a/docs/topics/index.rst b/docs/topics/index.rst index 6e06b8125..a02b35239 100644 --- a/docs/topics/index.rst +++ b/docs/topics/index.rst @@ -9,5 +9,6 @@ know: :maxdepth: 2 embedded-models + queryable-encryption transactions known-issues diff --git a/docs/topics/known-issues.rst b/docs/topics/known-issues.rst index 01259b49a..dd28e6fed 100644 --- a/docs/topics/known-issues.rst +++ b/docs/topics/known-issues.rst @@ -26,6 +26,8 @@ Model fields - :class:`~django.db.models.CompositePrimaryKey` - :class:`~django.db.models.GeneratedField` +.. _known-issues-limitations-querying: + Querying ======== diff --git a/docs/topics/queryable-encryption.rst b/docs/topics/queryable-encryption.rst new file mode 100644 index 000000000..0d6372f69 --- /dev/null +++ b/docs/topics/queryable-encryption.rst @@ -0,0 +1,140 @@ +==================== +Queryable Encryption +==================== + +.. versionadded:: 5.2.3 + +Once you have successfully set up MongoDB Queryable Encryption as described in +:doc:`the installation guide `, you can start +using encrypted fields in your Django models. + +Encrypted fields +================ + +The basics +---------- + +:doc:`Encrypted fields ` may be used to protect +sensitive data like social security numbers, credit card information, or +personal health information. With Queryable Encryption, you can also perform +queries on certain encrypted fields. To use encrypted fields in your models, +import the necessary field types from ``django_mongodb_backend.models`` and +define your models as usual. + +Here are models based on the `Python Queryable Encryption Tutorial`_:: + + # myapp/models.py + from django.db import models + from django_mongodb_backend.models import EmbeddedModel + from django_mongodb_backend.fields import ( + EmbeddedModelField, + EncryptedCharField, + EncryptedEmbeddedModelField, + ) + + + class Patient(models.Model): + patient_name = models.CharField(max_length=255) + patient_id = models.BigIntegerField() + patient_record = EmbeddedModelField("PatientRecord") + + def __str__(self): + return f"{self.patient_name} ({self.patient_id})" + + + class PatientRecord(EmbeddedModel): + ssn = EncryptedCharField(max_length=11) + billing = EncryptedEmbeddedModelField("Billing") + bill_amount = models.DecimalField(max_digits=10, decimal_places=2) + + + class Billing(EmbeddedModel): + cc_type = models.CharField(max_length=50) + cc_number = models.CharField(max_length=20) + +.. _Python Queryable Encryption Tutorial: https://github.com/mongodb/docs/tree/main/content/manual/manual/source/includes/qe-tutorials/python + +.. _qe-migrations: + +Migrations +---------- + +Once you have defined your models, create migrations with: + +.. code-block:: console + + $ python manage.py makemigrations + +Then run the migrations with: + +.. code-block:: console + + $ python manage.py migrate --database encrypted + +Now create and manipulate instances of the data just like any other Django +model data. The fields will automatically handle encryption and decryption, +ensuring that :ref:`sensitive data is stored securely in the database +`. + +Routers +------- + +The example above requires a :ref:`database router +` to direct operations on models with +encrypted fields to the appropriate database. It also requires the use of a +:ref:`router for embedded models `. Here +is an example that includes both:: + + # myproject/settings.py + DATABASE_ROUTERS = [ + "django_mongodb_backend.routers.MongoRouter", + "myproject.routers.EncryptedRouter", + ] + +Querying encrypted fields +------------------------- + +In order to query encrypted fields, you must define the queryable encryption +query type in the model field definition. For example, if you want to query the +``ssn`` field for equality, you can define it as follows:: + + class PatientRecord(EmbeddedModel): + ssn = EncryptedCharField(max_length=11, queries={"queryType": "equality"}) + billing = EncryptedEmbeddedModelField("Billing") + bill_amount = models.DecimalField(max_digits=10, decimal_places=2) + +Then you can perform a query like this: + +.. code-block:: console + + >>> patient = Patient.objects.get(patient_record__ssn="123-45-6789") + >>> patient.name + 'John Doe' + +.. _qe-available-query-types: + +Available query types +~~~~~~~~~~~~~~~~~~~~~ + +The ``queries`` option should be a dictionary that specifies the type of queries +that can be performed on the field. Of the :ref:`available query types +` Django MongoDB Backend currently +supports: + +- ``equality`` +- ``range`` + +.. admonition:: Query types vs. Django lookups + + Range queries in Queryable Encryption are different from Django's + :ref:`range lookups `. Range queries allow you to + perform comparisons on encrypted fields, while Django's range lookups are + used for filtering based on a range of values. + +QuerySet limitations +~~~~~~~~~~~~~~~~~~~~ + +In addition to :ref:`Django MongoDB Backend's QuerySet limitations +`, + +.. TODO diff --git a/pyproject.toml b/pyproject.toml index acecf20fb..0d9f1d981 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ docs = [ "furo>=2025.7.19", "sphinx-copybutton", ] +encryption = ["pymongo[encryption]"] [project.urls] Homepage = "https://www.mongodb.org" diff --git a/tests/backend_/test_features.py b/tests/backend_/test_features.py index 05959fa70..f36b591b1 100644 --- a/tests/backend_/test_features.py +++ b/tests/backend_/test_features.py @@ -44,3 +44,84 @@ def mocked_command(command): with patch("pymongo.synchronous.database.Database.command", wraps=mocked_command): self.assertIs(connection.features._supports_transactions, False) + + +class SupportsQueryableEncryptionTests(TestCase): + def setUp(self): + # Clear the cached property. + connection.features.__dict__.pop("supports_queryable_encryption", None) + # Must initialize cached properties before patching them. + connection.features._supports_transactions # noqa: B018 + connection.features.mongodb_version # noqa: B018 + + def tearDown(self): + del connection.features.supports_queryable_encryption + + @staticmethod + def enterprise_response(command): + if command == "buildInfo": + return {"modules": ["enterprise"]} + raise Exception("Unexpected command") + + @staticmethod + def non_enterprise_response(command): + if command == "buildInfo": + return {"modules": []} + raise Exception("Unexpected command") + + def test_supported_on_atlas(self): + """Supported on MongoDB 8.0+ Atlas replica set or sharded cluster.""" + with ( + patch( + "pymongo.synchronous.database.Database.command", wraps=self.non_enterprise_response + ), + patch("django.db.connection.features.supports_atlas_search", True), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_8_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, True) + + def test_supported_on_enterprise(self): + """Supported on MongoDB 8.0+ Enterprise replica set or sharded cluster.""" + with ( + patch("pymongo.synchronous.database.Database.command", wraps=self.enterprise_response), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_8_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, True) + + def test_atlas_or_enterprise_required(self): + """Not supported on MongoDB Community Edition.""" + with ( + patch( + "pymongo.synchronous.database.Database.command", wraps=self.non_enterprise_response + ), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_8_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, False) + + def test_transactions_required(self): + """ + Not supported if database isn't a replica set or sharded cluster + (i.e. DatabaseFeatures._supports_transactions = False). + """ + with ( + patch("pymongo.synchronous.database.Database.command", wraps=self.enterprise_response), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", False), + patch("django.db.connection.features.is_mongodb_8_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, False) + + def test_mongodb_8_0_required(self): + """Not supported on MongoDB < 8.0""" + with ( + patch("pymongo.synchronous.database.Database.command", wraps=self.enterprise_response), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_8_0", False), + ): + self.assertIs(connection.features.supports_queryable_encryption, False) diff --git a/tests/encryption_/__init__.py b/tests/encryption_/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/encryption_/models.py b/tests/encryption_/models.py new file mode 100644 index 000000000..995e4760c --- /dev/null +++ b/tests/encryption_/models.py @@ -0,0 +1,184 @@ +from django.db import models + +from django_mongodb_backend.fields import ( + EmbeddedModelField, + EncryptedArrayField, + EncryptedBigIntegerField, + EncryptedBinaryField, + EncryptedBooleanField, + EncryptedCharField, + EncryptedDateField, + EncryptedDateTimeField, + EncryptedDecimalField, + EncryptedDurationField, + EncryptedEmailField, + EncryptedEmbeddedModelArrayField, + EncryptedEmbeddedModelField, + EncryptedFloatField, + EncryptedGenericIPAddressField, + EncryptedIntegerField, + EncryptedObjectIdField, + EncryptedPositiveBigIntegerField, + EncryptedPositiveIntegerField, + EncryptedPositiveSmallIntegerField, + EncryptedSmallIntegerField, + EncryptedTextField, + EncryptedTimeField, + EncryptedURLField, + EncryptedUUIDField, +) +from django_mongodb_backend.models import EmbeddedModel + + +class Author(models.Model): + name = models.CharField(max_length=255) + + +class Book(models.Model): + title = models.CharField(max_length=255) + author = models.ForeignKey(Author, models.CASCADE) + + +class EncryptedTestModel(models.Model): + class Meta: + abstract = True + required_db_features = {"supports_queryable_encryption"} + + +# Array models +class ArrayModel(EncryptedTestModel): + values = EncryptedArrayField( + models.IntegerField(), + size=5, + ) + + +# Embedded models +class Patient(EncryptedTestModel): + patient_name = models.CharField(max_length=255) + patient_id = models.BigIntegerField() + patient_record = EmbeddedModelField("PatientRecord") + + def __str__(self): + return f"{self.patient_name} ({self.patient_id})" + + +class PatientRecord(EmbeddedModel): + ssn = EncryptedCharField(max_length=11, queries={"queryType": "equality"}) + billing = EncryptedEmbeddedModelField("Billing") + bill_amount = models.DecimalField(max_digits=10, decimal_places=2) + + +class Billing(EmbeddedModel): + cc_type = models.CharField(max_length=50) + cc_number = models.CharField(max_length=20) + + +# Embedded array models +class Actor(EmbeddedModel): + name = models.CharField(max_length=100) + + +class Movie(EncryptedTestModel): + title = models.CharField(max_length=200) + plot = models.TextField(blank=True) + runtime = models.IntegerField(default=0) + released = models.DateTimeField("release date") + cast = EncryptedEmbeddedModelArrayField(Actor) + + def __str__(self): + return self.title + + +# Equality-queryable field models +class BinaryModel(EncryptedTestModel): + value = EncryptedBinaryField(queries={"queryType": "equality"}) + + +class BooleanModel(EncryptedTestModel): + value = EncryptedBooleanField(queries={"queryType": "equality"}) + + +class CharModel(EncryptedTestModel): + value = EncryptedCharField(max_length=255, queries={"queryType": "equality"}) + + +class EmailModel(EncryptedTestModel): + value = EncryptedEmailField(max_length=255, queries={"queryType": "equality"}) + + +class GenericIPAddressModel(EncryptedTestModel): + value = EncryptedGenericIPAddressField(queries={"queryType": "equality"}) + + +class ObjectIdModel(EncryptedTestModel): + value = EncryptedObjectIdField(queries={"queryType": "equality"}) + + +class TextModel(EncryptedTestModel): + value = EncryptedTextField(queries={"queryType": "equality"}) + + +class URLModel(EncryptedTestModel): + value = EncryptedURLField(max_length=500, queries={"queryType": "equality"}) + + +class UUIDModel(EncryptedTestModel): + value = EncryptedUUIDField(queries={"queryType": "equality"}) + + +# Range-queryable field models +class BigIntegerModel(EncryptedTestModel): + value = EncryptedBigIntegerField(queries={"queryType": "range"}) + + +class DateModel(EncryptedTestModel): + value = EncryptedDateField(queries={"queryType": "range"}) + + +class DateTimeModel(EncryptedTestModel): + value = EncryptedDateTimeField(queries={"queryType": "range"}) + + +class DecimalModel(EncryptedTestModel): + value = EncryptedDecimalField(max_digits=10, decimal_places=2, queries={"queryType": "range"}) + + +class DurationModel(EncryptedTestModel): + value = EncryptedDurationField(queries={"queryType": "range"}) + + +class FloatModel(EncryptedTestModel): + value = EncryptedFloatField(queries={"queryType": "range"}) + + +class IntegerModel(EncryptedTestModel): + value = EncryptedIntegerField(queries={"queryType": "range"}) + + +class PositiveBigIntegerModel(EncryptedTestModel): + value = EncryptedPositiveBigIntegerField(queries={"queryType": "range"}) + + +class PositiveIntegerModel(EncryptedTestModel): + value = EncryptedPositiveIntegerField(queries={"queryType": "range"}) + + +class PositiveSmallIntegerModel(EncryptedTestModel): + value = EncryptedPositiveSmallIntegerField(queries={"queryType": "range"}) + + +class SmallIntegerModel(EncryptedTestModel): + value = EncryptedSmallIntegerField(queries={"queryType": "range"}) + + +class TimeModel(EncryptedTestModel): + value = EncryptedTimeField(queries={"queryType": "range"}) + + +class EncryptionKey(models.Model): + key_alt_name = models.CharField(max_length=500, db_column="keyAltNames") + + class Meta: + db_table = "__keyVault" + managed = False diff --git a/tests/encryption_/test_base.py b/tests/encryption_/test_base.py new file mode 100644 index 000000000..0c165d19a --- /dev/null +++ b/tests/encryption_/test_base.py @@ -0,0 +1,21 @@ +import pymongo +from bson.binary import Binary +from django.conf import settings +from django.db import connections +from django.test import TestCase, skipUnlessDBFeature + + +@skipUnlessDBFeature("supports_queryable_encryption") +class EncryptionTestCase(TestCase): + databases = {"default", "encrypted"} + maxDiff = None + + def assertEncrypted(self, model, field): + # Access encrypted database from an unencrypted connection + conn_params = connections["default"].get_connection_params() + db_name = settings.DATABASES["encrypted"]["NAME"] + with pymongo.MongoClient(**conn_params) as new_connection: + db = new_connection[db_name] + collection = db[model._meta.db_table] + data = collection.find_one({}, {field: 1, "_id": 0}) + self.assertIsInstance(data[field], Binary) diff --git a/tests/encryption_/test_fields.py b/tests/encryption_/test_fields.py new file mode 100644 index 000000000..56f271c3f --- /dev/null +++ b/tests/encryption_/test_fields.py @@ -0,0 +1,399 @@ +import datetime +import uuid +from decimal import Decimal +from operator import attrgetter + +from bson import ObjectId +from django.db import DatabaseError +from django.db.models import Avg, F + +from django_mongodb_backend.fields import ( + EncryptedArrayField, + EncryptedCharField, + EncryptedEmbeddedModelArrayField, + EncryptedEmbeddedModelField, + EncryptedIntegerField, +) + +from .models import ( + Actor, + ArrayModel, + Author, + BigIntegerModel, + Billing, + BinaryModel, + Book, + BooleanModel, + CharModel, + DateModel, + DateTimeModel, + DecimalModel, + DurationModel, + EmailModel, + FloatModel, + GenericIPAddressModel, + IntegerModel, + Movie, + ObjectIdModel, + Patient, + PatientRecord, + PositiveBigIntegerModel, + PositiveIntegerModel, + PositiveSmallIntegerModel, + SmallIntegerModel, + TextModel, + TimeModel, + URLModel, + UUIDModel, +) +from .test_base import EncryptionTestCase + + +class ArrayModelTests(EncryptionTestCase): + def setUp(self): + self.array_model = ArrayModel.objects.create(values=[1, 2, 3, 4, 5]) + + def test_array(self): + array_model = ArrayModel.objects.get(id=self.array_model.id) + self.assertEqual(array_model.values, [1, 2, 3, 4, 5]) + self.assertEncrypted(self.array_model, "values") + + +class EmbeddedModelTests(EncryptionTestCase): + def setUp(self): + self.billing = Billing(cc_type="Visa", cc_number="4111111111111111") + self.patient_record = PatientRecord(ssn="123-45-6789", billing=self.billing) + self.patient = Patient.objects.create( + patient_name="John Doe", patient_id=123456789, patient_record=self.patient_record + ) + + def test_object(self): + patient = Patient.objects.get(id=self.patient.id) + self.assertEqual(patient.patient_record.ssn, "123-45-6789") + self.assertEqual(patient.patient_record.billing.cc_type, "Visa") + self.assertEqual(patient.patient_record.billing.cc_number, "4111111111111111") + + +class EmbeddedModelArrayTests(EncryptionTestCase): + def setUp(self): + self.actor1 = Actor(name="Actor One") + self.actor2 = Actor(name="Actor Two") + self.movie = Movie.objects.create( + title="Sample Movie", + cast=[self.actor1, self.actor2], + released=datetime.date(2024, 6, 1), + ) + + def test_array(self): + movie = Movie.objects.get(id=self.movie.id) + self.assertEqual(len(movie.cast), 2) + self.assertEqual(movie.cast[0].name, "Actor One") + self.assertEqual(movie.cast[1].name, "Actor Two") + self.assertEncrypted(movie, "cast") + + +class FieldTests(EncryptionTestCase): + def assertEquality(self, model_cls, val): + model_cls.objects.create(value=val) + fetched = model_cls.objects.get(value=val) + self.assertEqual(fetched.value, val) + + def assertRange(self, model_cls, *, low, high, threshold): + model_cls.objects.create(value=low) + model_cls.objects.create(value=high) + self.assertEqual(model_cls.objects.get(value=low).value, low) + self.assertEqual(model_cls.objects.get(value=high).value, high) + objs = list(model_cls.objects.filter(value__gt=threshold)) + self.assertEqual(len(objs), 1) + self.assertEqual(objs[0].value, high) + + # Equality-only fields + def test_binary(self): + self.assertEquality(BinaryModel, b"\x00\x01\x02") + self.assertEncrypted(BinaryModel, "value") + + def test_boolean(self): + self.assertEquality(BooleanModel, True) + self.assertEncrypted(BooleanModel, "value") + + def test_char(self): + self.assertEquality(CharModel, "hello") + self.assertEncrypted(CharModel, "value") + + def test_email(self): + self.assertEquality(EmailModel, "test@example.com") + self.assertEncrypted(EmailModel, "value") + + def test_ip(self): + self.assertEquality(GenericIPAddressModel, "192.168.0.1") + self.assertEncrypted(GenericIPAddressModel, "value") + + def test_objectid(self): + self.assertEquality(ObjectIdModel, ObjectId()) + self.assertEncrypted(ObjectIdModel, "value") + + def test_text(self): + self.assertEquality(TextModel, "some text") + self.assertEncrypted(TextModel, "value") + + def test_url(self): + self.assertEquality(URLModel, "https://example.com") + self.assertEncrypted(URLModel, "value") + + def test_uuid(self): + self.assertEquality(UUIDModel, uuid.uuid4()) + self.assertEncrypted(UUIDModel, "value") + + # Range fields + def test_big_integer(self): + self.assertRange(BigIntegerModel, low=100, high=200, threshold=150) + self.assertEncrypted(BigIntegerModel, "value") + + def test_date(self): + self.assertRange( + DateModel, + low=datetime.date(2024, 6, 1), + high=datetime.date(2024, 6, 10), + threshold=datetime.date(2024, 6, 5), + ) + self.assertEncrypted(DateModel, "value") + + def test_datetime(self): + self.assertRange( + DateTimeModel, + low=datetime.datetime(2024, 6, 1, 12, 0), + high=datetime.datetime(2024, 6, 2, 12, 0), + threshold=datetime.datetime(2024, 6, 2, 0, 0), + ) + self.assertEncrypted(DateTimeModel, "value") + + def test_decimal(self): + self.assertRange( + DecimalModel, + low=Decimal("123.45"), + high=Decimal("200.50"), + threshold=Decimal("150"), + ) + self.assertEncrypted(DecimalModel, "value") + + def test_duration(self): + self.assertRange( + DurationModel, + low=datetime.timedelta(days=3), + high=datetime.timedelta(days=10), + threshold=datetime.timedelta(days=5), + ) + self.assertEncrypted(DurationModel, "value") + + def test_float(self): + self.assertRange(FloatModel, low=1.23, high=4.56, threshold=3.0) + self.assertEncrypted(FloatModel, "value") + + def test_integer(self): + self.assertRange(IntegerModel, low=5, high=10, threshold=7) + self.assertEncrypted(IntegerModel, "value") + + def test_positive_big_integer(self): + self.assertRange(PositiveBigIntegerModel, low=100, high=500, threshold=200) + self.assertEncrypted(PositiveBigIntegerModel, "value") + + def test_positive_integer(self): + self.assertRange(PositiveIntegerModel, low=10, high=20, threshold=15) + self.assertEncrypted(PositiveIntegerModel, "value") + + def test_positive_small_integer(self): + self.assertRange(PositiveSmallIntegerModel, low=5, high=8, threshold=6) + self.assertEncrypted(PositiveSmallIntegerModel, "value") + + def test_small_integer(self): + self.assertRange(SmallIntegerModel, low=-5, high=2, threshold=0) + self.assertEncrypted(SmallIntegerModel, "value") + + def test_time(self): + self.assertRange( + TimeModel, + low=datetime.time(10, 0), + high=datetime.time(15, 0), + threshold=datetime.time(12, 0), + ) + self.assertEncrypted(TimeModel, "value") + + +class QueryTests(EncryptionTestCase): + def test_aggregate(self): + msg = ( + "Aggregation stage $internalFacetTeeConsumer is not allowed or " + "supported with automatic encryption." + ) + with self.assertRaisesMessage(DatabaseError, msg): + list(IntegerModel.objects.aggregate(Avg("value"))) + + def test_alias(self): + msg = ( + "Cannot group on field '_id.value' which is encrypted with the " + "random algorithm or whose encryption properties are not known " + "until runtime" + ) + with self.assertRaisesMessage(DatabaseError, msg): + list(IntegerModel.objects.alias(avg=Avg("value"))) + + def test_annotate(self): + msg = ( + "Cannot group on field '_id.value' which is encrypted with the " + "random algorithm or whose encryption properties are not known " + "until runtime" + ) + with self.assertRaisesMessage(DatabaseError, msg): + list(IntegerModel.objects.annotate(avg=Avg("value"))) + + def test_bulk_create(self): + CharModel.objects.bulk_create([CharModel(value="abc"), CharModel(value="xyz")]) + self.assertQuerySetEqual( + CharModel.objects.order_by("pk"), ["abc", "xyz"], attrgetter("value") + ) + + def test_bulk_update(self): + objs = [ + CharModel.objects.create(value="abc"), + CharModel.objects.create(value="xyz"), + ] + objs[0].value = "def" + objs[1].value = "mno" + msg = "Multi-document updates are not allowed with Queryable Encryption" + with self.assertRaisesMessage(DatabaseError, msg): + CharModel.objects.bulk_update(objs, ["value"]) + + def test_contains(self): + obj = CharModel.objects.create(value="abc") + self.assertIs(CharModel.objects.contains(obj), True) + + def test_count(self): + msg = ( + "Aggregation stage $internalFacetTeeConsumer is not allowed or " + "supported with automatic encryption." + ) + with self.assertRaisesMessage(DatabaseError, msg): + list(CharModel.objects.count()) + + def test_dates(self): + msg = ( + "If the value type is a date, the type of the index must also be date (and vice versa)." + ) + with self.assertRaisesMessage(DatabaseError, msg): + list(DateModel.objects.dates("value", "year")) + + def test_datetimes(self): + msg = ( + "If the value type is a date, the type of the index must also be date (and vice versa)." + ) + with self.assertRaisesMessage(DatabaseError, msg): + list(DateTimeModel.objects.datetimes("value", "year")) + + def test_distinct(self): + msg = ( + "Cannot group on field '_id.value' which is encrypted with the " + "random algorithm or whose encryption properties are not known " + "until runtime" + ) + with self.assertRaisesMessage(DatabaseError, msg): + list(CharModel.objects.distinct("value")) + + def test_exclude(self): + obj1 = CharModel.objects.create(value="abc") + obj2 = CharModel.objects.create(value="xyz") + self.assertSequenceEqual(CharModel.objects.exclude(value=obj1.value), [obj2]) + + def test_exists(self): + self.assertIs(CharModel.objects.exists(), False) + + def test_get_or_create(self): + obj1, created1 = CharModel.objects.get_or_create(value="abc") + self.assertIs(created1, True) + obj2, created2 = CharModel.objects.get_or_create(value="abc") + self.assertIs(created2, False) + self.assertEqual(obj1, obj2) + + def test_join(self): + book = Book.objects.create(title="Book", author=Author.objects.create(name="Bob")) + self.assertSequenceEqual(Book.objects.filter(author__name="Bob"), [book]) + + def test_join_with_let(self): + msg = ( + "Non-empty 'let' field is not allowed in the $lookup aggregation " + "stage over an encrypted collection." + ) + with self.assertRaisesMessage(DatabaseError, msg): + list(Book.objects.filter(author__name=F("title"))) + + def test_order_by(self): + msg = "Cannot add an encrypted field as a prefix of another encrypted field" + with self.assertRaisesMessage(DatabaseError, msg): + list(CharModel.objects.order_by("value")) + + def test_select_related(self): + Book.objects.create(title="Book", author=Author.objects.create(name="Bob")) + with self.assertNumQueries(1, using="encrypted"): + books = Book.objects.select_related("author") + self.assertEqual(books[0].author.name, "Bob") + + def test_update(self): + msg = "Multi-document updates are not allowed with Queryable Encryption" + with self.assertRaisesMessage(DatabaseError, msg): + self.assertEqual(CharModel.objects.update(value="xyz"), 1) + + def test_update_or_create(self): + CharModel.objects.create(value="xyz") + msg = "Multi-document updates are not allowed with Queryable Encryption" + with self.assertRaisesMessage(DatabaseError, msg): + CharModel.objects.update_or_create(value="xyz", defaults={"plain": "abc"}) + + def test_union(self): + msg = "Aggregation stage $unionWith is not allowed or supported with automatic encryption." + qs1 = IntegerModel.objects.filter(value__gt=1) + qs2 = IntegerModel.objects.filter(value__gte=8) + with self.assertRaisesMessage(DatabaseError, msg): + list(qs1.union(qs2)) + + def test_values(self): + list(CharModel.objects.values("value")) + + def test_values_list(self): + list(CharModel.objects.values_list("value")) + + +class FieldMixinTests(EncryptionTestCase): + def test_db_index(self): + msg = "'db_index=True' is not supported on encrypted fields." + with self.assertRaisesMessage(ValueError, msg): + EncryptedIntegerField(db_index=True) + + def test_null(self): + msg = "'null=True' is not supported on encrypted fields." + with self.assertRaisesMessage(ValueError, msg): + EncryptedIntegerField(null=True) + + def test_unique(self): + msg = "'unique=True' is not supported on encrypted fields." + with self.assertRaisesMessage(ValueError, msg): + EncryptedIntegerField(unique=True) + + def test_deconstruct(self): + field = EncryptedCharField(max_length=50, queries={"field": "value"}) + field.name = "ssn" + name, path, args, kwargs = field.deconstruct() + self.assertEqual(name, "ssn") + self.assertEqual(path, "django_mongodb_backend.fields.EncryptedCharField") + self.assertEqual(args, []) + self.assertEqual(kwargs["queries"], {"field": "value"}) + + def test_fields_without_queries(self): + """Some field types (array, object) can't be queried.""" + for field in ( + EncryptedArrayField, + EncryptedEmbeddedModelField, + EncryptedEmbeddedModelArrayField, + ): + with self.subTest(field=field): + msg = f"{field.__name__} does not support the queries argument." + with self.assertRaisesMessage(ValueError, msg): + field(Actor, queries={}) diff --git a/tests/encryption_/test_management.py b/tests/encryption_/test_management.py new file mode 100644 index 000000000..096ccab3a --- /dev/null +++ b/tests/encryption_/test_management.py @@ -0,0 +1,131 @@ +from io import StringIO + +from bson import json_util +from django.core.exceptions import ImproperlyConfigured +from django.core.management import call_command +from django.db import connections +from django.test import modify_settings + +from .models import EncryptionKey +from .test_base import EncryptionTestCase + + +@modify_settings(INSTALLED_APPS={"prepend": "django_mongodb_backend"}) +class CommandTests(EncryptionTestCase): + # Expected encrypted field maps for all Encrypted* models + expected_maps = { + "encryption__patient": { + "fields": [ + { + "bsonType": "string", + "path": "patient_record.ssn", + "queries": {"queryType": "equality"}, + }, + {"bsonType": "object", "path": "patient_record.billing"}, + ] + }, + # Equality-queryable fields + "encryption__binarymodel": { + "fields": [ + {"bsonType": "binData", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "encryption__booleanmodel": { + "fields": [{"bsonType": "bool", "path": "value", "queries": {"queryType": "equality"}}] + }, + "encryption__charmodel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "encryption__emailmodel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "encryption__genericipaddressmodel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "encryption__textmodel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "encryption__urlmodel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + # Range-queryable fields + "encryption__bigintegermodel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__datemodel": { + "fields": [{"bsonType": "date", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__datetimemodel": { + "fields": [{"bsonType": "date", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__decimalmodel": { + "fields": [{"bsonType": "decimal", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__durationmodel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__floatmodel": { + "fields": [{"bsonType": "double", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__integermodel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__positivebigintegermodel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__positiveintegermodel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__positivesmallintegermodel": { + "fields": [{"bsonType": "int", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__smallintegermodel": { + "fields": [{"bsonType": "int", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__timemodel": { + "fields": [{"bsonType": "date", "path": "value", "queries": {"queryType": "range"}}] + }, + } + + def _compare_output(self, expected, actual): + for field in actual["fields"]: + del field["keyId"] # Can't compare dynamic value + self.assertEqual(expected, actual) + + def test_show_encrypted_fields_map(self): + out = StringIO() + call_command("showencryptedfieldsmap", "--database", "encrypted", verbosity=0, stdout=out) + command_output = json_util.loads(out.getvalue()) + + # Loop through each expected model + for model_key, expected in self.expected_maps.items(): + with self.subTest(model=model_key): + self.assertIn(model_key, command_output) + self._compare_output(expected, command_output[model_key]) + + def test_missing_key(self): + test_key = "encryption__patient.patient_record.ssn" + msg = ( + f"Encryption key {test_key} not found. Have migrated the " + " model?" + ) + EncryptionKey.objects.filter(key_alt_name=test_key).delete() + try: + with self.assertRaisesMessage(ImproperlyConfigured, msg): + call_command("showencryptedfieldsmap", "--database", "encrypted", verbosity=0) + finally: + # Replace the deleted key. + connections["encrypted"].client_encryption.create_data_key( + kms_provider="local", + key_alt_names=[test_key], + ) diff --git a/tests/encryption_/test_schema.py b/tests/encryption_/test_schema.py new file mode 100644 index 000000000..ae1803389 --- /dev/null +++ b/tests/encryption_/test_schema.py @@ -0,0 +1,152 @@ +from bson.binary import Binary +from django.core.exceptions import ImproperlyConfigured +from django.db import connections + +from . import models +from .models import EncryptionKey +from .test_base import EncryptionTestCase + + +class SchemaTests(EncryptionTestCase): + # Expected encrypted fields map per model + expected_map = { + "Patient": { + "fields": [ + { + "bsonType": "string", + "path": "patient_record.ssn", + "queries": {"queryType": "equality"}, + }, + {"bsonType": "object", "path": "patient_record.billing"}, + ] + }, + "BinaryModel": { + "fields": [ + {"bsonType": "binData", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "BooleanModel": { + "fields": [{"bsonType": "bool", "path": "value", "queries": {"queryType": "equality"}}] + }, + "CharModel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "EmailModel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "GenericIPAddressModel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "TextModel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "URLModel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "BigIntegerModel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "DateModel": { + "fields": [{"bsonType": "date", "path": "value", "queries": {"queryType": "range"}}] + }, + "DateTimeModel": { + "fields": [{"bsonType": "date", "path": "value", "queries": {"queryType": "range"}}] + }, + "DecimalModel": { + "fields": [{"bsonType": "decimal", "path": "value", "queries": {"queryType": "range"}}] + }, + "DurationModel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "FloatModel": { + "fields": [{"bsonType": "double", "path": "value", "queries": {"queryType": "range"}}] + }, + "IntegerModel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "PositiveBigIntegerModel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "PositiveIntegerModel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "PositiveSmallIntegerModel": { + "fields": [{"bsonType": "int", "path": "value", "queries": {"queryType": "range"}}] + }, + "SmallIntegerModel": { + "fields": [{"bsonType": "int", "path": "value", "queries": {"queryType": "range"}}] + }, + "TimeModel": { + "fields": [{"bsonType": "date", "path": "value", "queries": {"queryType": "range"}}] + }, + } + + def test_get_encrypted_fields_all_models(self): + """ + Loops through all models, + checks their encrypted fields map from the schema editor, + and compares to expected BSON type & queries mapping. + """ + # Deleting all keys is only correct only if this test includes all + # test models. This test may not be needed since it's tested when the + # test runner migrates all models. If any subTest fails, the key vault + # will be left in an inconsistent state. + EncryptionKey.objects.all().delete() + connection = connections["encrypted"] + for model_name, expected in self.expected_map.items(): + with self.subTest(model=model_name): + model_class = getattr(models, model_name) + with connection.schema_editor() as editor: + encrypted_fields = editor._get_encrypted_fields(model_class) + for field in encrypted_fields["fields"]: + del field["keyId"] # Can't compare dynamic value + self.assertEqual(encrypted_fields, expected) + + def test_key_creation_and_lookup(self): + """ + Use _get_encrypted_fields to + generate and store a data key in the vault, then + query the vault with the keyAltName. + """ + model_class = models.CharModel + test_key_alt_name = f"{model_class._meta.db_table}.value" + # Delete the test key and verify it's gone. + EncryptionKey.objects.filter(key_alt_name=test_key_alt_name).delete() + with self.assertRaises(EncryptionKey.DoesNotExist): + EncryptionKey.objects.get(key_alt_name=test_key_alt_name) + # Regenerate the keyId. + with connections["encrypted"].schema_editor() as editor: + encrypted_fields = editor._get_encrypted_fields(model_class) + # Validate schema contains a keyId for the field. + field_info = encrypted_fields["fields"][0] + self.assertEqual(field_info["path"], "value") + self.assertIsInstance(field_info["keyId"], Binary) + # Lookup in key vault by the keyAltName. + key = EncryptionKey.objects.get(key_alt_name=test_key_alt_name) + self.assertEqual(key.id, field_info["keyId"]) + self.assertEqual(key.key_alt_name, [test_key_alt_name]) + + def test_missing_auto_encryption_opts(self): + connection = connections["default"] + msg = ( + "Tried to create model encryption_.Patient in 'default' database. " + "The model has encrypted fields but DATABASES['default']['OPTIONS'] " + 'is missing the "auto_encryption_opts" parameter. If the model ' + "should not be created in this database, adjust your database " + "routers." + ) + with ( + self.assertRaisesMessage(ImproperlyConfigured, msg), + connection.schema_editor() as editor, + ): + editor.create_model(models.Patient) diff --git a/tests/raw_query_/test_raw_aggregate.py b/tests/raw_query_/test_raw_aggregate.py index 99dcd5faf..96df2f925 100644 --- a/tests/raw_query_/test_raw_aggregate.py +++ b/tests/raw_query_/test_raw_aggregate.py @@ -111,7 +111,7 @@ def assertAnnotations(self, results, expected_annotations): self.assertEqual(getattr(result, annotation), value) def test_rawqueryset_repr(self): - queryset = RawQuerySet(pipeline=[]) + queryset = RawQuerySet(pipeline=[], model=Book) self.assertEqual(repr(queryset), "") self.assertEqual(repr(queryset.query), "") From d4a555cc505b34595f49ada8af4550030fdeb23a Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 5 Nov 2025 09:24:51 -0500 Subject: [PATCH 03/26] use shared library instead of mongocryptd --- .github/workflows/encrypted_settings.py | 3 +++ .github/workflows/test-python-atlas.yml | 22 +++------------------- docs/howto/queryable-encryption.rst | 24 ++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/.github/workflows/encrypted_settings.py b/.github/workflows/encrypted_settings.py index 02dc3447d..0436b04f7 100644 --- a/.github/workflows/encrypted_settings.py +++ b/.github/workflows/encrypted_settings.py @@ -4,6 +4,8 @@ from mongodb_settings import * # noqa: F403 from pymongo.encryption import AutoEncryptionOpts +os.environ["LD_LIBRARY_PATH"] = os.environ["GITHUB_WORKSPACE"] + "/lib/" + DATABASES["encrypted"] = { # noqa: F405 "ENGINE": "django_mongodb_backend", "NAME": "djangotests_encrypted", @@ -11,6 +13,7 @@ "auto_encryption_opts": AutoEncryptionOpts( key_vault_namespace="djangotests_encrypted.__keyVault", kms_providers={"local": {"key": os.urandom(96)}}, + crypt_shared_lib_path=os.environ["GITHUB_WORKSPACE"] + "/lib/mongo_crypt_v1.so", ), "directConnection": True, }, diff --git a/.github/workflows/test-python-atlas.yml b/.github/workflows/test-python-atlas.yml index 33f1c1b3f..5f1900b70 100644 --- a/.github/workflows/test-python-atlas.yml +++ b/.github/workflows/test-python-atlas.yml @@ -52,26 +52,10 @@ jobs: - name: Start local Atlas working-directory: . run: bash .github/workflows/start_local_atlas.sh mongodb/mongodb-atlas-local:8.0.15 - - name: Install mongosh + - name: Download crypt shared run: | - wget -q https://downloads.mongodb.com/compass/mongosh-2.2.10-linux-x64.tgz - tar -xzf mongosh-*-linux-x64.tgz - sudo cp mongosh-*-linux-x64/bin/mongosh /usr/local/bin/ - mongosh --version - - name: Install mongocryptd from Enterprise tarball - run: | - curl -sSL -o mongodb-enterprise.tgz "https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-ubuntu2204-8.0.15.tgz" - tar -xzf mongodb-enterprise.tgz - sudo cp mongodb-linux-x86_64-enterprise-ubuntu2204-8.0.15/bin/mongocryptd /usr/local/bin/ - - name: Start mongocryptd - run: | - nohup mongocryptd --logpath=/tmp/mongocryptd.log & - - name: Verify MongoDB installation - run: | - mongosh --eval 'db.runCommand({ connectionStatus: 1 })' - - name: Verify mongocryptd is running - run: | - pgrep mongocryptd + wget https://downloads.mongodb.com/linux/mongo_crypt_shared_v1-linux-x86_64-enterprise-ubuntu2404-8.2.1.tgz + tar -xvzf mongo_crypt_shared_v1-linux-x86_64-enterprise-ubuntu2404-8.2.1.tgz lib/mongo_crypt_v1.so - name: Run tests run: python3 django_repo/tests/runtests_.py permissions: diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index 4c72f554d..ab827fb0e 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -285,5 +285,29 @@ settings:: }, } + +.. admonition:: Dynamic library path configuration + + The Automatic Encryption Shared Library is platform‑specific. Make sure to + download the correct version for your operating system and architecture, + and configure your environment so the system can locate it. + + Use the following variables depending on your platform: + + +---------------+---------------------------------+ + | **Platform** | **Environment Variable** | + +---------------+---------------------------------+ + | Windows | ``PATH`` | + +---------------+---------------------------------+ + | macOS | ``DYLD_FALLBACK_LIBRARY_PATH`` | + +---------------+---------------------------------+ + | Linux | ``LD_LIBRARY_PATH`` | + +---------------+---------------------------------+ + + For example on macOS, you can set the ``DYLD_FALLBACK_LIBRARY_PATH`` + environment variable in your shell before starting your Django application:: + + $ export DYLD_FALLBACK_LIBRARY_PATH="/path/to/mongo_crypt_shared_v1.dylib:$DYLD_FALLBACK_LIBRARY_PATH" + You are now ready to :doc:`start developing applications ` with Queryable Encryption! From 6e53f6d3169b235da0d8ee2fbfd0e4ae74fc9852 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Mon, 10 Nov 2025 20:27:07 -0500 Subject: [PATCH 04/26] polish howto --- docs/howto/queryable-encryption.rst | 157 +++++++++++++--------------- 1 file changed, 70 insertions(+), 87 deletions(-) diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index ab827fb0e..81709b61f 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -35,21 +35,11 @@ Configuring the ``DATABASES`` setting ===================================== In addition to the :ref:`database settings ` -required to use Django MongoDB Backend, Queryable Encryption requires you to -configure a separate encrypted database connection in your -:setting:`django:DATABASES` setting. +required to use Django MongoDB Backend, Queryable Encryption requires +configuring a separate database connection that uses use PyMongo's +:class:`~pymongo.encryption_options.AutoEncryptionOpts`. -.. admonition:: Encrypted database - - An encrypted database is a separate database connection in your - :setting:`django:DATABASES` setting that is configured to use PyMongo's - :class:`automatic encryption - `. - -Here's how to configure an encrypted database using a local KMS provider and -encryption keys stored in the ``encryption.__keyVault`` collection:: - - import os +Here's a sample configuration using a local KMS provider:: from pymongo.encryption_options import AutoEncryptionOpts @@ -63,25 +53,33 @@ encryption keys stored in the ``encryption.__keyVault`` collection:: "encrypted": { "ENGINE": "django_mongodb_backend", "HOST": "mongodb+srv://cluster0.example.mongodb.net", - "NAME": "my_database_encrypted", - "USER": "my_user", - "PASSWORD": "my_password", - "PORT": 27017, + "NAME": "my_encrypted_database", + # ... "OPTIONS": { "auto_encryption_opts": AutoEncryptionOpts( - key_vault_namespace="encryption.__keyVault", - kms_providers={"local": {"key": os.urandom(96)}}, + key_vault_namespace="my_encrypted_database.__keyVault", + kms_providers={ + "local": { + # Generated by os.urandom(96) + "key": ( + b'-\xc3\x0c\xe3\x93\xc3\x8b\xc0\xf8\x12\xc5#b' + b'\x19\xf3\xbc\xccR\xc8\xedI\xda\\ \xfb\x9cB' + b'\x7f\xab5\xe7\xb5\xc9x\xb8\xd4d\xba\xdc\x9c' + b'\x9a\xdb9J]\xe6\xce\x104p\x079q.=\xeb\x9dK*' + b'\x97\xea\xf8\x1e\xc3\xd49K\x18\x81\xc3\x1a"' + b'\xdc\x00U\xc4u"X\xe7xy\xa5\xb2\x0e\xbc\xd6+-' + b'\x80\x03\xef\xc2\xc4\x9bU' + }, + }, ) }, }, } -.. admonition:: Local KMS provider key - - In the example above, a random key is generated for the local KMS provider - using ``os.urandom(96)``. In a production environment, you should securely - :ref:`store and manage your encryption keys - `. +``key_vault_namespace`` specifies where to store the data encryption keys. +The database name of the key vault must be the same as in ``"NAME"``. The +vault's collection name can be whatever you wish, but by convention, it's often +``__keyVault``. .. _qe-configuring-database-routers-setting: @@ -90,11 +88,11 @@ Configuring the ``DATABASE_ROUTERS`` setting Similar to configuring the :ref:`DATABASE_ROUTERS ` setting for -:doc:`embedded models `, Queryable Encryption -requires a :setting:`DATABASE_ROUTERS ` setting to -route database operations to the encrypted database. +:doc:`embedded models `, Queryable Encryption requires +a :setting:`DATABASE_ROUTERS` setting to route database operations to the +encrypted database. -The following example shows how to configure a router for the "myapp" +The following example shows how to configure a router for the ``"myapp"`` application that routes database operations to the encrypted database for all models in that application:: @@ -119,37 +117,37 @@ Then in your Django settings, add the custom database router to the :setting:`django:DATABASE_ROUTERS` setting:: # settings.py - DATABASE_ROUTERS = ["myapp.routers.EncryptedRouter"] + DATABASE_ROUTERS = [ + "django_mongodb_backend.routers.MongoRouter", + "myapp.routers.EncryptedRouter", + ] .. _qe-configuring-kms: Configuring the Key Management Service (KMS) ============================================ +A local KMS provider with a hardcoded key is suitable for local development and +testing, but production environment, you should securely :ref:`store and manage your +encryption keys `. + To use Queryable Encryption, you must configure a Key Management Service (KMS) -to store and manage your encryption keys. Django MongoDB Backend allows you to -configure multiple KMS providers and select the appropriate provider for each -model using a custom database router. - -The KMS is responsible for managing the encryption keys used to encrypt and -decrypt data. The following table summarizes the available KMS configuration -options followed by an example of how to use them. - -+-------------------------------------------------------------------------+--------------------------------------------------------+ -| :setting:`KMS_CREDENTIALS ` | A dictionary of Key Management Service (KMS) | -| | credentials configured in the | -| | :setting:`django:DATABASES` setting. | -+-------------------------------------------------------------------------+--------------------------------------------------------+ -| :class:`kms_providers ` | A dictionary of KMS provider credentials used to | -| | access the KMS with ``kms_provider``. | -+-------------------------------------------------------------------------+--------------------------------------------------------+ -| :ref:`kms_provider ` | A single KMS provider name | -| | configured in your custom database | -| | router. | -+-------------------------------------------------------------------------+--------------------------------------------------------+ - -Example of KMS configuration with ``aws`` in your :class:`kms_providers -` setting:: +to store and manage the encryption keys used to encrypt and decrypt data. + +There are two primary configuration points: + +#. The ``kms_providers`` parameter of + :class:`~pymongo.encryption_options.AutoEncryptionOpts` (see the + ``kms_providers`` parameter in + :class:`~pymongo.encryption_options.AutoEncryptionOpts` for the available + providers (``aws``, ``azure``, ``gcp``, etc.) and provider options). + +#. The :setting:`KMS_CREDENTIALS ` inner option of + :setting:`DATABASES`. The keys for each provider are documented under the + ``master_key`` parameter of + :meth:`~pymongo.encryption.ClientEncryption.create_data_key`. + +Here's an example of KMS configuration with ``aws``:: from pymongo.encryption_options import AutoEncryptionOpts @@ -169,8 +167,8 @@ Example of KMS configuration with ``aws`` in your :class:`kms_providers }, "KMS_CREDENTIALS": { "aws": { - "key": os.getenv("AWS_KEY_ARN", ""), - "region": os.getenv("AWS_KEY_REGION", ""), + "key": "...", # Amazon Resource Name + "region": "...", # AWS region }, }, }, @@ -193,22 +191,17 @@ the provider for each model in your :ref:`database router Configuring the ``encrypted_fields_map`` option =============================================== -When you configure the :ref:`DATABASES ` -setting for Queryable Encryption *without* specifying an -``encrypted_fields_map``, Django MongoDB Backend will create encrypted -collections, including encryption keys, when you :ref:`run migrations for models -that have encrypted fields `. +Encryption keys are created when you :ref:`run migrations for models that have +encrypted fields `. -Encryption keys for encrypted fields are stored in the key vault specified in -the :ref:`DATABASES ` setting. To see the keys created by -Django MongoDB Backend, along with the entire schema, you can run the -:djadmin:`showencryptedfieldsmap` command:: +To see the encrypted fields map for your models (which includes the encryption +key IDs), run the :djadmin:`showencryptedfieldsmap` command:: $ python manage.py showencryptedfieldsmap --database encrypted -Use the output of :djadmin:`showencryptedfieldsmap` to set the -``encrypted_fields_map`` in :class:`AutoEncryptionOpts -` in your Django settings:: +In a production environment, it's recommended to include this map in your +settings to protect against a malicious server advertising a false encrypted +fields map:: from bson import json_util from pymongo.encryption_options import AutoEncryptionOpts @@ -244,12 +237,6 @@ Use the output of :djadmin:`showencryptedfieldsmap` to set the }, } -.. admonition:: Security consideration - - Supplying an encrypted fields map provides more security than relying on an - encrypted fields map obtained from the server. It protects against a - malicious server advertising a false encrypted fields map. - Configuring the Automatic Encryption Shared Library =================================================== @@ -263,12 +250,13 @@ recommended for use with Queryable Encryption. You can :ref:`download the shared library ` from the -:ref:`manual:enterprise-official-packages` and configure it in your Django -settings using the ``crypt_shared_lib_path`` option in -:class:`AutoEncryptionOpts `. +:ref:`manual:enterprise-official-packages`. The shared library is +platform‑specific. Make sure to download the correct version for your operating +system and architecture. -The following example shows how to configure the shared library in your Django -settings:: +To configure it in your Django settings, use +:class:`~pymongo.encryption_options.AutoEncryptionOpts`\'s +``crypt_shared_lib_path`` parameter:: from pymongo.encryption_options import AutoEncryptionOpts @@ -281,18 +269,13 @@ settings:: crypt_shared_lib_path="/path/to/mongo_crypt_shared_v1.dylib", ) }, - # ... }, } - .. admonition:: Dynamic library path configuration - The Automatic Encryption Shared Library is platform‑specific. Make sure to - download the correct version for your operating system and architecture, - and configure your environment so the system can locate it. - - Use the following variables depending on your platform: + You may also need to configure an environment variable so that your system + can locate the library: +---------------+---------------------------------+ | **Platform** | **Environment Variable** | @@ -304,7 +287,7 @@ settings:: | Linux | ``LD_LIBRARY_PATH`` | +---------------+---------------------------------+ - For example on macOS, you can set the ``DYLD_FALLBACK_LIBRARY_PATH`` + For example, on macOS you can set the ``DYLD_FALLBACK_LIBRARY_PATH`` environment variable in your shell before starting your Django application:: $ export DYLD_FALLBACK_LIBRARY_PATH="/path/to/mongo_crypt_shared_v1.dylib:$DYLD_FALLBACK_LIBRARY_PATH" From b5a02cd9a798eaf29f704fb0b5d6616c8268a9f2 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Tue, 11 Nov 2025 20:38:38 -0500 Subject: [PATCH 05/26] doc query limitations + docs polish + todos --- django_mongodb_backend/features.py | 8 ++- docs/howto/queryable-encryption.rst | 2 +- docs/ref/django-admin.rst | 16 ++---- docs/ref/settings.rst | 2 - docs/topics/queryable-encryption.rst | 73 +++++++++++++++++++++------- 5 files changed, 67 insertions(+), 34 deletions(-) diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index 7cc0b9ebb..be452c646 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -635,8 +635,12 @@ def _supports_transactions(self): @cached_property def supports_queryable_encryption(self): """ - Queryable Encryption requires a MongoDB 8.0 or later replica set or sharded - cluster, as well as MongoDB Atlas or Enterprise. + For testing purposes, Queryable Encryption requires a MongoDB 8.0 or + later replica set or sharded cluster, as well as MongoDB Atlas or + Enterprise. This flag must not guard any non-test functionality since + it would prevent MongoDB 7.0 from being used, which also supports + Queryable Encryption. The models in tests/encryption_ aren't compatible + with MongoDB 7.0 because {"queryType": "range"} being "rangePreview". """ self.connection.ensure_connection() build_info = self.connection.connection.admin.command("buildInfo") diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index 81709b61f..4489831f3 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -14,7 +14,7 @@ Encryption in your Django project. .. admonition:: MongoDB requirements Queryable Encryption can be used with MongoDB replica sets or sharded - clusters running version 8.0 or later. Standalone instances are not + clusters running version 7.0 or later. Standalone instances are not supported. The :ref:`manual:qe-compatibility-reference` table summarizes which MongoDB server products support Queryable Encryption. diff --git a/docs/ref/django-admin.rst b/docs/ref/django-admin.rst index 1e111eee8..3f4858939 100644 --- a/docs/ref/django-admin.rst +++ b/docs/ref/django-admin.rst @@ -21,18 +21,12 @@ Available commands .. django-admin:: showencryptedfieldsmap - This command shows the mapping of encrypted fields to attributes including - data type, data keys and query types. Its output can be used to set the - :ref:`encrypted_fields_map ` argument - in :class:`AutoEncryptionOpts - `. + This command generates output for includision in + :class:`~pymongo.encryption_options.AutoEncryptionOpts`\'s + ``encrypted_fields_map`` argument. + + See :ref:`qe-configuring-encrypted-fields-map`. .. django-admin-option:: --database DATABASE Specifies the database to use. Defaults to ``default``. - - To show the encrypted fields map for a database named ``encrypted``, run: - - .. code-block:: console - - $ python manage.py showencryptedfieldsmap --database encrypted diff --git a/docs/ref/settings.rst b/docs/ref/settings.rst index 233515262..6aba8a75a 100644 --- a/docs/ref/settings.rst +++ b/docs/ref/settings.rst @@ -2,8 +2,6 @@ Settings ======== -.. _queryable-encryption-settings: - Queryable Encryption ==================== diff --git a/docs/topics/queryable-encryption.rst b/docs/topics/queryable-encryption.rst index 0d6372f69..d641ad156 100644 --- a/docs/topics/queryable-encryption.rst +++ b/docs/topics/queryable-encryption.rst @@ -5,8 +5,8 @@ Queryable Encryption .. versionadded:: 5.2.3 Once you have successfully set up MongoDB Queryable Encryption as described in -:doc:`the installation guide `, you can start -using encrypted fields in your Django models. +:doc:`/howto/queryable-encryption`, you can start using encrypted fields in +your Django models. Encrypted fields ================ @@ -59,6 +59,8 @@ Here are models based on the `Python Queryable Encryption Tutorial`_:: Migrations ---------- +# TODO: this is coupled to the howto + Once you have defined your models, create migrations with: .. code-block:: console @@ -76,9 +78,14 @@ model data. The fields will automatically handle encryption and decryption, ensuring that :ref:`sensitive data is stored securely in the database `. +# TODO: once encrypted models are migrated, encrypted fields cannot be added or +modified. + Routers ------- +# TODO: this is redundant with the howto. + The example above requires a :ref:`database router ` to direct operations on models with encrypted fields to the appropriate database. It also requires the use of a @@ -96,16 +103,15 @@ Querying encrypted fields In order to query encrypted fields, you must define the queryable encryption query type in the model field definition. For example, if you want to query the -``ssn`` field for equality, you can define it as follows:: +``ssn`` field for equality, you can add the ``queries`` argument:: class PatientRecord(EmbeddedModel): ssn = EncryptedCharField(max_length=11, queries={"queryType": "equality"}) - billing = EncryptedEmbeddedModelField("Billing") - bill_amount = models.DecimalField(max_digits=10, decimal_places=2) -Then you can perform a query like this: +Then you can perform a equality query just like you would on a non-encrypted +field: -.. code-block:: console +.. code-block:: pycon >>> patient = Patient.objects.get(patient_record__ssn="123-45-6789") >>> patient.name @@ -116,13 +122,13 @@ Then you can perform a query like this: Available query types ~~~~~~~~~~~~~~~~~~~~~ -The ``queries`` option should be a dictionary that specifies the type of queries -that can be performed on the field. Of the :ref:`available query types -` Django MongoDB Backend currently -supports: +The ``queries`` option should be a dictionary that specifies the type of +queries that can be performed on the field, as well as any query options. The +:ref:`available query types ` depend on +your version of MongoDB. -- ``equality`` -- ``range`` +For example, in MongoDB 8.0, the supported types are ``equality`` and +``range``. .. admonition:: Query types vs. Django lookups @@ -131,10 +137,41 @@ supports: perform comparisons on encrypted fields, while Django's range lookups are used for filtering based on a range of values. -QuerySet limitations -~~~~~~~~~~~~~~~~~~~~ +``QuerySet`` limitations +~~~~~~~~~~~~~~~~~~~~~~~~ In addition to :ref:`Django MongoDB Backend's QuerySet limitations -`, - -.. TODO +`, some ``QuerySet`` methods aren't +supported on encrypted fields. Each unsupported method is followed by a sample +error message from the database. Depending on the exact query, error messages +may vary. + +- :meth:`~django.db.models.query.QuerySet.order_by`: Cannot add an encrypted + field as a prefix of another encrypted field. +- :meth:`~django.db.models.query.QuerySet.alias`, + :meth:`~django.db.models.query.QuerySet.annotate`, + :meth:`~django.db.models.query.QuerySet.distinct`: Cannot group on field + '_id.value' which is encrypted with the random algorithm or whose encryption + properties are not known until runtime. +- :meth:`~django.db.models.query.QuerySet.dates`, + :meth:`~django.db.models.query.QuerySet.datetimes`: If the value type is a + date, the type of the index must also be date (and vice versa). +- :meth:`~django.db.models.query.QuerySet.in_bulk`: Encrypted fields can't have + unique constraints. + +# TODO: add details about joined queries after +https://github.com/mongodb/django-mongodb-backend/pull/443 is finalized. + +There are also several ``QuerySet`` methods that aren't permitted on any models +(regardless of whether or not they have encrypted fields) that use a database +connection with Automatic Encryption. Each unsupported method is followed by a +sample error message from the database. + +- :meth:`~django.db.models.query.QuerySet.update`: Multi-document updates are + not allowed with Queryable Encryption. +- :meth:`~django.db.models.query.QuerySet.aggregate`, + :meth:`~django.db.models.query.QuerySet.count`: Aggregation stage + $internalFacetTeeConsumer is not allowed or supported with automatic + encryption. +- :meth:`~django.db.models.query.QuerySet.union`: Aggregation stage $unionWith + is not allowed or supported with automatic encryption. From f33d0a45daf3e1b05d6bb8319f22675f5ee78db4 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 12 Nov 2025 17:27:43 -0500 Subject: [PATCH 06/26] edit "Dynamic library path configuration" --- docs/howto/queryable-encryption.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index 4489831f3..a261457ec 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -274,8 +274,11 @@ To configure it in your Django settings, use .. admonition:: Dynamic library path configuration - You may also need to configure an environment variable so that your system - can locate the library: + If you encounter ``Pymongocrypt.errors.MongoCryptError: An existing + crypt_shared library is loaded by the application at + [/path/to/mongo_crypt_v1.so], but the current call to mongocrypt_init() + failed to find that same library.``, you probably need to configure an + environment variable so that your system can locate the library: +---------------+---------------------------------+ | **Platform** | **Environment Variable** | @@ -290,7 +293,7 @@ To configure it in your Django settings, use For example, on macOS you can set the ``DYLD_FALLBACK_LIBRARY_PATH`` environment variable in your shell before starting your Django application:: - $ export DYLD_FALLBACK_LIBRARY_PATH="/path/to/mongo_crypt_shared_v1.dylib:$DYLD_FALLBACK_LIBRARY_PATH" + $ export DYLD_FALLBACK_LIBRARY_PATH="/path/to/mongo_crypt_shared/:$DYLD_FALLBACK_LIBRARY_PATH" You are now ready to :doc:`start developing applications ` with Queryable Encryption! From b1ad80e1eb6558c23fb335841024e01f2305512a Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 12 Nov 2025 20:36:43 -0500 Subject: [PATCH 07/26] combine topic guide with howto --- docs/howto/queryable-encryption.rst | 96 ++++++++++++++- docs/index.rst | 1 - docs/ref/models/encrypted-fields.rst | 131 +++++++++++++++----- docs/topics/index.rst | 1 - docs/topics/queryable-encryption.rst | 177 --------------------------- 5 files changed, 188 insertions(+), 218 deletions(-) delete mode 100644 docs/topics/queryable-encryption.rst diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index a261457ec..34ec345bb 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -122,14 +122,103 @@ Then in your Django settings, add the custom database router to the "myapp.routers.EncryptedRouter", ] +Encrypted fields +================ + +Now you can start using encrypted fields in your Django models. + +:doc:`Encrypted fields ` may be used to protect +sensitive data like social security numbers, credit card information, or +personal health information. With Queryable Encryption, you can also perform +queries on encrypted fields. To use encrypted fields in your models, +import the necessary field types from ``django_mongodb_backend.models`` and +define your models as usual. + +Here are models based on the `Python Queryable Encryption Tutorial`_:: + + # myapp/models.py + from django.db import models + from django_mongodb_backend.models import EmbeddedModel + from django_mongodb_backend.fields import ( + EmbeddedModelField, + EncryptedCharField, + EncryptedEmbeddedModelField, + ) + + + class PatientRecord(EmbeddedModel): + ssn = EncryptedCharField(max_length=11, queries={"queryType": "equality"}) + billing = EncryptedEmbeddedModelField("Billing") + bill_amount = models.DecimalField(max_digits=10, decimal_places=2) + + class Patient(models.Model): + patient_name = models.CharField(max_length=255) + patient_id = models.BigIntegerField() + patient_record = EmbeddedModelField("PatientRecord") + + def __str__(self): + return f"{self.patient_name} ({self.patient_id})" + + class Billing(EmbeddedModel): + cc_type = models.CharField(max_length=50) + cc_number = models.CharField(max_length=20) + +.. _Python Queryable Encryption Tutorial: https://github.com/mongodb/docs/tree/main/content/manual/manual/source/includes/qe-tutorials/python + +.. _qe-migrations: + +Migrations +========== + +Once you have defined your models, create a migration as usual: + +.. code-block:: console + + $ python manage.py makemigrations + +Then run the migrations with: + +.. code-block:: console + + $ python manage.py migrate --database encrypted + +.. warning:: + + Be aware that you cannot add encrypted fields to existing models, nor can + you change the definition of an encrypted field, for example, to make it + queryable. + +Now create and manipulate instances of the data just like any other Django +model data. The fields will automatically handle encryption and decryption, +ensuring that :ref:`sensitive data is stored securely in the database +`. + +Querying encrypted fields +========================= + +In order to query encrypted fields, you must include the :ref:`queries +` argument. For example, notice ``PatientRecord``\'s +``ssn`` field:: + + class PatientRecord(EmbeddedModel): + ssn = EncryptedCharField(max_length=11, queries={"queryType": "equality"}) + +You can perform a equality query just like you would on a non-encrypted field: + +.. code-block:: pycon + + >>> patient = Patient.objects.get(patient_record__ssn="123-45-6789") + >>> patient.name + 'John Doe' + .. _qe-configuring-kms: Configuring the Key Management Service (KMS) ============================================ A local KMS provider with a hardcoded key is suitable for local development and -testing, but production environment, you should securely :ref:`store and manage your -encryption keys `. +testing, but production environment, you should securely :ref:`store and manage +your encryption keys `. To use Queryable Encryption, you must configure a Key Management Service (KMS) to store and manage the encryption keys used to encrypt and decrypt data. @@ -294,6 +383,3 @@ To configure it in your Django settings, use environment variable in your shell before starting your Django application:: $ export DYLD_FALLBACK_LIBRARY_PATH="/path/to/mongo_crypt_shared/:$DYLD_FALLBACK_LIBRARY_PATH" - -You are now ready to :doc:`start developing applications -` with Queryable Encryption! diff --git a/docs/index.rst b/docs/index.rst index 7e3690249..d332e9822 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -51,7 +51,6 @@ Models **Topic guides:** - :doc:`topics/embedded-models` -- :doc:`topics/queryable-encryption` - :doc:`topics/transactions` Forms diff --git a/docs/ref/models/encrypted-fields.rst b/docs/ref/models/encrypted-fields.rst index f09dfa168..c102ef03a 100644 --- a/docs/ref/models/encrypted-fields.rst +++ b/docs/ref/models/encrypted-fields.rst @@ -2,15 +2,12 @@ Encrypted fields ================ -.. versionadded:: 5.2.3 - -Django MongoDB Backend supports :doc:`manual:core/queryable-encryption`. +.. currentmodule:: django_mongodb_backend.fields -See :doc:`/howto/queryable-encryption` for more information on how to use -Queryable Encryption with Django MongoDB Backend. +.. versionadded:: 5.2.3 -See the :doc:`/topics/queryable-encryption` topic guide for -more information on developing applications with Queryable Encryption. +To use encrypted fields, you must :doc:`configure Queryable Encryption +`. The following tables detailed which fields have encrypted counterparts. In all cases, the encrypted field names are simply prefixed with ``Encrypted``, e.g. @@ -48,19 +45,62 @@ cases, the encrypted field names are simply prefixed with ``Encrypted``, e.g. .. csv-table:: ``django_mongodb_backend.fields`` :header: "Model Field", "Encrypted version available?" - :class:`~.fields.ArrayField`, Yes - :class:`~.fields.EmbeddedModelArrayField`, Yes - :class:`~.fields.EmbeddedModelField`, Yes - :class:`~.fields.ObjectIdField`, Yes - :class:`~.fields.PolymorphicEmbeddedModelField`, No: may be implemented in the future. - :class:`~.fields.PolymorphicEmbeddedModelArrayField`, No: may be implemented in the future. + :class:`ArrayField`, Yes + :class:`EmbeddedModelArrayField`, Yes + :class:`EmbeddedModelField`, Yes + :class:`ObjectIdField`, Yes + :class:`PolymorphicEmbeddedModelField`, No: may be implemented in the future. + :class:`PolymorphicEmbeddedModelArrayField`, No: may be implemented in the future. + +.. _encrypted-fields-queries: + +``EncryptedField.queries`` +-------------------------- + +Most encrypted fields* take an optional ``queries`` argument. It's a dictionary +that specifies the type of queries that can be performed on the field, as well +as any query options. -These fields don't support the ``queries`` argument: +The :ref:`available query types ` depend +on your version of MongoDB. For example, in MongoDB 8.0, the supported types +are ``equality`` and ``range``. + +.. admonition:: Query types vs. Django lookups + + Range queries in Queryable Encryption are different from Django's + :ref:`range lookups `. Range queries allow you to + perform comparisons on encrypted fields, while Django's range lookups are + used for filtering based on a range of values. + +\* These fields don't support the ``queries`` argument: - ``EncryptedArrayField`` - ``EncryptedEmbeddedModelArrayField`` - ``EncryptedEmbeddedModelField`` +Embedded model encryption +========================= + +There are two ways to encrypt embedded models. You can either encrypt the +entire subdocument, in which case you can't query any the subdocuments fields, +or you can encrypt only selected fields of the subdocument. + +Encrypting the entire subdocument +--------------------------------- + +To encrypt a subdocument, use ``EncryptedEmbeddedModelField`` or +``EncryptedEmbeddedModelArrayField``. In this case, the field's embedded model +cannot have any encrypted fields. + +Encrypting selected fields of a subdocument +------------------------------------------- + +To encrypt only select fields of a subdocument, use :class:`EmbeddedModelField` +and any of the other encrypted fields on the embedded model. + +MongoDB doesn't support encrypting selected fields of +``EmbeddedModelArrayField``. + Limitations =========== @@ -70,6 +110,45 @@ MongoDB imposes some restrictions on encrypted fields: * They cannot be part of a unique constraint. * They cannot be null. +``QuerySet`` limitations +------------------------ + +In addition to :ref:`Django MongoDB Backend's QuerySet limitations +`, some ``QuerySet`` methods aren't +supported on encrypted fields. Each unsupported method is followed by a sample +error message from the database. Depending on the exact query, error messages +may vary. + +- :meth:`~django.db.models.query.QuerySet.order_by`: Cannot add an encrypted + field as a prefix of another encrypted field. +- :meth:`~django.db.models.query.QuerySet.alias`, + :meth:`~django.db.models.query.QuerySet.annotate`, + :meth:`~django.db.models.query.QuerySet.distinct`: Cannot group on field + '_id.value' which is encrypted with the random algorithm or whose encryption + properties are not known until runtime. +- :meth:`~django.db.models.query.QuerySet.dates`, + :meth:`~django.db.models.query.QuerySet.datetimes`: If the value type is a + date, the type of the index must also be date (and vice versa). +- :meth:`~django.db.models.query.QuerySet.in_bulk`: Encrypted fields can't have + unique constraints. + +# TODO: add details about joined queries after +https://github.com/mongodb/django-mongodb-backend/pull/443 is finalized. + +There are also several ``QuerySet`` methods that aren't permitted on any models +(regardless of whether or not they have encrypted fields) that use a database +connection with Automatic Encryption. Each unsupported method is followed by a +sample error message from the database. + +- :meth:`~django.db.models.query.QuerySet.update`: Multi-document updates are + not allowed with Queryable Encryption. +- :meth:`~django.db.models.query.QuerySet.aggregate`, + :meth:`~django.db.models.query.QuerySet.count`: Aggregation stage + $internalFacetTeeConsumer is not allowed or supported with automatic + encryption. +- :meth:`~django.db.models.query.QuerySet.union`: Aggregation stage $unionWith + is not allowed or supported with automatic encryption. + ``EncryptedFieldMixin`` ======================= @@ -77,13 +156,8 @@ MongoDB imposes some restrictions on encrypted fields: .. versionadded:: 5.2.3 - A mixin that can be used to create custom encrypted fields with Queryable - Encryption. - - To create an encrypted field, inherit from ``EncryptedFieldMixin`` and - your custom field class: - - .. code-block:: python + Use this mixin to create encrypted versions of your own custom fields. For + example, to create an encrypted version of ``MyField``:: from django.db import models from django_mongodb_backend.fields import EncryptedFieldMixin @@ -93,16 +167,5 @@ MongoDB imposes some restrictions on encrypted fields: class MyEncryptedField(EncryptedFieldMixin, MyField): pass - - You can then use your custom encrypted field in a model, specifying the - desired query types: - - .. code-block:: python - - class MyModel(models.Model): - my_encrypted_field = MyEncryptedField( - queries={"queryType": "equality"}, - ) - my_encrypted_field_too = MyEncryptedField( - queries={"queryType": "range"}, - ) + This adds the :ref:`queries ` argument to the + field. diff --git a/docs/topics/index.rst b/docs/topics/index.rst index a02b35239..6e06b8125 100644 --- a/docs/topics/index.rst +++ b/docs/topics/index.rst @@ -9,6 +9,5 @@ know: :maxdepth: 2 embedded-models - queryable-encryption transactions known-issues diff --git a/docs/topics/queryable-encryption.rst b/docs/topics/queryable-encryption.rst deleted file mode 100644 index d641ad156..000000000 --- a/docs/topics/queryable-encryption.rst +++ /dev/null @@ -1,177 +0,0 @@ -==================== -Queryable Encryption -==================== - -.. versionadded:: 5.2.3 - -Once you have successfully set up MongoDB Queryable Encryption as described in -:doc:`/howto/queryable-encryption`, you can start using encrypted fields in -your Django models. - -Encrypted fields -================ - -The basics ----------- - -:doc:`Encrypted fields ` may be used to protect -sensitive data like social security numbers, credit card information, or -personal health information. With Queryable Encryption, you can also perform -queries on certain encrypted fields. To use encrypted fields in your models, -import the necessary field types from ``django_mongodb_backend.models`` and -define your models as usual. - -Here are models based on the `Python Queryable Encryption Tutorial`_:: - - # myapp/models.py - from django.db import models - from django_mongodb_backend.models import EmbeddedModel - from django_mongodb_backend.fields import ( - EmbeddedModelField, - EncryptedCharField, - EncryptedEmbeddedModelField, - ) - - - class Patient(models.Model): - patient_name = models.CharField(max_length=255) - patient_id = models.BigIntegerField() - patient_record = EmbeddedModelField("PatientRecord") - - def __str__(self): - return f"{self.patient_name} ({self.patient_id})" - - - class PatientRecord(EmbeddedModel): - ssn = EncryptedCharField(max_length=11) - billing = EncryptedEmbeddedModelField("Billing") - bill_amount = models.DecimalField(max_digits=10, decimal_places=2) - - - class Billing(EmbeddedModel): - cc_type = models.CharField(max_length=50) - cc_number = models.CharField(max_length=20) - -.. _Python Queryable Encryption Tutorial: https://github.com/mongodb/docs/tree/main/content/manual/manual/source/includes/qe-tutorials/python - -.. _qe-migrations: - -Migrations ----------- - -# TODO: this is coupled to the howto - -Once you have defined your models, create migrations with: - -.. code-block:: console - - $ python manage.py makemigrations - -Then run the migrations with: - -.. code-block:: console - - $ python manage.py migrate --database encrypted - -Now create and manipulate instances of the data just like any other Django -model data. The fields will automatically handle encryption and decryption, -ensuring that :ref:`sensitive data is stored securely in the database -`. - -# TODO: once encrypted models are migrated, encrypted fields cannot be added or -modified. - -Routers -------- - -# TODO: this is redundant with the howto. - -The example above requires a :ref:`database router -` to direct operations on models with -encrypted fields to the appropriate database. It also requires the use of a -:ref:`router for embedded models `. Here -is an example that includes both:: - - # myproject/settings.py - DATABASE_ROUTERS = [ - "django_mongodb_backend.routers.MongoRouter", - "myproject.routers.EncryptedRouter", - ] - -Querying encrypted fields -------------------------- - -In order to query encrypted fields, you must define the queryable encryption -query type in the model field definition. For example, if you want to query the -``ssn`` field for equality, you can add the ``queries`` argument:: - - class PatientRecord(EmbeddedModel): - ssn = EncryptedCharField(max_length=11, queries={"queryType": "equality"}) - -Then you can perform a equality query just like you would on a non-encrypted -field: - -.. code-block:: pycon - - >>> patient = Patient.objects.get(patient_record__ssn="123-45-6789") - >>> patient.name - 'John Doe' - -.. _qe-available-query-types: - -Available query types -~~~~~~~~~~~~~~~~~~~~~ - -The ``queries`` option should be a dictionary that specifies the type of -queries that can be performed on the field, as well as any query options. The -:ref:`available query types ` depend on -your version of MongoDB. - -For example, in MongoDB 8.0, the supported types are ``equality`` and -``range``. - -.. admonition:: Query types vs. Django lookups - - Range queries in Queryable Encryption are different from Django's - :ref:`range lookups `. Range queries allow you to - perform comparisons on encrypted fields, while Django's range lookups are - used for filtering based on a range of values. - -``QuerySet`` limitations -~~~~~~~~~~~~~~~~~~~~~~~~ - -In addition to :ref:`Django MongoDB Backend's QuerySet limitations -`, some ``QuerySet`` methods aren't -supported on encrypted fields. Each unsupported method is followed by a sample -error message from the database. Depending on the exact query, error messages -may vary. - -- :meth:`~django.db.models.query.QuerySet.order_by`: Cannot add an encrypted - field as a prefix of another encrypted field. -- :meth:`~django.db.models.query.QuerySet.alias`, - :meth:`~django.db.models.query.QuerySet.annotate`, - :meth:`~django.db.models.query.QuerySet.distinct`: Cannot group on field - '_id.value' which is encrypted with the random algorithm or whose encryption - properties are not known until runtime. -- :meth:`~django.db.models.query.QuerySet.dates`, - :meth:`~django.db.models.query.QuerySet.datetimes`: If the value type is a - date, the type of the index must also be date (and vice versa). -- :meth:`~django.db.models.query.QuerySet.in_bulk`: Encrypted fields can't have - unique constraints. - -# TODO: add details about joined queries after -https://github.com/mongodb/django-mongodb-backend/pull/443 is finalized. - -There are also several ``QuerySet`` methods that aren't permitted on any models -(regardless of whether or not they have encrypted fields) that use a database -connection with Automatic Encryption. Each unsupported method is followed by a -sample error message from the database. - -- :meth:`~django.db.models.query.QuerySet.update`: Multi-document updates are - not allowed with Queryable Encryption. -- :meth:`~django.db.models.query.QuerySet.aggregate`, - :meth:`~django.db.models.query.QuerySet.count`: Aggregation stage - $internalFacetTeeConsumer is not allowed or supported with automatic - encryption. -- :meth:`~django.db.models.query.QuerySet.union`: Aggregation stage $unionWith - is not allowed or supported with automatic encryption. From 7404fa9050dd528feb85928f78e042f486569267 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Fri, 14 Nov 2025 20:47:12 -0500 Subject: [PATCH 08/26] Add "start csfle servers" func to evergreen config Via https://github.com/mongodb-labs/drivers-evergreen-tools/tree/master/.evergreen/csfle --- .evergreen/config.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 40046f4a8..366ad27d8 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -12,6 +12,17 @@ post_error_fails_task: true post_timeout_secs: 1800 # 5 minutes functions: + "start csfle servers": + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} + - command: subprocess.exec + params: + binary: bash + include_expansions_in_env: ["AWS_SECRET_ACCESS_KEY", "AWS_ACCESS_KEY_ID", "AWS_SESSION_TOKEN"] + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/setup.sh + "setup": - command: git.get_project params: @@ -54,6 +65,7 @@ functions: binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/teardown.sh + - ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh pre: - func: setup @@ -65,6 +77,7 @@ post: tasks: - name: run-tests commands: + - func: "start csfle servers" - func: "run unit tests" buildvariants: From 6fc5f8d5aae60ff175bc332a2e0f7d2e52d2216b Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 14 Nov 2025 20:16:11 -0500 Subject: [PATCH 09/26] remove support for multiple kms providers --- django_mongodb_backend/__init__.py | 2 -- django_mongodb_backend/routers.py | 21 --------------------- django_mongodb_backend/schema.py | 11 +++++++---- docs/howto/queryable-encryption.rst | 12 ------------ tests/encryption_/test_schema.py | 22 +++++++++++++++++++++- 5 files changed, 28 insertions(+), 40 deletions(-) diff --git a/django_mongodb_backend/__init__.py b/django_mongodb_backend/__init__.py index d1d9cee5d..ed0deadef 100644 --- a/django_mongodb_backend/__init__.py +++ b/django_mongodb_backend/__init__.py @@ -13,7 +13,6 @@ from .indexes import register_indexes # noqa: E402 from .lookups import register_lookups # noqa: E402 from .query import register_nodes # noqa: E402 -from .routers import register_routers # noqa: E402 register_aggregates() register_expressions() @@ -22,4 +21,3 @@ register_indexes() register_lookups() register_nodes() -register_routers() diff --git a/django_mongodb_backend/routers.py b/django_mongodb_backend/routers.py index b17f4b021..4b75efbe8 100644 --- a/django_mongodb_backend/routers.py +++ b/django_mongodb_backend/routers.py @@ -1,6 +1,4 @@ from django.apps import apps -from django.core.exceptions import ImproperlyConfigured -from django.db.utils import ConnectionRouter class MongoRouter: @@ -18,22 +16,3 @@ def allow_migrate(self, db, app_label, model_name=None, **hints): except LookupError: return None return False if issubclass(model, EmbeddedModel) else None - - -# This function is intended to be monkey-patched as a method of ConnectionRouter. -def kms_provider(self, model, *args, **kwargs): - """ - Return the Key Management Service (KMS) provider for a given model. - - Call each router's kms_provider() method (if present), and return the - first non-None result. Raise ImproperlyConfigured if no provider is found. - """ - for router in self.routers: - func = getattr(router, "kms_provider", None) - if func and callable(func) and (result := func(model, *args, **kwargs)): - return result - raise ImproperlyConfigured("No kms_provider found in database routers.") - - -def register_routers(): - ConnectionRouter.kms_provider = kms_provider diff --git a/django_mongodb_backend/schema.py b/django_mongodb_backend/schema.py index 4ee0009fe..17d836f87 100644 --- a/django_mongodb_backend/schema.py +++ b/django_mongodb_backend/schema.py @@ -1,7 +1,7 @@ from time import monotonic, sleep from django.core.exceptions import ImproperlyConfigured -from django.db import router +from django.db import NotSupportedError from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.models import Index, UniqueConstraint from pymongo.operations import SearchIndexModel @@ -508,9 +508,12 @@ def _get_encrypted_fields( if len(kms_providers) == 1: # If one provider is configured, no need to consult the router. kms_provider = next(iter(kms_providers.keys())) - else: - # Otherwise, call the user-defined router.kms_provider(). - kms_provider = router.kms_provider(model) + else: # (Since PyMongo requires at least one KMS provider.) + raise NotSupportedError( + "Multiple KMS providers per database aren't supported. " + "Please create a feature request with details about your " + "use case." + ) if kms_provider == "local": master_key = None else: diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index 34ec345bb..e9b98122a 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -263,18 +263,6 @@ Here's an example of KMS configuration with ``aws``:: }, } -(TODO: If there's a use case for multiple providers, motivate with a use case -and add a test.) - -If you've configured multiple KMS providers, you must define logic to determine -the provider for each model in your :ref:`database router -`:: - - class EncryptedRouter: - # ... - def kms_provider(self, model, **hints): - return "aws" - .. _qe-configuring-encrypted-fields-map: Configuring the ``encrypted_fields_map`` option diff --git a/tests/encryption_/test_schema.py b/tests/encryption_/test_schema.py index ae1803389..05b54aefd 100644 --- a/tests/encryption_/test_schema.py +++ b/tests/encryption_/test_schema.py @@ -1,6 +1,6 @@ from bson.binary import Binary from django.core.exceptions import ImproperlyConfigured -from django.db import connections +from django.db import NotSupportedError, connections from . import models from .models import EncryptionKey @@ -150,3 +150,23 @@ def test_missing_auto_encryption_opts(self): connection.schema_editor() as editor, ): editor.create_model(models.Patient) + + def test_multiple_kms_providers(self): + connection = connections["encrypted"] + auto_encryption_opts = connection.connection._options.auto_encryption_opts + kms_providers = auto_encryption_opts._kms_providers + # Mock multiple kms_providers by using a list of length > 1. + auto_encryption_opts._kms_providers = [{}, {}] + msg = ( + "Multiple KMS providers per database aren't supported. Please " + "create a feature request with details about your use case." + ) + try: + with ( + self.assertRaisesMessage(NotSupportedError, msg), + connection.schema_editor() as editor, + ): + editor.create_model(models.Patient) + finally: + # Restore the original value. + auto_encryption_opts._kms_providers = kms_providers From c4bb896627da05159ff566275ae0d4a13d07b446 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Sun, 16 Nov 2025 22:49:03 -0500 Subject: [PATCH 10/26] Add tests-8-qe to evergreen buildvariants --- .evergreen/config.yml | 12 +++++++++++- .evergreen/run-tests.sh | 2 +- .github/workflows/encrypted_settings.py | 5 +++-- .github/workflows/test-python-atlas.yml | 1 + 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 366ad27d8..919ad6ced 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -55,7 +55,7 @@ functions: params: binary: bash working_dir: "src" - include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI"] + include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI", "DJANGO_SETTINGS_MODULE", "CRYPT_SHARED_LIB_PATH"] args: - ./.evergreen/run-tests.sh @@ -124,3 +124,13 @@ buildvariants: SSL: "ssl" tasks: - name: run-tests + + - name: tests-8-qe + display_name: Run Tests 8.0 QE + run_on: rhel94-perf-atlas + expansions: + MONGODB_VERSION: "8.2" + TOPOLOGY: replica_set + DJANGO_SETTINGS_MODULE: "encrypted_settings" + tasks: + - name: run-tests diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 46f02be16..a8579648e 100644 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -6,7 +6,7 @@ set -eux /opt/python/3.12/bin/python3 -m venv venv . venv/bin/activate python -m pip install -U pip -pip install -e . +pip install -e '.[encryption]' # Install django and test dependencies git clone --branch mongodb-6.0.x https://github.com/mongodb-forks/django django_repo diff --git a/.github/workflows/encrypted_settings.py b/.github/workflows/encrypted_settings.py index 0436b04f7..1efdb38b4 100644 --- a/.github/workflows/encrypted_settings.py +++ b/.github/workflows/encrypted_settings.py @@ -1,10 +1,11 @@ # Settings for django_mongodb_backend/tests when encryption is supported. import os +from pathlib import Path from mongodb_settings import * # noqa: F403 from pymongo.encryption import AutoEncryptionOpts -os.environ["LD_LIBRARY_PATH"] = os.environ["GITHUB_WORKSPACE"] + "/lib/" +os.environ["LD_LIBRARY_PATH"] = str(Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent) DATABASES["encrypted"] = { # noqa: F405 "ENGINE": "django_mongodb_backend", @@ -13,7 +14,7 @@ "auto_encryption_opts": AutoEncryptionOpts( key_vault_namespace="djangotests_encrypted.__keyVault", kms_providers={"local": {"key": os.urandom(96)}}, - crypt_shared_lib_path=os.environ["GITHUB_WORKSPACE"] + "/lib/mongo_crypt_v1.so", + crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"], ), "directConnection": True, }, diff --git a/.github/workflows/test-python-atlas.yml b/.github/workflows/test-python-atlas.yml index 5f1900b70..dda600bb7 100644 --- a/.github/workflows/test-python-atlas.yml +++ b/.github/workflows/test-python-atlas.yml @@ -62,3 +62,4 @@ jobs: contents: read env: DJANGO_SETTINGS_MODULE: "encrypted_settings" + CRYPT_SHARED_LIB_PATH: "${{ github.workspace }}/lib/mongo_crypt_v1.so" From abe29b711d91730460f92edd4ac5ab1db6e92fd5 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Mon, 17 Nov 2025 21:00:09 -0500 Subject: [PATCH 11/26] fix less than lookup on encrypted fields --- django_mongodb_backend/lookups.py | 24 ++++++++++++++++ docs/ref/models/encrypted-fields.rst | 10 +++---- tests/encryption_/test_fields.py | 41 ++++++++++++++++++++++------ 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/django_mongodb_backend/lookups.py b/django_mongodb_backend/lookups.py index 0174ff0ee..e4afa2842 100644 --- a/django_mongodb_backend/lookups.py +++ b/django_mongodb_backend/lookups.py @@ -4,6 +4,8 @@ BuiltinLookup, FieldGetDbPrepValueIterableMixin, IsNull, + LessThan, + LessThanOrEqual, Lookup, PatternLookup, UUIDTextMixin, @@ -88,6 +90,26 @@ def is_null_path(self, compiler, connection): return connection.mongo_operators["isnull"](lhs_mql, self.rhs) +def less_than_path(self, compiler, connection): + lhs_mql = process_lhs(self, compiler, connection) + value = process_rhs(self, compiler, connection) + # Encrypted fields don't support null and Automatic Encryption cannot + # handle it ("csfle "analyze_query" failed: typenull type isn't supported + # for the range encrypted index.), so omit the null check. + if getattr(self.lhs.output_field, "encrypted", False): + return {lhs_mql: {"$lt": value}} + return connection.mongo_operators[self.lookup_name](lhs_mql, value) + + +def less_than_or_equal_path(self, compiler, connection): + lhs_mql = process_lhs(self, compiler, connection) + value = process_rhs(self, compiler, connection) + # Same comment as less_than_path. + if getattr(self.lhs.output_field, "encrypted", False): + return {lhs_mql: {"$lte": value}} + return connection.mongo_operators[self.lookup_name](lhs_mql, value) + + # from https://www.pcre.org/current/doc/html/pcre2pattern.html#SEC4 REGEX_MATCH_ESCAPE_CHARS = ( ("\\", r"\\"), # general escape character @@ -144,6 +166,8 @@ def register_lookups(): In.get_subquery_wrapping_pipeline = get_subquery_wrapping_pipeline IsNull.as_mql_expr = is_null_expr IsNull.as_mql_path = is_null_path + LessThan.as_mql_path = less_than_path + LessThanOrEqual.as_mql_path = less_than_or_equal_path Lookup.can_use_path = lookup_can_use_path PatternLookup.prep_lookup_value_mongo = pattern_lookup_prep_lookup_value UUIDTextMixin.as_mql = uuid_text_mixin diff --git a/docs/ref/models/encrypted-fields.rst b/docs/ref/models/encrypted-fields.rst index c102ef03a..34b891f1f 100644 --- a/docs/ref/models/encrypted-fields.rst +++ b/docs/ref/models/encrypted-fields.rst @@ -65,12 +65,12 @@ The :ref:`available query types ` depend on your version of MongoDB. For example, in MongoDB 8.0, the supported types are ``equality`` and ``range``. -.. admonition:: Query types vs. Django lookups +The supported lookups for ``equality`` queries are: :lookup:`exact` and +lookup:`in`. The supported operators are AND (``&``) and OR (``|``). - Range queries in Queryable Encryption are different from Django's - :ref:`range lookups `. Range queries allow you to - perform comparisons on encrypted fields, while Django's range lookups are - used for filtering based on a range of values. +The supported lookups for ``range`` queries include those of ``equality`` +queries as well as :lookup:`lt`, :lookup:`lte`, :lookup:`gt`, and +:lookup:`gte`. \* These fields don't support the ``queries`` argument: diff --git a/tests/encryption_/test_fields.py b/tests/encryption_/test_fields.py index 56f271c3f..685987fa1 100644 --- a/tests/encryption_/test_fields.py +++ b/tests/encryption_/test_fields.py @@ -5,7 +5,7 @@ from bson import ObjectId from django.db import DatabaseError -from django.db.models import Avg, F +from django.db.models import Avg, F, Q from django_mongodb_backend.fields import ( EncryptedArrayField, @@ -94,18 +94,41 @@ def test_array(self): class FieldTests(EncryptionTestCase): def assertEquality(self, model_cls, val): - model_cls.objects.create(value=val) - fetched = model_cls.objects.get(value=val) - self.assertEqual(fetched.value, val) + obj = model_cls.objects.create(value=val) + self.assertEqual(model_cls.objects.get(value=val), obj) + self.assertEqual(model_cls.objects.get(value__in=[val]), obj) + self.assertQuerySetEqual(model_cls.objects.exclude(value=val), []) def assertRange(self, model_cls, *, low, high, threshold): - model_cls.objects.create(value=low) - model_cls.objects.create(value=high) + obj1 = model_cls.objects.create(value=low) + obj2 = model_cls.objects.create(value=high) self.assertEqual(model_cls.objects.get(value=low).value, low) self.assertEqual(model_cls.objects.get(value=high).value, high) - objs = list(model_cls.objects.filter(value__gt=threshold)) - self.assertEqual(len(objs), 1) - self.assertEqual(objs[0].value, high) + self.assertEqual(model_cls.objects.exclude(value=high).get().value, low) + self.assertCountEqual(model_cls.objects.filter(Q(value=high) | Q(value=low)), [obj1, obj2]) + self.assertQuerySetEqual( + model_cls.objects.filter(value__gt=threshold), [high], attrgetter("value") + ) + self.assertQuerySetEqual( + model_cls.objects.filter(value__gte=threshold), [high], attrgetter("value") + ) + self.assertQuerySetEqual( + model_cls.objects.filter(value__lt=threshold), [low], attrgetter("value") + ) + self.assertQuerySetEqual( + model_cls.objects.filter(value__lte=threshold), [low], attrgetter("value") + ) + self.assertQuerySetEqual( + model_cls.objects.filter(value__in=[low]), [low], attrgetter("value") + ) + msg = ( + "Comparison disallowed between Queryable Encryption encrypted " + "fields and non-constant expressions; field 'value' is encrypted." + ) + with self.assertRaisesMessage(DatabaseError, msg): + self.assertQuerySetEqual( + model_cls.objects.filter(value__lte=F("value")), [low], attrgetter("value") + ) # Equality-only fields def test_binary(self): From c157f04d5166f0ae3af853b99bc3ca6475e3ae34 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 19 Nov 2025 20:04:07 -0500 Subject: [PATCH 12/26] simplify "Configuring the Automatic Encryption Shared Library" to remove mongocryptd --- docs/howto/queryable-encryption.rst | 38 ++++++++++++++--------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index e9b98122a..a56a7eec4 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -317,34 +317,26 @@ fields map:: Configuring the Automatic Encryption Shared Library =================================================== -The :ref:`manual:qe-reference-shared-library` is a preferred alternative to -:ref:`manual:qe-mongocryptd` and does not require you to start another process -to perform automatic encryption. +Next, you'll need to configure the :ref:`manual:qe-reference-shared-library`. -In practice, if you use Atlas or Enterprise MongoDB, ``mongocryptd`` is already -configured for you, however in such cases the shared library is still -recommended for use with Queryable Encryption. +First, :ref:`download the shared library +`. You can choose the latest version, +even if it doesn't match your MongoDB server version. -You can :ref:`download the shared library -` from the -:ref:`manual:enterprise-official-packages`. The shared library is -platform‑specific. Make sure to download the correct version for your operating -system and architecture. - -To configure it in your Django settings, use -:class:`~pymongo.encryption_options.AutoEncryptionOpts`\'s -``crypt_shared_lib_path`` parameter:: - - from pymongo.encryption_options import AutoEncryptionOpts +After extracting the shared library archive, configure your Django settings to +point to the shared library. For example, on macOS, the name of the shared +library is ``mongo_crypt_shared_v1.dylib``:: DATABASES = { + # ... "encrypted": { # ... "OPTIONS": { "auto_encryption_opts": AutoEncryptionOpts( # ... crypt_shared_lib_path="/path/to/mongo_crypt_shared_v1.dylib", - ) + crypt_shared_lib_required=True, + ), }, }, } @@ -367,7 +359,13 @@ To configure it in your Django settings, use | Linux | ``LD_LIBRARY_PATH`` | +---------------+---------------------------------+ - For example, on macOS you can set the ``DYLD_FALLBACK_LIBRARY_PATH`` - environment variable in your shell before starting your Django application:: + For example, on macOS you can set ``DYLD_FALLBACK_LIBRARY_PATH`` in your + shell before starting your Django application: + + .. code-block:: console $ export DYLD_FALLBACK_LIBRARY_PATH="/path/to/mongo_crypt_shared/:$DYLD_FALLBACK_LIBRARY_PATH" + + Unlike ``crypt_shared_lib_path`` earlier, the environment variable points + to the directory that contains the shared library, not the shared library + itself. From 483784f85cc28b831ca0a69c473a3ee3ac069734 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 19 Nov 2025 20:05:05 -0500 Subject: [PATCH 13/26] reorder "Configuring the Automatic Encryption Shared Library" to make it part of the normal flow --- docs/howto/queryable-encryption.rst | 112 ++++++++++++++-------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index a56a7eec4..3e5f96552 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -81,6 +81,62 @@ The database name of the key vault must be the same as in ``"NAME"``. The vault's collection name can be whatever you wish, but by convention, it's often ``__keyVault``. +Configuring the Automatic Encryption Shared Library +=================================================== + +Next, you'll need to configure the :ref:`manual:qe-reference-shared-library`. + +First, :ref:`download the shared library +`. You can choose the latest version, +even if it doesn't match your MongoDB server version. + +After extracting the shared library archive, configure your Django settings to +point to the shared library. For example, on macOS, the name of the shared +library is ``mongo_crypt_shared_v1.dylib``:: + + DATABASES = { + # ... + "encrypted": { + # ... + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + # ... + crypt_shared_lib_path="/path/to/mongo_crypt_shared_v1.dylib", + crypt_shared_lib_required=True, + ), + }, + }, + } + +.. admonition:: Dynamic library path configuration + + If you encounter ``Pymongocrypt.errors.MongoCryptError: An existing + crypt_shared library is loaded by the application at + [/path/to/mongo_crypt_v1.so], but the current call to mongocrypt_init() + failed to find that same library.``, you probably need to configure an + environment variable so that your system can locate the library: + + +---------------+---------------------------------+ + | **Platform** | **Environment Variable** | + +---------------+---------------------------------+ + | Windows | ``PATH`` | + +---------------+---------------------------------+ + | macOS | ``DYLD_FALLBACK_LIBRARY_PATH`` | + +---------------+---------------------------------+ + | Linux | ``LD_LIBRARY_PATH`` | + +---------------+---------------------------------+ + + For example, on macOS you can set ``DYLD_FALLBACK_LIBRARY_PATH`` in your + shell before starting your Django application: + + .. code-block:: console + + $ export DYLD_FALLBACK_LIBRARY_PATH="/path/to/mongo_crypt_shared/:$DYLD_FALLBACK_LIBRARY_PATH" + + Unlike ``crypt_shared_lib_path`` earlier, the environment variable points + to the directory that contains the shared library, not the shared library + itself. + .. _qe-configuring-database-routers-setting: Configuring the ``DATABASE_ROUTERS`` setting @@ -313,59 +369,3 @@ fields map:: }, }, } - -Configuring the Automatic Encryption Shared Library -=================================================== - -Next, you'll need to configure the :ref:`manual:qe-reference-shared-library`. - -First, :ref:`download the shared library -`. You can choose the latest version, -even if it doesn't match your MongoDB server version. - -After extracting the shared library archive, configure your Django settings to -point to the shared library. For example, on macOS, the name of the shared -library is ``mongo_crypt_shared_v1.dylib``:: - - DATABASES = { - # ... - "encrypted": { - # ... - "OPTIONS": { - "auto_encryption_opts": AutoEncryptionOpts( - # ... - crypt_shared_lib_path="/path/to/mongo_crypt_shared_v1.dylib", - crypt_shared_lib_required=True, - ), - }, - }, - } - -.. admonition:: Dynamic library path configuration - - If you encounter ``Pymongocrypt.errors.MongoCryptError: An existing - crypt_shared library is loaded by the application at - [/path/to/mongo_crypt_v1.so], but the current call to mongocrypt_init() - failed to find that same library.``, you probably need to configure an - environment variable so that your system can locate the library: - - +---------------+---------------------------------+ - | **Platform** | **Environment Variable** | - +---------------+---------------------------------+ - | Windows | ``PATH`` | - +---------------+---------------------------------+ - | macOS | ``DYLD_FALLBACK_LIBRARY_PATH`` | - +---------------+---------------------------------+ - | Linux | ``LD_LIBRARY_PATH`` | - +---------------+---------------------------------+ - - For example, on macOS you can set ``DYLD_FALLBACK_LIBRARY_PATH`` in your - shell before starting your Django application: - - .. code-block:: console - - $ export DYLD_FALLBACK_LIBRARY_PATH="/path/to/mongo_crypt_shared/:$DYLD_FALLBACK_LIBRARY_PATH" - - Unlike ``crypt_shared_lib_path`` earlier, the environment variable points - to the directory that contains the shared library, not the shared library - itself. From 8fcc3c30a3cd53ca16525ea3a2f1a2e54dc20140 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Sat, 22 Nov 2025 14:23:09 -0500 Subject: [PATCH 14/26] update docs/tests for $facet removal --- docs/ref/models/encrypted-fields.rst | 16 +++++++--------- tests/encryption_/test_fields.py | 21 +++++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/docs/ref/models/encrypted-fields.rst b/docs/ref/models/encrypted-fields.rst index 34b891f1f..24b1d2caa 100644 --- a/docs/ref/models/encrypted-fields.rst +++ b/docs/ref/models/encrypted-fields.rst @@ -124,16 +124,16 @@ may vary. - :meth:`~django.db.models.query.QuerySet.alias`, :meth:`~django.db.models.query.QuerySet.annotate`, :meth:`~django.db.models.query.QuerySet.distinct`: Cannot group on field - '_id.value' which is encrypted with the random algorithm or whose encryption - properties are not known until runtime. + '' which is encrypted with the random algorithm or whose + encryption properties are not known until runtime. - :meth:`~django.db.models.query.QuerySet.dates`, :meth:`~django.db.models.query.QuerySet.datetimes`: If the value type is a date, the type of the index must also be date (and vice versa). - :meth:`~django.db.models.query.QuerySet.in_bulk`: Encrypted fields can't have unique constraints. - -# TODO: add details about joined queries after -https://github.com/mongodb/django-mongodb-backend/pull/443 is finalized. +- Queries that join multiple collections and require the ``let`` operator. Such + queries usually involve expressions or subqueries: Non-empty 'let' field is + not allowed in the $lookup aggregation stage over an encrypted collection. There are also several ``QuerySet`` methods that aren't permitted on any models (regardless of whether or not they have encrypted fields) that use a database @@ -142,10 +142,8 @@ sample error message from the database. - :meth:`~django.db.models.query.QuerySet.update`: Multi-document updates are not allowed with Queryable Encryption. -- :meth:`~django.db.models.query.QuerySet.aggregate`, - :meth:`~django.db.models.query.QuerySet.count`: Aggregation stage - $internalFacetTeeConsumer is not allowed or supported with automatic - encryption. +- :meth:`~django.db.models.query.QuerySet.aggregate`: Invalid reference to an + encrypted field within aggregate expression. - :meth:`~django.db.models.query.QuerySet.union`: Aggregation stage $unionWith is not allowed or supported with automatic encryption. diff --git a/tests/encryption_/test_fields.py b/tests/encryption_/test_fields.py index 685987fa1..3769f6725 100644 --- a/tests/encryption_/test_fields.py +++ b/tests/encryption_/test_fields.py @@ -5,7 +5,7 @@ from bson import ObjectId from django.db import DatabaseError -from django.db.models import Avg, F, Q +from django.db.models import Avg, Count, F, Q from django_mongodb_backend.fields import ( EncryptedArrayField, @@ -243,14 +243,18 @@ def test_time(self): class QueryTests(EncryptionTestCase): - def test_aggregate(self): + def test_aggregate_avg(self): msg = ( - "Aggregation stage $internalFacetTeeConsumer is not allowed or " - "supported with automatic encryption." + "csfle \"analyze_query\" failed: Accumulator '$avg' cannot aggregate encrypted fields." ) with self.assertRaisesMessage(DatabaseError, msg): list(IntegerModel.objects.aggregate(Avg("value"))) + def test_aggregate_count(self): + msg = "Invalid reference to an encrypted field within aggregate expression: value" + with self.assertRaisesMessage(DatabaseError, msg): + list(IntegerModel.objects.aggregate(Count("value"))) + def test_alias(self): msg = ( "Cannot group on field '_id.value' which is encrypted with the " @@ -291,12 +295,9 @@ def test_contains(self): self.assertIs(CharModel.objects.contains(obj), True) def test_count(self): - msg = ( - "Aggregation stage $internalFacetTeeConsumer is not allowed or " - "supported with automatic encryption." - ) - with self.assertRaisesMessage(DatabaseError, msg): - list(CharModel.objects.count()) + CharModel.objects.create(value="a") + CharModel.objects.create(value="b") + self.assertEqual(CharModel.objects.count(), 2) def test_dates(self): msg = ( From aaa3d4ffdf9c324be1dbb1226ceca66160a7d0b3 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Tue, 2 Dec 2025 20:54:51 -0500 Subject: [PATCH 15/26] Combine crypt shared w/installation & db setup Move crypt shared installation instructions to "Installation" section and crypt shared configuration steps to "Configuring the DATABASES setting" section. --- docs/howto/queryable-encryption.rst | 42 ++++++++++------------------- 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index 3e5f96552..41780ad08 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -23,12 +23,21 @@ Installation In addition to Django MongoDB Backend's regular :doc:`installation ` and :doc:`configuration ` steps, Queryable -Encryption has additional Python dependencies: +Encryption requires installing optional Python dependencies and the +:ref:`manual:csfle-reference-install-shared-lib`. + +To install the optional dependencies, use pip with the ``encryption`` extra: .. code-block:: console $ pip install django-mongodb-backend[encryption] +Next, :ref:`download the shared library +`. You can choose the latest version, +even if it doesn't match your MongoDB server version. After extracting the +shared library archive, :ref:`configure your Django settings +` with the path to the shared library. + .. _qe-configuring-databases-setting: Configuring the ``DATABASES`` setting @@ -69,8 +78,11 @@ Here's a sample configuration using a local KMS provider:: b'\x97\xea\xf8\x1e\xc3\xd49K\x18\x81\xc3\x1a"' b'\xdc\x00U\xc4u"X\xe7xy\xa5\xb2\x0e\xbc\xd6+-' b'\x80\x03\xef\xc2\xc4\x9bU' + ) }, }, + crypt_shared_lib_path="/path/to/mongo_crypt_shared_v1.dylib", + crypt_shared_lib_required=True, ) }, }, @@ -81,39 +93,13 @@ The database name of the key vault must be the same as in ``"NAME"``. The vault's collection name can be whatever you wish, but by convention, it's often ``__keyVault``. -Configuring the Automatic Encryption Shared Library -=================================================== - -Next, you'll need to configure the :ref:`manual:qe-reference-shared-library`. - -First, :ref:`download the shared library -`. You can choose the latest version, -even if it doesn't match your MongoDB server version. - -After extracting the shared library archive, configure your Django settings to -point to the shared library. For example, on macOS, the name of the shared -library is ``mongo_crypt_shared_v1.dylib``:: - - DATABASES = { - # ... - "encrypted": { - # ... - "OPTIONS": { - "auto_encryption_opts": AutoEncryptionOpts( - # ... - crypt_shared_lib_path="/path/to/mongo_crypt_shared_v1.dylib", - crypt_shared_lib_required=True, - ), - }, - }, - } .. admonition:: Dynamic library path configuration If you encounter ``Pymongocrypt.errors.MongoCryptError: An existing crypt_shared library is loaded by the application at [/path/to/mongo_crypt_v1.so], but the current call to mongocrypt_init() - failed to find that same library.``, you probably need to configure an + failed to find that same library.``, you may need to configure an environment variable so that your system can locate the library: +---------------+---------------------------------+ From 6ebc3a8f029371f97fed2a5046785b18190d527a Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Tue, 2 Dec 2025 21:05:11 -0500 Subject: [PATCH 16/26] Update versionadded to 6.0.1 --- .github/workflows/mongodb_settings.py | 2 +- docs/howto/queryable-encryption.rst | 2 +- docs/ref/django-admin.rst | 2 +- docs/ref/models/encrypted-fields.rst | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/mongodb_settings.py b/.github/workflows/mongodb_settings.py index 619bdcd95..3cb1db18a 100644 --- a/.github/workflows/mongodb_settings.py +++ b/.github/workflows/mongodb_settings.py @@ -1,4 +1,4 @@ -# Settings for django_mongodb_backend/tests when encryption isn't supported. +# Settings for django_mongodb_backend/tests. from django_settings import * # noqa: F403 DATABASES["encrypted"] = {} # noqa: F405 diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index 41780ad08..b08dd6528 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -2,7 +2,7 @@ Configuring Queryable Encryption ================================ -.. versionadded:: 5.2.3 +.. versionadded:: 6.0.1 :doc:`manual:core/queryable-encryption` is a powerful MongoDB feature that allows you to encrypt sensitive fields in your database while still supporting diff --git a/docs/ref/django-admin.rst b/docs/ref/django-admin.rst index 3f4858939..8fe4f0edf 100644 --- a/docs/ref/django-admin.rst +++ b/docs/ref/django-admin.rst @@ -17,7 +17,7 @@ Available commands ``showencryptedfieldsmap`` -------------------------- -.. versionadded:: 5.2.3 +.. versionadded:: 6.0.1 .. django-admin:: showencryptedfieldsmap diff --git a/docs/ref/models/encrypted-fields.rst b/docs/ref/models/encrypted-fields.rst index 24b1d2caa..9a2ab8123 100644 --- a/docs/ref/models/encrypted-fields.rst +++ b/docs/ref/models/encrypted-fields.rst @@ -4,7 +4,7 @@ Encrypted fields .. currentmodule:: django_mongodb_backend.fields -.. versionadded:: 5.2.3 +.. versionadded:: 6.0.1 To use encrypted fields, you must :doc:`configure Queryable Encryption `. @@ -152,7 +152,7 @@ sample error message from the database. .. class:: EncryptedFieldMixin - .. versionadded:: 5.2.3 + .. versionadded:: 6.0.1 Use this mixin to create encrypted versions of your own custom fields. For example, to create an encrypted version of ``MyField``:: From e9a748c051a4b03a00187cb7b72866bdfe643b08 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Fri, 5 Dec 2025 16:34:14 -0500 Subject: [PATCH 17/26] Configure AWS KMS for testing on evergreen --- .evergreen/config.yml | 6 +++--- .evergreen/run-tests.sh | 3 +++ .evergreen/setup.sh | 8 ++++---- .github/workflows/encrypted_settings.py | 24 ++++++++++++++++++++++-- tests/encryption_/test_base.py | 17 +++++++++++++++++ tests/encryption_/test_management.py | 6 +++++- 6 files changed, 54 insertions(+), 10 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 919ad6ced..4eb1d0210 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -69,6 +69,7 @@ functions: pre: - func: setup + - func: start csfle servers - func: bootstrap mongo-orchestration post: @@ -77,7 +78,6 @@ post: tasks: - name: run-tests commands: - - func: "start csfle servers" - func: "run unit tests" buildvariants: @@ -126,8 +126,8 @@ buildvariants: - name: run-tests - name: tests-8-qe - display_name: Run Tests 8.0 QE - run_on: rhel94-perf-atlas + display_name: Run Tests 8.2 QE + run_on: rhel87-small expansions: MONGODB_VERSION: "8.2" TOPOLOGY: replica_set diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index a8579648e..8e980526c 100644 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -2,6 +2,9 @@ set -eux +# Export secrets as environment variables +. ../secrets-export.sh + # Install django-mongodb-backend /opt/python/3.12/bin/python3 -m venv venv . venv/bin/activate diff --git a/.evergreen/setup.sh b/.evergreen/setup.sh index 4709ed9bd..f1d552f47 100644 --- a/.evergreen/setup.sh +++ b/.evergreen/setup.sh @@ -16,8 +16,8 @@ DRIVERS_TOOLS="$(dirname "$(pwd)")/drivers-tools" PROJECT_DIRECTORY="$(pwd)" if [ "Windows_NT" = "${OS:-}" ]; then - DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) - PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY) + DRIVERS_TOOLS=$(cygpath -m "$DRIVERS_TOOLS") + PROJECT_DIRECTORY=$(cygpath -m "$PROJECT_DIRECTORY") fi export PROJECT_DIRECTORY export DRIVERS_TOOLS @@ -37,8 +37,8 @@ PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" EOT # Set up drivers-tools with a .env file. -git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS} -cat < ${DRIVERS_TOOLS}/.env +git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git "${DRIVERS_TOOLS}" +cat < "${DRIVERS_TOOLS}/.env" CURRENT_VERSION="$CURRENT_VERSION" DRIVERS_TOOLS="$DRIVERS_TOOLS" MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" diff --git a/.github/workflows/encrypted_settings.py b/.github/workflows/encrypted_settings.py index 1efdb38b4..de0b5509d 100644 --- a/.github/workflows/encrypted_settings.py +++ b/.github/workflows/encrypted_settings.py @@ -7,18 +7,38 @@ os.environ["LD_LIBRARY_PATH"] = str(Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent) +AWS_CREDS = { + "accessKeyId": os.environ.get("FLE_AWS_KEY", ""), + "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""), +} + +_USE_AWS_KMS = any(AWS_CREDS.values()) + +if _USE_AWS_KMS: + _AWS_REGION = os.environ.get("FLE_AWS_KMS_REGION", "us-east-1") + _AWS_KEY_ARN = os.environ.get( + "FLE_AWS_KMS_KEY_ARN", + "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + ) + KMS_PROVIDERS = {"aws": AWS_CREDS} + KMS_CREDENTIALS = {"aws": {"key": _AWS_KEY_ARN, "region": _AWS_REGION}} +else: + KMS_PROVIDERS = {"local": {"key": os.urandom(96)}} + KMS_CREDENTIALS = {"local": {}} + DATABASES["encrypted"] = { # noqa: F405 "ENGINE": "django_mongodb_backend", "NAME": "djangotests_encrypted", "OPTIONS": { "auto_encryption_opts": AutoEncryptionOpts( key_vault_namespace="djangotests_encrypted.__keyVault", - kms_providers={"local": {"key": os.urandom(96)}}, + kms_providers=KMS_PROVIDERS, crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"], + crypt_shared_lib_required=True, ), "directConnection": True, }, - "KMS_CREDENTIALS": {}, + "KMS_CREDENTIALS": KMS_CREDENTIALS, } diff --git a/tests/encryption_/test_base.py b/tests/encryption_/test_base.py index 0c165d19a..8bc183781 100644 --- a/tests/encryption_/test_base.py +++ b/tests/encryption_/test_base.py @@ -1,3 +1,5 @@ +import os + import pymongo from bson.binary import Binary from django.conf import settings @@ -19,3 +21,18 @@ def assertEncrypted(self, model, field): collection = db[model._meta.db_table] data = collection.find_one({}, {field: 1, "_id": 0}) self.assertIsInstance(data[field], Binary) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + AWS_CREDS = { + "accessKeyId": os.environ.get("FLE_AWS_KEY", ""), + "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""), + } + _USE_AWS_KMS = any(AWS_CREDS.values()) + + if _USE_AWS_KMS: + self.DEFAULT_KMS_PROVIDER = "aws" + else: + # Local-only fallback + self.DEFAULT_KMS_PROVIDER = "local" diff --git a/tests/encryption_/test_management.py b/tests/encryption_/test_management.py index 096ccab3a..63721e229 100644 --- a/tests/encryption_/test_management.py +++ b/tests/encryption_/test_management.py @@ -125,7 +125,11 @@ def test_missing_key(self): call_command("showencryptedfieldsmap", "--database", "encrypted", verbosity=0) finally: # Replace the deleted key. + master_key = connections["encrypted"].settings_dict["KMS_CREDENTIALS"][ + self.DEFAULT_KMS_PROVIDER + ] connections["encrypted"].client_encryption.create_data_key( - kms_provider="local", + kms_provider=self.DEFAULT_KMS_PROVIDER, + master_key=master_key, key_alt_names=[test_key], ) From 10014f462c231a4a487a2aa40fbb4f2590800a91 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Sun, 7 Dec 2025 18:03:35 -0500 Subject: [PATCH 18/26] doc edits --- docs/howto/queryable-encryption.rst | 82 ++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 26 deletions(-) diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index b08dd6528..d894575d8 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -32,11 +32,11 @@ To install the optional dependencies, use pip with the ``encryption`` extra: $ pip install django-mongodb-backend[encryption] -Next, :ref:`download the shared library +Next, download the :ref:`Automatic Encryption Shared Library `. You can choose the latest version, even if it doesn't match your MongoDB server version. After extracting the -shared library archive, :ref:`configure your Django settings -` with the path to the shared library. +archive, configure the :ref:`crypt_shared_lib_path +`. .. _qe-configuring-databases-setting: @@ -81,7 +81,7 @@ Here's a sample configuration using a local KMS provider:: ) }, }, - crypt_shared_lib_path="/path/to/mongo_crypt_shared_v1.dylib", + crypt_shared_lib_path="/path/to/mongo_crypt_shared_v1", crypt_shared_lib_required=True, ) }, @@ -93,35 +93,65 @@ The database name of the key vault must be the same as in ``"NAME"``. The vault's collection name can be whatever you wish, but by convention, it's often ``__keyVault``. - .. admonition:: Dynamic library path configuration - If you encounter ``Pymongocrypt.errors.MongoCryptError: An existing - crypt_shared library is loaded by the application at - [/path/to/mongo_crypt_v1.so], but the current call to mongocrypt_init() - failed to find that same library.``, you may need to configure an - environment variable so that your system can locate the library: + If you encounter the following error: + + .. code-block:: text + + Pymongocrypt.errors.MongoCryptError: An existing crypt_shared library is + loaded by the application at [/path/to/mongo_crypt_v1.so], but the current + call to mongocrypt_init() failed to find that same library. + + add the directory that contains the shared library to your platform’s dynamic + library search path: + + +---------------+------------------------------+ + | **Platform** | **Environment variable** | + +---------------+------------------------------+ + | Windows | PATH | + +---------------+------------------------------+ + | macOS | DYLD_FALLBACK_LIBRARY_PATH | + +---------------+------------------------------+ + | Linux | LD_LIBRARY_PATH | + +---------------+------------------------------+ + + Examples: + + macOS (bash): + + .. code-block:: console + + $ export DYLD_FALLBACK_LIBRARY_PATH="/path/to/mongo_crypt_shared:${DYLD_FALLBACK_LIBRARY_PATH}" + + Linux (bash): + + .. code-block:: console + + $ export LD_LIBRARY_PATH="/path/to/mongo_crypt_shared:${LD_LIBRARY_PATH}" + + Windows (PowerShell): + + .. code-block:: powershell + + $env:Path = "C:\path\to\mongo_crypt_shared" + ";" + $env:Path + + Windows (Command Prompt): - +---------------+---------------------------------+ - | **Platform** | **Environment Variable** | - +---------------+---------------------------------+ - | Windows | ``PATH`` | - +---------------+---------------------------------+ - | macOS | ``DYLD_FALLBACK_LIBRARY_PATH`` | - +---------------+---------------------------------+ - | Linux | ``LD_LIBRARY_PATH`` | - +---------------+---------------------------------+ + .. code-block:: bat - For example, on macOS you can set ``DYLD_FALLBACK_LIBRARY_PATH`` in your - shell before starting your Django application: + set PATH=C:\path\to\mongo_crypt_shared;%PATH% - .. code-block:: console + Notes: - $ export DYLD_FALLBACK_LIBRARY_PATH="/path/to/mongo_crypt_shared/:$DYLD_FALLBACK_LIBRARY_PATH" + * Set the variable to the directory that contains the shared library file + (for example, ``mongo_crypt_shared_v1.dylib`` on macOS, + ``mongo_crypt_v1.so`` on Linux, or ``mongo_crypt_v1.dll`` on Windows), not + the file itself. - Unlike ``crypt_shared_lib_path`` earlier, the environment variable points - to the directory that contains the shared library, not the shared library - itself. + * This environment variable is separate from the ``crypt_shared_lib_path`` + option: the environment variable points to a directory, while + ``crypt_shared_lib_path`` is the explicit path to the library file. .. _qe-configuring-database-routers-setting: From 1caf6323747481e0ee2f381aeb72d376215f7bfb Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Tue, 9 Dec 2025 21:38:58 -0500 Subject: [PATCH 19/26] Address review - avoid encryption tasks on non-encrypted builds - retrieve configured provider from conn settings - fail loudly & avoid uppercased values in settings --- .evergreen/config.yml | 33 ++++++++++++++++--- .evergreen/run-tests.sh | 14 ++++++-- .../workflows/aws_kms_encrypted_settings.py | 26 +++++++++++++++ ...ngs.py => local_kms_encrypted_settings.py} | 24 ++------------ .github/workflows/test-python-atlas.yml | 2 +- tests/encryption_/test_base.py | 17 ---------- tests/encryption_/test_management.py | 12 ++++--- 7 files changed, 76 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/aws_kms_encrypted_settings.py rename .github/workflows/{encrypted_settings.py => local_kms_encrypted_settings.py} (63%) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4eb1d0210..93723aec4 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -59,6 +59,17 @@ functions: args: - ./.evergreen/run-tests.sh + "run encryption tests": + - command: subprocess.exec + type: test + params: + binary: bash + working_dir: "src" + include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI", "DJANGO_SETTINGS_MODULE", "CRYPT_SHARED_LIB_PATH"] + args: + - ./.evergreen/run-tests.sh + - encryption + "teardown": - command: subprocess.exec params: @@ -80,6 +91,10 @@ tasks: commands: - func: "run unit tests" + - name: run-encryption-tests + commands: + - func: "run encryption tests" + buildvariants: - name: tests-7-noauth-nossl display_name: Run Tests 7.0 NoAuth NoSSL @@ -125,12 +140,22 @@ buildvariants: tasks: - name: run-tests - - name: tests-8-qe - display_name: Run Tests 8.2 QE + - name: tests-8-qe-local + display_name: Run Tests 8.2 QE local KMS run_on: rhel87-small expansions: MONGODB_VERSION: "8.2" TOPOLOGY: replica_set - DJANGO_SETTINGS_MODULE: "encrypted_settings" + DJANGO_SETTINGS_MODULE: "local_kms_encrypted_settings" tasks: - - name: run-tests + - name: run-encryption-tests + + - name: tests-8-qe-aws + display_name: Run Tests 8.2 QE aws KMS + run_on: rhel87-small + expansions: + MONGODB_VERSION: "8.2" + TOPOLOGY: replica_set + DJANGO_SETTINGS_MODULE: "aws_kms_encrypted_settings" + tasks: + - name: run-encryption-tests diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 8e980526c..7ebf4ece4 100644 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -3,13 +3,21 @@ set -eux # Export secrets as environment variables -. ../secrets-export.sh +if [[ "${1:-}" == "encryption" ]]; then + . ../secrets-export.sh +fi -# Install django-mongodb-backend +# Set up virtual environment /opt/python/3.12/bin/python3 -m venv venv . venv/bin/activate python -m pip install -U pip -pip install -e '.[encryption]' + +# Conditionally install encryption extra if "encryption" arg is passed +if [[ "${1:-}" == "encryption" ]]; then + pip install -e '.[encryption]' +else + pip install -e . +fi # Install django and test dependencies git clone --branch mongodb-6.0.x https://github.com/mongodb-forks/django django_repo diff --git a/.github/workflows/aws_kms_encrypted_settings.py b/.github/workflows/aws_kms_encrypted_settings.py new file mode 100644 index 000000000..58f1fa230 --- /dev/null +++ b/.github/workflows/aws_kms_encrypted_settings.py @@ -0,0 +1,26 @@ +from local_kms_encrypted_settings import * # noqa: F403 + +DATABASES["encrypted"] = { # noqa: F405 + "ENGINE": "django_mongodb_backend", + "NAME": "djangotests_encrypted", + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( # noqa: F405 + key_vault_namespace="djangotests_encrypted.__keyVault", + kms_providers={ + "aws": { + "accessKeyId": os.environ.get("FLE_AWS_KEY"), # noqa: F405 + "secretAccessKey": os.environ.get("FLE_AWS_SECRET"), # noqa: F405 + } + }, + crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"], # noqa: F405 + crypt_shared_lib_required=True, + ), + "directConnection": True, + }, + "KMS_CREDENTIALS": { + "aws": { + "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "region": "us-east-1", + } + }, +} diff --git a/.github/workflows/encrypted_settings.py b/.github/workflows/local_kms_encrypted_settings.py similarity index 63% rename from .github/workflows/encrypted_settings.py rename to .github/workflows/local_kms_encrypted_settings.py index de0b5509d..a0d25075e 100644 --- a/.github/workflows/encrypted_settings.py +++ b/.github/workflows/local_kms_encrypted_settings.py @@ -7,38 +7,18 @@ os.environ["LD_LIBRARY_PATH"] = str(Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent) -AWS_CREDS = { - "accessKeyId": os.environ.get("FLE_AWS_KEY", ""), - "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""), -} - -_USE_AWS_KMS = any(AWS_CREDS.values()) - -if _USE_AWS_KMS: - _AWS_REGION = os.environ.get("FLE_AWS_KMS_REGION", "us-east-1") - _AWS_KEY_ARN = os.environ.get( - "FLE_AWS_KMS_KEY_ARN", - "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", - ) - KMS_PROVIDERS = {"aws": AWS_CREDS} - KMS_CREDENTIALS = {"aws": {"key": _AWS_KEY_ARN, "region": _AWS_REGION}} -else: - KMS_PROVIDERS = {"local": {"key": os.urandom(96)}} - KMS_CREDENTIALS = {"local": {}} - DATABASES["encrypted"] = { # noqa: F405 "ENGINE": "django_mongodb_backend", "NAME": "djangotests_encrypted", "OPTIONS": { "auto_encryption_opts": AutoEncryptionOpts( key_vault_namespace="djangotests_encrypted.__keyVault", - kms_providers=KMS_PROVIDERS, + kms_providers={"local": {"key": os.urandom(96)}}, crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"], - crypt_shared_lib_required=True, ), "directConnection": True, }, - "KMS_CREDENTIALS": KMS_CREDENTIALS, + "KMS_CREDENTIALS": {"local": {}}, } diff --git a/.github/workflows/test-python-atlas.yml b/.github/workflows/test-python-atlas.yml index dda600bb7..ebcc0b0e5 100644 --- a/.github/workflows/test-python-atlas.yml +++ b/.github/workflows/test-python-atlas.yml @@ -61,5 +61,5 @@ jobs: permissions: contents: read env: - DJANGO_SETTINGS_MODULE: "encrypted_settings" + DJANGO_SETTINGS_MODULE: "local_kms_encrypted_settings" CRYPT_SHARED_LIB_PATH: "${{ github.workspace }}/lib/mongo_crypt_v1.so" diff --git a/tests/encryption_/test_base.py b/tests/encryption_/test_base.py index 8bc183781..0c165d19a 100644 --- a/tests/encryption_/test_base.py +++ b/tests/encryption_/test_base.py @@ -1,5 +1,3 @@ -import os - import pymongo from bson.binary import Binary from django.conf import settings @@ -21,18 +19,3 @@ def assertEncrypted(self, model, field): collection = db[model._meta.db_table] data = collection.find_one({}, {field: 1, "_id": 0}) self.assertIsInstance(data[field], Binary) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - AWS_CREDS = { - "accessKeyId": os.environ.get("FLE_AWS_KEY", ""), - "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""), - } - _USE_AWS_KMS = any(AWS_CREDS.values()) - - if _USE_AWS_KMS: - self.DEFAULT_KMS_PROVIDER = "aws" - else: - # Local-only fallback - self.DEFAULT_KMS_PROVIDER = "local" diff --git a/tests/encryption_/test_management.py b/tests/encryption_/test_management.py index 63721e229..180325675 100644 --- a/tests/encryption_/test_management.py +++ b/tests/encryption_/test_management.py @@ -114,6 +114,9 @@ def test_show_encrypted_fields_map(self): self._compare_output(expected, command_output[model_key]) def test_missing_key(self): + connection = connections["encrypted"] + auto_encryption_opts = connection.connection._options.auto_encryption_opts + kms_providers = auto_encryption_opts._kms_providers test_key = "encryption__patient.patient_record.ssn" msg = ( f"Encryption key {test_key} not found. Have migrated the " @@ -125,11 +128,10 @@ def test_missing_key(self): call_command("showencryptedfieldsmap", "--database", "encrypted", verbosity=0) finally: # Replace the deleted key. - master_key = connections["encrypted"].settings_dict["KMS_CREDENTIALS"][ - self.DEFAULT_KMS_PROVIDER - ] - connections["encrypted"].client_encryption.create_data_key( - kms_provider=self.DEFAULT_KMS_PROVIDER, + kms_provider = next(iter(kms_providers.keys())) + master_key = connection.settings_dict["KMS_CREDENTIALS"][kms_provider] + connection.client_encryption.create_data_key( + kms_provider=kms_provider, master_key=master_key, key_alt_names=[test_key], ) From be1d3cb2e80a091bfdd241f63dab0d6c31c55a5b Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Wed, 10 Dec 2025 16:54:30 -0500 Subject: [PATCH 20/26] Address UAT feedback --- docs/howto/queryable-encryption.rst | 16 ++++++++++++---- docs/ref/django-admin.rst | 2 ++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index d894575d8..4ed11d13b 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -30,7 +30,7 @@ To install the optional dependencies, use pip with the ``encryption`` extra: .. code-block:: console - $ pip install django-mongodb-backend[encryption] + $ pip install 'django-mongodb-backend[encryption]' Next, download the :ref:`Automatic Encryption Shared Library `. You can choose the latest version, @@ -62,11 +62,11 @@ Here's a sample configuration using a local KMS provider:: "encrypted": { "ENGINE": "django_mongodb_backend", "HOST": "mongodb+srv://cluster0.example.mongodb.net", - "NAME": "my_encrypted_database", + "NAME": "encrypted", # ... "OPTIONS": { "auto_encryption_opts": AutoEncryptionOpts( - key_vault_namespace="my_encrypted_database.__keyVault", + key_vault_namespace="encrypted.__keyVault", kms_providers={ "local": { # Generated by os.urandom(96) @@ -280,7 +280,7 @@ You can perform a equality query just like you would on a non-encrypted field: .. code-block:: pycon >>> patient = Patient.objects.get(patient_record__ssn="123-45-6789") - >>> patient.name + >>> patient.patient_name 'John Doe' .. _qe-configuring-kms: @@ -340,6 +340,14 @@ Here's an example of KMS configuration with ``aws``:: Configuring the ``encrypted_fields_map`` option =============================================== + +.. admonition:: Required configuration + + As :ref:`described here `, + ensure ``django_mongodb_backend`` is listed in + :setting:`django:INSTALLED_APPS` to enable the + :djadmin:`showencryptedfieldsmap` command. + Encryption keys are created when you :ref:`run migrations for models that have encrypted fields `. diff --git a/docs/ref/django-admin.rst b/docs/ref/django-admin.rst index 8fe4f0edf..567aeaab1 100644 --- a/docs/ref/django-admin.rst +++ b/docs/ref/django-admin.rst @@ -5,6 +5,8 @@ Management commands Django MongoDB Backend includes some :doc:`Django management commands `. +.. _qe-showencryptedfieldsmap-required-configuration: + Required configuration ====================== From 417f1efac770a37c6e8f86847ca6162e9767135a Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Thu, 11 Dec 2025 10:52:07 -0500 Subject: [PATCH 21/26] Address UAT feedback --- docs/howto/queryable-encryption.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index 4ed11d13b..7027cdd84 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -260,11 +260,24 @@ Then run the migrations with: you change the definition of an encrypted field, for example, to make it queryable. +Creating encrypted data +======================= + Now create and manipulate instances of the data just like any other Django model data. The fields will automatically handle encryption and decryption, ensuring that :ref:`sensitive data is stored securely in the database `. +Here's an example of creating a new ``Patient`` instance with encrypted fields: + +.. code-block:: pycon + + >>> from myapp.models import Patient, PatientRecord, Billing + >>> billing = Billing(cc_type="Visa", cc_number="4111111111111111") + >>> record = PatientRecord(ssn="123-45-6789", billing=billing, bill_amount=250.75) + >>> patient = Patient(patient_name="John Doe", patient_id=1001, patient_record=record) + >>> patient.save() + Querying encrypted fields ========================= From 070c2a66a9f501717f8230bce56a07118b50be31 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 17 Dec 2025 13:05:33 -0500 Subject: [PATCH 22/26] add encryption-compatible aggregation wrap --- django_mongodb_backend/base.py | 15 +++++----- django_mongodb_backend/query.py | 51 ++++++++++++++++++++++----------- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/django_mongodb_backend/base.py b/django_mongodb_backend/base.py index b1afc1b03..179939f95 100644 --- a/django_mongodb_backend/base.py +++ b/django_mongodb_backend/base.py @@ -244,13 +244,14 @@ def get_database(self): @cached_property def client_encryption(self): - auto_encryption_opts = self.connection._options.auto_encryption_opts - return ClientEncryption( - auto_encryption_opts._kms_providers, - auto_encryption_opts._key_vault_namespace, - self.connection, - self.connection.codec_options, - ) + if auto_encryption_opts := self.connection._options.auto_encryption_opts: + return ClientEncryption( + auto_encryption_opts._kms_providers, + auto_encryption_opts._key_vault_namespace, + self.connection, + self.connection.codec_options, + ) + return None @cached_property def database(self): diff --git a/django_mongodb_backend/query.py b/django_mongodb_backend/query.py index 72a552945..49668fd79 100644 --- a/django_mongodb_backend/query.py +++ b/django_mongodb_backend/query.py @@ -93,24 +93,41 @@ def get_pipeline(self): if self.aggregation_pipeline: pipeline.extend(self.aggregation_pipeline) if self.needs_wrap_aggregation: - # Add the aggregation stage for queries without a GROUP BY. - # e.g. SQL equivalent of SELECT avg(col) FROM table - pipeline.extend( - [ - # Workaround for https://jira.mongodb.org/browse/SERVER-114196: - # $$NOW becomes unavailable after $unionWith, so it must be - # stored beforehand to ensure it remains accessible later - # in the pipeline. - {"$addFields": {"__now": "$$NOW"}}, - # Add an empty extra document to handle default values on - # empty results. - {"$unionWith": {"pipeline": [{"$documents": [{}]}]}}, - # Limiting to one document ensures the original result - # takes precedence when present, otherwise the injected - # empty document is used. - {"$limit": 1}, + if self.compiler.connection.client_encryption: + pipeline = [ + {"$collStats": {}}, + { + "$lookup": { + "from": self.compiler.collection_name, + "as": "wrapped", + "pipeline": pipeline, + } + }, + { + "$replaceWith": { + "$cond": [{"$eq": ["$wrapped", []]}, {}, {"$first": "$wrapped"}] + } + }, ] - ) + else: + # Add the aggregation stage for queries without a GROUP BY. + # e.g. SQL equivalent of SELECT avg(col) FROM table + pipeline.extend( + [ + # Workaround for https://jira.mongodb.org/browse/SERVER-114196: + # $$NOW becomes unavailable after $unionWith, so it + # must be stored beforehand to ensure it remains + # accessible later in the pipeline. + {"$addFields": {"__now": "$$NOW"}}, + # Add an empty extra document to handle default values + # on empty results. + {"$unionWith": {"pipeline": [{"$documents": [{}]}]}}, + # Limiting to one document ensures the original result + # takes precedence when present, otherwise the injected + # empty document is used. + {"$limit": 1}, + ] + ) if self.project_fields: pipeline.append({"$project": self.project_fields}) if self.combinator_pipeline: From 47e064f9e951e31af9911510b8429a5c07e7f01b Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 17 Dec 2025 15:01:45 -0500 Subject: [PATCH 23/26] remove KMS_CREDENTIALS for "local" provider --- .github/workflows/local_kms_encrypted_settings.py | 1 - tests/encryption_/test_management.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/local_kms_encrypted_settings.py b/.github/workflows/local_kms_encrypted_settings.py index a0d25075e..b70a28594 100644 --- a/.github/workflows/local_kms_encrypted_settings.py +++ b/.github/workflows/local_kms_encrypted_settings.py @@ -18,7 +18,6 @@ ), "directConnection": True, }, - "KMS_CREDENTIALS": {"local": {}}, } diff --git a/tests/encryption_/test_management.py b/tests/encryption_/test_management.py index 180325675..0e6470587 100644 --- a/tests/encryption_/test_management.py +++ b/tests/encryption_/test_management.py @@ -129,7 +129,8 @@ def test_missing_key(self): finally: # Replace the deleted key. kms_provider = next(iter(kms_providers.keys())) - master_key = connection.settings_dict["KMS_CREDENTIALS"][kms_provider] + credentials = connection.settings_dict.get("KMS_CREDENTIALS") + master_key = credentials[kms_provider] if credentials else None connection.client_encryption.create_data_key( kms_provider=kms_provider, master_key=master_key, From 2e87e1db03e673a83cfbfde4dec4e151d3bbe2d4 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 17 Dec 2025 15:02:28 -0500 Subject: [PATCH 24/26] Relax "cannot aggregate encrypted fields" error message, possibly for MongoDB 8.2.2 --- tests/encryption_/test_fields.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/encryption_/test_fields.py b/tests/encryption_/test_fields.py index 3769f6725..689aa163a 100644 --- a/tests/encryption_/test_fields.py +++ b/tests/encryption_/test_fields.py @@ -244,9 +244,7 @@ def test_time(self): class QueryTests(EncryptionTestCase): def test_aggregate_avg(self): - msg = ( - "csfle \"analyze_query\" failed: Accumulator '$avg' cannot aggregate encrypted fields." - ) + msg = "Accumulator '$avg' cannot aggregate encrypted fields." with self.assertRaisesMessage(DatabaseError, msg): list(IntegerModel.objects.aggregate(Avg("value"))) From be2ac3b40d1dc1460c15f2a428828ddd6ba81c5d Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Thu, 18 Dec 2025 18:03:22 -0500 Subject: [PATCH 25/26] Fix link to QE docs in release notes --- docs/releases/6.0.x.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/releases/6.0.x.rst b/docs/releases/6.0.x.rst index a446ac809..db047546d 100644 --- a/docs/releases/6.0.x.rst +++ b/docs/releases/6.0.x.rst @@ -10,7 +10,7 @@ Django MongoDB Backend 6.0.x New features ------------ -- Added support for :doc:`Queryable Encryption`. +- Added support for :doc:`Queryable Encryption`. Bug fixes --------- From 65906058b8e0b96070aafdfea3431f5b71d28fda Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Fri, 19 Dec 2025 09:33:38 -0500 Subject: [PATCH 26/26] Be able to link to the MongoCryptError workaround We may not be able to resolve this in PyMongo prior to releasing QE so let's make this as good as it can be before we can remove it. --- docs/howto/queryable-encryption.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index 7027cdd84..1bf3bfd5c 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -93,7 +93,10 @@ The database name of the key vault must be the same as in ``"NAME"``. The vault's collection name can be whatever you wish, but by convention, it's often ``__keyVault``. -.. admonition:: Dynamic library path configuration +Dynamic library path configuration +---------------------------------- + +.. admonition:: Pymongocrypt.errors.MongoCryptError If you encounter the following error: