Fixes compatibility with Elasticsearch 8.18.1

.github/workflows/ci.yml

@@ -13,14 +13,21 @@ jobs:
       fail-fast: false
 
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
-        django-version: ["3.2", "4.1", "4.2"]
-        es-dsl-version: ["6.4", "7.4"]
-        es-version: ["8.10.2"]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ]
+        django-version: [ "3.2", "4.1", "4.2", "5.0", "5.1", "5.2" ]
+        es-py-version: [ "6.4", "7.4", "8.18.1"]
 
         exclude:
           - python-version: "3.11"
             django-version: "3.2"
+          - python-version: "3.12"
+            django-version: "3.2"
+          - python-version: "3.13"
+            django-version: "3.2"
+          - es-py-version: "6.4"
+            es-version: "8.10.2"
+          - es-py-version: "7.4"
+            es-version: "8.10.2"
 
     steps:
       - name: Install and Run Elasticsearch
@@ -47,12 +54,12 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python -m pip install "Django==${{ matrix.django-version }}"
-          python -m pip install "elasticsearch-dsl==${{ matrix.es-dsl-version }}"
+          python -m pip install "elasticsearch==${{ matrix.es-py-version }}"
           python -m pip install -r requirements_test.txt
 
-      - name: Run tests with Python ${{ matrix.python-version }} and Django ${{ matrix.django-version }} and elasticsearch-dsl-py ${{ matrix.es-dsl-version }}
+      - name: Run tests with Python ${{ matrix.python-version }} and Django ${{ matrix.django-version }} and elasticsearch-dsl-py ${{ matrix.es-py-version }}
         run: |
-          TOX_ENV=$(echo "py${{ matrix.python-version }}-django-${{ matrix.django-version }}-es${{ matrix.es-dsl-version }}" | tr -d .)
+          TOX_ENV=$(echo "py${{ matrix.python-version }}-django-${{ matrix.django-version }}-es${{ matrix.es-py-version }}" | tr -d .)
           python -m tox -e $TOX_ENV -- --elasticsearch
           python -m tox -e $TOX_ENV -- --elasticsearch --signal-processor celery
 

README.rst

@@ -46,7 +46,7 @@ The library is compatible with all Elasticsearch versions since 5.x
 .. code-block:: python
 
     # Elasticsearch 8.x
-    elasticsearch-dsl>=8.0.0,<9.0.0
+    elasticsearch-dsl>=8.18.1,<9.0.0
 
     # Elasticsearch 7.x
     elasticsearch-dsl>=7.0.0,<8.0.0

django_elasticsearch_dsl/apps.py

@@ -2,7 +2,7 @@
 from django.conf import settings
 from django.utils.module_loading import import_string
 
-from elasticsearch_dsl.connections import connections
+from elasticsearch.dsl.connections import connections
 
 
 class DEDConfig(AppConfig):

django_elasticsearch_dsl/documents.py

@@ -7,7 +7,7 @@
 from django import VERSION as DJANGO_VERSION
 from django.db import models
 from elasticsearch.helpers import bulk, parallel_bulk
-from elasticsearch_dsl import Document as DSLDocument
+from elasticsearch.dsl import Document as DSLDocument
 from six import iteritems
 
 from .exceptions import ModelFieldNotMappedError

django_elasticsearch_dsl/fields.py

@@ -10,7 +10,7 @@
 else:
     from django.utils.encoding import force_str
 from django.utils.functional import Promise
-from elasticsearch_dsl.field import (
+from elasticsearch.dsl.field import (
     Boolean,
     Byte,
     Completion,
@@ -100,36 +100,24 @@ class ObjectField(DEDField, Object):
     def _get_inner_field_data(self, obj, field_value_to_ignore=None):
         data = {}
 
-        if hasattr(self, 'properties'):
-            for name, field in self.properties.to_dict().items():
-                if not isinstance(field, DEDField):
-                    continue
+        doc_instance = self._doc_class()
+        for name, field in self._doc_class._doc_type.mapping.properties._params.get(
+            'properties', {}).items():  # noqa
+            if not isinstance(field, DEDField):
+                continue
 
-                if field._path == []:
-                    field._path = [name]
+            if field._path == []:
+                field._path = [name]
 
+            # This allows for retrieving data from an InnerDoc with prepare_field_name functions.
+            prep_func = getattr(doc_instance, 'prepare_%s' % name, None)
+
+            if prep_func:
+                data[name] = prep_func(obj)
+            else:
                 data[name] = field.get_value_from_instance(
                     obj, field_value_to_ignore
                 )
-        else:
-            doc_instance = self._doc_class()
-            for name, field in self._doc_class._doc_type.mapping.properties._params.get(
-                'properties', {}).items():  # noqa
-                if not isinstance(field, DEDField):
-                    continue
-
-                if field._path == []:
-                    field._path = [name]
-
-                # This allows for retrieving data from an InnerDoc with prepare_field_name functions.
-                prep_func = getattr(doc_instance, 'prepare_%s' % name, None)
-
-                if prep_func:
-                    data[name] = prep_func(obj)
-                else:
-                    data[name] = field.get_value_from_instance(
-                        obj, field_value_to_ignore
-                    )
 
         # This allows for ObjectFields to be indexed from dicts with
         # dynamic keys (i.e. keys/fields not defined in 'properties')

django_elasticsearch_dsl/indices.py

@@ -1,6 +1,6 @@
 from copy import deepcopy
 
-from elasticsearch_dsl import Index as DSLIndex
+from elasticsearch.dsl import Index as DSLIndex
 from six import python_2_unicode_compatible
 
 from .apps import DEDConfig

django_elasticsearch_dsl/management/commands/search_index.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals, absolute_import
 from datetime import datetime
 
-from elasticsearch_dsl import connections
+from elasticsearch.dsl import connections
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 from six.moves import input

django_elasticsearch_dsl/registries.py

@@ -5,7 +5,7 @@
 
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.exceptions import ImproperlyConfigured
-from elasticsearch_dsl import AttrDict
+from elasticsearch.dsl import AttrDict
 from six import itervalues, iterkeys, iteritems
 
 from django_elasticsearch_dsl.exceptions import RedeclaredFieldError

django_elasticsearch_dsl/search.py

@@ -1,7 +1,7 @@
 from django.db.models import Case, When
 from django.db.models.fields import IntegerField
 
-from elasticsearch_dsl import Search as DSLSearch
+from elasticsearch.dsl import Search as DSLSearch
 
 
 class Search(DSLSearch):

django_elasticsearch_dsl/test/testcases.py

@@ -1,7 +1,7 @@
 import re
 
 from django.test.utils import captured_stderr
-from elasticsearch_dsl.connections import connections
+from elasticsearch.dsl.connections import connections
 
 from ..registries import registry
 

docs/source/es_index.rst

@@ -4,7 +4,7 @@ Index
 In typical scenario using `class Index` on a `Document` class is sufficient to perform any action.
 In a few cases though it can be useful to manipulate an Index object directly.
 
-To define an Elasticsearch index you must instantiate a ``elasticsearch_dsl.Index`` class
+To define an Elasticsearch index you must instantiate a ``elasticsearch.dsl.Index`` class
 and set the name and settings of the index.
 After you instantiate your class,
 you need to associate it with the Document you want to put in this Elasticsearch index
@@ -14,7 +14,7 @@ and also add the `registry.register_document` decorator.
 .. code-block:: python
 
     # documents.py
-    from elasticsearch_dsl import Index
+    from elasticsearch.dsl import Index
     from django_elasticsearch_dsl import Document
     from .models import Car, Manufacturer
 

docs/source/quickstart.rst

@@ -12,20 +12,20 @@ Install Django Elasticsearch DSL::
 
 Then add ``django_elasticsearch_dsl`` to the INSTALLED_APPS
 
-You must define ``ELASTICSEARCH_DSL`` in your django settings.
+You must define ``elasticsearch.dsl`` in your django settings.
 
 For example:
 
 .. code-block:: python
 
-    ELASTICSEARCH_DSL={
+    elasticsearch.dsl={
         'default': {
             'hosts': 'localhost:9200',
             'http_auth': ('username', 'password')
         }
     }
 
-``ELASTICSEARCH_DSL`` is then passed to ``elasticsearch-dsl-py.connections.configure`` (see here_).
+``elasticsearch.dsl`` is then passed to ``elasticsearch-dsl-py.connections.configure`` (see here_).
 
 .. _here: http://elasticsearch-dsl.readthedocs.io/en/stable/configuration.html#multiple-clusters
 

example/README.rst

@@ -18,7 +18,7 @@ connections setting in example/settings.py.
 
 .. code:: python
 
-    ELASTICSEARCH_DSL={
+    elasticsearch.dsl={
         'default': {
             'hosts': 'localhost:9200'
         },

example/example/settings.py

@@ -33,7 +33,6 @@
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
-
     'django_elasticsearch_dsl',
 
     # if your app has other dependencies that need to be added to the site
@@ -82,9 +81,7 @@
 }
 
 ELASTICSEARCH_DSL = {
-    'default': {
-        'hosts': 'localhost:9200'
-    },
+    "default": {"host": "localhost", "port": 9200, "timeout": 60},
 }
 
 ELASTICSEARCH_DSL_INDEX_SETTINGS = {

example/test_app/documents.py

@@ -1,4 +1,4 @@
-from elasticsearch_dsl import analyzer
+from elasticsearch.dsl import analyzer
 from django_elasticsearch_dsl import Document, Index, fields
 from django_elasticsearch_dsl.registries import registry
 

requirements.txt

@@ -1,2 +1,2 @@
 django>=3.2
-elasticsearch-dsl>=8.0.0,<9.0.0
+elasticsearch>=8.18.1,<9.0.0

requirements_dev.txt

@@ -1,7 +1,7 @@
 bumpversion==0.6.0
 wheel==0.41.2
 django>=3.2
-elasticsearch-dsl>=7.0.0,<8.0.0
+elasticsearch>=8.18.1,<9.0.0
 twine
 sphinx
 -e .

runtests.py

@@ -12,17 +12,15 @@ def get_settings(signal_processor):
         elasticsearch_dsl_default_settings = {
             'hosts': os.environ.get(
                 'ELASTICSEARCH_URL',
-                'https://127.0.0.1:9200'
+                'http://127.0.0.1:9200'
             ),
-            'basic_auth': (
-                os.environ.get('ELASTICSEARCH_USERNAME'),
-                os.environ.get('ELASTICSEARCH_PASSWORD')
-            )
+
         }
 
         elasticsearch_certs_path = os.environ.get(
             'ELASTICSEARCH_CERTS_PATH'
         )
+
         if elasticsearch_certs_path:
             elasticsearch_dsl_default_settings['ca_certs'] = (
                 elasticsearch_certs_path
@@ -120,18 +118,20 @@ def make_parser():
 def run_tests(*test_args):
     args, test_args = make_parser().parse_known_args(test_args)
     if args.elasticsearch:
-        os.environ.setdefault('ELASTICSEARCH_URL', "https://127.0.0.1:9200")
+        os.environ.setdefault('ELASTICSEARCH_URL', "http://127.0.0.1:9200")
 
-        username = args.elasticsearch_username or "elastic"
-        password = args.elasticsearch_password or "changeme"
-        os.environ.setdefault(
-            'ELASTICSEARCH_USERNAME', username
-        )
-        os.environ.setdefault(
-            'ELASTICSEARCH_PASSWORD', password
-        )
+        username = args.elasticsearch_username
+        password = args.elasticsearch_password
+        if username and password:
+            os.environ.setdefault(
+                'ELASTICSEARCH_USERNAME', username
+            )
+            os.environ.setdefault(
+                'ELASTICSEARCH_PASSWORD', password
+            )
 
     if args.elasticsearch_certs_path:
+        os.environ.setdefault('ELASTICSEARCH_URL', "https://127.0.0.1:9200")
         os.environ.setdefault(
             'ELASTICSEARCH_CERTS_PATH', args.elasticsearch_certs_path
         )

setup.py

@@ -8,7 +8,7 @@
 except ImportError:
     from distutils.core import setup
 
-version = '8.0'
+version = '8.18.1'
 
 if sys.argv[-1] == 'publish':
     try:
@@ -43,7 +43,7 @@
     ],
     include_package_data=True,
     install_requires=[
-        'elasticsearch-dsl>=8.9.0,<9.0.0',
+        'elasticsearch>=8.18.1,<9.0.0',
         'six',
     ],
     license="Apache Software License 2.0",

tests/__init__.py

@@ -1,3 +1,3 @@
-from elasticsearch_dsl import VERSION
+from elasticsearch import VERSION
 
 ES_MAJOR_VERSION = VERSION[0]

tests/documents.py

@@ -1,4 +1,4 @@
-from elasticsearch_dsl import analyzer
+from elasticsearch.dsl import analyzer
 from django_elasticsearch_dsl import Document, fields
 from django_elasticsearch_dsl.registries import registry
 

tests/test_documents.py

@@ -10,7 +10,7 @@
     from django.utils.translation import ugettext_lazy as _
 else:
     from django.utils.translation import gettext_lazy as _
-from elasticsearch_dsl import GeoPoint, InnerDoc
+from elasticsearch.dsl import GeoPoint, InnerDoc
 from mock import patch, Mock
 
 from django_elasticsearch_dsl import fields
@@ -453,7 +453,7 @@ def test_init_prepare_results(self):
     # got iterated and generate_id called.
     # If we mock the bulk in django_elasticsearch_dsl.document
     # the actual bulk will be never called and the test will fail
-    @patch('elasticsearch_dsl.connections.Elasticsearch.bulk')
+    @patch('elasticsearch.dsl.connections.Elasticsearch.bulk')
     def test_default_generate_id_is_called(self, _):
         article = Article(
             id=124594,
@@ -481,7 +481,7 @@ class Index:
             d.update(article)
             patched_method.assert_called()
 
-    @patch('elasticsearch_dsl.connections.Elasticsearch.bulk')
+    @patch('elasticsearch.dsl.connections.Elasticsearch.bulk')
     def test_custom_generate_id_is_called(self, mock_bulk):
         article = Article(
             id=54218,
@@ -511,7 +511,7 @@ def generate_id(cls, article):
         data = json.loads(mock_bulk.call_args[1]['operations'][1])
         assert data['slug'] == article.slug
 
-    @patch('elasticsearch_dsl.connections.Elasticsearch.bulk')
+    @patch('elasticsearch.dsl.connections.Elasticsearch.bulk')
     def test_should_index_object_is_called(self, mock_bulk):
         doc = CarDocument()
         car1 = Car()
@@ -525,7 +525,7 @@ def test_should_index_object_is_called(self, mock_bulk):
             self.assertEqual(mock_should_index_object.call_count, 3,
                              "should_index_object is called")
 
-    @patch('elasticsearch_dsl.connections.Elasticsearch.bulk')
+    @patch('elasticsearch.dsl.connections.Elasticsearch.bulk')
     def test_should_index_object_working_perfectly(self, mock_bulk):
         article1 = Article(slug='article1')
         article2 = Article(slug='article2')

tests/test_integration.py

@@ -11,7 +11,7 @@
 from six import StringIO
 
 from elasticsearch.exceptions import NotFoundError
-from elasticsearch_dsl import Index as DSLIndex
+from elasticsearch.dsl import Index as DSLIndex
 from django_elasticsearch_dsl.test import ESTestCase, is_es_online
 from tests import ES_MAJOR_VERSION