From 9e88c99a1d6218d37e06310bcea2d1c4607187c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mehmet=20Ali=20SARA=C3=87?= Date: Wed, 28 May 2025 02:32:31 +0300 Subject: [PATCH] Fixes compatibility with Elasticsearch 8.18.1 Updates dependencies and code to ensure compatibility with Elasticsearch version 8.18.1. This involves: - Replacing `elasticsearch-dsl` with `elasticsearch` in requirements and code. - Updating the GitHub Actions CI configuration to test against newer versions of Python and Django. - Adjusting settings and test configurations. This resolves issues arising from API changes in the newer Elasticsearch client. Check the note on top of the elasticsearch dsl (https://elasticsearch-dsl.readthedocs.io/en/latest/) --- .github/workflows/ci.yml | 21 ++++++---- README.rst | 2 +- django_elasticsearch_dsl/apps.py | 2 +- django_elasticsearch_dsl/documents.py | 2 +- django_elasticsearch_dsl/fields.py | 40 +++++++------------ django_elasticsearch_dsl/indices.py | 2 +- .../management/commands/search_index.py | 2 +- django_elasticsearch_dsl/registries.py | 2 +- django_elasticsearch_dsl/search.py | 2 +- django_elasticsearch_dsl/test/testcases.py | 2 +- docs/source/es_index.rst | 4 +- docs/source/quickstart.rst | 6 +-- example/README.rst | 2 +- example/example/settings.py | 5 +-- example/test_app/documents.py | 2 +- requirements.txt | 2 +- requirements_dev.txt | 2 +- runtests.py | 28 ++++++------- setup.py | 4 +- tests/__init__.py | 2 +- tests/documents.py | 2 +- tests/test_documents.py | 10 ++--- tests/test_integration.py | 2 +- tox.ini | 10 ++++- 24 files changed, 78 insertions(+), 80 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b384d056..686fca41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,14 +13,21 @@ jobs: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - django-version: ["3.2", "4.1", "4.2"] - es-dsl-version: ["6.4", "7.4"] - es-version: ["8.10.2"] + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ] + django-version: [ "3.2", "4.1", "4.2", "5.0", "5.1", "5.2" ] + es-py-version: [ "6.4", "7.4", "8.18.1"] exclude: - python-version: "3.11" django-version: "3.2" + - python-version: "3.12" + django-version: "3.2" + - python-version: "3.13" + django-version: "3.2" + - es-py-version: "6.4" + es-version: "8.10.2" + - es-py-version: "7.4" + es-version: "8.10.2" steps: - name: Install and Run Elasticsearch @@ -47,12 +54,12 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install "Django==${{ matrix.django-version }}" - python -m pip install "elasticsearch-dsl==${{ matrix.es-dsl-version }}" + python -m pip install "elasticsearch==${{ matrix.es-py-version }}" python -m pip install -r requirements_test.txt - - name: Run tests with Python ${{ matrix.python-version }} and Django ${{ matrix.django-version }} and elasticsearch-dsl-py ${{ matrix.es-dsl-version }} + - name: Run tests with Python ${{ matrix.python-version }} and Django ${{ matrix.django-version }} and elasticsearch-dsl-py ${{ matrix.es-py-version }} run: | - TOX_ENV=$(echo "py${{ matrix.python-version }}-django-${{ matrix.django-version }}-es${{ matrix.es-dsl-version }}" | tr -d .) + TOX_ENV=$(echo "py${{ matrix.python-version }}-django-${{ matrix.django-version }}-es${{ matrix.es-py-version }}" | tr -d .) python -m tox -e $TOX_ENV -- --elasticsearch python -m tox -e $TOX_ENV -- --elasticsearch --signal-processor celery diff --git a/README.rst b/README.rst index eb29ad89..376e136d 100644 --- a/README.rst +++ b/README.rst @@ -46,7 +46,7 @@ The library is compatible with all Elasticsearch versions since 5.x .. code-block:: python # Elasticsearch 8.x - elasticsearch-dsl>=8.0.0,<9.0.0 + elasticsearch-dsl>=8.18.1,<9.0.0 # Elasticsearch 7.x elasticsearch-dsl>=7.0.0,<8.0.0 diff --git a/django_elasticsearch_dsl/apps.py b/django_elasticsearch_dsl/apps.py index 73e451c3..884bedc5 100644 --- a/django_elasticsearch_dsl/apps.py +++ b/django_elasticsearch_dsl/apps.py @@ -2,7 +2,7 @@ from django.conf import settings from django.utils.module_loading import import_string -from elasticsearch_dsl.connections import connections +from elasticsearch.dsl.connections import connections class DEDConfig(AppConfig): diff --git a/django_elasticsearch_dsl/documents.py b/django_elasticsearch_dsl/documents.py index 4671064d..913656ae 100644 --- a/django_elasticsearch_dsl/documents.py +++ b/django_elasticsearch_dsl/documents.py @@ -7,7 +7,7 @@ from django import VERSION as DJANGO_VERSION from django.db import models from elasticsearch.helpers import bulk, parallel_bulk -from elasticsearch_dsl import Document as DSLDocument +from elasticsearch.dsl import Document as DSLDocument from six import iteritems from .exceptions import ModelFieldNotMappedError diff --git a/django_elasticsearch_dsl/fields.py b/django_elasticsearch_dsl/fields.py index 2652b68e..5056810c 100644 --- a/django_elasticsearch_dsl/fields.py +++ b/django_elasticsearch_dsl/fields.py @@ -10,7 +10,7 @@ else: from django.utils.encoding import force_str from django.utils.functional import Promise -from elasticsearch_dsl.field import ( +from elasticsearch.dsl.field import ( Boolean, Byte, Completion, @@ -100,36 +100,24 @@ class ObjectField(DEDField, Object): def _get_inner_field_data(self, obj, field_value_to_ignore=None): data = {} - if hasattr(self, 'properties'): - for name, field in self.properties.to_dict().items(): - if not isinstance(field, DEDField): - continue + doc_instance = self._doc_class() + for name, field in self._doc_class._doc_type.mapping.properties._params.get( + 'properties', {}).items(): # noqa + if not isinstance(field, DEDField): + continue - if field._path == []: - field._path = [name] + if field._path == []: + field._path = [name] + # This allows for retrieving data from an InnerDoc with prepare_field_name functions. + prep_func = getattr(doc_instance, 'prepare_%s' % name, None) + + if prep_func: + data[name] = prep_func(obj) + else: data[name] = field.get_value_from_instance( obj, field_value_to_ignore ) - else: - doc_instance = self._doc_class() - for name, field in self._doc_class._doc_type.mapping.properties._params.get( - 'properties', {}).items(): # noqa - if not isinstance(field, DEDField): - continue - - if field._path == []: - field._path = [name] - - # This allows for retrieving data from an InnerDoc with prepare_field_name functions. - prep_func = getattr(doc_instance, 'prepare_%s' % name, None) - - if prep_func: - data[name] = prep_func(obj) - else: - data[name] = field.get_value_from_instance( - obj, field_value_to_ignore - ) # This allows for ObjectFields to be indexed from dicts with # dynamic keys (i.e. keys/fields not defined in 'properties') diff --git a/django_elasticsearch_dsl/indices.py b/django_elasticsearch_dsl/indices.py index 115874d5..b2a1ef79 100644 --- a/django_elasticsearch_dsl/indices.py +++ b/django_elasticsearch_dsl/indices.py @@ -1,6 +1,6 @@ from copy import deepcopy -from elasticsearch_dsl import Index as DSLIndex +from elasticsearch.dsl import Index as DSLIndex from six import python_2_unicode_compatible from .apps import DEDConfig diff --git a/django_elasticsearch_dsl/management/commands/search_index.py b/django_elasticsearch_dsl/management/commands/search_index.py index 06bf8519..4e31697c 100644 --- a/django_elasticsearch_dsl/management/commands/search_index.py +++ b/django_elasticsearch_dsl/management/commands/search_index.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals, absolute_import from datetime import datetime -from elasticsearch_dsl import connections +from elasticsearch.dsl import connections from django.conf import settings from django.core.management.base import BaseCommand, CommandError from six.moves import input diff --git a/django_elasticsearch_dsl/registries.py b/django_elasticsearch_dsl/registries.py index e2623ddd..b7b1d519 100644 --- a/django_elasticsearch_dsl/registries.py +++ b/django_elasticsearch_dsl/registries.py @@ -5,7 +5,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ImproperlyConfigured -from elasticsearch_dsl import AttrDict +from elasticsearch.dsl import AttrDict from six import itervalues, iterkeys, iteritems from django_elasticsearch_dsl.exceptions import RedeclaredFieldError diff --git a/django_elasticsearch_dsl/search.py b/django_elasticsearch_dsl/search.py index bc66d82e..be0093ab 100644 --- a/django_elasticsearch_dsl/search.py +++ b/django_elasticsearch_dsl/search.py @@ -1,7 +1,7 @@ from django.db.models import Case, When from django.db.models.fields import IntegerField -from elasticsearch_dsl import Search as DSLSearch +from elasticsearch.dsl import Search as DSLSearch class Search(DSLSearch): diff --git a/django_elasticsearch_dsl/test/testcases.py b/django_elasticsearch_dsl/test/testcases.py index 12c3a802..43787b22 100644 --- a/django_elasticsearch_dsl/test/testcases.py +++ b/django_elasticsearch_dsl/test/testcases.py @@ -1,7 +1,7 @@ import re from django.test.utils import captured_stderr -from elasticsearch_dsl.connections import connections +from elasticsearch.dsl.connections import connections from ..registries import registry diff --git a/docs/source/es_index.rst b/docs/source/es_index.rst index 618856dc..963d3db9 100644 --- a/docs/source/es_index.rst +++ b/docs/source/es_index.rst @@ -4,7 +4,7 @@ Index In typical scenario using `class Index` on a `Document` class is sufficient to perform any action. In a few cases though it can be useful to manipulate an Index object directly. -To define an Elasticsearch index you must instantiate a ``elasticsearch_dsl.Index`` class +To define an Elasticsearch index you must instantiate a ``elasticsearch.dsl.Index`` class and set the name and settings of the index. After you instantiate your class, you need to associate it with the Document you want to put in this Elasticsearch index @@ -14,7 +14,7 @@ and also add the `registry.register_document` decorator. .. code-block:: python # documents.py - from elasticsearch_dsl import Index + from elasticsearch.dsl import Index from django_elasticsearch_dsl import Document from .models import Car, Manufacturer diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index f3b02b84..97e87569 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -12,20 +12,20 @@ Install Django Elasticsearch DSL:: Then add ``django_elasticsearch_dsl`` to the INSTALLED_APPS -You must define ``ELASTICSEARCH_DSL`` in your django settings. +You must define ``elasticsearch.dsl`` in your django settings. For example: .. code-block:: python - ELASTICSEARCH_DSL={ + elasticsearch.dsl={ 'default': { 'hosts': 'localhost:9200', 'http_auth': ('username', 'password') } } -``ELASTICSEARCH_DSL`` is then passed to ``elasticsearch-dsl-py.connections.configure`` (see here_). +``elasticsearch.dsl`` is then passed to ``elasticsearch-dsl-py.connections.configure`` (see here_). .. _here: http://elasticsearch-dsl.readthedocs.io/en/stable/configuration.html#multiple-clusters diff --git a/example/README.rst b/example/README.rst index 27335c34..2d842b65 100644 --- a/example/README.rst +++ b/example/README.rst @@ -18,7 +18,7 @@ connections setting in example/settings.py. .. code:: python - ELASTICSEARCH_DSL={ + elasticsearch.dsl={ 'default': { 'hosts': 'localhost:9200' }, diff --git a/example/example/settings.py b/example/example/settings.py index 9367615f..b526c30f 100644 --- a/example/example/settings.py +++ b/example/example/settings.py @@ -33,7 +33,6 @@ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', - 'django_elasticsearch_dsl', # if your app has other dependencies that need to be added to the site @@ -82,9 +81,7 @@ } ELASTICSEARCH_DSL = { - 'default': { - 'hosts': 'localhost:9200' - }, + "default": {"host": "localhost", "port": 9200, "timeout": 60}, } ELASTICSEARCH_DSL_INDEX_SETTINGS = { diff --git a/example/test_app/documents.py b/example/test_app/documents.py index e1e449d9..693d665f 100644 --- a/example/test_app/documents.py +++ b/example/test_app/documents.py @@ -1,4 +1,4 @@ -from elasticsearch_dsl import analyzer +from elasticsearch.dsl import analyzer from django_elasticsearch_dsl import Document, Index, fields from django_elasticsearch_dsl.registries import registry diff --git a/requirements.txt b/requirements.txt index 53d79b32..c2b3608f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ django>=3.2 -elasticsearch-dsl>=8.0.0,<9.0.0 +elasticsearch>=8.18.1,<9.0.0 diff --git a/requirements_dev.txt b/requirements_dev.txt index 43b5c3fe..e9689985 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,7 +1,7 @@ bumpversion==0.6.0 wheel==0.41.2 django>=3.2 -elasticsearch-dsl>=7.0.0,<8.0.0 +elasticsearch>=8.18.1,<9.0.0 twine sphinx -e . diff --git a/runtests.py b/runtests.py index 0b613389..843f0c60 100644 --- a/runtests.py +++ b/runtests.py @@ -12,17 +12,15 @@ def get_settings(signal_processor): elasticsearch_dsl_default_settings = { 'hosts': os.environ.get( 'ELASTICSEARCH_URL', - 'https://127.0.0.1:9200' + 'http://127.0.0.1:9200' ), - 'basic_auth': ( - os.environ.get('ELASTICSEARCH_USERNAME'), - os.environ.get('ELASTICSEARCH_PASSWORD') - ) + } elasticsearch_certs_path = os.environ.get( 'ELASTICSEARCH_CERTS_PATH' ) + if elasticsearch_certs_path: elasticsearch_dsl_default_settings['ca_certs'] = ( elasticsearch_certs_path @@ -120,18 +118,20 @@ def make_parser(): def run_tests(*test_args): args, test_args = make_parser().parse_known_args(test_args) if args.elasticsearch: - os.environ.setdefault('ELASTICSEARCH_URL', "https://127.0.0.1:9200") + os.environ.setdefault('ELASTICSEARCH_URL', "http://127.0.0.1:9200") - username = args.elasticsearch_username or "elastic" - password = args.elasticsearch_password or "changeme" - os.environ.setdefault( - 'ELASTICSEARCH_USERNAME', username - ) - os.environ.setdefault( - 'ELASTICSEARCH_PASSWORD', password - ) + username = args.elasticsearch_username + password = args.elasticsearch_password + if username and password: + os.environ.setdefault( + 'ELASTICSEARCH_USERNAME', username + ) + os.environ.setdefault( + 'ELASTICSEARCH_PASSWORD', password + ) if args.elasticsearch_certs_path: + os.environ.setdefault('ELASTICSEARCH_URL', "https://127.0.0.1:9200") os.environ.setdefault( 'ELASTICSEARCH_CERTS_PATH', args.elasticsearch_certs_path ) diff --git a/setup.py b/setup.py index a3c60e1b..960787c4 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ except ImportError: from distutils.core import setup -version = '8.0' +version = '8.18.1' if sys.argv[-1] == 'publish': try: @@ -43,7 +43,7 @@ ], include_package_data=True, install_requires=[ - 'elasticsearch-dsl>=8.9.0,<9.0.0', + 'elasticsearch>=8.18.1,<9.0.0', 'six', ], license="Apache Software License 2.0", diff --git a/tests/__init__.py b/tests/__init__.py index 7f06df26..4e452429 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,3 +1,3 @@ -from elasticsearch_dsl import VERSION +from elasticsearch import VERSION ES_MAJOR_VERSION = VERSION[0] diff --git a/tests/documents.py b/tests/documents.py index 08ef83f1..30748868 100644 --- a/tests/documents.py +++ b/tests/documents.py @@ -1,4 +1,4 @@ -from elasticsearch_dsl import analyzer +from elasticsearch.dsl import analyzer from django_elasticsearch_dsl import Document, fields from django_elasticsearch_dsl.registries import registry diff --git a/tests/test_documents.py b/tests/test_documents.py index d7dd2ac7..484dc0a3 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -10,7 +10,7 @@ from django.utils.translation import ugettext_lazy as _ else: from django.utils.translation import gettext_lazy as _ -from elasticsearch_dsl import GeoPoint, InnerDoc +from elasticsearch.dsl import GeoPoint, InnerDoc from mock import patch, Mock from django_elasticsearch_dsl import fields @@ -453,7 +453,7 @@ def test_init_prepare_results(self): # got iterated and generate_id called. # If we mock the bulk in django_elasticsearch_dsl.document # the actual bulk will be never called and the test will fail - @patch('elasticsearch_dsl.connections.Elasticsearch.bulk') + @patch('elasticsearch.dsl.connections.Elasticsearch.bulk') def test_default_generate_id_is_called(self, _): article = Article( id=124594, @@ -481,7 +481,7 @@ class Index: d.update(article) patched_method.assert_called() - @patch('elasticsearch_dsl.connections.Elasticsearch.bulk') + @patch('elasticsearch.dsl.connections.Elasticsearch.bulk') def test_custom_generate_id_is_called(self, mock_bulk): article = Article( id=54218, @@ -511,7 +511,7 @@ def generate_id(cls, article): data = json.loads(mock_bulk.call_args[1]['operations'][1]) assert data['slug'] == article.slug - @patch('elasticsearch_dsl.connections.Elasticsearch.bulk') + @patch('elasticsearch.dsl.connections.Elasticsearch.bulk') def test_should_index_object_is_called(self, mock_bulk): doc = CarDocument() car1 = Car() @@ -525,7 +525,7 @@ def test_should_index_object_is_called(self, mock_bulk): self.assertEqual(mock_should_index_object.call_count, 3, "should_index_object is called") - @patch('elasticsearch_dsl.connections.Elasticsearch.bulk') + @patch('elasticsearch.dsl.connections.Elasticsearch.bulk') def test_should_index_object_working_perfectly(self, mock_bulk): article1 = Article(slug='article1') article2 = Article(slug='article2') diff --git a/tests/test_integration.py b/tests/test_integration.py index f01d5781..abb0b237 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -11,7 +11,7 @@ from six import StringIO from elasticsearch.exceptions import NotFoundError -from elasticsearch_dsl import Index as DSLIndex +from elasticsearch.dsl import Index as DSLIndex from django_elasticsearch_dsl.test import ESTestCase, is_es_online from tests import ES_MAJOR_VERSION diff --git a/tox.ini b/tox.ini index 54fd05f8..d75c9f80 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = - py{38,39,310}-django-{32,41,42}-{es64,es74} - py{311}-django-{41,42}-{es64,es74} + py{38,39,310,311}-django-{32,41,42}-{es64,es74,es9} + py{311,312,313}-django-{41,42,50,51,52}-{es64,es74,es818} [testenv] @@ -13,8 +13,12 @@ deps = django-32: Django>=3.2,<3.3 django-41: Django>=4.1,<4.2 django-42: Django>=4.2,<4.3 + django-50: Django>=5.0,<5.1 + django-51: Django>=5.1,<5.2 + django-52: Django>=5.2,<5.3 es64: elasticsearch-dsl>=6.4.0,<7.0.0 es74: elasticsearch-dsl>=7.4.0,<8 + es818: elasticsearch>=8.18.0,<9 -r{toxinidir}/requirements_test.txt basepython = @@ -22,3 +26,5 @@ basepython = py39: python3.9 py310: python3.10 py311: python3.11 + py312: python3.12 + py313: python3.13