Migrate to Elasticsearch 8.19, Python 3.12 and SQLAlchemy 2.0

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-slim-bookworm
+FROM python:3.12-slim-bookworm
 
 # Avoid prompts from apt
 ENV DEBIAN_FRONTEND=noninteractive

.github/workflows/ci.yml

@@ -44,7 +44,7 @@ jobs:
         uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: 3.9
+          python-version: "3.12"
       - name: Cache pip
         uses: actions/cache@v4.3.0
         with:
@@ -70,19 +70,21 @@ jobs:
           PGPORT: 5432
       - name: Create config from templates
         run: make CONFIG=config/env.test load-env
-      - name: Install java 8 for (old) elasticsearch
-        uses: actions/setup-java@v5
-        with:
-          distribution: "adopt"
-          java-version: 8
-      - name: Run (old) elasticsearch
+      - name: Install elasticsearch & plugin & 1st start
         run: |
           mkdir /tmp/elasticsearch
-          wget -O - https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.4.6/elasticsearch-2.4.6.tar.gz | tar xz --directory=/tmp/elasticsearch --strip-components=1
-          /tmp/elasticsearch/bin/plugin install analysis-icu
-          /tmp/elasticsearch/bin/elasticsearch --daemonize --path.data /tmp
+          wget -O - https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-8.19.18-linux-x86_64.tar.gz | tar xz --directory=/tmp/elasticsearch --strip-components=1
+          /tmp/elasticsearch/bin/elasticsearch-plugin install analysis-icu
+          /tmp/elasticsearch/bin/elasticsearch --daemonize
           sleep 30 # ElasticSearch takes few seconds to start, make sure it is available when the build script runs
-      - run: curl -v http://localhost:9200
+      - name: ElasticSearch settings update after auto configuration at 1st start & restart
+        run: |
+          sed -i 's/true/false/g' /tmp/elasticsearch/config/elasticsearch.yml
+          pgrep -f elasticsearch | xargs kill -9
+          /tmp/elasticsearch/bin/elasticsearch --daemonize
+          sleep 30 # Password change requires time
+      - name: Check elasticsearch
+        run: curl -v http://localhost:9200
       - name: Run tests (with secrets)
         # Accept running on pull requests except if from dependabot
         if: ${{ github.event_name != 'pull_request' || github.actor != 'dependabot[bot]' }}

.gitignore

@@ -22,3 +22,5 @@ config/docker
 coverage.xml
 /config/env.local
 project.tar
+scripts/anonymize-database/*/
+config/env.sandbox

alembic_migration/versions/2c90b2e5ca7e_add_chinese.py

@@ -48,7 +48,7 @@ def upgrade():
     # if there is some value in the table. If yes, it's not a test DB and we have to complete them.
 
     conn = op.get_bind()
-    res = conn.execute("SELECT count(1) FROM guidebook.langs").fetchall()
+    res = conn.execute(sa.text("SELECT count(1) FROM guidebook.langs")).fetchall()
 
     if res[0][0] != 0:
         op.execute("INSERT INTO guidebook.langs VALUES ('zh');")

alembic_migration/versions/305b064bdf66_add_slovenian.py

@@ -21,7 +21,7 @@
 
 def upgrade():
     conn = op.get_bind()
-    res = conn.execute("SELECT count(1) FROM guidebook.langs").fetchall()
+    res = conn.execute(sa.text("SELECT count(1) FROM guidebook.langs")).fetchall()
 
     if res[0][0] != 0:
         op.execute("INSERT INTO guidebook.langs VALUES ('sl');")

alembic_migration/versions/38df9393c9a9_init.py

@@ -702,18 +702,27 @@ def upgrade():
     op.create_table('documents_geometries',
     sa.Column('version', sa.Integer(), nullable=False),
     sa.Column('document_id', sa.Integer(), nullable=False),
-    sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='POINT', srid=3857, management=True), nullable=True),
-    sa.Column('geom_detail', geoalchemy2.types.Geometry(srid=3857, management=True, use_typmod=False), nullable=True),
+    sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='POINT', srid=3857), nullable=True),
+    sa.Column('geom_detail', geoalchemy2.types.Geometry(srid=3857, use_typmod=False, spatial_index=False), nullable=True),
     sa.ForeignKeyConstraint(['document_id'], ['guidebook.documents.document_id'], ),
     sa.PrimaryKeyConstraint('document_id'),
     schema='guidebook'
     )
+    # geom_detail uses use_typmod=False (see below, dimension constraint is dropped to allow
+    # mixed 2D/3D/4D geometries), which routes it through geoalchemy2's legacy "managed"
+    # AddGeometryColumn() codepath. That codepath's automatic spatial-index creation collides
+    # with alembic's own index creation (the Index object re-attaches itself to the table),
+    # raising a DuplicateTable error - so spatial_index is disabled above and the index is
+    # created explicitly here instead.
+    op.create_index(
+        'idx_documents_geometries_geom_detail', 'documents_geometries', ['geom_detail'],
+        unique=False, schema='guidebook', postgresql_using='gist')
     op.create_table('documents_geometries_archives',
     sa.Column('version', sa.Integer(), nullable=False),
     sa.Column('id', sa.Integer(), nullable=False),
     sa.Column('document_id', sa.Integer(), nullable=False),
-    sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='POINT', srid=3857, spatial_index=False, management=True), nullable=True),
-    sa.Column('geom_detail', geoalchemy2.types.Geometry(srid=3857, spatial_index=False, management=True, use_typmod=False), nullable=True),
+    sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='POINT', srid=3857, spatial_index=False), nullable=True),
+    sa.Column('geom_detail', geoalchemy2.types.Geometry(srid=3857, spatial_index=False, use_typmod=False), nullable=True),
     sa.ForeignKeyConstraint(['document_id'], ['guidebook.documents.document_id'], ),
     sa.PrimaryKeyConstraint('id'),
     sa.UniqueConstraint('version', 'document_id', name='uq_documents_geometries_archives_document_id_version_lang'),

alembic_migration/versions/bb61456d557f_create_stops_and_waypoints_stops.py

@@ -23,7 +23,7 @@ def upgrade():
         sa.Column('stoparea_name', sa.String(), nullable=False),
         sa.Column('line', sa.String(), nullable=False),
         sa.Column('operator', sa.String(), nullable=False),
-        sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='POINT', srid=3857, management=True), nullable=True),
+        sa.Column('geom', geoalchemy2.types.Geometry(geometry_type='POINT', srid=3857), nullable=True),
         schema='guidebook'
     )
 

c2corg_api/ext/colander_ext.py

@@ -1,14 +1,19 @@
 # Inspired from the c2cgeoform colander extension
 # https://github.com/camptocamp/c2cgeoform/blob/master/c2cgeoform/ext/
 
+import struct
+
 from colander import (null, Invalid, SchemaType)
 
 from geoalchemy2 import WKBElement
 from geomet import wkb
-from geoalchemy2.compat import buffer, bytes
 import geojson
 from numbers import Number
 
+# EWKB SRID flag, see https://git.osgeo.org/gitea/postgis/postgis/raw/
+# master/doc/ZMSgeoms.txt
+_EWKB_SRID_FLAG = 0x20000000
+
 
 class Geometry(SchemaType):
     """ A Colander type meant to be used with GeoAlchemy 2 geometry columns.
@@ -78,16 +83,35 @@ def cstruct_children(self, node, cstruct):
 
 def wkbelement_from_geojson(geojson, srid):
     geometry = wkb.dumps(geojson, big_endian=False)
+    if srid and srid > 0:
+        # Embed the SRID directly in the WKB header, turning it into EWKB.
+        # geoalchemy2 binds a non-extended WKBElement by converting it
+        # through Shapely, which does not support M (4th dimension)
+        # coordinates and silently drops them. A WKBElement built from real
+        # EWKB is detected as already-extended and bound as a hex string
+        # directly instead, preserving all dimensions.
+        geometry = _to_ewkb(geometry, srid)
     return from_wkb(geometry, srid)
 
 
+def _to_ewkb(wkb_bytes, srid):
+    byte_order = wkb_bytes[0]
+    fmt = '<I' if byte_order == 1 else '>I'
+    geom_type = struct.unpack(fmt, wkb_bytes[1:5])[0]
+    ewkb_type = geom_type | _EWKB_SRID_FLAG
+    return (
+        wkb_bytes[0:1] + struct.pack(fmt, ewkb_type) +
+        struct.pack(fmt, srid) + wkb_bytes[5:]
+    )
+
+
 def geojson_from_wkbelement(wkb_element):
     geometry = wkb.loads(bytes(wkb_element.data))
     return geojson.dumps(geometry)
 
 
 def from_wkb(wkb, srid=-1):
-    return WKBElement(buffer(wkb), srid=srid)
+    return WKBElement(memoryview(wkb), srid=srid)
 
 
 def is_valid_geometry(obj):

c2corg_api/models/__init__.py

@@ -1,6 +1,5 @@
-from sqlalchemy.ext.declarative import declarative_base
-
 from sqlalchemy.orm import (
+    declarative_base,
     scoped_session,
     sessionmaker,
     )

c2corg_api/models/area_association.py

@@ -66,7 +66,7 @@ def update_area(area, reset=False):
         # ignore forwarded areas
         return
 
-    area_geom = select([DocumentGeometry.geom_detail]). \
+    area_geom = select(DocumentGeometry.geom_detail). \
         where(DocumentGeometry.document_id == area.document_id)
     intersecting_documents = DBSession. \
         query(
@@ -111,9 +111,9 @@ def update_areas_for_document(document, reset=False):
         # ignore forwarded areas
         return
 
-    document_geom = select([DocumentGeometry.geom]). \
+    document_geom = select(DocumentGeometry.geom). \
         where(DocumentGeometry.document_id == document.document_id)
-    document_geom_detail = select([DocumentGeometry.geom_detail]). \
+    document_geom_detail = select(DocumentGeometry.geom_detail). \
         where(DocumentGeometry.document_id == document.document_id)
     intersecting_areas = DBSession. \
         query(

c2corg_api/models/association_views.py

@@ -30,21 +30,21 @@ def _get_select_waypoints_for_routes():
     route_type = text('\'' + ROUTE_TYPE + '\'')
 
     select_linked_waypoints = \
-        select([
+        select(
             Association.child_document_id.label('route_id'),
             Association.parent_document_id.label('waypoint_id')
-        ]). \
+        ). \
         where(
             and_(
                 Association.parent_document_type == waypoint_type,
                 Association.child_document_type == route_type)). \
         cte('linked_waypoints')
 
     select_waypoint_parents = \
-        select([
+        select(
             select_linked_waypoints.c.route_id,
             Association.parent_document_id.label('waypoint_id')
-        ]). \
+        ). \
         select_from(join(
             select_linked_waypoints,
             Association,
@@ -56,10 +56,10 @@ def _get_select_waypoints_for_routes():
         cte('waypoint_parents')
 
     select_waypoint_grandparents = \
-        select([
+        select(
             select_waypoint_parents.c.route_id,
             Association.parent_document_id.label('waypoint_id')
-        ]). \
+        ). \
         select_from(join(
             select_waypoint_parents,
             Association,
@@ -89,12 +89,12 @@ def _get_select_waypoints_for_routes_aggregated():
     """
     all_waypoints = _get_select_waypoints_for_routes()
     return \
-        select([
+        select(
             all_waypoints.c.route_id.label('route_id'),
             func.array_agg(
                 all_waypoints.c.waypoint_id,
                 type_=postgresql.ARRAY(Integer)).label('waypoint_ids')
-        ]). \
+        ). \
         select_from(all_waypoints). \
         group_by(all_waypoints.c.route_id)
 
@@ -135,10 +135,10 @@ def _get_select_waypoints_for_outings_aggregated():
     route_type = text('\'' + ROUTE_TYPE + '\'')
     all_waypoints_for_routes = _get_select_waypoints_for_routes()
     waypoints_for_outings = \
-        select([
+        select(
             Association.child_document_id.label('outing_id'),
             all_waypoints_for_routes.c.waypoint_id
-        ]). \
+        ). \
         select_from(join(
             Association,
             all_waypoints_for_routes,
@@ -150,12 +150,12 @@ def _get_select_waypoints_for_outings_aggregated():
             ))). \
         cte('waypoints_for_outings')
     return \
-        select([
+        select(
             waypoints_for_outings.c.outing_id.label('outing_id'),
             func.array_agg(
                 waypoints_for_outings.c.waypoint_id,
                 type_=postgresql.ARRAY(Integer)).label('waypoint_ids')
-        ]). \
+        ). \
         select_from(waypoints_for_outings). \
         group_by(waypoints_for_outings.c.outing_id)
 
@@ -189,12 +189,12 @@ def _get_select_users_for_outings_aggregated():
     outing_type = text('\'' + OUTING_TYPE + '\'')
     user_type = text('\'' + USERPROFILE_TYPE + '\'')
     return \
-        select([
+        select(
             Association.child_document_id.label('outing_id'),
             func.array_agg(
                 Association.parent_document_id,
                 type_=postgresql.ARRAY(Integer)).label('user_ids')
-        ]). \
+        ). \
         select_from(Association). \
         where(and_(
             Association.parent_document_type == user_type,
@@ -232,12 +232,12 @@ def _get_select_routes_for_outings_aggregated():
     outing_type = text('\'' + OUTING_TYPE + '\'')
     route_type = text('\'' + ROUTE_TYPE + '\'')
     return \
-        select([
+        select(
             Association.child_document_id.label('outing_id'),
             func.array_agg(
                 Association.parent_document_id,
                 type_=postgresql.ARRAY(Integer)).label('route_ids')
-        ]). \
+        ). \
         select_from(Association). \
         where(and_(
             Association.parent_document_type == route_type,
@@ -273,12 +273,12 @@ def _get_select_users_for_routes_aggregated():
     associated users.
     """
     return \
-        select([
+        select(
             DocumentTag.document_id.label('route_id'),
             func.array_agg(
                 DocumentTag.user_id,
                 type_=postgresql.ARRAY(Integer)).label('user_ids')
-        ]). \
+        ). \
         select_from(DocumentTag). \
         group_by(DocumentTag.document_id)
 

c2corg_api/models/document.py

@@ -118,13 +118,14 @@ def update(self, other):
         """
         copy_attributes(other, self, Document._ATTRIBUTES_WHITELISTED)
 
-        for locale_in in other.locales:
-            locale = self.get_locale(locale_in.lang)
-            if locale:
-                locale.update(locale_in)
-                locale.document_id = self.document_id
-            else:
-                self.locales.append(locale_in)
+        with DBSession.no_autoflush:
+            for locale_in in other.locales:
+                locale = self.get_locale(locale_in.lang)
+                if locale:
+                    locale.update(locale_in)
+                    locale.document_id = self.document_id
+                else:
+                    self.locales.append(locale_in)
 
         if other.geometry:
             if self.geometry:
@@ -295,7 +296,6 @@ def geom(self):
         return Column(
             Geometry(
                 geometry_type='POINT', srid=3857, dimension=2,
-                management=True,
                 spatial_index=self.__name__ != 'ArchiveDocumentGeometry'),
             info={
                 'colanderalchemy': {
@@ -309,7 +309,6 @@ def geom_detail(self):
         return Column(
             Geometry(
                 geometry_type='GEOMETRY', srid=3857,
-                management=True,
                 use_typmod=False,
                 spatial_index=self.__name__ != 'ArchiveDocumentGeometry'),
             info={