Feature: Add support for ES8 Persistence

.dockerignore

@@ -1,4 +1,47 @@
-# General
+# Git/VCS
+.git
+.gitignore
+.gitattributes
+.github
 
-# UI dependencies
-**/node_modules
+# IDE/editor
+.idea
+.vscode
+.classpath
+.project
+.settings
+*.iml
+
+# OS/filesystem noise
+.DS_Store
+
+# Caches & temp
+**/.gradle
+**/.cache
+**/tmp
+**/logs
+**/*.log
+
+# Build outputs (keep source in docker build)
+**/build
+**/out
+**/target
+**/dist
+**/coverage
+
+# Python
+venv
+**/__pycache__/
+**/.pytest_cache/
+
+# JS tooling
+**/node_modules
+**/.npm
+**/.yarn
+**/.pnpm-store
+**/.eslintcache
+**/.parcel-cache
+**/.next
+
+# UI build artifacts (explicit)
+ui/build

README.md

@@ -117,6 +117,7 @@ conductor-skills install <skill-name>
 | Backend | Configuration |
 |---------|---------------|
 | Redis + ES7 (default) | [config-redis.properties](docker/server/config/config-redis.properties) |
+| Redis + ES8 | [config-redis-es8.properties](docker/server/config/config-redis-es8.properties) |
 | Redis + OpenSearch | [config-redis-os.properties](docker/server/config/config-redis-os.properties) |
 | Postgres | [config-postgres.properties](docker/server/config/config-postgres.properties) |
 | Postgres + ES7 | [config-postgres-es7.properties](docker/server/config/config-postgres-es7.properties) |

build.gradle

@@ -61,8 +61,8 @@ allprojects {
             exclude group: 'org.slf4j', module: 'slf4j-log4j12'
 
             resolutionStrategy.eachDependency { details ->
-                if (details.requested.group.startsWith('com.fasterxml.jackson.') ) {
-                    details.useVersion "2.18.0"
+                if (details.requested.group.startsWith('com.fasterxml.jackson.')) {
+                    details.useVersion "2.17.0"
                 }
                 // Force commons-lang3 to 3.18.0+ for compatibility with commons-compress used by Testcontainers
                 if (details.requested.group == 'org.apache.commons' && details.requested.name == 'commons-lang3') {
@@ -143,6 +143,8 @@ allprojects {
 
     test {
         useJUnitPlatform()
+        // Prefer provider auto-detection and ignore user-level forced client strategy pins.
+        systemProperty 'dockerconfig.source', 'autoIgnoringUserProperties'
         testLogging {
             events = ["SKIPPED", "FAILED"]
             exceptionFormat = "full"

dependencies.gradle

@@ -53,13 +53,14 @@ ext {
     revRxJava = '1.2.2'
     revSpock = '2.4-M4-groovy-4.0'
     revSpotifyCompletableFutures = '0.3.3'
-    revTestContainer = '1.19.3'
+    revTestContainer = '1.21.4'
     revFasterXml = '2.15.3'
     revAmqpClient = '5.13.0'
     revKafka = '2.6.0'
     revMicrometer = '1.14.6'
     revPrometheus = '0.9.0'
     revElasticSearch7 = '7.17.11'
+    revElasticSearch8 = '8.19.11'
     revCodec = '1.15'
     revAzureStorageBlobSdk = '12.25.1'
     revNatsStreaming = '2.6.5'

docker/docker-compose-es8.yaml

@@ -0,0 +1,79 @@
+services:
+  conductor-server:
+    environment:
+      - CONFIG_PROP=config-redis-es8.properties
+      - JAVA_OPTS=-Dpolyglot.engine.WarnInterpreterOnly=false
+      - conductor.app.ownerEmailMandatory=false
+    image: conductor:server
+    container_name: conductor-server
+    build:
+      context: ../
+      dockerfile: docker/server/Dockerfile
+      args:
+        YARN_OPTS: ${YARN_OPTS}
+        INDEXING_BACKEND: elasticsearch8
+    networks:
+      - internal
+    ports:
+      - 8080:8080
+      - 8127:5000
+    healthcheck:
+      test: ["CMD", "curl","-I" ,"-XGET", "http://localhost:8080/health"]
+      interval: 60s
+      timeout: 30s
+      retries: 12
+    links:
+      - conductor-elasticsearch:es
+      - conductor-redis:rs
+    depends_on:
+      conductor-elasticsearch:
+        condition: service_healthy
+      conductor-redis:
+        condition: service_healthy
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "1k"
+        max-file: "3"
+
+  conductor-redis:
+    image: redis:6.2.3-alpine
+    volumes:
+      - ../server/config/redis.conf:/usr/local/etc/redis/redis.conf
+    networks:
+      - internal
+    ports:
+      - 7379:6379
+    healthcheck:
+      test: [ "CMD", "redis-cli","ping" ]
+
+  conductor-elasticsearch:
+    # Keep ES image aligned with elasticsearch-java client version (8.19.11).
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.11
+    environment:
+      - "ES_JAVA_OPTS=-Xms512m -Xmx1024m"
+      - xpack.security.enabled=false
+      - discovery.type=single-node
+    volumes:
+      - esdata-conductor:/usr/share/elasticsearch/data
+    networks:
+      - internal
+    ports:
+      - 9201:9200
+    healthcheck:
+      test: curl http://localhost:9200/_cluster/health -o /dev/null
+      interval: 5s
+      timeout: 5s
+      retries: 12
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "1k"
+        max-file: "3"
+
+volumes:
+  esdata-conductor:
+    driver: local
+
+networks:
+  internal:

docker/server/Dockerfile

@@ -30,6 +30,7 @@ RUN if [ "$PREBUILT" = "true" ]; then \
         --no-daemon --no-parallel; \
     fi
 
+
 # ===========================================================================================================
 # 1. UI builder stage
 # ===========================================================================================================
@@ -40,7 +41,7 @@ ARG YARN_OPTS
 
 LABEL maintainer="Orkes OSS <oss@orkes.io>"
 
-COPY . /conductor
+COPY ui /conductor/ui
 WORKDIR /conductor/ui
 
 RUN if [ "$PREBUILT" = "false" ]; then \
@@ -77,7 +78,7 @@ COPY --from=builder /conductor/server/build/libs/*boot*.jar /app/libs/conductor-
 WORKDIR /usr/share/nginx/html
 RUN rm -rf ./*
 COPY --from=ui-builder /conductor/ui/build .
-COPY --from=ui-builder /conductor/docker/server/nginx/nginx.conf /etc/nginx/conf.d/default.conf
+COPY docker/server/nginx/nginx.conf  /etc/nginx/conf.d/default.conf
 
 # Copy the files for the server into the app folders
 RUN chmod +x /app/startup.sh

docker/server/config/config-redis-es8.properties

@@ -0,0 +1,37 @@
+# Database persistence type.
+# Below are the properties for redis
+conductor.db.type=redis_standalone
+conductor.queue.type=redis_standalone
+
+conductor.redis.hosts=rs:6379:us-east-1c
+conductor.redis-lock.serverAddress=redis://rs:6379
+conductor.redis.taskDefCacheRefreshInterval=1
+conductor.redis.workflowNamespacePrefix=conductor
+conductor.redis.queueNamespacePrefix=conductor_queues
+
+conductor.workflow-execution-lock.type=redis
+conductor.app.workflowExecutionLockEnabled=true
+conductor.app.lockTimeToTry=500
+
+conductor.app.systemTaskWorkerThreadCount=20
+conductor.app.systemTaskMaxPollCount=20
+
+# Elastic search instance indexing is enabled.
+conductor.indexing.enabled=true
+conductor.indexing.type=elasticsearch8
+conductor.elasticsearch.url=http://es:9200
+conductor.elasticsearch.indexName=conductor
+conductor.elasticsearch.clusterHealthColor=yellow
+
+# Additional modules for metrics collection exposed to Prometheus (optional)
+conductor.metrics-prometheus.enabled=true
+management.endpoints.web.exposure.include=health,prometheus
+
+# Redis health indicator
+management.health.redis.enabled=true
+
+# Load sample kitchen sink workflow
+loadSample=true
+
+# Redis cluster connection in SSL mode
+conductor.redis.ssl=false

docs/devguide/running/deploy.md

@@ -51,6 +51,7 @@ Pre-built compose files for other backend combinations:
 | Compose file | Database | Queue | Index |
 |:--|:--|:--|:--|
 | `docker-compose.yaml` | Redis | Redis | Elasticsearch 7 |
+| `docker-compose-es8.yaml` | Redis | Redis | Elasticsearch 8 |
 | `docker-compose-postgres.yaml` | PostgreSQL | PostgreSQL | PostgreSQL |
 | `docker-compose-postgres-es7.yaml` | PostgreSQL | PostgreSQL | Elasticsearch 7 |
 | `docker-compose-mysql.yaml` | MySQL | Redis | Elasticsearch 7 |
@@ -61,10 +62,16 @@ Pre-built compose files for other backend combinations:
 # Example: PostgreSQL for everything
 docker compose -f docker/docker-compose-postgres.yaml up
 
+# Example: Redis + Elasticsearch 8
+docker compose -f docker/docker-compose-es8.yaml up
+
 # Example: Redis + OpenSearch 3
 docker compose -f docker/docker-compose-redis-os3.yaml up
 ```
 
+For Elasticsearch 8, set `conductor.indexing.type=elasticsearch8` and use
+`config-redis-es8.properties` or an equivalent custom config.
+
 ---
 
 ## Production configuration
@@ -180,6 +187,7 @@ conductor.indexing.type=postgres
 |:--|:--|:--|:--|
 | PostgreSQL | `postgres` | Simplest stack when database is also PostgreSQL. | Set `conductor.elasticsearch.version=0` to disable ES client. |
 | Elasticsearch 7 | `elasticsearch` | Best search performance at scale. Full-text search. | Set `conductor.elasticsearch.version=7`. |
+| Elasticsearch 8 | `elasticsearch8` | Use when running the ES8 persistence module. | Set `conductor.elasticsearch.version=8`. |
 | OpenSearch 2 | `opensearch2` | Open-source ES alternative. | Compatible with ES 7 queries. |
 | OpenSearch 3 | `opensearch3` | Latest OpenSearch. | |
 | SQLite | `sqlite` | Local development only. | |
@@ -216,6 +224,17 @@ conductor.elasticsearch.username=elastic
 conductor.elasticsearch.password=<password>
 ```
 
+#### Elasticsearch 8
+
+```properties
+conductor.indexing.enabled=true
+conductor.indexing.type=elasticsearch8
+conductor.elasticsearch.url=http://es-host:9200
+conductor.elasticsearch.version=8
+conductor.elasticsearch.indexName=conductor
+conductor.elasticsearch.clusterHealthColor=yellow
+```
+
 #### OpenSearch
 
 ```properties

es7-persistence/README.md

@@ -2,6 +2,8 @@
 
 This module provides ES7 persistence when indexing workflows and tasks.
 
+If you need Elasticsearch 8.x, use the `es8-persistence` module instead.
+
 ### ES Breaking changes
 
 From ES6 to ES7 there were significant breaking changes which affected ES7-persistence module implementation.

es8-persistence/README.md

@@ -0,0 +1,82 @@
+# ES8 Persistence
+
+This module provides Elasticsearch 8.x persistence for indexing workflows and tasks.
+
+It uses the Elasticsearch Java API Client (`elasticsearch-java`) aligned with ES8.
+
+## Index management (ES8 best practices)
+
+This module uses composable index templates, write aliases, and an ILM policy:
+
+- **ILM policy:** `${indexPrefix}-default-ilm-policy`
+- **Rollover conditions (hot phase):** `max_primary_shard_size=50gb`
+- **Write aliases:** `${indexPrefix}_workflow`, `${indexPrefix}_task`, `${indexPrefix}_task_log`,
+  `${indexPrefix}_message`, `${indexPrefix}_event`
+- **Initial indices:** `${alias}-000001` (created automatically when index management is enabled)
+- **Refresh interval:** defaults to `30s` for all indices (configurable)
+
+## Build and usage
+
+### 1) Build-time module selection
+
+When building the server, select the ES8 persistence module to avoid Lucene conflicts:
+
+```sh
+./gradlew :conductor-server:bootJar -PindexingBackend=elasticsearch8
+```
+
+`-PindexingBackend=es8` is also accepted.
+
+### 2) Runtime configuration
+
+Select the Elasticsearch 8 backend in your configuration:
+
+```properties
+conductor.indexing.type=elasticsearch8
+```
+
+### Elasticsearch version compatibility
+
+The ES8 module uses `elasticsearch-java` client version `8.19.11`.
+For local Docker-based setups, use Elasticsearch `8.19.x` (the provided compose file pins
+`8.19.11`).
+
+All other `conductor.elasticsearch.*` properties are shared with the ES7 module.
+
+### Configuration
+
+(Default values shown below)
+
+```properties
+# A comma separated list of scheme/host/port of the ES nodes to communicate with.
+# Scheme can be `http` or `https`. If scheme is omitted then `http` will be used.
+conductor.elasticsearch.url=localhost:9200
+
+# The name of the workflow and task index.
+conductor.elasticsearch.indexPrefix=conductor
+
+# Default refresh interval applied via the component template.
+conductor.elasticsearch.indexRefreshInterval=30s
+
+# Path to a PEM-encoded certificate to trust for HTTPS connections.
+conductor.elasticsearch.trustCertPath=
+
+# Worker queue size used in executor service for async methods in IndexDao.
+conductor.elasticsearch.asyncWorkerQueueSize=100
+
+# Maximum thread pool size in executor service for async methods in IndexDao
+conductor.elasticsearch.asyncMaxPoolSize=12
+
+# Timeout (in seconds) for the in-memory to be flushed if not explicitly indexed
+conductor.elasticsearch.asyncBufferFlushTimeout=10
+```
+
+### BASIC Authentication
+
+If you need to pass user/password to connect to ES, add the following properties to your
+config file:
+
+```
+conductor.elasticsearch.username=someusername
+conductor.elasticsearch.password=somepassword
+```