Merge pull request #188 from AtomicMaps/add-opensearch

Opensearch 2.11.1 Support

Author: jonhealy1

.github/workflows/cicd.yml

@@ -46,6 +46,22 @@ jobs:
           ES_JAVA_OPTS: -Xms512m -Xmx1g
         ports:
           - 9400:9400
+
+      opensearch_2_11:
+        image: opensearchproject/opensearch:2.11.1
+        env:
+          cluster.name: stac-cluster
+          node.name: os01
+          network.host: 0.0.0.0
+          transport.host: 0.0.0.0
+          discovery.type: single-node
+          http.port: 9202
+          http.cors.enabled: true
+          plugins.security.disabled: true
+          plugins.security.ssl.http.enabled: true
+          OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m
+        ports:
+          - 9202:9202
     strategy:
       matrix:
         python-version: [ "3.8", "3.9", "3.10", "3.11"]
@@ -90,4 +106,15 @@ jobs:
           ES_PORT: 9400
           ES_HOST: 172.17.0.1
           ES_USE_SSL: false
-          ES_VERIFY_CERTS: false
+          ES_VERIFY_CERTS: false
+
+      - name: Run test suite against OpenSearch 2.11.1
+        run: |
+          cd stac_fastapi/elasticsearch && pipenv run pytest -svvv
+        env:
+          ENVIRONMENT: testing
+          ES_PORT: 9202
+          ES_HOST: 172.17.0.1
+          ES_USE_SSL: false
+          ES_VERIFY_CERTS: false
+          BACKEND: opensearch

CHANGELOG.md

@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 
 - Advanced comparison (LIKE, IN, BETWEEN) operators to the Filter extension [#178](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/178)
+- Collection update endpoint no longer delete all sub items [#177](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/177)
+- OpenSearch 2.11.1 support [#188](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/188)
 
 ### Changed
 

Makefile

@@ -1,20 +1,32 @@
 #!make
 APP_HOST ?= 0.0.0.0
-APP_PORT ?= 8080
+ES_APP_PORT ?= 8080
 EXTERNAL_APP_PORT ?= ${APP_PORT}
 
-APP_PORT ?= 8080
+ES_APP_PORT ?= 8080
 ES_HOST ?= docker.for.mac.localhost
 ES_PORT ?= 9200
 
+OS_APP_PORT ?= 8082
+ES_HOST ?= docker.for.mac.localhost
+OS_PORT ?= 9202
+
 run_es = docker-compose \
 	run \
-	-p ${EXTERNAL_APP_PORT}:${APP_PORT} \
+	-p ${EXTERNAL_APP_PORT}:${ES_APP_PORT} \
 	-e PY_IGNORE_IMPORTMISMATCH=1 \
 	-e APP_HOST=${APP_HOST} \
-	-e APP_PORT=${APP_PORT} \
+	-e APP_PORT=${ES_APP_PORT} \
 	app-elasticsearch
 
+run_os = docker-compose \
+	run \
+	-p ${EXTERNAL_APP_PORT}:${OS_APP_PORT} \
+	-e PY_IGNORE_IMPORTMISMATCH=1 \
+	-e APP_HOST=${APP_HOST} \
+	-e APP_PORT=${OS_APP_PORT} \
+	app-opensearch
+
 .PHONY: image-deploy
 image-deploy:
 	docker build -f Dockerfile.deploy -t stac-fastapi-elasticsearch:latest .
@@ -40,15 +52,32 @@ docker-run: image-dev
 docker-shell:
 	$(run_es) /bin/bash
 
+.PHONY: test-elasticsearch
+test:
+	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
+	docker-compose down
+
+.PHONY: test-opensearch
+test-opensearch:
+	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
+	docker-compose down
+
 .PHONY: test
 test:
 	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
 	docker-compose down
 
-.PHONY: run-database
-run-database:
+	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
+	docker-compose down
+
+.PHONY: run-database-es
+run-database-es:
 	docker-compose run --rm elasticsearch
 
+.PHONY: run-database-os
+run-database-os:
+	docker-compose run --rm opensearch
+
 .PHONY: pybase-install
 pybase-install:
 	pip install wheel && \

README.md

@@ -42,13 +42,15 @@ docker-compose build
 docker-compose up
 ```
 
-By default, docker-compose uses Elasticsearch 8.x. However, most recent 7.x versions should also work. 
+By default, docker-compose uses Elasticsearch 8.x and OpenSearch 2.11.1. 
 If you wish to use a different version, put the following in a 
 file named `.env` in the same directory you run docker-compose from:
 
 ```shell
 ELASTICSEARCH_VERSION=7.17.1
+OPENSEARCH_VERSION=2.11.0
 ```
+The most recent Elasticsearch 7.x versions should also work. See the [opensearch-py docs](https://github.com/opensearch-project/opensearch-py/blob/main/COMPATIBILITY.md) for compatibility information.
 
 To create a new Collection:
 
@@ -78,7 +80,18 @@ curl -X "GET" "http://localhost:8080/collections?limit=1&token=example_token"
 ```shell
 make test
 ```
-   
+Test against OpenSearch only
+
+```shell
+make test-opensearch
+```
+
+Test against Elasticsearch only
+
+```shell
+make test-elasticsearch
+```  
+
 ## Ingest sample data
 
 ```shell

data_loader/data_loader.py

@@ -1,12 +1,24 @@
 """Database ingestion script."""
 import json
 import os
+import sys
 
 import click
 import requests
 
+if len(sys.argv) != 2:
+    print("Usage: python data_loader.py <opensearch|elasticsearch>")
+    sys.exit(1)
+
 DATA_DIR = os.path.join(os.path.dirname(__file__), "setup_data/")
-STAC_API_BASE_URL = "http://localhost:8080"
+
+backend = sys.argv[1].lower()
+if backend == "opensearch":
+    STAC_API_BASE_URL = "http://localhost:8082"
+elif backend == "elasticsearch":
+    STAC_API_BASE_URL = "http://localhost:8080"
+else:
+    print("Invalid backend tag. Enter either 'opensearch' or 'elasticsearch'.")
 
 
 def load_data(filename):

docker-compose.yml

@@ -3,7 +3,7 @@ version: '3.9'
 services:
   app-elasticsearch:
     container_name: stac-fastapi-es
-    image: stac-utils/stac-fastapi
+    image: stac-utils/stac-fastapi-es
     restart: always
     build:
       context: .
@@ -18,6 +18,7 @@ services:
       - ES_PORT=9200
       - ES_USE_SSL=false
       - ES_VERIFY_CERTS=false
+      - BACKEND=elasticsearch
     ports:
       - "8080:8080"
     volumes:
@@ -29,6 +30,35 @@ services:
     command:
       bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elasticsearch.app"
 
+  app-opensearch:
+    container_name: stac-fastapi-os
+    image: stac-utils/stac-fastapi-os
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+    environment:
+      - APP_HOST=0.0.0.0
+      - APP_PORT=8082
+      - RELOAD=true
+      - ENVIRONMENT=local
+      - WEB_CONCURRENCY=10
+      - ES_HOST=opensearch
+      - ES_PORT=9202
+      - ES_USE_SSL=false
+      - ES_VERIFY_CERTS=false
+      - BACKEND=opensearch
+    ports:
+      - "8082:8082"
+    volumes:
+      - ./stac_fastapi:/app/stac_fastapi
+      - ./scripts:/app/scripts
+      - ./osdata:/usr/share/opensearch/data
+    depends_on:
+      - opensearch
+    command:
+      bash -c "./scripts/wait-for-it-es.sh os-container:9202 && python -m stac_fastapi.elasticsearch.app"
+
   elasticsearch:
     container_name: es-container
     image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.11.0}
@@ -40,3 +70,17 @@ services:
       - ./elasticsearch/snapshots:/usr/share/elasticsearch/snapshots
     ports:
       - "9200:9200"
+
+  opensearch:
+    container_name: os-container
+    image: opensearchproject/opensearch:${OPENSEARCH_VERSION:-2.11.1}
+    hostname: opensearch
+    environment:
+      - discovery.type=single-node
+      - plugins.security.disabled=true
+      - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m
+    volumes:
+      - ./opensearch/config/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
+      - ./opensearch/snapshots:/usr/share/opensearch/snapshots
+    ports:
+      - "9202:9202"

opensearch/config/opensearch.yml

@@ -0,0 +1,19 @@
+## Cluster Settings
+cluster.name: stac-cluster
+node.name: os01
+network.host: 0.0.0.0
+transport.host: 0.0.0.0
+discovery.type: single-node
+http.port: 9202
+http.cors.enabled: true
+http.cors.allow-headers: X-Requested-With,Content-Type,Content-Length,Accept,Authorization
+
+path:
+  repo:
+    - /usr/share/opensearch/snapshots
+
+# Security
+plugins.security.disabled: true
+plugins.security.ssl.http.enabled: true
+
+node.max_local_storage_nodes: 3

stac_fastapi/elasticsearch/setup.py

@@ -15,6 +15,8 @@
     "stac-fastapi.extensions==2.4.9",
     "elasticsearch[async]==8.11.0",
     "elasticsearch-dsl==8.11.0",
+    "opensearch-py==2.4.2",
+    "opensearch-py[async]==2.4.2",
     "pystac[validation]",
     "uvicorn",
     "orjson",

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py

@@ -1,14 +1,26 @@
 """FastAPI application."""
+import os
+
 from stac_fastapi.api.app import StacApi
 from stac_fastapi.api.models import create_get_request_model, create_post_request_model
-from stac_fastapi.elasticsearch.config import ElasticsearchSettings
 from stac_fastapi.elasticsearch.core import (
     BulkTransactionsClient,
     CoreClient,
     EsAsyncBaseFiltersClient,
     TransactionsClient,
 )
-from stac_fastapi.elasticsearch.database_logic import create_collection_index
+
+if os.getenv("BACKEND", "elasticsearch").lower() == "opensearch":
+    from stac_fastapi.elasticsearch.config.config_opensearch import SearchSettings
+    from stac_fastapi.elasticsearch.database_logic.database_logic_opensearch import (
+        create_collection_index,
+    )
+else:
+    from stac_fastapi.elasticsearch.config.config_elasticsearch import SearchSettings
+    from stac_fastapi.elasticsearch.database_logic.database_logic_elasticsearch import (
+        create_collection_index,
+    )
+
 from stac_fastapi.elasticsearch.extensions import QueryExtension
 from stac_fastapi.elasticsearch.session import Session
 from stac_fastapi.extensions.core import (
@@ -21,7 +33,7 @@
 )
 from stac_fastapi.extensions.third_party import BulkTransactionExtension
 
-settings = ElasticsearchSettings()
+settings = SearchSettings()
 session = Session.create_from_settings(settings)
 
 filter_extension = FilterExtension(client=EsAsyncBaseFiltersClient())

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config/__init__.py

@@ -0,0 +1 @@
+"""client config implementations."""

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py renamed to stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config/config_elasticsearch.py

@@ -54,7 +54,7 @@ def _es_config() -> Dict[str, Any]:
 _forbidden_fields: Set[str] = {"type"}
 
 
-class ElasticsearchSettings(ApiSettings):
+class SearchSettings(ApiSettings):
     """API settings."""
 
     # Fields which are defined by STAC but not included in the database model
@@ -67,7 +67,7 @@ def create_client(self):
         return Elasticsearch(**_es_config())
 
 
-class AsyncElasticsearchSettings(ApiSettings):
+class AsyncSearchSettings(ApiSettings):
     """API settings."""
 
     # Fields which are defined by STAC but not included in the database model