[US510035] add QMS Docker image for vector search (#26)

README.md

@@ -27,7 +27,7 @@ Run the `deploy.py` tool using Python.  (Much like when running `docker`, you
 may have to run it as a different user with sufficient permissions to manage Docker containers.)
 
 ```
-python3 deploy.py auth entity filestore analysis audit dataset-locations api ui
+python3 deploy.py auth entity entity-vector filestore analysis audit dataset-locations api ui
 ```
 
 With the default configuration, the Discover UI will be available at `https://localhost:8090` once the
@@ -93,7 +93,7 @@ ingest documents from an existing IDOL setup, and preserve the IDOL ACL fields,
 To use HTTP instead of HTTPS, for testing purposes only, run:
 
 ```
-python3 deploy.py --disable-encryption auth entity filestore analysis audit dataset-locations api ui
+python3 deploy.py --disable-encryption auth entity entity-vector filestore analysis audit dataset-locations api ui
 ```
 
 > note: changes to the encryption state of a deployed system require manual deletion of the realm in Keycloak before running `deploy.py` with the new state.
@@ -129,6 +129,7 @@ while others listen on 127.0.0.1 only):
 | entity            | 8021     | no         | ACI port of IDOL Content database backend for the Gremlin database         |
 | entity            | 8022     | no         | Index port of IDOL Content database backend for the Gremlin database       |
 | entity            | 8023     | no         | CQL port of Cassandra-compatible database backend for the Gremlin database |
+| entity-vector     | 8024     | no         | ACI port of IDOL QMS used for vector search                                |
 | filestore         | 8030     | no         | S3-compatible object storage                                               |
 | analysis          | 8040     | no         | NiFi server for media analysis (API and admin UI)                          |
 | audit             | 8050     | no         | PostgreSQL database storing audit logs                                     |
@@ -145,10 +146,12 @@ Docker volumes are created with the prefix `opentext-idol-discover_`, which can
 | entity            | entity-storagedb-data          | Application data                    |
 | entity            | entity-indexdb-data            | Search index for application data   |
 | entity            | entity-indexdb-license-data    | Cache for license information       |
+| entity-vector     | entity-vector-qms-license-data | Cache for license information       |
 | filestore         | filestore-service-data         | Uploaded and generated files        |
 | audit             | audit-db-data                  | Audit logs                          |
 | dataset-locations | dataset-locations-license-data | Cache for license information       |
 | -                 | entity-data                    | Schema for application data         |
+| -                 | entity-vector-data             | Configuration for vector search     |
 | -                 | security-data                  | Security-related data               |
 
 All containers connect to a Docker network called `opentext-idol-discover_main`.  The

config-fixed/entity.env

@@ -1,3 +1,4 @@
 ISOL_ENTITY_STORAGEDB_MAX_HEAP_SIZE=
 ISOL_ENTITY_STORAGEDB_HEAP_NEWSIZE=
 ISOL_ENTITY_INDEXDB_SAN=
+ISOL_ENTITY_VECTOR_QMS_SAN=

config/base.env

@@ -47,6 +47,9 @@ ISOL_ENTITY_INDEXDB_COMMUNITY_ACI_PORT=9030
 # when ISOL_ENTITY_IDOL_SECURITY_MODE=community, this should be set to the corresponding configuration values used in
 # the Community component (DocumentSecurityType)
 ISOL_ENTITY_IDOL_SECURITY_TYPE=Generic_V4
+ISOL_ENTITY_VECTOR_QMS_PROTOCOL=http
+ISOL_ENTITY_VECTOR_QMS_HOST=entity-vector-qms
+ISOL_ENTITY_VECTOR_QMS_ACI_PORT=16000
 
 # location of the `filestore` component, as accessed from Docker containers
 ISOL_FILESTORE_SERVICE_PROTOCOL=http
@@ -86,3 +89,9 @@ DISCOVER_UI_EXTERNAL_HOST=localhost
 DISCOVER_UI_EXTERNAL_PORT=8090
 # this must end in /
 DISCOVER_UI_EXTERNAL_PATH=/
+
+# HTTP proxy settings for requests made by backend components
+# this is only used by the entity.vector.qms component
+ISOL_HTTP_PROXY=
+ISOL_HTTPS_PROXY=
+ISOL_NO_PROXY=

config/entity-vector.env

@@ -0,0 +1,2 @@
+# address on the Docker host to expose the QMS ports on
+ISOL_ENTITY_VECTOR_QMS_ACI_DOCKER_PORT=127.0.0.1:8024

custom/data-entity-vector/Dockerfile

@@ -0,0 +1,20 @@
+#
+# Copyright 2021-2024 Open Text.
+#
+# Licensed under the MIT License (the "License"); you may not use this file
+# except in compliance with the License.
+#
+# The only warranties for products and services of Open Text and its affiliates
+# and licensors ("Open Text") are as may be set forth in the express warranty
+# statements accompanying such products and services. Nothing herein should be
+# construed as constituting an additional warranty. Open Text shall not be
+# liable for technical or editorial errors or omissions contained herein. The
+# information contained herein is subject to change without notice.
+#
+
+ARG BASE_IMAGE
+FROM $BASE_IMAGE
+
+COPY custom/ /source-data/
+
+ENTRYPOINT ["/startup.sh"]

custom/data-entity-vector/custom/README.md

@@ -0,0 +1,3 @@
+# Vector search configuration
+
+The files in this directory are used to configure models used for vector search.

custom/data-entity-vector/custom/config.yaml

@@ -0,0 +1,21 @@
+#
+# Copyright 2021-2024 Open Text.
+#
+# The only warranties for products and services of Open Text and its
+# affiliates and licensors ("Open Text") are as may be set forth in the
+# express warranty statements accompanying such products and services.
+# Nothing herein should be construed as constituting an additional
+# warranty. Open Text shall not be liable for technical or editorial
+# errors or omissions contained herein. The information contained herein
+# is subject to change without notice.
+#
+# Except as specifically indicated otherwise, this document contains
+# confidential information and a valid license is required for possession,
+# use or copying. If this work is provided to the U.S. Government,
+# consistent with FAR 12.211 and 12.212, Commercial Computer Software,
+# Computer Software Documentation, and Technical Data for Commercial Items
+# are licensed to the U.S. Government under vendor's standard commercial
+# license.
+#
+
+models: {}

custom/data-security/custom/README.md

@@ -9,3 +9,7 @@ encryption.
   component
 * **securityinfo/entity-indexdb/security-info-keys**: SecurityInfoKeys used to encrypt secured documents in
   entity-indexdb component
+* **tls/entity-vector-qms/server-https.key.pem**: private key used for internal HTTPS connections to entity-vector-qms
+  component
+* **tls/entity-vector-qms/server-https.cert.pem**: certificate used for internal HTTPS connections to entity-vector-qms
+  component

deploy.py

@@ -52,6 +52,7 @@ def wrap(text):
 ''') + '''
 
 - entity: storage for application data
+- entity-vector: vector search support
 - analysis: media analysis system
 - analysis-live: live media analysis system
 - dataset-locations: database of global locations
@@ -192,7 +193,7 @@ def main():
     component_paths = get_component_paths(program_args)
 
     if program_args.component:
-        run_compose(['data-entity', 'data-security'], component_paths, program_args,
+        run_compose(['data-entity', 'data-security', 'data-entity-vector'], component_paths, program_args,
                     skip_deploy=program_args.skip_deploy, detach=False, remove=False, log_level='error')
         components = program_args.component
         validate_components(components, component_paths)

docker-compose/docker-compose.analysis.yml

@@ -32,6 +32,7 @@ services:
     volumes:
       - security-data:/data/security
       - entity-data:/data/entity
+      - entity-vector-data:/data/entity-vector
     ports:
       - ${ISOL_ANALYSIS_SERVICE_DOCKER_PORT}:8090
     mac_address: 06-00-00-0A-00-01
@@ -54,6 +55,9 @@ services:
       - ISOL_ENTITY_INDEXDB_COMMUNITY_PROTOCOL=${ISOL_ENTITY_INDEXDB_COMMUNITY_PROTOCOL}
       - ISOL_ENTITY_INDEXDB_COMMUNITY_HOST=${ISOL_ENTITY_INDEXDB_COMMUNITY_HOST}
       - ISOL_ENTITY_INDEXDB_COMMUNITY_ACI_PORT=${ISOL_ENTITY_INDEXDB_COMMUNITY_ACI_PORT}
+      - ISOL_ENTITY_VECTOR_QMS_PROTOCOL=${ISOL_ENTITY_VECTOR_QMS_PROTOCOL}
+      - ISOL_ENTITY_VECTOR_QMS_HOST=${ISOL_ENTITY_VECTOR_QMS_HOST}
+      - ISOL_ENTITY_VECTOR_QMS_ACI_PORT=${ISOL_ENTITY_VECTOR_QMS_ACI_PORT}
       - ISOL_ENTITY_DATA_VOLUME_PATH=/data/entity
       - ISOL_ENTITY_IDOL_SECURITY_TYPE=${ISOL_ENTITY_IDOL_SECURITY_TYPE}
       - ISOL_FILE_STORE_URL=${ISOL_FILESTORE_SERVICE_PROTOCOL}://${ISOL_FILESTORE_SERVICE_HOST}:${ISOL_FILESTORE_SERVICE_PORT}${ISOL_FILESTORE_SERVICE_PATH}
@@ -114,3 +118,4 @@ services:
 volumes:
   security-data:
   entity-data:
+  entity-vector-data:

docker-compose/docker-compose.api.yml

@@ -22,6 +22,7 @@ services:
       - ${ISOL_API_SERVICE_TLS_CERT_DIR}:/https
       - security-data:/data/security
       - entity-data:/data/entity
+      - entity-vector-data:/data/entity-vector
     ports:
       - ${ISOL_API_SERVICE_DOCKER_PORT}:${ISOL_API_SERVICE_INTERNAL_PORT}
     environment:
@@ -37,7 +38,11 @@ services:
       - ISOL_ENTITY_INDEXDB_COMMUNITY_PROTOCOL=${ISOL_ENTITY_INDEXDB_COMMUNITY_PROTOCOL}
       - ISOL_ENTITY_INDEXDB_COMMUNITY_HOST=${ISOL_ENTITY_INDEXDB_COMMUNITY_HOST}
       - ISOL_ENTITY_INDEXDB_COMMUNITY_ACI_PORT=${ISOL_ENTITY_INDEXDB_COMMUNITY_ACI_PORT}
+      - ISOL_ENTITY_VECTOR_QMS_PROTOCOL=${ISOL_ENTITY_VECTOR_QMS_PROTOCOL}
+      - ISOL_ENTITY_VECTOR_QMS_HOST=${ISOL_ENTITY_VECTOR_QMS_HOST}
+      - ISOL_ENTITY_VECTOR_QMS_ACI_PORT=${ISOL_ENTITY_VECTOR_QMS_ACI_PORT}
       - ISOL_ENTITY_DATA_VOLUME_PATH=/data/entity
+      - ISOL_ENTITY_VECTOR_DATA_VOLUME_PATH=/data/entity
       - ISOL_AUTH_SERVICE_PROTOCOL=${ISOL_AUTH_SERVICE_PROTOCOL}
       - ISOL_AUTH_SERVICE_HOST=${ISOL_AUTH_SERVICE_HOST}
       - ISOL_AUTH_SERVICE_PORT=${ISOL_AUTH_SERVICE_PORT}
@@ -95,3 +100,4 @@ services:
 volumes:
   security-data:
   entity-data:
+  entity-vector-data:

docker-compose/docker-compose.data-entity-vector.yml

@@ -0,0 +1,36 @@
+#
+# Copyright 2021-2024 Open Text.
+#
+# Licensed under the MIT License (the "License"); you may not use this file
+# except in compliance with the License.
+#
+# The only warranties for products and services of Open Text and its affiliates
+# and licensors ("Open Text") are as may be set forth in the express warranty
+# statements accompanying such products and services. Nothing herein should be
+# construed as constituting an additional warranty. Open Text shall not be
+# liable for technical or editorial errors or omissions contained herein. The
+# information contained herein is subject to change without notice.
+#
+
+services:
+
+  data-entity-vector:
+    image: ${ISOL_DOCKER_REGISTRY}${ISOL_DOCKER_NAME_SEP}solutions-data-entity-vector${ISOL_DOCKER_VERSION_SEP}${ISOL_API_VERSION}
+    # create a container but don't do anything, so the container is available to list to discover all required images
+    entrypoint: "true"
+    networks:
+      - main
+
+  data-entity-vector-custom:
+    image: ${ISOL_DOCKER_REGISTRY}${ISOL_DOCKER_NAME_SEP}solutions-data-entity-vector-custom${ISOL_DOCKER_VERSION_SEP}${ISOL_API_VERSION}
+    build:
+      context: ../custom/data-entity-vector
+      args:
+        BASE_IMAGE: ${ISOL_DOCKER_REGISTRY}${ISOL_DOCKER_NAME_SEP}solutions-data-entity-vector${ISOL_DOCKER_VERSION_SEP}${ISOL_API_VERSION}
+    networks:
+      - main
+    volumes:
+      - entity-vector-data:/target-volume
+
+volumes:
+  entity-vector-data:

docker-compose/docker-compose.data-security.yml

@@ -32,7 +32,7 @@ services:
     volumes:
       - security-data:/target-volume
     environment:
-      - ISOL_SECURITY_TLS=entity-indexdb ${ISOL_ENTITY_INDEXDB_HOST} ${ISOL_ENTITY_INDEXDB_SAN}
+      - ISOL_SECURITY_TLS=entity-indexdb ${ISOL_ENTITY_INDEXDB_HOST} ${ISOL_ENTITY_INDEXDB_SAN},entity-vector-qms ${ISOL_ENTITY_VECTOR_QMS_HOST} ${ISOL_ENTITY_VECTOR_QMS_SAN}
       - ISOL_SECURITY_SECURITYINFO=entity-indexdb
 
 volumes:

docker-compose/docker-compose.entity-vector.yml

@@ -0,0 +1,40 @@
+#
+# Copyright 2021-2023 Open Text.
+#
+# Licensed under the MIT License (the "License"); you may not use this file
+# except in compliance with the License.
+#
+# The only warranties for products and services of Open Text and its affiliates
+# and licensors ("Open Text") are as may be set forth in the express warranty
+# statements accompanying such products and services. Nothing herein should be
+# construed as constituting an additional warranty. Open Text shall not be
+# liable for technical or editorial errors or omissions contained herein. The
+# information contained herein is subject to change without notice.
+#
+
+services:
+
+  entity-vector-qms:
+    image: ${ISOL_DOCKER_REGISTRY}${ISOL_DOCKER_NAME_SEP}solutions-entity-vector-qms${ISOL_DOCKER_VERSION_SEP}${ISOL_API_VERSION}
+    networks:
+      - main
+    volumes:
+      - security-data:/data/security
+      - entity-vector-data:/data/entity-vector
+      - entity-vector-qms-license-data:/qms/license-cache
+    ports:
+      - ${ISOL_ENTITY_VECTOR_QMS_ACI_DOCKER_PORT}:16000
+    mac_address: 06-00-00-18-00-01
+    environment:
+      - ISOL_LICENSE_SERVER_HOST=${ISOL_LICENSE_SERVER_HOST}
+      - ISOL_LICENSE_SERVER_PORT=${ISOL_LICENSE_SERVER_PORT}
+      - ISOL_ENTITY_INDEXDB_HOST=${ISOL_ENTITY_INDEXDB_HOST}
+      - ISOL_ENTITY_INDEXDB_ACI_PORT=${ISOL_ENTITY_INDEXDB_ACI_PORT}
+      - http_proxy=${ISOL_HTTP_PROXY}
+      - https_proxy=${ISOL_HTTPS_PROXY}
+      - no_proxy=${ISOL_NO_PROXY}
+
+volumes:
+  security-data:
+  entity-vector-data:
+  entity-vector-qms-license-data:

helm/custom/build.env

@@ -6,5 +6,6 @@ ISOL_DOCKER_REGISTRY=microfocusidolserver
 ISOL_DOCKER_NAME_SEP=/
 ISOL_DOCKER_VERSION_SEP=:
 
+ISOL_BASE_VERSION=24.3.0
 ISOL_API_VERSION=24.3.0
 ISOL_ANALYSIS_NIFI_VERSION=24.3.0

helm/custom/data-entity-vector/Dockerfile

@@ -0,0 +1,20 @@
+#
+# Copyright 2021-2024 Open Text.
+#
+# Licensed under the MIT License (the "License"); you may not use this file
+# except in compliance with the License.
+#
+# The only warranties for products and services of Open Text and its affiliates
+# and licensors ("Open Text") are as may be set forth in the express warranty
+# statements accompanying such products and services. Nothing herein should be
+# construed as constituting an additional warranty. Open Text shall not be
+# liable for technical or editorial errors or omissions contained herein. The
+# information contained herein is subject to change without notice.
+#
+
+ARG BASE_IMAGE
+FROM $BASE_IMAGE
+
+COPY custom/ /source-data/
+
+ENTRYPOINT ["/startup.sh"]

helm/custom/data-entity-vector/custom/README.md

@@ -0,0 +1,3 @@
+# Vector search configuration
+
+The files in this directory are used to configure models used for vector search.

helm/custom/data-entity-vector/custom/config.yaml

@@ -0,0 +1,21 @@
+#
+# Copyright 2021-2024 Open Text.
+#
+# The only warranties for products and services of Open Text and its
+# affiliates and licensors ("Open Text") are as may be set forth in the
+# express warranty statements accompanying such products and services.
+# Nothing herein should be construed as constituting an additional
+# warranty. Open Text shall not be liable for technical or editorial
+# errors or omissions contained herein. The information contained herein
+# is subject to change without notice.
+#
+# Except as specifically indicated otherwise, this document contains
+# confidential information and a valid license is required for possession,
+# use or copying. If this work is provided to the U.S. Government,
+# consistent with FAR 12.211 and 12.212, Commercial Computer Software,
+# Computer Software Documentation, and Technical Data for Commercial Items
+# are licensed to the U.S. Government under vendor's standard commercial
+# license.
+#
+
+models: {}

helm/custom/data-entity-vector/docker-compose.data-entity-vector.yml

@@ -0,0 +1,22 @@
+#
+# Copyright 2021-2024 Open Text.
+#
+# Licensed under the MIT License (the "License"); you may not use this file
+# except in compliance with the License.
+#
+# The only warranties for products and services of Open Text and its affiliates
+# and licensors ("Open Text") are as may be set forth in the express warranty
+# statements accompanying such products and services. Nothing herein should be
+# construed as constituting an additional warranty. Open Text shall not be
+# liable for technical or editorial errors or omissions contained herein. The
+# information contained herein is subject to change without notice.
+#
+
+services:
+
+  data-entity-vector-custom:
+    image: ${CUSTOM_DOCKER_REGISTRY}${CUSTOM_DOCKER_NAME_SEP}solutions-data-entity-vector-custom${CUSTOM_DOCKER_VERSION_SEP}${ISOL_API_VERSION}
+    build:
+      context: .
+      args:
+        BASE_IMAGE: ${ISOL_DOCKER_REGISTRY}${ISOL_DOCKER_NAME_SEP}solutions-data-entity-vector${ISOL_DOCKER_VERSION_SEP}${ISOL_API_VERSION}

helm/custom/data-security/custom/README.md

@@ -9,3 +9,7 @@ encryption.
   component
 * **securityinfo/entity-indexdb/security-info-keys**: SecurityInfoKeys used to encrypt secured documents in
   entity-indexdb component
+* **tls/entity-vector-qms/server-https.key.pem**: private key used for internal HTTPS connections to entity-vector-qms
+  component
+* **tls/entity-vector-qms/server-https.cert.pem**: certificate used for internal HTTPS connections to entity-vector-qms
+  component

helm/discover/templates/configMap/analysis.yml

@@ -18,6 +18,10 @@ data:
   ISOL_ENTITY_INDEXDB_COMMUNITY_PROTOCOL: "{{ .Values.entity.index.community.protocol }}"
   ISOL_ENTITY_INDEXDB_COMMUNITY_HOST: "{{ .Values.entity.index.community.host }}"
   ISOL_ENTITY_INDEXDB_COMMUNITY_ACI_PORT: "{{ .Values.entity.index.community.aciPort }}"
+  ISOL_ENTITY_VECTOR_QMS_PROTOCOL: "{{ .Values.entity.vector.qms.service.protocol }}"
+  ISOL_ENTITY_VECTOR_QMS_HOST: "{{ .Values.entity.vector.qms.service.name }}"
+  ISOL_ENTITY_VECTOR_QMS_ACI_PORT: "{{ .Values.entity.vector.qms.service.aciPort }}"
+
   ISOL_SECURITY_DATA_VOLUME_PATH: "/data/security"
   ISOL_ENTITY_DATA_VOLUME_PATH: "/data/entity"
   ISOL_ENTITY_IDOL_SECURITY_TYPE: "{{ .Values.api.idolSecurity.type }}"

helm/discover/templates/configMap/api.yml

@@ -14,6 +14,9 @@ data:
   ISOL_ENTITY_INDEXDB_COMMUNITY_PROTOCOL: "{{ .Values.entity.index.community.protocol }}"
   ISOL_ENTITY_INDEXDB_COMMUNITY_HOST: "{{ .Values.entity.index.community.host }}"
   ISOL_ENTITY_INDEXDB_COMMUNITY_ACI_PORT: "{{ .Values.entity.index.community.aciPort }}"
+  ISOL_ENTITY_VECTOR_QMS_PROTOCOL: "{{ .Values.entity.vector.qms.service.protocol }}"
+  ISOL_ENTITY_VECTOR_QMS_HOST: "{{ .Values.entity.vector.qms.service.name }}"
+  ISOL_ENTITY_VECTOR_QMS_ACI_PORT: "{{ .Values.entity.vector.qms.service.aciPort }}"
 
   ISOL_AUTH_SERVICE_PROTOCOL: "http"
   ISOL_AUTH_SERVICE_HOST: "{{ .Values.auth.service.name }}"

helm/discover/templates/configMap/data-security.yml

@@ -3,5 +3,5 @@ kind: ConfigMap
 metadata:
   name: {{ .Values.dataSecurity.config }}
 data:
-  ISOL_SECURITY_TLS: "entity-indexdb {{ .Values.entity.index.service.name }}"
+  ISOL_SECURITY_TLS: "entity-indexdb {{ .Values.entity.index.service.name }},entity-vector-qms {{ .Values.entity.vector.qms.service.name }}"
   ISOL_SECURITY_SECURITYINFO: "entity-indexdb"

helm/discover/templates/configMap/entity-vector-qms.yml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.entity.vector.qms.config }}
+data:
+  ISOL_LICENSE_SERVER_HOST: "{{ .Values.licenseServer.service.name}}"
+  ISOL_LICENSE_SERVER_PORT: "{{ .Values.licenseServer.service.port }}"
+  ISOL_ENTITY_INDEXDB_HOST: "{{ .Values.entity.index.service.name }}"
+  ISOL_ENTITY_INDEXDB_ACI_PORT: "{{ .Values.entity.index.service.queryPort }}"
+  http_proxy: "{{ .Values.proxy.httpProxy }}"
+  https_proxy: "{{ .Values.proxy.httpsProxy }}"
+  no_proxy: "{{ .Values.proxy.noProxy }}"