[US125050] Expose NiFi concurreny settings (#5)

opentext-idol · Feb 2, 2024 · 44cfe71 · 44cfe71
1 parent 64a6742
commit 44cfe71
Show file tree

Hide file tree

Showing 6 changed files with 145 additions and 12 deletions.
diff --git a/config/analysis.env b/config/analysis.env
@@ -5,3 +5,35 @@ ISOL_ANALYSIS_SERVICE_DOCKER_PORT=127.0.0.1:8040
 ISOL_ANALYSIS_FACE_DEC_THREADS=1
 ISOL_ANALYSIS_VEHICLE_THREADS=1
 ISOL_ANALYSIS_OBJECT_THREADS=1
+
+# configure the maximum number of threads that can be utilized by NiFi across all NiFi processors
+ISOL_ANALYSIS_MAX_TIMER_DRIVEN_THREAD_COUNT=10
+
+# Configure the number of tasks that can be concurrently scheduled by each NiFi processor
+ISOL_ANALYSIS_CREATE_PREVIEW_TASK_COUNT=4
+ISOL_ANALYSIS_EXTRACT_METADATA_TASK_COUNT=4
+ISOL_ANALYSIS_FACE_DETECTION_TASK_COUNT=4
+ISOL_ANALYSIS_FACE_RECOGNITION_TASK_COUNT=4
+ISOL_ANALYSIS_FACE_TRAINING_TASK_COUNT=4
+ISOL_ANALYSIS_FETCH_S3_OBJECT_TASK_COUNT=4
+ISOL_ANALYSIS_IMAGE_EXTRACT_TASK_COUNT=4
+ISOL_ANALYSIS_KEY_VIEW_EXPORT_TO_HTML_TASK_COUNT=4
+ISOL_ANALYSIS_OBJECT_CLASS_RECOGNITION_TASK_COUNT=4
+ISOL_ANALYSIS_PUT_ISOL_TASK_COUNT=4
+ISOL_ANALYSIS_REMOVE_DOCUMENT_PART_TASK_COUNT=4
+ISOL_ANALYSIS_TRANSCODE_VIDEO_TASK_COUNT=4
+ISOL_ANALYSIS_VEHICLE_ANALYSIS_TASK_COUNT=4
+ISOL_ANALYSIS_VIDEO_CLIP_TASK_COUNT=4
+
+# The following tasks are internal and should normally be left at the default of 1
+ISOL_ANALYSIS_ANALYSIS_ROUTE_ON_ATTRIBUTE_TASK_COUNT=1
+ISOL_ANALYSIS_CREATE_PREVIEW_ROUTE_ON_ATTRIBUTE_TASK_COUNT=1
+ISOL_ANALYSIS_EXTRACT_JOB_ATTRIBUTES_TASK_COUNT=1
+ISOL_ANALYSIS_EXTRACT_JOB_OPTIONS_TASK_COUNT=1
+ISOL_ANALYSIS_FAIL_ISOL_TASK_COUNT=1
+ISOL_ANALYSIS_GET_LICENSE_TASK_COUNT=1
+ISOL_ANALYSIS_HANDLE_HTTP_REQUEST_TASK_COUNT=1
+ISOL_ANALYSIS_HANDLE_HTTP_RESPONSE_FAILURE_TASK_COUNT=1
+ISOL_ANALYSIS_HANDLE_HTTP_RESPONSE_SUCCESS_TASK_COUNT=1
+ISOL_ANALYSIS_HTML_FROM_DOCUMENT_ROUTE_ON_ATTRIBUTE_TASK_COUNT=1
+ISOL_ANALYSIS_SET_CONTENT_TASK_COUNT=1
diff --git a/docker-compose/docker-compose.analysis.yml b/docker-compose/docker-compose.analysis.yml
@@ -47,6 +47,32 @@ services:
       - ISOL_ANALYSIS_FACE_DEC_THREADS=${ISOL_ANALYSIS_FACE_DEC_THREADS}
       - ISOL_ANALYSIS_VEHICLE_THREADS=${ISOL_ANALYSIS_VEHICLE_THREADS}
       - ISOL_ANALYSIS_OBJECT_THREADS=${ISOL_ANALYSIS_OBJECT_THREADS}
+      - ISOL_ANALYSIS_MAX_TIMER_DRIVEN_THREAD_COUNT=${ISOL_ANALYSIS_MAX_TIMER_DRIVEN_THREAD_COUNT}
+      - ISOL_ANALYSIS_CREATE_PREVIEW_TASK_COUNT=${ISOL_ANALYSIS_CREATE_PREVIEW_TASK_COUNT}
+      - ISOL_ANALYSIS_EXTRACT_METADATA_TASK_COUNT=${ISOL_ANALYSIS_EXTRACT_METADATA_TASK_COUNT}
+      - ISOL_ANALYSIS_FACE_DETECTION_TASK_COUNT=${ISOL_ANALYSIS_FACE_DETECTION_TASK_COUNT}
+      - ISOL_ANALYSIS_FACE_RECOGNITION_TASK_COUNT=${ISOL_ANALYSIS_FACE_RECOGNITION_TASK_COUNT}
+      - ISOL_ANALYSIS_FACE_TRAINING_TASK_COUNT=${ISOL_ANALYSIS_FACE_TRAINING_TASK_COUNT}
+      - ISOL_ANALYSIS_FETCH_S3_OBJECT_TASK_COUNT=${ISOL_ANALYSIS_FETCH_S3_OBJECT_TASK_COUNT}
+      - ISOL_ANALYSIS_IMAGE_EXTRACT_TASK_COUNT=${ISOL_ANALYSIS_IMAGE_EXTRACT_TASK_COUNT}
+      - ISOL_ANALYSIS_KEY_VIEW_EXPORT_TO_HTML_TASK_COUNT=${ISOL_ANALYSIS_KEY_VIEW_EXPORT_TO_HTML_TASK_COUNT}
+      - ISOL_ANALYSIS_OBJECT_CLASS_RECOGNITION_TASK_COUNT=${ISOL_ANALYSIS_OBJECT_CLASS_RECOGNITION_TASK_COUNT}
+      - ISOL_ANALYSIS_PUT_ISOL_TASK_COUNT=${ISOL_ANALYSIS_PUT_ISOL_TASK_COUNT}
+      - ISOL_ANALYSIS_REMOVE_DOCUMENT_PART_TASK_COUNT=${ISOL_ANALYSIS_REMOVE_DOCUMENT_PART_TASK_COUNT}
+      - ISOL_ANALYSIS_TRANSCODE_VIDEO_TASK_COUNT=${ISOL_ANALYSIS_TRANSCODE_VIDEO_TASK_COUNT}
+      - ISOL_ANALYSIS_VEHICLE_ANALYSIS_TASK_COUNT=${ISOL_ANALYSIS_VEHICLE_ANALYSIS_TASK_COUNT}
+      - ISOL_ANALYSIS_VIDEO_CLIP_TASK_COUNT=${ISOL_ANALYSIS_VIDEO_CLIP_TASK_COUNT}
+      - ISOL_ANALYSIS_ANALYSIS_ROUTE_ON_ATTRIBUTE_TASK_COUNT=${ISOL_ANALYSIS_ANALYSIS_ROUTE_ON_ATTRIBUTE_TASK_COUNT}
+      - ISOL_ANALYSIS_CREATE_PREVIEW_ROUTE_ON_ATTRIBUTE_TASK_COUNT=${ISOL_ANALYSIS_CREATE_PREVIEW_ROUTE_ON_ATTRIBUTE_TASK_COUNT}
+      - ISOL_ANALYSIS_EXTRACT_JOB_ATTRIBUTES_TASK_COUNT=${ISOL_ANALYSIS_EXTRACT_JOB_ATTRIBUTES_TASK_COUNT}
+      - ISOL_ANALYSIS_EXTRACT_JOB_OPTIONS_TASK_COUNT=${ISOL_ANALYSIS_EXTRACT_JOB_OPTIONS_TASK_COUNT}
+      - ISOL_ANALYSIS_FAIL_ISOL_TASK_COUNT=${ISOL_ANALYSIS_FAIL_ISOL_TASK_COUNT}
+      - ISOL_ANALYSIS_GET_LICENSE_TASK_COUNT=${ISOL_ANALYSIS_GET_LICENSE_TASK_COUNT}
+      - ISOL_ANALYSIS_HANDLE_HTTP_REQUEST_TASK_COUNT=${ISOL_ANALYSIS_HANDLE_HTTP_REQUEST_TASK_COUNT}
+      - ISOL_ANALYSIS_HANDLE_HTTP_RESPONSE_FAILURE_TASK_COUNT=${ISOL_ANALYSIS_HANDLE_HTTP_RESPONSE_FAILURE_TASK_COUNT}
+      - ISOL_ANALYSIS_HANDLE_HTTP_RESPONSE_SUCCESS_TASK_COUNT=${ISOL_ANALYSIS_HANDLE_HTTP_RESPONSE_SUCCESS_TASK_COUNT}
+      - ISOL_ANALYSIS_HTML_FROM_DOCUMENT_ROUTE_ON_ATTRIBUTE_TASK_COUNT=${ISOL_ANALYSIS_HTML_FROM_DOCUMENT_ROUTE_ON_ATTRIBUTE_TASK_COUNT}
+      - ISOL_ANALYSIS_SET_CONTENT_TASK_COUNT=${ISOL_ANALYSIS_SET_CONTENT_TASK_COUNT}
 
 volumes:
   entity-data:
diff --git a/helm/discover/templates/configMap/analysis.yml b/helm/discover/templates/configMap/analysis.yml
@@ -17,9 +17,35 @@ data:
   ISOL_ENTITY_DATA_VOLUME_PATH: "/data/entity"
   ISOL_FILE_STORE_URL: "{{ .Values.filestore.service.protocol }}://{{ .Values.filestore.service.name }}:{{ .Values.filestore.service.port }}{{ .Values.filestore.service.path }}"
   ISOL_FILE_STORE_BUCKET: "{{ .Values.filestore.bucket }}"
-  ISOL_ANALYSIS_FACE_DEC_THREADS: "{{ .Values.analysis.concurrency.face }}"
-  ISOL_ANALYSIS_VEHICLE_THREADS: "{{ .Values.analysis.concurrency.vehicle }}"
-  ISOL_ANALYSIS_OBJECT_THREADS: "{{ .Values.analysis.concurrency.object }}"
+  ISOL_ANALYSIS_FACE_DEC_THREADS: "{{ .Values.analysis.concurrency.threads.face }}"
+  ISOL_ANALYSIS_VEHICLE_THREADS: "{{ .Values.analysis.concurrency.threads.vehicle }}"
+  ISOL_ANALYSIS_OBJECT_THREADS: "{{ .Values.analysis.concurrency.threads.object }}"
+  ISOL_ANALYSIS_MAX_TIMER_DRIVEN_THREAD_COUNT: "{{ .Values.analysis.concurrency.max }}"
+  ISOL_ANALYSIS_CREATE_PREVIEW_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.createPreview}}"
+  ISOL_ANALYSIS_EXTRACT_METADATA_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.extractMetadata}}"
+  ISOL_ANALYSIS_FACE_DETECTION_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.faceDetection}}"
+  ISOL_ANALYSIS_FACE_RECOGNITION_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.faceRecognition}}"
+  ISOL_ANALYSIS_FACE_TRAINING_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.faceTraining}}"
+  ISOL_ANALYSIS_FETCH_S3_OBJECT_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.fetchS3Object}}"
+  ISOL_ANALYSIS_IMAGE_EXTRACT_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.imageExtract}}"
+  ISOL_ANALYSIS_KEY_VIEW_EXPORT_TO_HTML_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.keyViewExportToHTML}}"
+  ISOL_ANALYSIS_OBJECT_CLASS_RECOGNITION_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.objectClassRecognition}}"
+  ISOL_ANALYSIS_PUT_ISOL_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.putIsol}}"
+  ISOL_ANALYSIS_REMOVE_DOCUMENT_PART_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.removeDocumentPart}}"
+  ISOL_ANALYSIS_TRANSCODE_VIDEO_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.transcodeVideo}}"
+  ISOL_ANALYSIS_VEHICLE_ANALYSIS_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.vehicleAnalysis}}"
+  ISOL_ANALYSIS_VIDEO_CLIP_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.videoClip}}"
+  ISOL_ANALYSIS_ANALYSIS_ROUTE_ON_ATTRIBUTE_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.analysisRouteOnAttribute}}"
+  ISOL_ANALYSIS_CREATE_PREVIEW_ROUTE_ON_ATTRIBUTE_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.createPreviewRouteOnAttribute}}"
+  ISOL_ANALYSIS_EXTRACT_JOB_ATTRIBUTES_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.extractJobAttributes}}"
+  ISOL_ANALYSIS_EXTRACT_JOB_OPTIONS_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.extractJobOptions}}"
+  ISOL_ANALYSIS_FAIL_ISOL_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.failIsol}}"
+  ISOL_ANALYSIS_GET_LICENSE_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.getLicense}}"
+  ISOL_ANALYSIS_HANDLE_HTTP_REQUEST_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.handleHTTPRequest}}"
+  ISOL_ANALYSIS_HANDLE_HTTP_RESPONSE_FAILURE_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.handleHTTPResponseFailure}}"
+  ISOL_ANALYSIS_HANDLE_HTTP_RESPONSE_SUCCESS_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.handleHTTPResponseSuccess}}"
+  ISOL_ANALYSIS_HTML_FROM_DOCUMENT_ROUTE_ON_ATTRIBUTE_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.htmlFromDocumentRouteOnAttribute}}"
+  ISOL_ANALYSIS_SET_CONTENT_TASK_COUNT: "{{ .Values.analysis.concurrency.taskCount.setContent}}"
 
 ---
 

diff --git a/helm/discover/values.dev.yaml b/helm/discover/values.dev.yaml
@@ -24,9 +24,10 @@ filestore:
   dataVolumeSize: 10Gi
 analysis:
   concurrency:
-    face: 2
-    vehicle: 2
-    object: 2
+    threads:
+      face: 2
+      vehicle: 2
+      object: 2
 datasets:
   locations:
     replicas: 1

diff --git a/helm/discover/values.user.yaml b/helm/discover/values.user.yaml
@@ -79,9 +79,27 @@ analysis:
   enabled: true
   # Number of analysis processes to run in parallel for each analysis type
   concurrency:
-    face: 4
-    vehicle: 4
-    object: 4
+    max: 10
+    threads:
+      face: 4
+      vehicle: 4
+      object: 4
+    taskCount:
+      # Configure the number of tasks that can be concurrently scheduled by each NiFi processor
+      createPreview: 4
+      extractMetadata: 4
+      faceDetection: 4
+      faceRecognition: 4
+      faceTraining: 4
+      fetchS3Object: 4
+      imageExtract: 4
+      keyViewExportToHTML: 4
+      objectClassRecognition: 4
+      putIsol: 4
+      removeDocumentPart: 4
+      transcodeVideo: 4
+      vehicleAnalysis: 4
+      videoClip: 4
 
 # PostgreSQL database server storing audit logs
 audit:

diff --git a/helm/discover/values.yaml b/helm/discover/values.yaml
@@ -126,9 +126,39 @@ analysis:
   secret: analysis-nifi-secret
   replicas: 1
   concurrency:
-    face: 4
-    vehicle: 4
-    object: 4
+    max: 10
+    threads:
+      face: 4
+      vehicle: 4
+      object: 4
+    taskCount:
+      # Configure the number of tasks that can be concurrently scheduled by each NiFi processor
+      createPreview: 4
+      extractMetadata: 4
+      faceDetection: 4
+      faceRecognition: 4
+      faceTraining: 4
+      fetchS3Object: 4
+      imageExtract: 4
+      keyViewExportToHTML: 4
+      objectClassRecognition: 4
+      putIsol: 4
+      removeDocumentPart: 4
+      transcodeVideo: 4
+      vehicleAnalysis: 4
+      videoClip: 4
+      # The following tasks are internal and should normally be left at the default of 1
+      analysisRouteOnAttribute: 1
+      createPreviewRouteOnAttribute: 1
+      extractJobAttributes: 1
+      extractJobOptions: 1
+      failIsol: 1
+      getLicense: 1
+      handleHTTPRequest: 1
+      handleHTTPResponseFailure: 1
+      handleHTTPResponseSuccess: 1
+      htmlFromDocumentRouteOnAttribute: 1
+      setContent: 1
   service:
     name: analysis-service
     port: 8090