Merge pull request #238 from DataONEorg/develop

3.1.4/chart-1.3.1 Release: merge develop to main

Author: artntek

RELEASE-NOTES.md

@@ -1,5 +1,48 @@
 # dataone-indexer Release Notes
 
+## dataone-indexer version 3.1.4 & helm chart version 1.3.1
+
+### Release date: 2025-05-20
+
+### dataone-indexer version 3.1.4
+
+This is a patch release with the following minor fixes and upgrades:
+
+* Use `org.apache.commons.logging` throughout; removed direct references to
+ `org.apache.log4j.Logger` ([Issue #178](https://github.com/DataONEorg/dataone-indexer/issues/178))
+* Remove the spring-web component from the dependency library. ([Issue #216](https://github.com/DataONEorg/dataone-indexer/issues/216))
+* Disable `livenessProbe` check for non-k8s deployments. ([Issue
+  #225](https://github.com/DataONEorg/dataone-indexer/issues/225))
+* Bump `log4j-layout-template-json` version to 2.24.3
+
+### helm chart version 1.3.1
+
+* Add a "warm up" postStart hook for Lustre/HPC Filesystems (disabled by default - see
+  `idxworker.enableMountWarmupHook` in values.yaml. Addresses visibility issues with Lustre and
+  similar HPC filesystems. ([Issue #227](https://github.com/DataONEorg/dataone-indexer/issues/227))
+* Remove `subPath` for log4j properties configmap to allow dynamic reloading ([Issue
+  #210](https://github.com/DataONEorg/dataone-indexer/issues/210#issuecomment-2859717382))
+* Allow configuration of Java Heap Memory for index workers, via values.yaml (`idxworker.javaMem`)
+  ([Issue #231](https://github.com/DataONEorg/dataone-indexer/issues/231))
+* Added `extraVolumes` and `extraVolumeMounts` to values.yaml, allowing users to specify additional
+  volumes and mounts in their deployments ([Issue
+  #223](https://github.com/DataONEorg/dataone-indexer/issues/223))
+* Full Support for granular configuration of security context in Containers and InitContainers
+  ([Issue #224](https://github.com/DataONEorg/dataone-indexer/issues/224))
+  * Add `podSecurityContext.runAsUser: 59997` (applies to dependencies initContainer and indexer
+      container)
+  * Add `runAsNonRoot: true` to dependencies initContainer (was previously applied only to indexer
+    container)
+* Introduced `idxworker.tripleDbStorageDefinition` in values.yaml to allow custom storage
+  configurations for the tripleDB volume. Supports alternative storage types such as hostPath and
+  emptyDir ([Issue #228](https://github.com/DataONEorg/dataone-indexer/issues/228)).
+* Bump indexer App version to 3.1.4
+
+> [!CAUTION]
+> **If upgrading from 1.2.0, note the section entitled:
+> `Caution - ENSURE THAT THE RABBITMQ QUEUE IS EMPTY,` [in the release notes for helm chart v1.3.0
+> below!](#helm-chart-version-130)**,
+
 ## dataone-indexer version 3.1.3 & helm chart version 1.3.0
 
 ### Release date: 2025-05-01

docker/Dockerfile

@@ -7,7 +7,7 @@ ARG TAG="NO-TAG"
 ENV TAG=${TAG}
 ENV DATAONE_INDEXER_CONFIG=/etc/dataone/dataone-indexer.properties
 
-# Set the working directory 
+# Set the working directory
 WORKDIR /var/lib/dataone-indexer
 
 RUN apt update && apt -y install \

docker/entrypoint.sh

@@ -6,9 +6,12 @@ then
     echo "Starting infinite loop, ctrl-c to interrupt..."
     sh -c 'trap "exit" TERM; while true; do sleep 1; done'
 else
+    echo "Starting DataONE Indexer from jar file: ./dataone-index-worker-${TAG}-shaded.jar,"
+    echo "using memory options: [${IDX_JAVA_MEM}]"
+
     java  -Dlog4j2.formatMsgNoLookups=true \
           -XX:+UnlockExperimentalVMOptions \
-          -XX:+UseContainerSupport \
+          -XX:+UseContainerSupport  $IDX_JAVA_MEM \
           -XX:+UseSerialGC \
           -cp /etc/dataone/:./dataone-index-worker-${TAG}-shaded.jar \
           org.dataone.cn.indexer.IndexWorker

helm/Chart.yaml

@@ -21,13 +21,13 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: "1.3.0"
+version: "1.3.1"
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "3.1.3"
+appVersion: "3.1.4"
 
 # Chart dependencies
 dependencies:

helm/templates/deployment.yaml

@@ -19,6 +19,7 @@ spec:
       {{- end }}
       labels:
         {{- include "idxworker.selectorLabels" . | nindent 8 }}
+      namespace: {{ .Release.Namespace }}
     spec:
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
@@ -47,8 +48,29 @@ spec:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       containers:
         - name: {{ .Chart.Name }}
+        {{- if .Values.idxworker.enableMountWarmupHook }}
+          lifecycle:
+            postStart:
+              exec:
+                command:
+                  - sh
+                  - -c
+                  - |
+                    {
+                      ## This is a workaround for the Lustre file system
+                      echo "$(date '+%Y-%m-%d %H:%M:%S.%3N') - Warming up mounted directories..." >> /tmp/postStart.log
+                      ls -1 {{ .Values.idxworker.storage.hashStoreRootDir }}/metadata/tmp > /dev/null 2>&1 || true
+                      ls -1 {{ .Values.idxworker.storage.hashStoreRootDir }}/refs/tmp > /dev/null 2>&1 || true
+                      ls -1 {{ .Values.idxworker.storage.hashStoreRootDir }}/objects > /dev/null 2>&1 || true
+                      ls -1 {{ .Values.idxworker.storage.hashStoreRootDir }}/refs/pids > /dev/null 2>&1 || true
+                      ls -1 {{ .Values.idxworker.storage.hashStoreRootDir }}/refs/cids > /dev/null 2>&1 || true
+                      echo "$(date '+%Y-%m-%d %H:%M:%S.%3N') - Warm-up complete." >> /tmp/postStart.log
+                    }
+        {{- end }}
+          {{- if .Values.securityContext }}
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
+          {{- end }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           {{- if .Values.idxworker.readinessProbe.enabled }}
@@ -74,12 +96,12 @@ spec:
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:
-            - mountPath: /etc/dataone/dataone-indexer.properties
-              subPath: dataone-indexer.properties
-              name: {{ .Release.Name }}-config-volume
-            - mountPath: /etc/dataone/log4j2.properties
-              subPath: log4j2.properties
+            {{- if .Values.extraVolumeMounts }}
+            {{- tpl (toYaml .Values.extraVolumeMounts) . | nindent 12 }}
+            {{- end }}
+            - mountPath: /etc/dataone
               name: {{ .Release.Name }}-config-volume
+              readOnly: true
             - mountPath: {{ .Values.persistence.mountPath }}
               name: indexer-metacat-pv
               {{- if .Values.persistence.subPath }}
@@ -91,6 +113,10 @@ spec:
           env:
             - name: DEBUG
               value: "{{ .Values.idxworker.debug }}"
+            {{- if .Values.idxworker.javaMem }}
+            - name: IDX_JAVA_MEM
+              value: {{ .Values.idxworker.javaMem | quote }}
+            {{- end }}
             - name: DATAONE_AUTH_TOKEN
               valueFrom:
                 secretKeyRef:
@@ -116,6 +142,10 @@ spec:
       initContainers:
         - name: dependencies
           image: busybox:latest
+          {{- if .Values.securityContext }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          {{- end }}
           command:
             - sh
             - -c
@@ -144,6 +174,9 @@ spec:
               echo waiting for Solr Schema to be accessible at http://{{ $solrHost }}:
               {{- $solrPort }}$SOLRURI; sleep 1; done;
       volumes:
+        {{- if .Values.extraVolumes }}
+        {{- tpl (toYaml .Values.extraVolumes) . | nindent 8 }}
+        {{- end }}
         - name: {{ .Release.Name }}-config-volume
           configMap:
             name: {{ .Release.Name }}-indexer-configfiles
@@ -159,6 +192,9 @@ spec:
             readOnly: true
           {{- end }}
         - name: {{ .Release.Name }}-temp-tripledb-volume
+          {{- if .Values.idxworker.tripleDbStorageDefinition }}
+          {{- tpl (toYaml .Values.idxworker.tripleDbStorageDefinition) . | nindent 10 }}
+          {{- else }}
           ephemeral:
             volumeClaimTemplate:
               spec:
@@ -169,3 +205,4 @@ spec:
                 resources:
                   requests:
                     storage: 1Gi
+         {{- end }}

helm/values.yaml

@@ -73,8 +73,10 @@ serviceAccount:
 podAnnotations: {}
 
 podSecurityContext:
+  runAsUser: 59997
   fsGroup: 1000 # must match metacat group for shared volume at /var/metacat
 
+## @param securityContext Security context for the container (This will be applied to all containers)
 securityContext:
   runAsNonRoot: true
 
@@ -106,6 +108,11 @@ tolerations: []
 
 affinity: {}
 
+## @param extraVolumes Additional volumes to be added to the pod
+extraVolumes: []
+## @param extraVolumeMounts Additional volume mounts to be added to the pod
+extraVolumeMounts: []
+
 persistence:
   ## @param persistence.claimName Name of existing PVC to use (typically shared with metacat)
   ## Set a value for 'claimName' only if you want to re-use a Persistent Volume Claim that has
@@ -203,6 +210,28 @@ idxworker:
   ##
   tripleDbDirectory: /etc/dataone/tdb-cache
 
+  ## @param idxworker.tripleDbStorageDefinition type of storage for the triple store (default is "ephemeral"). Other
+  ## options  are "hostPath" and "emptyDir".  Must give full definition for hostPath or emptyDir.
+  ## For example:
+  ##  tripleDbStorageDefinition:
+  ##    emptyDir: {}
+  ##
+  ##  For the emptyDir option you will need to think about specifying the ephemeral-storage limit in the
+  ##  resources section.
+  ##
+  ## For example:
+  ##    resources:
+  ##      requests:
+  ##        ephemeral-storage: 1Gi
+  ##      limits:
+  ##        ephemeral-storage: 2Gi
+  tripleDbStorageDefinition: {}
+
+  ## @param idxworker.enableMountWarmupHook Enable or disable the mount warmup hook.  This is a workaround for
+  ## specifically Lustre filesystems. This workaround addresses known behavior with Lustre and other HPC
+  ## file systems that require access within the container runtime to finalize mount visibility.
+  enableMountWarmupHook: false
+
   storage:
     hashStoreClassName: "org.dataone.hashstore.filehashstore.FileHashStore"
     hashStoreRootDir: "/var/metacat/hashstore"
@@ -219,6 +248,12 @@ idxworker:
   readinessProbe:
     enabled: true
 
+  ## @param idxworker.javaMem Java memory options to pass to the index worker container
+  ## (format: "-Xms512m -Xmx2g"). If you do not set a value, the JVM will default to using 1/4 of
+  ## the available memory as the maximum
+  ##
+  javaMem: ""
+
 ## @section RabbitMQ Bitnami Sub-Chart Configuration
 ##
 rabbitmq:

pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>org.dataone</groupId>
     <artifactId>dataone-index-worker</artifactId>
-    <version>3.1.3</version>
+    <version>3.1.4</version>
     <packaging>jar</packaging>
     <name>dataone-index-worker</name>
     <url>http://maven.apache.org</url>
@@ -184,11 +184,6 @@
             <artifactId>spring-jdbc</artifactId>
             <version>${spring.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-web</artifactId>
-            <version>${spring.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-test</artifactId>
@@ -283,7 +278,7 @@
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-layout-template-json</artifactId>
-            <version>2.17.2</version>
+            <version>${log4j.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.solr</groupId>

src/main/java/org/dataone/cn/indexer/IndexWorker.java

@@ -106,16 +106,21 @@ public class IndexWorker {
     public static void main(String[] args) {
         logger.info("IndexWorker.main - Starting index worker...");
         String propertyFile = null;
+        if (args != null && args.length == 1) {
+            // The args should be a property file which the dataone-indexer will use
+            propertyFile = args[0];
+            logger.debug("The external property file specified in the argument is " + propertyFile);
+        }
         loadExternalPropertiesFile(propertyFile);
         try {
             IndexWorker worker = new IndexWorker();
             worker.start();
             worker.startReadinessProbe();
+            worker.startLivenessProbe();
         } catch (Exception e) {
             logger.fatal("IndexWorker.main() exiting due to fatal error: " + e.getMessage(), e);
             System.exit(1);
         }
-        startLivenessProbe();
     }
 
     /**
@@ -483,7 +488,11 @@ private void indexObject(IndexQueueMessageParser parser, boolean multipleThread)
         }
     }
 
-    private static void startLivenessProbe() {
+    private void startLivenessProbe() {
+        if (!isK8s) {
+            logger.debug("This is a non-k8s deployment and LivenessProbe do nothing.");
+            return;
+        }
         ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
         Path path = Paths.get("./livenessprobe");
         Runnable task = () -> {

src/main/java/org/dataone/cn/indexer/XmlDocumentUtility.java

@@ -31,14 +31,15 @@
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 
-import org.apache.log4j.Logger;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.w3c.dom.Document;
 import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;
 
 public class XmlDocumentUtility {
 
-    private static Logger log = Logger.getLogger(XmlDocumentUtility.class);
+    private static Log log = LogFactory.getLog(XmlDocumentUtility.class);
 
     private static final String INPUT_ENCODING = "UTF-8";
 

src/main/java/org/dataone/cn/indexer/annotation/AnnotatorSubprocessor.java

@@ -18,7 +18,8 @@
 
 import org.apache.commons.codec.EncoderException;
 import org.apache.commons.io.IOUtils;
-import org.apache.log4j.Logger;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.dataone.cn.indexer.parser.IDocumentSubprocessor;
 import org.dataone.cn.indexer.parser.ISolrDataField;
 import org.dataone.cn.indexer.parser.SubprocessorUtility;
@@ -45,7 +46,7 @@ public class AnnotatorSubprocessor implements IDocumentSubprocessor {
     public static final String FIELD_ANNOTATED_BY = "sem_annotated_by";
     public static final String FIELD_COMMENT = "sem_comment";
 
-    private static Logger log = Logger.getLogger(AnnotatorSubprocessor.class.getName());
+    private static Log log = LogFactory.getLog(AnnotatorSubprocessor.class.getName());
 
     private SubprocessorUtility processorUtility;