Merge pull request #254 from DataONEorg/develop

Release 3.1.5 develop->main

Author: artntek

README.md

@@ -284,6 +284,14 @@ vhost = /         # Used as default for declare / delete / list
 for n in $(seq 1 30); do echo $n; rabbitmqadmin -c rmq.conf -N default -U rmq -p $RMQPW publish exchange=testexchange routing_key=testqueue payload="Message: ${n}" --vhost=/; done
 ```
 
+## Switching the Storage System
+The Dataone Indexer can be configured to use different storage systems by setting the environmental
+variable `DATAONE_INDEXER_OBJECT_MANAGER_CLASS_NAME`.
+By default, this variable is not set, and the indexer uses
+`org.dataone.cn.indexer.object.hashstore.HashStoreObjManager`, which enables support for Hashstore.
+To use the legacy storage system instead, set the variable to
+`org.dataone.cn.indexer.object.legacystore.LegacyStoreObjManager`.
+
 ## History
 
 This is a refactored version of the original DataONE [d1_cn_index_processor](https://github.com/DataONEorg/d1_cn_index_processor) that runs

RELEASE-NOTES.md

@@ -1,5 +1,27 @@
 # dataone-indexer Release Notes
 
+> [!CAUTION]
+> **If upgrading from Helm chart v1.2.0 or earlier, note the section entitled:
+> `Caution - ENSURE THAT THE RABBITMQ QUEUE IS EMPTY,` [in the release notes for helm chart v1.3.0
+> below!](#helm-chart-version-130)**,
+
+## dataone-indexer version 3.1.5 & helm chart version 1.3.2
+
+### Release date: 2025-06-26
+
+### dataone-indexer version 3.1.5
+
+This is a patch release with the following minor fixes and upgrades
+
+- Dataone-indexer can handle legacy Metacat object repository ([Issue #222](https://github.com/DataONEorg/dataone-indexer/issues/222))
+- Remove some extra log statements (for version conflict retries) that are confusing to users ([Issue #243](https://github.com/DataONEorg/dataone-indexer/issues/243))
+- Indexer performance improvement: Decrease the re-try waiting time for a version conflict error ([Issue #245](https://github.com/DataONEorg/dataone-indexer/issues/245))
+- Remove unnecessary dependency on PostrgeSQL jar ([Issue #247](https://github.com/DataONEorg/dataone-indexer/issues/247))
+
+### helm chart version 1.3.2
+- Bump indexer App version to 3.1.5
+
+
 ## dataone-indexer version 3.1.4 & helm chart version 1.3.1
 
 ### Release date: 2025-05-20

helm/Chart.yaml

@@ -21,13 +21,13 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: "1.3.1"
+version: "1.3.2"
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "3.1.4"
+appVersion: "3.1.5"
 
 # Chart dependencies
 dependencies:

helm/config/dataone-indexer.properties

@@ -38,7 +38,7 @@ dataone.mn.registration.serviceType.url={{ .Values.idxworker.d1_serviceType_url
 
 index.resourcemap.waitingComponent.time={{ default 800 .Values.idxworker.resourcemapWaitMs }}
 index.resourcemap.waitingComponent.max.attempts={{ default 25 .Values.idxworker.resourcemapMaxTries }}
-index.solr.versionConflict.waiting.time={{ default 1000 .Values.idxworker.solrVerConflictWaitMs }}
+index.solr.versionConflict.waiting.time={{ default 10 .Values.idxworker.solrVerConflictWaitMs }}
 index.solr.versionConflict.max.attempts={{ default 25000 .Values.idxworker.solrVerConflictMaxTries }}
 
 # Storage properties

helm/templates/deployment.yaml

@@ -117,7 +117,7 @@ spec:
             - name: IDX_JAVA_MEM
               value: {{ .Values.idxworker.javaMem | quote }}
             {{- end }}
-            - name: DATAONE_AUTH_TOKEN
+            - name: DATAONE_INDEXER_AUTH_TOKEN
               valueFrom:
                 secretKeyRef:
                   name: {{ .Release.Name }}-indexer-token

helm/values.yaml

@@ -167,7 +167,7 @@ idxworker:
   ## @param idxworker.solrVerConflictWaitMs wait time (mS) before indexer grabs a newer version
   ## of solr doc after a version conflict
   ##
-  solrVerConflictWaitMs: 1000
+  solrVerConflictWaitMs: 10
 
   ## @param idxworker.solrVerConflictMaxTries Number of tries to get a newer version of solr doc
   ## after a version conflict

pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>org.dataone</groupId>
     <artifactId>dataone-index-worker</artifactId>
-    <version>3.1.4</version>
+    <version>3.1.5</version>
     <packaging>jar</packaging>
     <name>dataone-index-worker</name>
     <url>http://maven.apache.org</url>
@@ -306,6 +306,12 @@
             <groupId>org.dataone</groupId>
             <artifactId>hashstore</artifactId>
             <version>1.1.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.postgresql</groupId>
+                    <artifactId>postgresql</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>uk.org.webcompere</groupId>

src/main/java/org/dataone/cn/indexer/IndexWorker.java

@@ -2,6 +2,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -18,14 +19,13 @@
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.xpath.XPathExpressionException;
 
-import com.rabbitmq.client.ShutdownSignalException;
 import org.apache.commons.codec.EncoderException;
 import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.dataone.cn.indexer.annotation.OntologyModelService;
-import org.dataone.cn.indexer.object.ObjectManager;
+import org.dataone.cn.indexer.object.ObjectManagerFactory;
 import org.dataone.configuration.Settings;
 import org.dataone.exceptions.MarshallingException;
 import org.dataone.indexer.queue.IndexQueueMessageParser;
@@ -59,7 +59,7 @@ public class IndexWorker {
     //The create is the index task type for the action when a new object was created. So the solr index will be generated.
     //delete is the index task type for the action when an object was deleted. So the solr index will be deleted
     //sysmeta is the index task type for the action when the system metadata of an existing object was updated.
-    public final static String CREATE_INDEXT_TYPE = "create";
+    public final static String CREATE_INDEX_TYPE = "create";
     public final static String DELETE_INDEX_TYPE = "delete";
     public final static String SYSMETA_CHANGE_TYPE = "sysmeta"; //this handle for resource map only
 
@@ -219,7 +219,10 @@ public static void loadAdditionalPropertyFile(String propertyFile) {
      * @throws TimeoutException
      * @throws ServiceFailure
      */
-    public IndexWorker() throws IOException, TimeoutException, ServiceFailure {
+    public IndexWorker()
+        throws IOException, TimeoutException, ServiceFailure, ClassNotFoundException,
+        InvocationTargetException, NoSuchMethodException, InstantiationException,
+        IllegalAccessException {
         this(true);
     }
 
@@ -231,7 +234,9 @@ public IndexWorker() throws IOException, TimeoutException, ServiceFailure {
      * @throws TimeoutException
      * @throws ServiceFailure
      */
-    public IndexWorker(Boolean initialize) throws IOException, TimeoutException {
+    public IndexWorker(Boolean initialize)
+        throws IOException, TimeoutException, ClassNotFoundException, InvocationTargetException,
+        NoSuchMethodException, InstantiationException, IllegalAccessException {
         String value = System.getenv("KUBERNETES_SERVICE_HOST");
         // Java doc says: the string value of the variable, or null if the variable is not defined
         // in the system environment
@@ -243,7 +248,7 @@ public IndexWorker(Boolean initialize) throws IOException, TimeoutException {
             initExecutorService();//initialize the executor first
             initIndexQueue();
             initIndexParsers();
-            ObjectManager.getInstance();
+            ObjectManagerFactory.getObjectManager();
             OntologyModelService.getInstance();
         }
     }
@@ -385,9 +390,11 @@ public void run() {
                         indexObject(parser, multipleThread);
                     }
                 } catch (InvalidRequest e) {
-                    logger.error(
-                        "cannot index the task for identifier " + parser.getIdentifier().getValue()
-                            + " since " + e.getMessage());
+                    String error = "Cannot index the task for the object since " + e.getMessage();
+                    if (parser.getIdentifier() != null) {
+                        error = error + " with the identifier " + parser.getIdentifier().getValue();
+                    }
+                    logger.error(error);
                     boolean requeue = false;
                     rabbitMQchannel.basicReject(envelope.getDeliveryTag(), requeue);
                 }
@@ -449,21 +456,22 @@ private void indexObject(IndexQueueMessageParser parser, boolean multipleThread)
         Identifier pid = parser.getIdentifier();
         String indexType = parser.getIndexType();
         int priority = parser.getPriority();
+        String docId = parser.getDocId();// It can be null.
         try {
             long threadId = Thread.currentThread().getId();
             logger.info("IndexWorker.consumer.indexObject by multiple thread? " + multipleThread
                             + ", with the thread id " + threadId
                     + " - Received the index task from the index queue with the identifier: "
                             + pid.getValue() + " , the index type: " + indexType
-                            + ", the priority: " + priority);
+                            + ", the priority: " + priority + ", the docId(can be null): " + docId);
             switch (indexType) {
-                case CREATE_INDEXT_TYPE -> {
+                case CREATE_INDEX_TYPE -> {
                     boolean sysmetaOnly = false;
-                    solrIndex.update(pid, sysmetaOnly);
+                    solrIndex.update(pid, sysmetaOnly, docId);
                 }
                 case SYSMETA_CHANGE_TYPE -> {
                     boolean sysmetaOnly = true;
-                    solrIndex.update(pid, sysmetaOnly);
+                    solrIndex.update(pid, sysmetaOnly, docId);
                 }
                 case DELETE_INDEX_TYPE -> solrIndex.remove(pid);
                 default -> throw new InvalidRequest(
@@ -482,7 +490,8 @@ private void indexObject(IndexQueueMessageParser parser, boolean multipleThread)
                  ServiceFailure | XPathExpressionException | UnsupportedType | SAXException |
                  ParserConfigurationException | SolrServerException | MarshallingException |
                  EncoderException | InterruptedException | IOException | InstantiationException |
-                 IllegalAccessException e) {
+                 IllegalAccessException | ClassNotFoundException | InvocationTargetException |
+                 NoSuchMethodException e) {
             logger.error("Cannot index the task for identifier " + pid.getValue()
                              + " since " + e.getMessage(), e);
         }