Skip to content

Commit eae4917

Browse files
josegar74fxprunayre
josegar74
and
fxprunayre
authored
Support for translation providers in harvesters (geonetwork#7849)
* Support for translation providers in harvesters * Support for translation providers / fix artifact in services/pom.xml * Support for translation providers / directive to select multiple languages * Support for translation providers / update harvester translation and use ACE Editor widget for translation fields configuration * Support for translation providers / refactor directive to select multiple languages to be generic * Support for translation providers / directive to select multiple languages - use a watcher to initialize typeahead when the data is available * Translation providers improvements (geonetwork#98) * Support for translation providers / Better handle record which may contains other language, add translate all, translate only empty. Also experiment support in suggestion (disabled for now - list of xpath is not supported, only one xpath at a time). * Support for translation providers / Add google translate service * Create a script ```js var mock = { parameter:{ q:'hello', source:'en', target:'fr' } }; function doGet(e) { e = e || mock; var sourceText = '' if (e.parameter.q){ sourceText = e.parameter.q; } var sourceLang = ''; if (e.parameter.source){ sourceLang = e.parameter.source; } var targetLang = 'en'; if (e.parameter.target){ targetLang = e.parameter.target; } var translatedText = LanguageApp.translate(sourceText, sourceLang, targetLang, {contentType: 'html'}); return ContentService.createTextOutput(translatedText).setMimeType(ContentService.MimeType.JSON); } ``` * Deploy it as webapp for anonymous user * Configure the service URL in the translation provider * Support for translation providers / Add Anchor support. * Support for translation providers / Create one directive for the configuration. Add examples for easier XPath configuration. * Support for translation providers / Improve translation. * Support for translation providers / Fix bad xpath. * Update translationproviders/src/main/java/org/fao/geonet/translations/googletranslate/GoogleTranslateService.java * Update translationproviders/src/main/java/org/fao/geonet/translations/libretranslate/LibreTranslateClientException.java * Update web-ui/src/main/resources/catalog/components/admin/harvester/partials/translate.html * Update web-ui/src/main/resources/catalog/components/admin/harvester/partials/translate.html --------- Co-authored-by: Jose García <[email protected]> * Translation providers - support additional harvesters: GeoNetwork, Local filesystem, OAI-PHM, WebDav --------- Co-authored-by: François Prunayre <[email protected]>
1 parent e98fa93 commit eae4917

File tree

56 files changed

+1888
-60
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+1888
-60
lines changed

‎core/src/main/java/org/fao/geonet/kernel/setting/Settings.java

+3
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,9 @@ public class Settings {
155155
public static final String SYSTEM_SECURITY_PASSWORDENFORCEMENT_USEPATTERN = "system/security/passwordEnforcement/usePattern";
156156
public static final String SYSTEM_SECURITY_PASSWORDENFORCEMENT_PATTERN = "system/security/passwordEnforcement/pattern";
157157
public static final String SYSTEM_SECURITY_PASSWORD_ALLOWADMINRESET = "system/security/password/allowAdminReset";
158+
public static final String SYSTEM_TRANSLATION_PROVIDER = "system/translation/provider";
159+
public static final String SYSTEM_TRANSLATION_SERVICEURL = "system/translation/serviceUrl";
160+
public static final String SYSTEM_TRANSLATION_APIKEY = "system/translation/apiKey";
158161

159162
public static final String MICROSERVICES_ENABLED = "microservices/enabled";
160163

‎harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java

+70-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (C) 2001-2016 Food and Agriculture Organization of the
2+
* Copyright (C) 2001-2024 Food and Agriculture Organization of the
33
* United Nations (FAO-UN), United Nations World Food Programme (WFP)
44
* and United Nations Environment Programme (UNEP)
55
*
@@ -28,21 +28,28 @@
2828
import org.fao.geonet.domain.AbstractMetadata;
2929
import org.fao.geonet.domain.MetadataCategory;
3030
import org.fao.geonet.kernel.DataManager;
31+
import org.fao.geonet.kernel.SchemaManager;
3132
import org.fao.geonet.kernel.datamanager.IMetadataManager;
3233
import org.fao.geonet.kernel.harvest.harvester.AbstractHarvester;
3334
import org.fao.geonet.kernel.harvest.harvester.AbstractParams;
3435
import org.fao.geonet.kernel.harvest.harvester.CategoryMapper;
3536
import org.fao.geonet.kernel.harvest.harvester.GroupMapper;
3637
import org.fao.geonet.kernel.harvest.harvester.Privileges;
3738
import org.fao.geonet.kernel.setting.SettingManager;
39+
import org.fao.geonet.kernel.setting.Settings;
3840
import org.fao.geonet.repository.MetadataCategoryRepository;
3941
import org.fao.geonet.repository.OperationAllowedRepository;
42+
import org.fao.geonet.utils.Xml;
4043
import org.jdom.Element;
4144
import org.slf4j.Logger;
4245
import org.slf4j.LoggerFactory;
43-
import org.springframework.beans.factory.annotation.Autowired;
46+
import org.springframework.util.StringUtils;
4447

48+
import java.nio.file.Files;
49+
import java.nio.file.Path;
50+
import java.util.Arrays;
4551
import java.util.HashMap;
52+
import java.util.List;
4653
import java.util.Map;
4754
import java.util.concurrent.atomic.AtomicBoolean;
4855

@@ -62,7 +69,7 @@ public abstract class BaseAligner<P extends AbstractParams> extends AbstractAlig
6269

6370
public final AtomicBoolean cancelMonitor;
6471

65-
public BaseAligner(AtomicBoolean cancelMonitor) {
72+
protected BaseAligner(AtomicBoolean cancelMonitor) {
6673
this.cancelMonitor = cancelMonitor;
6774
}
6875

@@ -71,7 +78,7 @@ public void addCategories(AbstractMetadata metadata, Iterable<String> categories
7178
String serverCategory, boolean saveMetadata) {
7279

7380
MetadataCategoryRepository metadataCategoryRepository = context.getBean(MetadataCategoryRepository.class);
74-
Map<String, MetadataCategory> nameToCategoryMap = new HashMap<String, MetadataCategory>();
81+
Map<String, MetadataCategory> nameToCategoryMap = new HashMap<>();
7582
for (MetadataCategory metadataCategory : metadataCategoryRepository.findAll()) {
7683
nameToCategoryMap.put("" + metadataCategory.getId(), metadataCategory);
7784
}
@@ -133,4 +140,63 @@ public void addPrivileges(String id, Iterable<Privileges> privilegesIterable, Gr
133140
}
134141
}
135142
}
143+
144+
/**
145+
* Applies a xslt process (schema_folder/process/translate.xsl) to translate create the metadata
146+
* fields configured in the harvester to the languages configured, using the translation provider
147+
* configured in the application settings.
148+
*
149+
* If no translation provider is configured or if the schema doesn't have the translation xslt,
150+
* the translation process is not applied to the metadata.
151+
*
152+
* @param context
153+
* @param md
154+
* @param schema
155+
* @return
156+
*/
157+
public Element translateMetadataContent(ServiceContext context,
158+
Element md,
159+
String schema) {
160+
161+
SettingManager settingManager = context.getBean(SettingManager.class);
162+
163+
String translationProvider = settingManager.getValue(Settings.SYSTEM_TRANSLATION_PROVIDER);
164+
165+
if (!StringUtils.hasLength(translationProvider)) {
166+
LOGGER.warn(" metadata content can't be translated. Translation provider not configured.");
167+
return md;
168+
}
169+
170+
if (!StringUtils.hasLength(params.getTranslateContentLangs()) ||
171+
!StringUtils.hasLength(params.getTranslateContentFields())) {
172+
LOGGER.warn(" metadata content can't be translated. No languages or fields provided to translate.");
173+
return md;
174+
}
175+
176+
SchemaManager schemaManager = context.getBean(SchemaManager.class);
177+
178+
Path filePath = schemaManager.getSchemaDir(schema).resolve("process").resolve( "translate.xsl");
179+
180+
if (!Files.exists(filePath)) {
181+
LOGGER.debug(String.format(" metadata content translation process not available for schema %s", schema));
182+
} else {
183+
Element processedMetadata;
184+
try {
185+
Map<String, Object> processParams = new HashMap<>();
186+
List<String> langs = Arrays.asList(params.getTranslateContentLangs().split(","));
187+
processParams.put("languages", langs);
188+
189+
List<String> fields = Arrays.asList(params.getTranslateContentFields().split("\\n"));
190+
processParams.put("fieldsToTranslate", fields);
191+
192+
processedMetadata = Xml.transform(md, filePath, processParams);
193+
LOGGER.debug(" metadata content translated.");
194+
md = processedMetadata;
195+
} catch (Exception e) {
196+
LOGGER.warn(String.format(" metadata content translated error: %s", e.getMessage()));
197+
}
198+
}
199+
return md;
200+
}
201+
136202
}

‎harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java

+32-34
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
//=============================================================================
2-
//=== Copyright (C) 2001-2020 Food and Agriculture Organization of the
2+
//=== Copyright (C) 2001-2024 Food and Agriculture Organization of the
33
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
44
//=== and United Nations Environment Programme (UNEP)
55
//===
@@ -85,15 +85,7 @@
8585
import java.time.OffsetDateTime;
8686
import java.time.ZoneOffset;
8787
import java.time.format.DateTimeFormatter;
88-
import java.util.Arrays;
89-
import java.util.Collections;
90-
import java.util.HashSet;
91-
import java.util.LinkedList;
92-
import java.util.List;
93-
import java.util.Map;
94-
import java.util.Set;
95-
import java.util.TimeZone;
96-
import java.util.UUID;
88+
import java.util.*;
9789
import java.util.concurrent.TimeUnit;
9890
import java.util.concurrent.atomic.AtomicBoolean;
9991
import java.util.concurrent.locks.ReentrantLock;
@@ -286,9 +278,9 @@ public void destroy() throws Exception {
286278

287279
final Specification<? extends AbstractMetadata> ownedByHarvester = Specification.where(MetadataSpecs.hasHarvesterUuid(getParams().getUuid()));
288280
Set<String> sources = new HashSet<>();
289-
for (Integer id : metadataRepository.findAllIdsBy(ownedByHarvester)) {
290-
sources.add(metadataUtils.findOne(id).getSourceInfo().getSourceId());
291-
metadataManager.deleteMetadata(context, "" + id);
281+
for (Integer metadataId : metadataRepository.findAllIdsBy(ownedByHarvester)) {
282+
sources.add(metadataUtils.findOne(metadataId).getSourceInfo().getSourceId());
283+
metadataManager.deleteMetadata(context, "" + metadataId);
292284
}
293285

294286
// Remove all sources related to the harvestUuid if they are not linked to any record anymore
@@ -573,7 +565,10 @@ private void login() throws Exception {
573565
UserRepository repository = this.context.getBean(UserRepository.class);
574566
User user = null;
575567
if (StringUtils.isNotEmpty(ownerId)) {
576-
user = repository.findById(Integer.parseInt(ownerId)).get();
568+
Optional<User> userOptional = repository.findById(Integer.parseInt(ownerId));
569+
if (userOptional.isPresent()) {
570+
user = userOptional.get();
571+
}
577572
}
578573

579574
// for harvesters created before owner was added to the harvester code,
@@ -697,21 +692,21 @@ protected OperResult harvest() {
697692
private void logHarvest(String logfile, Logger logger, String nodeName, String lastRun, long elapsedTime) {
698693
try {
699694
// record the results/errors for this harvest in the database
700-
Element result = getResult();
695+
Element resultEl = getResult();
701696
if (error != null) {
702-
result = JeevesException.toElement(error);
697+
resultEl = JeevesException.toElement(error);
703698
}
704-
Element priorLogfile_ = result.getChild("logfile");
705-
if (priorLogfile_ != null) {
699+
Element priorLogfileEl = resultEl.getChild("logfile");
700+
if (priorLogfileEl != null) {
706701
// removing prior logfile
707-
logger.warning("Detected duplicate logfile: " + priorLogfile_.getText());
708-
result.getChildren().remove(priorLogfile_);
702+
logger.warning("Detected duplicate logfile: " + priorLogfileEl.getText());
703+
resultEl.getChildren().remove(priorLogfileEl);
709704
}
710-
Element logfile_ = new Element("logfile");
711-
logfile_.setText(logfile);
712-
result.addContent(logfile_);
705+
Element logfileEl = new Element("logfile");
706+
logfileEl.setText(logfile);
707+
resultEl.addContent(logfileEl);
713708

714-
result.addContent(toElement(errors));
709+
resultEl.addContent(toElement(errors));
715710
final HarvestHistoryRepository historyRepository = context.getBean(HarvestHistoryRepository.class);
716711
final HarvestHistory history = new HarvestHistory()
717712
.setHarvesterType(getType())
@@ -720,7 +715,7 @@ private void logHarvest(String logfile, Logger logger, String nodeName, String l
720715
.setElapsedTime((int) elapsedTime)
721716
.setHarvestDate(new ISODate(lastRun))
722717
.setParams(getParams().getNodeElement())
723-
.setInfo(result);
718+
.setInfo(resultEl);
724719
historyRepository.save(history);
725720

726721

@@ -746,18 +741,18 @@ private void logHarvest(String logfile, Logger logger, String nodeName, String l
746741
*/
747742
private Element toElement(List<HarvestError> errors) {
748743
Element res = new Element("errors");
749-
for (HarvestError error : errors) {
744+
for (HarvestError harvestError : errors) {
750745
Element herror = new Element("error");
751746

752747
Element desc = new Element("description");
753-
desc.setText(error.getDescription());
748+
desc.setText(harvestError.getDescription());
754749
herror.addContent(desc);
755750

756751
Element hint = new Element("hint");
757-
hint.setText(error.getHint());
752+
hint.setText(harvestError.getHint());
758753
herror.addContent(hint);
759754

760-
herror.addContent(JeevesException.toElement(error.getOrigin()));
755+
herror.addContent(JeevesException.toElement(harvestError.getOrigin()));
761756
res.addContent(herror);
762757
}
763758
return res;
@@ -814,8 +809,8 @@ private final String doAdd(Element node) throws BadInputEx, SQLException {
814809
//--- force the creation of a new uuid
815810
params.setUuid(UUID.randomUUID().toString());
816811

817-
String id = harvesterSettingsManager.add("harvesting", "node", getType());
818-
storeNode(params, "id:" + id);
812+
String nodeId = harvesterSettingsManager.add("harvesting", "node", getType());
813+
storeNode(params, "id:" + nodeId);
819814

820815
Source source = new Source(params.getUuid(), params.getName(), params.getTranslations(), SourceType.harvester);
821816
final String icon = params.getIcon();
@@ -826,7 +821,7 @@ private final String doAdd(Element node) throws BadInputEx, SQLException {
826821
}
827822
context.getBean(SourceRepository.class).save(source);
828823

829-
return id;
824+
return nodeId;
830825
}
831826

832827
private void doUpdate(String id, Element node) throws BadInputEx, SQLException {
@@ -919,6 +914,9 @@ private void storeNode(P params, String path) throws SQLException {
919914
harvesterSettingsManager.add(ID_PREFIX + contentId, "importxslt", params.getImportXslt());
920915
harvesterSettingsManager.add(ID_PREFIX + contentId, "batchEdits", params.getBatchEdits());
921916
harvesterSettingsManager.add(ID_PREFIX + contentId, "validate", params.getValidate());
917+
harvesterSettingsManager.add(ID_PREFIX + contentId, "translateContent", params.isTranslateContent());
918+
harvesterSettingsManager.add(ID_PREFIX + contentId, "translateContentLangs", params.getTranslateContentLangs());
919+
harvesterSettingsManager.add(ID_PREFIX + contentId, "translateContentFields", params.getTranslateContentFields());
922920

923921
//--- setup stats node ----------------------------------------
924922

@@ -952,8 +950,8 @@ private void storePrivileges(P params, String path) {
952950
private void storeCategories(P params, String path) {
953951
String categId = harvesterSettingsManager.add(path, "categories", "");
954952

955-
for (String id : params.getCategories()) {
956-
harvesterSettingsManager.add(ID_PREFIX + categId, "category", id);
953+
for (String cId : params.getCategories()) {
954+
harvesterSettingsManager.add(ID_PREFIX + categId, "category", cId);
957955
}
958956
}
959957

‎harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractParams.java

+37-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
//=============================================================================
2-
//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the
2+
//=== Copyright (C) 2001-2024 Food and Agriculture Organization of the
33
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
44
//=== and United Nations Environment Programme (UNEP)
55
//===
@@ -95,6 +95,10 @@ public enum OverrideUuid {
9595
private String ownerIdUser;
9696
private OverrideUuid overrideUuid;
9797

98+
private boolean translateContent;
99+
private String translateContentLangs;
100+
private String translateContentFields;
101+
98102
/**
99103
* When more than one harvester harvest the same record, then record is usually rejected.
100104
* It can override existing, but the privileges are not preserved. This option
@@ -200,6 +204,9 @@ public void create(Element node) throws BadInputEx {
200204

201205
setImportXslt(Util.getParam(content, "importxslt", "none"));
202206
setBatchEdits(Util.getParam(content, "batchEdits", ""));
207+
setTranslateContent(Util.getParam(content, "translateContent", false));
208+
setTranslateContentLangs(Util.getParam(content, "translateContentLangs", ""));
209+
setTranslateContentFields(Util.getParam(content, "translateContentFields", ""));
203210

204211
this.setValidate(readValidateFromParams(content));
205212

@@ -280,6 +287,9 @@ public void update(Element node) throws BadInputEx {
280287

281288
setImportXslt(Util.getParam(content, "importxslt", "none"));
282289
setBatchEdits(Util.getParam(content, "batchEdits", getBatchEdits()));
290+
setTranslateContent(Util.getParam(content, "translateContent", false));
291+
setTranslateContentLangs(Util.getParam(content, "translateContentLangs", ""));
292+
setTranslateContentFields(Util.getParam(content, "translateContentFields", ""));
283293
this.setValidate(readValidateFromParams(content));
284294

285295
if (privil != null) {
@@ -330,7 +340,9 @@ protected void copyTo(AbstractParams copy) {
330340

331341
copy.setImportXslt(getImportXslt());
332342
copy.setBatchEdits(getBatchEdits());
343+
copy.setTranslateContent(isTranslateContent());
333344
copy.setValidate(getValidate());
345+
copy.setTranslateContent(isTranslateContent());
334346

335347
for (Privileges p : alPrivileges) {
336348
copy.addPrivilege(p.copy());
@@ -643,4 +655,28 @@ public String getBatchEdits() {
643655
public void setBatchEdits(String batchEdits) {
644656
this.batchEdits = batchEdits;
645657
}
658+
659+
public boolean isTranslateContent() {
660+
return translateContent;
661+
}
662+
663+
public void setTranslateContent(boolean translateContent) {
664+
this.translateContent = translateContent;
665+
}
666+
667+
public String getTranslateContentLangs() {
668+
return translateContentLangs;
669+
}
670+
671+
public void setTranslateContentLangs(String translateContentLangs) {
672+
this.translateContentLangs = translateContentLangs;
673+
}
674+
675+
public String getTranslateContentFields() {
676+
return translateContentFields;
677+
}
678+
679+
public void setTranslateContentFields(String translateContentFields) {
680+
this.translateContentFields = translateContentFields;
681+
}
646682
}

‎harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java

+5
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,11 @@ private void addMetadata(RecordInfo ri, String uuidToAssign) throws Exception {
326326

327327
}
328328

329+
// Translate metadata
330+
if (params.isTranslateContent()) {
331+
md = translateMetadataContent(context, md, schema);
332+
}
333+
329334
//
330335
// insert metadata
331336
//

‎harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java

+10
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,11 @@ private String addMetadata(RecordInfo ri, Element md, Element info, boolean loca
463463

464464
if (log.isDebugEnabled()) log.debug(" - Adding metadata with remote uuid:" + ri.uuid);
465465

466+
// Translate metadata
467+
if (params.isTranslateContent()) {
468+
md = translateMetadataContent(context, md, schema);
469+
}
470+
466471
try {
467472
Integer groupIdVal = null;
468473
if (StringUtils.isNotEmpty(params.getOwnerIdGroup())) {
@@ -744,6 +749,11 @@ private void updateMetadata(RecordInfo ri, String id, Element md,
744749
String date = localUuids.getChangeDate(ri.uuid);
745750

746751

752+
// Translate metadata
753+
if (params.isTranslateContent()) {
754+
md = translateMetadataContent(context, md, ri.schema);
755+
}
756+
747757
try {
748758
Integer groupIdVal = null;
749759
if (StringUtils.isNotEmpty(params.getOwnerIdGroup())) {

0 commit comments

Comments
 (0)