Added validation process for #126

Author: fbanados

lib/convert/dlx2importjson.js

@@ -3,6 +3,7 @@ import createSpinner        from 'ora';
 import fs                   from 'fs-extra';
 import parseCategory        from '../utilities/parseCategory.js';
 import readNDJSON           from '../utilities/readNDJSON.js';
+import writeNDJSON           from '../utilities/writeNDJSON.js';
 import removeParentheticals from '../utilities/removeParentheticals.js';
 import sortKeys             from 'sort-keys';
 
@@ -107,7 +108,7 @@ function convertEntry({
     wn_domains,
   };
 
-  if(dataSources.CW.htmlInfo) {
+  if(dataSources.CW?.htmlInfo) {
     linguistInfo.analysis = dataSources.CW.htmlInfo;
   }
 
@@ -279,6 +280,65 @@ export default async function dlx2importjson(inputPath, outputPath = `out.json`)
 
   }
 
+  await validateImportJson(entries, createSpinner('Verifying correct structure of importjson file.').start());
+  
+
   return entries;
 
 }
+
+function getDuplicateEntries( list ) {
+  const counts = new Map();
+  list.forEach((entry, index) => {
+    if (counts.has(entry)) {
+      counts.get(entry).push(index)
+    } else {
+      counts.set(entry, [index])
+    }
+  });
+  return counts.entries().filter(entry => entry[1].length > 1)
+}
+
+/**
+ * Verify that the entries generated in the importjson are consistent with the constraints expected by morphodict.
+ */
+async function validateImportJson ( entries, spinner) {
+  /** in "lexicon_sourcelanguagekeyword" CONSTRAINT "source_kw_text_and_wordform" UNIQUE ("text", "wordform_id")); */
+  /** in  "lexicon_targetlanguagekeyword" CONSTRAINT "target_kw_text_and_wordform" UNIQUE ("text", "wordform_id")) */
+  /** slugs must be unique */
+
+  const duplicateSlugs = Array.from(getDuplicateEntries(entries.map(entry => entry.slug).filter(entry=>entry)));
+
+  if (duplicateSlugs.length > 0 ) {
+    console.log(`ERROR: MULTIPLE ENTRIES WITH SAME KEY PRESENT\n${duplicateSlugs.toString()}`)  
+    return spinner.fail('Importjson will not import correctly into morphodict.')
+  }
+
+  const groupByWordform = Array.from(getDuplicateEntries(
+    entries.map(entry => entry.slug || (entry.formOf ? entry.head+' of '+entry.formOf : entry.head))
+  )).map(entry => [entry[0], entry[1].map(i => entries[i])]);
+
+  function targetLanguageKeywordsNotUnique ( entries ) {
+    return Array.from(getDuplicateEntries(
+      entries.map(entry => entry.senses.map(sense => (sense.semanticDefinition || sense.definition).split(' ')).flat()).flat()
+    )).length > 1;
+  }
+
+  const guaranteedToFailConstraints = groupByWordform
+    .filter(entry => targetLanguageKeywordsNotUnique(entry[1]));
+
+  await writeNDJSON(`data/brittle-entries.json`, groupByWordform);
+  if (guaranteedToFailConstraints.length > 0 ){
+    console.log(`\n\nERROR: SOME ENTRIES HAVE DEFINITIONS WHOSE TargetLanguageKeyword ENTRIES ARE NOT UNIQUE!\n${guaranteedToFailConstraints.toString()}`);
+    spinner.fail('Importjson will not import correctly into morphodict')
+    throw new Error();
+  } else {
+    console.log([
+      'Warning:  There are brittle entries in this importjson where small changes in definitions may break the import process',
+      'Please see the documentation and the data/brittle_entries.json file'
+    ].join("."));
+  }
+
+  return spinner.succeed('Importjson should work with morphodict.')
+
+}