Avoiding importing multiple entries with same contents

Author: fbanados

lib/import/AECD.js

@@ -164,11 +164,15 @@ export default async function importAECD(aecdPath, dbPath, fstPath, { report } =
 
   }
 
-  for (const aecdEntry of unmatched) {
-    if (aecdEntry.senses.length > 0) {
-      dbIndex.add(new DatabaseEntry(aecdEntry))
+  const candidates = Map.groupBy(unmatched, entry => entry.head.ae);
+
+  for (const aecdEntry of candidates.values()) {
+    const possibleEntries = aecdEntry.filter(entry => entry.senses.length > 0)
+    if (possibleEntries.length > 0) {
+      dbIndex.add(new DatabaseEntry(possibleEntries[0]));
+      entriesSkipped+=(aecdEntry.length-1);
     } else {
-      entriesSkipped++;
+      entriesSkipped+=aecdEntry.length;
     }
   }
 
@@ -185,9 +189,9 @@ export default async function importAECD(aecdPath, dbPath, fstPath, { report } =
   console.info(`\n`);
   console.table({
     'Entries updated:':         entriesUpdated,
-    'Entries without a match (new):': unmatched.length,
-    'Entries skipped (match without senses):': entriesSkipped,
-    'Entries without a standardized lemma:': unstandardized.length,
+    'Entries without a match (new):': unmatched.length-entriesSkipped,
+    'Entries skipped:': entriesSkipped,
+    'Entries without a standardized lemma:': unstandardized.length
   });
 
   if (report) {