accelerate comparison

Author: jheinecke

src/main/java/com/orange/labs/comparison/Analyser.java

@@ -32,6 +32,7 @@
  */
 package com.orange.labs.comparison;
 
+import static java.lang.Math.abs;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -97,7 +98,7 @@ public void run() {
                         identical("FORM", cursent, othersent);
                     }
                 } else if (form > 0) {
-                    int dist = calculateDistance(cursent.sentence, othersent.sentence);
+                    int dist = calculateDistance(cursent.sentence, othersent.sentence, form);
                     if (dist == 0) {
                         identical("FORM", cursent, othersent);
                     } else  if (dist <= form) {
@@ -111,7 +112,7 @@ public void run() {
                         identical("LEMMA", cursent, othersent);
                     }
                 } else if (lemma > 0) {
-                    int dist = calculateDistance(cursent.lemmas, othersent.lemmas);
+                    int dist = calculateDistance(cursent.lemmas, othersent.lemmas, lemma);
                     if (dist == 0) {
                         identical("LEMMA", cursent, othersent);
                     } else if (dist <= lemma) {
@@ -125,7 +126,7 @@ public void run() {
                         identical("UPOS", cursent, othersent);
                     }
                 } else if (upos > 0) {
-                    int dist = calculateDistance(cursent.uposs, othersent.uposs);
+                    int dist = calculateDistance(cursent.uposs, othersent.uposs, upos);
                     if (dist == 0) {
                         identical("UPOS", cursent, othersent);
                     } else if (dist <= upos) {
@@ -139,7 +140,7 @@ public void run() {
                         identical("XPOS", cursent, othersent);
                     }
                 } else if (xpos > 0) {
-                    int dist = calculateDistance(cursent.xposs, othersent.xposs);
+                    int dist = calculateDistance(cursent.xposs, othersent.xposs, xpos);
                     if (dist == 0) {
                         identical("XPOS", cursent, othersent);
                     } else if (dist <= xpos) {
@@ -153,7 +154,7 @@ public void run() {
                         identical("FEATS", cursent, othersent);
                     }
                 } else if (feats > 0) {
-                    int dist = calculateDistance(cursent.feats, othersent.feats);
+                    int dist = calculateDistance(cursent.feats, othersent.feats, feats);
                     if (dist == 0) {
                         identical("FEATS", cursent, othersent);
                     } else if (dist <= feats) {
@@ -167,7 +168,7 @@ public void run() {
                         identical("DEPREL", cursent, othersent);
                     }
                 } else if (deprel > 0) {
-                    int dist = calculateDistance(cursent.deprels, othersent.deprels);
+                    int dist = calculateDistance(cursent.deprels, othersent.deprels, deprel);
                     if (dist == 0) {
                         identical("DEPREL", cursent, othersent);
                     } else if (dist <= deprel) {
@@ -182,16 +183,20 @@ public void run() {
     // inspired by https://github.com/crwohlfeil/damerau-levenshtein
     /**
      * calculate the levenshtein-damerau distance between two lists of objects (characters or strings)
+     * levenstein_distance(a,b) >= |len(a) - len(b)|
      * @param source
      * @param target
      * @return
      */
-    private int calculateDistance(List<? extends Object> source, List<? extends Object> target) {
+    private int calculateDistance(List<? extends Object> source, List<? extends Object> target, int maxdist) {
         //if (source == null || target == null) {
         //    throw new IllegalArgumentException("Parameter must not be null");
         //}
         int sourceLength = source.size();
         int targetLength = target.size();
+        // if the length of the two sentences differs more than maxdist, we stop here
+        if (abs(sourceLength - targetLength) > maxdist) return abs(sourceLength - targetLength);
+
         if (sourceLength == 0) {
             return targetLength;
         }

src/main/java/com/orange/labs/comparison/ConlluComparator.java

@@ -99,7 +99,7 @@ public ConlluComparator(List<? extends Object> objects, int numberOfThreads) thr
      */
     public void analyse(int form, int lemma, int upos, int xpos, int feats, int deprel) throws InterruptedException {
         List<String> keys = Arrays.asList(csents.keySet().toArray(new String[0]));
-        int comps = 0;
+        long comps = 0;
         for(int x = 1;  x< keys.size(); ++x) comps += x;
         System.err.println(comps + " comparisons needed");
         List<Thread> thrs = new ArrayList<>();
@@ -137,7 +137,7 @@ public void analyse(int form, int lemma, int upos, int xpos, int feats, int depr
         }
         // output identical sentences
         for (String sentence : identical.keySet()) {
-            System.out.format("FORM\t0\t%s\t%s\n", sentence, String.join("\t", identical.get(sentence)));
+            System.out.format("FORM\t0\t%s\t%d\t%s\n", sentence, identical.get(sentence).size(), String.join("\t", identical.get(sentence)));
         }
         for (String [] sim: similar) {
             System.out.println(String.join("\t", sim));