more readable output

Author: jheinecke

src/main/java/com/orange/labs/comparison/Analyser.java

@@ -32,6 +32,8 @@
  */
 package com.orange.labs.comparison;
 
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -51,6 +53,9 @@ public class Analyser implements Runnable {
     private int feats;
     private int deprel;
     Map<String, ConlluComparator.Signatures> csents; // unique id (filename#id#number): sentence
+    //Map<String, List<ConlluComparator.Result>> results; // store similar sentences: {id1: [(id2,dit)]}
+    List<String []> results; // store similar sentences: "column <TAB> dist <TAB> id1 <TAB> id2"
+    private boolean aggregate; // collect results and return
 
     public Analyser(int modulo, int totalthreads, List<String> keys, Map<String, ConlluComparator.Signatures> csents,
             int form, int lemma, int upos, int xpos, int feats, int deprel) {
@@ -66,13 +71,18 @@ public Analyser(int modulo, int totalthreads, List<String> keys, Map<String, Con
         this.xpos = xpos;
         this.feats = feats;
         this.deprel = deprel;
-
+        aggregate = true;
+        if (aggregate) {
+            results = new ArrayList<>();
+        }
     }
 
     @Override
     public void run() {
         for (int i = 0; i < len; ++i) {
-            //System.err.println(modulo + ": Checking " + i);
+            if (modulo == 0 && i % 7 == 0) {
+                System.err.format("Checking %d/%d\r", i, len);
+            }
             ConlluComparator.Signatures cursent = csents.get(keys.get(i));
             for (int j = i + 1; j < len; ++j) {
                 if (j % totalthreads != modulo) {
@@ -90,7 +100,7 @@ public void run() {
                     int dist = calculateDistance(cursent.sentence, othersent.sentence);
                     if (dist == 0) {
                         identical("FORM", cursent, othersent);
-                    } else if (dist <= form) {
+                    } else  if (dist <= form) {
                         similar("FORM", dist, cursent, othersent);
                     }
                 }
@@ -166,6 +176,7 @@ public void run() {
                 }
             }
         }
+        if (modulo == 0) System.err.println();
     }
 
     // inspired by https://github.com/crwohlfeil/damerau-levenshtein
@@ -209,19 +220,34 @@ private int calculateDistance(List<? extends Object> source, List<? extends Obje
         return dist[sourceLength][targetLength];
     }
 
+    public List<String []> getResults() {
+        return results;
+    }
 
     private void identical(String column, ConlluComparator.Signatures s1, ConlluComparator.Signatures s2) {
+        if (aggregate) {
+            //results.add(String.format("%s\t0\t%s\t%s\t%s", column, s1.id, s2.id, s1.sent));
+            String [] e = { column, "0", s1.id, s2.id, s1.sent};
+            results.add(e);
+        } else {
         System.out.format("%s identical\t%s\t%s\n", column, s1.id, s2.id);
         System.out.format("# %s\n", s1.sent);
         printColumn(column, s1);
+        }
     }
 
     private void similar(String column, int dist, ConlluComparator.Signatures s1, ConlluComparator.Signatures s2) {
-        System.out.format("%s similar %d\t%s\t%s\n", column, dist, s1.id, s2.id);
-        System.out.format("# %s\n", s1.sent);
-        printColumn(column, s1);
-        System.out.format("# %s\n", s2.sent);
-        printColumn(column, s2);
+        if (aggregate) {
+            //results.add(String.format("%s\t%d\t%s\t%s\t%s\t%s", column, dist, s1.id, s2.id, s1.sent, s2.sent));
+            String [] e = {column, ""+dist, s1.id, s2.id, s1.sent, s2.sent};
+            results.add(e);
+        } else {
+            System.out.format("%s similar %d\t%s\t%s\n", column, dist, s1.id, s2.id);
+            System.out.format("# %s\n", s1.sent);
+            printColumn(column, s1);
+            System.out.format("# %s\n", s2.sent);
+            printColumn(column, s2);
+        }
     }
 
 

src/main/java/com/orange/labs/comparison/ConlluComparator.java

@@ -43,6 +43,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Set;
 
 
 /**
@@ -81,7 +83,8 @@ public ConlluComparator(List<? extends Object> objects, int numberOfThreads) thr
         for (ConllFile cf : cdocs) {
             for (ConllSentence csent : cf.getSentences()) {
                 ct += 1;
-                String id = String.format("%s#%s#%d", cf.getFile(), csent.getSentid(), ct);
+                //String id = String.format("%s#%s#%d", cf.getFile(), csent.getSentid(), ct);
+                String id = String.format("%s#%s", cf.getFile(), csent.getSentid());
                 csents.put(id, new Signatures(csent, id));
             }
         }
@@ -100,20 +103,55 @@ public void analyse(int form, int lemma, int upos, int xpos, int feats, int depr
         for(int x = 1;  x< keys.size(); ++x) comps += x;
         System.err.println(comps + " comparisons needed");
         List<Thread> thrs = new ArrayList<>();
+        List<Analyser>analysers = new ArrayList<>();
 
         for (int th = 0; th < numberOfThreads; ++th) {
             Analyser a = new Analyser(th, numberOfThreads, keys, csents, form, lemma, upos, xpos, feats, deprel);
             Thread thr = new Thread(a);
             thr.start();
             thrs.add(thr);
+            analysers.add(a);
         }
 
         for(Thread thr : thrs) {
             thr.join();
         }
 
+        Map<String, Set<String>>identical = new LinkedHashMap<>(); // sentence: [ids]
+        List<String []>similar = new ArrayList<>();
+        // aggregate identical
+        for(Analyser a : analysers) {
+            for (String [] elems : a.getResults()) {
+                if (elems[0].equals("FORM") && elems[1].equals("0")) {
+                    Set<String>ids = identical.get(elems[4]);
+                    if (ids == null) {
+                        ids = new LinkedHashSet<>();
+                        identical.put(elems[4], ids);
+                    }
+                    ids.add(elems[2]);
+                    ids.add(elems[3]);
+                } else {
+                    similar.add(elems);
+                }
+            }
+        }
+        // output identical sentences
+        for (String sentence : identical.keySet()) {
+            System.out.format("FORM\t0\t%s\t%s\n", sentence, String.join("\t", identical.get(sentence)));
+        }
+        for (String [] sim: similar) {
+            System.out.println(String.join("\t", sim));
+        }
+    }
+    /** comparison results */
+    class Result {
+        public String id;
+        public int dist;
+        public String col;
     }
 
+
+    /** preprocess the sentences to speed up the comparison */
     class Signatures {
 
         public ConllSentence cs;
@@ -136,7 +174,7 @@ public Signatures(ConllSentence cs, String id) {
             xposs = new ArrayList<>();
             deprels = new ArrayList<>();
             feats = new ArrayList<>();
-            sent = cs.getSentence();
+            sent = cs.getSentence().strip();
             sentence = new ArrayList<>();
             for (char c : sent.toCharArray()) {
                 sentence.add(c);