add some stats

Author: jheinecke

src/main/java/com/orange/labs/comparison/Analyser.java

@@ -34,7 +34,6 @@
 
 import static java.lang.Math.abs;
 import java.util.ArrayList;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -46,7 +45,7 @@ public class Analyser implements Runnable {
     private int modulo; // current thread number
     private int totalthreads; // number of threads used (here we process one out of totalthread
     private List<String> keys;
-    private int len;
+    private int len; // number of sentences
     private int form;
     private int lemma;
     private int upos;
@@ -177,7 +176,9 @@ public void run() {
                 }
             }
         }
-        if (modulo == 0) System.err.println();
+        if (modulo == 0) {
+            System.err.format("Checked %d      \n", len);
+        }
     }
 
     // inspired by https://github.com/crwohlfeil/damerau-levenshtein
@@ -228,16 +229,17 @@ private int calculateDistance(List<? extends Object> source, List<? extends Obje
     public List<String []> getResults() {
         return results;
     }
-
+    
+    
     private void identical(String column, ConlluComparator.Signatures s1, ConlluComparator.Signatures s2) {
         if (aggregate) {
             //results.add(String.format("%s\t0\t%s\t%s\t%s", column, s1.id, s2.id, s1.sent));
             String [] e = { column, "0", s1.id, s2.id, s1.sent};
             results.add(e);
         } else {
-        System.out.format("%s identical\t%s\t%s\n", column, s1.id, s2.id);
-        System.out.format("# %s\n", s1.sent);
-        printColumn(column, s1);
+            System.out.format("%s identical\t%s\t%s\n", column, s1.id, s2.id);
+            System.out.format("# %s\n", s1.sent);
+            printColumn(column, s1);
         }
     }
 

src/main/java/com/orange/labs/comparison/ConlluComparator.java

@@ -40,6 +40,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.TreeMap;
 import java.util.List;
 import java.util.Map;
 import java.util.LinkedHashMap;
@@ -58,6 +59,7 @@ public class ConlluComparator {
 
     private Map<String, Signatures> csents; // unique id (filename#id#number): sentence
     private int numberOfThreads;
+    Map<Integer, Integer>sentencelengths;  // sentencelength: number of sentences
 
     public ConlluComparator(List<? extends Object> objects, int numberOfThreads) throws ConllException, IOException {
         //public ConlluComparator(List<String> objects) throws ConllException, IOException {
@@ -79,15 +81,25 @@ public ConlluComparator(List<? extends Object> objects, int numberOfThreads) thr
 
         this.numberOfThreads = numberOfThreads;
         csents = new LinkedHashMap<>();
+        sentencelengths = new TreeMap<>(); // sentencelength: number of sentences
         int ct = 0;
         for (ConllFile cf : cdocs) {
             for (ConllSentence csent : cf.getSentences()) {
                 ct += 1;
                 //String id = String.format("%s#%s#%d", cf.getFile(), csent.getSentid(), ct);
                 String id = String.format("%s#%s", cf.getFile(), csent.getSentid());
                 csents.put(id, new Signatures(csent, id));
+                int tokens = csent.getAllWords().size();
+                Integer occ = sentencelengths.get(tokens);
+                if (occ == null) {
+                    sentencelengths.put(tokens, 1);
+                } else {
+                     sentencelengths.put(tokens, occ+1);
+                }
+                
             }
         }
+        
     }
 
     /**
@@ -120,6 +132,8 @@ public void analyse(int form, int lemma, int upos, int xpos, int feats, int depr
         Map<String, Set<String>>identical = new LinkedHashMap<>(); // sentence: [ids]
         List<String []>similar = new ArrayList<>();
         // aggregate identical
+
+
         for(Analyser a : analysers) {
             for (String [] elems : a.getResults()) {
                 if (elems[0].equals("FORM") && elems[1].equals("0")) {
@@ -135,6 +149,22 @@ public void analyse(int form, int lemma, int upos, int xpos, int feats, int depr
                 }
             }
         }
+
+
+
+        System.out.println("# sentence lenghts");
+        for (int slen : sentencelengths.keySet()) {
+            System.out.format("#  %3d tokens: %4d sentences\n", slen, sentencelengths.get(slen));
+        }
+
+        if (form >= 0) {
+            int identical_form = 0;
+            for (String sentence : identical.keySet()) {
+                identical_form += identical.get(sentence).size();
+            }
+            System.out.format("# identical sentences (Form) %d/%d  %.1f%%\n", identical_form, keys.size(), (100.0*identical_form/keys.size()));
+        }
+
         // output identical sentences
         for (String sentence : identical.keySet()) {
             System.out.format("FORM\t0\t%s\t%d\t%s\n", sentence, identical.get(sentence).size(), String.join("\t", identical.get(sentence)));