compute different analysis for fitness function

Author: mangpo

Diff for: GA/GA-stochastic.rkt

@@ -88,8 +88,12 @@
           (set! a (+ a (arithmetic-shift a -16)))
           (bitwise-and a #x3f))
 
-        (pop-count (bitwise-xor (bitwise-and x #x3ffff) 
-                                (bitwise-and y #x3ffff))))
+        ;; (pop-count (bitwise-xor (bitwise-and x #x3ffff) 
+        ;;                         (bitwise-and y #x3ffff))))
+
+        (/
+         (log (add1 (abs (- (bitwise-and x #x3ffff) (bitwise-and y #x3ffff)))))
+         (log 2)))
 
       (define-syntax-rule (accum x) (set! correctness (+ correctness x)))
 

Diff for: GA/main.rkt

@@ -48,8 +48,15 @@
   (generate-outputs-steps (send printer encode code) dir subdir
                           machine printer simulator backward-stochastic))
 
-#|(define (GA-calculate-cost dir name live-out)
-  (calculate-cost dir name live-out machine backward-stochastic))|#
+(define (GA-calculate-cost dir name live-out)
+  (calculate-cost dir name live-out machine backward-stochastic))
+
+(define (GA-calculate-cost2 dir name live-out n)
+  (calculate-cost2 dir name live-out n machine backward-stochastic))
+
+(define (GA-generate-tree code dir subdir degree n)
+  (generate-tree (send printer encode code) dir subdir degree n
+                 machine printer simulator backward-stochastic))
 
 #|
 (GA-generate-inputs 
@@ -61,12 +68,15 @@
 ;; (GA-generate-outputs-steps 
 ;;  (send (new GA-parser%) ast-from-string 
 ;;        "2 b! @b 3 b! !b 1 b! @b 2 b! !b")
-;;  "data-ex" "prog")
+;;  "data-ex" "n100-s3")
 
-#|
-(GA-calculate-cost "data-ex/spec-n100-1" "v1" '((data . 2) memory))
-(GA-calculate-cost "data-ex/spec-n100-2" "v1" '((data . 2) memory))
-(GA-calculate-cost "data-ex/spec-n100-3" "v1" '((data . 2) memory))|#
 
+;(GA-calculate-cost "data-ex/n100-s3" "v1" '((data . 2) memory))
+
+#|
+(GA-generate-tree 
+ (send (new GA-parser%) ast-from-string 
+       "2 b! @b 3 b! !b 1 b! @b 2 b! !b")
+ "data-ex" "tree-n5000" 4 5000)|#
 
-  
+(GA-calculate-cost2 "data-ex/tree-n5000" "v3" '((data . 2) memory) 5005)

Diff for: GA/run.rkt

@@ -11,14 +11,15 @@
 ;;           #:assume '((<= . 65535) (<= . 65535) (<= . 65535)))
 
 ;; mem
-;; (optimize (send parser ast-from-string "2 b! @b 3 b! !b 1 b! @b 2 b! !b")
-;;           '((data . 2) memory)
-;;           #t #t 0 #:cores 8 #:time-limit 3600 #:size 8)
 (optimize (send parser ast-from-string "2 b! @b 3 b! !b 1 b! @b 2 b! !b")
           '((data . 2) memory)
-          #t #t 0 #:cores 2 #:time-limit 3600 #:size 8
-          #:input-file "data-ex/inputs" 
-          #:start-prog "2 b! @b 3 b! !b 1 !b @b 2 b! b!")
+          #t #t 0 #:cores 4 #:time-limit 60 #:size 8
+          #:input-file "data-ex/inputs")
+;; (optimize (send parser ast-from-string "2 b! @b 3 b! !b 1 b! @b 2 b! !b")
+;;           '((data . 2) memory)
+;;           #t #t 0 #:cores 2 #:time-limit 3600 #:size 8
+;;           #:input-file "data-ex/inputs" 
+;;           #:start-prog "2 b! @b 3 b! !b 1 !b @b 2 b! b!")
 
 ;; has zero byte?
 ;; (optimize (send parser ast-from-string 

Diff for: fitness-learner.rkt

@@ -1,10 +1,11 @@
 #lang racket
 
 (require "ast.rkt")
-(provide generate-inputs generate-outputs-steps)
+(provide generate-inputs generate-outputs-steps calculate-cost
+         generate-tree calculate-cost2)
 
-(define nsteps 10)
-(define repeat 10)
+(define nsteps 3)
+(define repeat 100)
 
 (define (get-states-from-file machine file)
   (send machine get-states-from-file file))
@@ -27,65 +28,6 @@
        (send machine display-state-text (cons #t input))))))
 
 
-;; (define (generate-outputs-steps code live-out dir subdir
-;;                                 machine printer simulator stochastic)
-;;   (define (interpret code input)
-;;     (with-handlers*
-;;      ([exn:state? (lambda (e)
-;;                     (cons #f (exn:state-state e)))])
-;;      (cons #t (send simulator interpret code input))))
-
-;;   (define inputs (get-states-from-file machine (format "~a/inputs" dir)))
-;;   (define correct-outputs (map (lambda (x) (interpret code (cdr x))) inputs))
-;;   (define constraint (send machine output-constraint live-out))
-;;   (system (format "rm -r ~a/~a" dir subdir))
-;;   (system (format "mkdir ~a/~a" dir subdir))
-
-;;   (define (compare states)
-;;     (min
-;;      10000
-;;      (for/sum ([state1 correct-outputs]
-;;                [state2 states])
-;;               (if (car state2)
-;;                   (send stochastic correctness-cost (cdr state1) (cdr state2) constraint)
-;;                   10000))))
-  
-;;   (define (print-state-step code outputs cost step count round)
-;;     (when (>= step 0)
-;;     (with-output-to-file #:exists 'append (format "~a/~a/cost_~a" dir subdir count)
-;;       (thunk (pretty-display (format "~a,~a,~a" step cost round)))))
-;;     (with-output-to-file #:exists 'append (format "~a/~a/program_~a" dir subdir count)
-;;       (thunk (send printer print-syntax (send printer decode code))
-;;              (newline)))
-;;     (with-output-to-file (format "~a/~a/outputs_~a_~a" dir subdir step count)
-;;       (thunk
-;;        (for ([output outputs])
-;;          (send machine display-state-text output)))))
-
-;;   (define (get-new-code code cost [round 0])
-;;     (define new-code (send stochastic mutate code))
-;;     (define outputs (map (lambda (x) (interpret new-code (cdr x))) inputs))
-;;     (define new-cost (compare outputs))
-;;     (pretty-display `(get-new-code ,new-cost ,(andmap car outputs)))
-;;     (send printer print-syntax (send printer decode new-code)) (newline)
-;;     (if (or (and (>= new-cost cost) (andmap car outputs)) (> round 10))
-;;         (values new-code outputs new-cost round)
-;;         (get-new-code code cost (add1 round))))
-    
-;;   (define (iter code cost step count)
-;;     (pretty-display `(iter ,count ,step))
-;;     (define-values (new-code outputs new-cost round) (get-new-code code cost))
-;;     (print-state-step new-code outputs new-cost step count round)
-;;     (when (< step nsteps)
-;;           (iter new-code new-cost (add1 step) count)))
-
-;;   (send printer print-syntax (send printer decode code)) (newline)
-;;   (pretty-display ">>> Phase 2: generate output states")
-;;   (print-state-step code correct-outputs 0 -1 0 0)
-;;   (for ([i (in-range repeat)])
-;;        (iter code 0 1 i)))
-
-
 (define (generate-outputs-steps code dir subdir
                                 machine printer simulator stochastic)
   (define (interpret code input)
@@ -113,9 +55,9 @@
     (define outputs (map (lambda (x) (interpret new-code (cdr x))) inputs))
     ;; (pretty-display `(get-new-code ,(andmap car outputs)))
     ;; (send printer print-syntax (send printer decode new-code)) (newline)
-    (if (or (not (send machine syntax-equal? code new-code)) 
-            (> round 10))
-         ;;(or (andmap car outputs) (> round 10))
+    (if ;;(or (not (send machine syntax-equal? code new-code)) 
+        ;;    (> round 10))
+        (or (andmap car outputs) (> round 10))
         (values new-code outputs)
         (get-new-code code (add1 round))))
 
@@ -132,29 +74,101 @@
   (for ([i (in-range 1 (add1 repeat))])
        (iter code 1 i)))
 
-;; (define (calculate-cost dir name live-out
-;;                         machine stochastic)
+(define (calculate-cost dir name live-out
+                        machine stochastic)
+
+  (define constraint (send machine output-constraint live-out))
+  (define ref-states (get-states-from-file machine (format "~a/outputs_~a_~a" dir 0 0)))
+  
+  (define (compare states)
+    (min
+     10000
+     (for/sum ([state1 ref-states]
+               [state2 states])
+              (if (car state2)
+                  (send stochastic correctness-cost (cdr state1) (cdr state2) constraint)
+                  10000))))
+
+  (system (format "mkdir ~a/~a" dir name))
+  (for ([count (in-range 1 (add1 repeat))])
+       (with-output-to-file (format "~a/~a/cost-~a.csv" dir name count)
+         (thunk
+          (for* ([step (range 1 (add1 nsteps))])
+                (let ([states (get-states-from-file 
+                               machine 
+                               (format "~a/outputs_~a_~a" dir step count))])
+                  (pretty-display (format "~a,~a" step (compare states)))))))))
+
+(define (calculate-cost2 dir name live-out n
+                         machine stochastic)
+
+  (define constraint (send machine output-constraint live-out))
+  (define ref-states (get-states-from-file machine (format "~a/outputs_0" dir)))
+  
+  (define (compare states)
+    (min
+     10000
+     (for/sum ([state1 ref-states]
+               [state2 states])
+              (if (car state2)
+                  (send stochastic correctness-cost (cdr state1) (cdr state2) constraint)
+                  10000))))
+
+  (with-output-to-file (format "~a/costs-~a" dir name)
+    (thunk
+     (for* ([id n])
+           (let ([states (get-states-from-file  machine (format "~a/outputs_~a" dir id))])
+             (pretty-display (compare states)))))))
+  
+
+(define (generate-tree code dir subdir degree n
+                       machine printer simulator stochastic)
+  (define (interpret code input)
+    (with-handlers*
+     ([exn:state? (lambda (e)
+                    (cons #f (exn:state-state e)))])
+     (cons #t (send simulator interpret code input))))
 
-;;   (define constraint (send machine output-constraint live-out))
-;;   (define ref-states (get-states-from-file machine (format "~a/outputs_~a_~a" dir 0 0)))
+  (system (format "rm -r ~a/~a" dir subdir))
+  (system (format "mkdir ~a/~a" dir subdir))
+  (define inputs (get-states-from-file machine (format "~a/inputs" dir)))
+  (define correct-outputs (map (lambda (x) (interpret code (cdr x))) inputs))
 
-;;   (define (compare states)
-;;     (min
-;;      10000
-;;      (for/sum ([state1 ref-states]
-;;                [state2 states])
-;;               (if (car state2)
-;;                   (send stochastic correctness-cost (cdr state1) (cdr state2) constraint)
-;;                   10000))))
-
-;;   (system (format "mkdir ~a/~a" dir name))
-;;   (for ([count (in-range 1 (add1 repeat))])
-;;        (with-output-to-file (format "~a/~a/cost-~a.csv" dir name count)
-;;          (thunk
-;;           (for* ([step (range 1 (add1 nsteps))])
-;;                 (let ([states (get-states-from-file 
-;;                                machine 
-;;                                (format "~a/outputs_~a_~a" dir step count))])
-;;                   (pretty-display (format "~a,~a" step (compare states)))))))))
+  (define (print-state code outputs id)
+    (with-output-to-file #:exists 'append (format "~a/~a/programs" dir subdir)
+      (thunk (send printer print-syntax (send printer decode code))
+             (newline)))
+    (with-output-to-file (format "~a/~a/outputs_~a" dir subdir id)
+      (thunk
+       (for ([output outputs])
+         (send machine display-state-text output)))))
+
+  (define (get-new-code code id [round 0])
+    (define new-code (send stochastic mutate code))
+    (define outputs (map (lambda (x) (interpret new-code (cdr x))) inputs))
+    ;; (pretty-display `(get-new-code ,(andmap car outputs)))
+    ;; (send printer print-syntax (send printer decode new-code)) (newline)
+    (if ;;(or (not (send machine syntax-equal? code new-code)) 
+        ;;    (> round 10))
+        (or (andmap car outputs) (> round 10))
+        (begin
+          (print-state new-code outputs id)
+          new-code)
+        (get-new-code code id (add1 round))))
 
+  (define (loop code-list id-list count)
+    (define my-code (car code-list))
+    (define my-id (car id-list))
+    (define new-codes (for/list ([i degree]) (get-new-code my-code (+ count i))))
+    (define new-ids (for/list ([i degree]) (+ count i)))
+    (with-output-to-file #:exists 'append (format "~a/~a/tree" dir subdir)
+      (thunk 
+       (for ([i degree]) (pretty-display (format "~a ~a" my-id (+ count i))))))
+    (when (< count n)
+          (loop (append (cdr code-list) new-codes)
+                (append (cdr id-list) new-ids)
+                (+ count degree))))
+
+  (print-state code correct-outputs 0)
+  (loop (list code) (list 0) 1))
 

Diff for: stat.rkt

@@ -36,7 +36,10 @@
 
     (super-new)
 
-    (define/public (inc-iter)
+    (define/public (inc-iter current-cost)
+      (with-output-to-file #:exists 'append (format "~a.csv" name)
+        (thunk
+         (pretty-display (format "~a,~a" iter-count current-cost))))
       (set! iter-count (add1 iter-count))
       (when (= (modulo iter-count 1000) 0)
             (print-stat-to-file)
@@ -75,6 +78,9 @@
          ;; (pretty-display (format "best-correct-time: ~a" best-correct-time))
          (send printer print-syntax (send printer decode program))))
       )
+    
+    (define/public (update-best-correct-program program)
+      (set! best-correct-program program))
 
     (define/public (update-best-correct program cost)
       (set! best-correct-program program)

Diff for: stat/correlation.py

@@ -0,0 +1,40 @@
+from scipy.stats import spearmanr, kendalltau
+import numpy
+import commands
+
+def correlation(dirname):
+  status, files = commands.getstatusoutput('ls ' + dirname)
+  files = files.split('\n')
+  steps = []
+  costs = []
+  count = 0
+  correlation = []
+  for filename in files:
+    f = open(dirname + '/' + filename,'r')
+    last_cost = 0
+    my_costs = []
+    my_steps = []
+    for line in f:
+      tokens = line.split(',')
+      step = int(tokens[0])
+      cost = int(tokens[1])
+      if step < 5:
+        steps.append(step)
+        costs.append(cost)
+        my_steps.append(step)
+        my_costs.append(cost)
+    f.close()
+    #corr, p = spearmanr(my_steps,my_costs)
+    #correlation.append(corr)
+    if my_costs[0] <= my_costs[1] and my_costs[1] <= my_costs[2]:
+      count = count + 1
+  
+  #print dirname, spearmanr(steps,costs)
+  print dirname, count
+  #print numpy.median(correlation), numpy.percentile(correlation, 25), numpy.mean(correlation)
+  #print filename, kendalltau(steps,costs)
+
+
+#print spearmanr([1,2,3,4,5],[5,6,7,8,7])
+correlation("../GA/data-ex/n100-s3/v1")
+correlation("../GA/data-ex/n100-s3/v2")