zero rollout buffer

Author: jsuarez5341

cache_data.py

@@ -12,7 +12,7 @@
     #'tetris',
     #'g2048',
     #'moba',
-    #'pong',
+    'pong',
     #'tower_climb',
     #'grid',
     #'nmmo3',
@@ -43,12 +43,14 @@
     'vec/total_agents',
 ]
 
-ALL_KEYS = [
+METRICS = [
     'agent_steps',
-    'cost',
+    'uptime',
     'environment/score',
     'environment/perf'
-] + HYPERS
+]
+
+ALL_KEYS = METRICS + HYPERS
 
 def pareto_idx(steps, costs, scores):
     idxs = []
@@ -63,53 +65,46 @@ def pareto_idx(steps, costs, scores):
 
 def load_sweep_data(path):
     data = {}
-    keys = None
+    sweep_metadata = {}
     for fpath in glob.glob(path):
         if 'cache.json' in fpath:
             continue
 
         with open(fpath, 'r') as f:
             exp = json.load(f)
 
-        if not data:
-            for kk in exp.keys():
-                if kk == 'data':
-                    for k, v in exp[kk][-1].items():
-                        data[k] = []
-                else:
-                    data[kk] = []
-
-        discard = False
-        for kk in list(data.keys()):
-            if kk not in exp and kk not in exp['data'][-1]:
-                discard = True
-                break
-
-        if discard:
-            continue
+        if not sweep_metadata:
+            for k, v in exp.items():
+                if k.startswith('sweep/'):
+                    sweep_metadata[k.replace('sweep/', '')] = v
+
+        for k, v in exp.items():
+            if k.startswith('sweep/'):
+                continue
+
+            if isinstance(v, dict):
+                continue
+
+            if k not in data:
+                data[k] = []
+
+            data[k].append(v)
 
-        for kk in list(data.keys()):
-            if kk in exp:
-                v = exp[kk]
-                sweep_key = f'sweep/{kk}/distribution'
-                if sweep_key in data and exp[sweep_key] == 'logit_normal':
-                    v = 1 - v
-                elif kk in ('train/vtrace_rho_clip', 'train/vtrace_c_clip'):
-                    v = max(v, 0.1)
+        for k in METRICS:
+            if k not in data:
+                data[k] = []
 
-                data[kk].append(v)
-            else:
-                data[kk].append(exp['data'][-1][kk])
+            data[k].append(exp['data'][-1][k])
 
     steps = data['agent_steps']
-    costs = data['cost']
+    costs = data['uptime']
     scores = data['environment/score']
 
     idxs = pareto_idx(steps, costs, scores)
 
     # Filter to pareto
-    for k in data:
-        data[k] = [data[k][i] for i in idxs]
+    #for k in data:
+    #    data[k] = [data[k][i] for i in idxs]
 
     # Monkey patch: Cap performance
     data['environment/perf'] = [min(e, 1.0) for e in data['environment/perf']]
@@ -119,6 +114,7 @@ def load_sweep_data(path):
         skip = data['env/frameskip']
         data['agent_steps'] = [n*m for n, m in zip(data['agent_steps'], skip)]
 
+    data['sweep'] = sweep_metadata
     return data
 
 def cached_sweep_load(path, env_name):
@@ -136,20 +132,21 @@ def cached_sweep_load(path, env_name):
 
 def compute_tsne():
     data = {name: cached_sweep_load(f'experiments/logs/puffer_{name}', name) for name in env_names}
+    sweep_metadata = {name: data[name].pop('sweep') for name in env_names}
 
     flat = []
     flat_mmin = []
     flat_mmax = []
     for env in env_names:
         flat.append(np.stack([data[env][hyper] for hyper in HYPERS], axis=1))
-        flat_mmin.append(np.stack([data[env][f'sweep/{hyper}/min'] for hyper in HYPERS], axis=1))
-        flat_mmax.append(np.stack([data[env][f'sweep/{hyper}/max'] for hyper in HYPERS], axis=1))
+        flat_mmin.append(np.stack([sweep_metadata[env][f'{hyper}/min'] for hyper in HYPERS]))
+        flat_mmax.append(np.stack([sweep_metadata[env][f'{hyper}/max'] for hyper in HYPERS]))
 
-    flat_distribution = [data[env][f'sweep/{hyper}/distribution'] for env in env_names for hyper in HYPERS]
+    flat_distribution = [sweep_metadata[env][f'{hyper}/distribution'] for env in env_names for hyper in HYPERS]
 
     flat = np.concatenate(flat, axis=0)
-    flat_mmin = np.concatenate(flat_mmin, axis=0).min(axis=0)
-    flat_mmax = np.concatenate(flat_mmax, axis=0).max(axis=0)
+    flat_mmin = np.stack(flat_mmin, axis=0).min(axis=0)
+    flat_mmax = np.stack(flat_mmax, axis=0).max(axis=0)
 
     normed = flat.copy()
     for i in range(len(HYPERS)):
@@ -178,7 +175,7 @@ def compute_tsne():
         '''
         sz = len(data[env]['agent_steps'])
 
-        data[env] = {k: v for k, v in data[env].items() if k in ALL_KEYS}
+        #data[env] = {k: v for k, v in data[env].items() if k in ALL_KEYS}
         if reduced is not None:
             data[env]['tsne1'] = reduced[row:row+sz, 0].tolist()
             data[env]['tsne2'] = reduced[row:row+sz, 1].tolist()

pufferlib/config/ocean/grid.ini

@@ -17,39 +17,37 @@ num_maps = 8192
 map_size = -1
 
 [train]
-# Best params
-#total_timesteps = 435_000_000
-#adam_beta1 = 0.9801350114303844
-#adam_beta2 = 0.9931056135397744
-#adam_eps = 6.024885743259763e-8
-#clip_coef = 0.283658795325587
-#ent_coef = 0.007885530106105381
-#gae_lambda = 0.9574676436577135
-#gamma = 0.9961782334639131
-#learning_rate = 0.0007890771333884192
-#max_grad_norm = 2.5271346931510053
-#minibatch_size = 8192
-#prio_alpha = 0.8735470630752789
-#prio_beta0 = 0.6533958384978629
-#vf_clip_coef = 1.9338563232919095
-#vf_coef = 3.915248046963283
-#vtrace_c_clip = 1.018588814067991
-#vtrace_rho_clip = 2.4215244529216466
+#beta1 = 0.9581470335001424
+#beta2 = 0.9939188445458832
+#clip_coef = 0.4056152781082447
+#ent_coef = 0.0005788152954138715
+#eps = 0.00000000017742923522
+#gae_lambda = 0.9701506766299144
+#gamma = 0.986452277080654
+#learning_rate = 0.0029890509569117014
+#max_grad_norm = 3.235365412303803
+#minibatch_size = 32768
+#prio_alpha = 0.7936109272717734
+#prio_beta0 = 0.95653936979398
+#replay_ratio = 2.168572638529935
+#total_timesteps = 195748615
+#vf_clip_coef = 1.6903254428390564
+#vf_coef = 0.8650810043551426
+#vtrace_c_clip = 1.3763043127146155
+#vtrace_rho_clip = 4.674513875485618
 
-# New sweep best params
 total_timesteps = 435_000_000
 beta1 = 0.9493079570168755
 beta2 = 0.9998213228757207
 eps = 2.16720639574209e-8
-horizon = 64
+bptt_horizon = 64
 clip_coef = 0.399530686596841
 ent_coef = 0.0017271288609381147
 gae_lambda = 0.9491722822649111
 gamma = 0.9877360824574745
 learning_rate = 0.0012892859713461897
 max_grad_norm = 3.016348031602564
-#minibatch_size = 8192
-minibatch_size = 32768
+minibatch_size = 8192
 prio_alpha = 0.8219794821639037
 prio_beta0 = 0.9447478232810274
 vf_clip_coef = 0.6051579400844748

pufferlib/config/ocean/pong.ini

@@ -43,10 +43,6 @@ vf_coef = 1.6832989594296321
 vtrace_c_clip = 2.878171091654008
 vtrace_rho_clip = 0.7876748061547312
 
-[sweep]
-downsample = 5
-max_cost = 10
-
 [sweep.train.total_timesteps]
 distribution = log_normal
 min = 5e6