Updated benchmark results (no seed in deap).

Author: Alessandro Lucantonio

bench/bench.py

@@ -2,11 +2,9 @@
 from deap import gp
 
 from alpine.gp import regressor as gps
-from alpine.data import Dataset
 from alpine.gp import util
 import numpy as np
 import ray
-import yaml
 
 import time
 
@@ -33,15 +31,15 @@ def check_nested_trig_fn(ind):
     return util.detect_nested_trigonometric_functions(str(ind))
 
 
-def eval_model(individual, D, consts=[]):
+def eval_model(individual, X, consts=[]):
     warnings.filterwarnings("ignore")
-    y_pred = individual(*D.X, consts)
+    y_pred = individual(*X, consts)
     return y_pred
 
 
-def compute_MSE(individual, D, consts=[]):
-    y_pred = eval_model(individual, D, consts)
-    MSE = np.mean((D.y - y_pred) ** 2)
+def compute_MSE(individual, X, y, consts=[]):
+    y_pred = eval_model(individual, X, consts)
+    MSE = np.mean((y - y_pred) ** 2)
 
     if np.isnan(MSE) or np.isinf(MSE):
         MSE = 1e8
@@ -66,7 +64,7 @@ def compile_individual_with_consts(tree, toolbox, special_term_name="a"):
     return individual, const_idx
 
 
-def eval_MSE_and_tune_constants(tree, toolbox, D):
+def eval_MSE_and_tune_constants(tree, toolbox, X, y):
     individual, num_consts = compile_individual_with_consts(tree, toolbox)
 
     if num_consts > 0:
@@ -75,8 +73,8 @@ def eval_MSE_and_tune_constants(tree, toolbox, D):
         # outside?
         def eval_MSE(consts):
             warnings.filterwarnings("ignore")
-            y_pred = individual(*D.X, consts)
-            total_err = np.mean((D.y - y_pred) ** 2)
+            y_pred = individual(*X, consts)
+            total_err = np.mean((y - y_pred) ** 2)
 
             return total_err
 
@@ -113,7 +111,7 @@ def get_bounds(self):
         if np.isinf(MSE) or np.isnan(MSE):
             MSE = 1e8
     else:
-        MSE = compute_MSE(individual, D)
+        MSE = compute_MSE(individual, X, y)
         consts = []
     return MSE, consts
 
@@ -133,31 +131,31 @@ def get_features_batch(
 
 
 @ray.remote(num_cpus=num_cpus)
-def predict(individuals_str_batch, toolbox, dataset, penalty, fitness_scale):
+def predict(individuals_str_batch, toolbox, X, penalty, fitness_scale):
 
     predictions = [None] * len(individuals_str_batch)
 
     for i, tree in enumerate(individuals_str_batch):
         callable, _ = compile_individual_with_consts(tree, toolbox)
-        predictions[i] = eval_model(callable, dataset, consts=tree.consts)
+        predictions[i] = eval_model(callable, X, consts=tree.consts)
 
     return predictions
 
 
 @ray.remote(num_cpus=num_cpus)
-def compute_MSEs(individuals_str_batch, toolbox, dataset, penalty, fitness_scale):
+def compute_MSEs(individuals_str_batch, toolbox, X, y, penalty, fitness_scale):
 
     total_errs = [None] * len(individuals_str_batch)
 
     for i, tree in enumerate(individuals_str_batch):
         callable, _ = compile_individual_with_consts(tree, toolbox)
-        total_errs[i] = compute_MSE(callable, dataset, consts=tree.consts)
+        total_errs[i] = compute_MSE(callable, X, y, consts=tree.consts)
 
     return total_errs
 
 
 @ray.remote(num_cpus=num_cpus)
-def compute_attributes(individuals_str_batch, toolbox, dataset, penalty, fitness_scale):
+def compute_attributes(individuals_str_batch, toolbox, X, y, penalty, fitness_scale):
 
     attributes = [None] * len(individuals_str_batch)
 
@@ -170,7 +168,7 @@ def compute_attributes(individuals_str_batch, toolbox, dataset, penalty, fitness
             consts = None
             fitness = (1e8,)
         else:
-            MSE, consts = eval_MSE_and_tune_constants(tree, toolbox, dataset)
+            MSE, consts = eval_MSE_and_tune_constants(tree, toolbox, X, y)
             fitness = (
                 fitness_scale
                 * (
@@ -192,8 +190,7 @@ def assign_attributes(individuals, attributes):
 
 def eval(problem, cfgfile, seed=42):
 
-    with open(cfgfile) as config_file:
-        config_file_data = yaml.safe_load(config_file)
+    regressor_params, config_file_data = util.load_config_data(cfgfile)
 
     scaleXy = config_file_data["gp"]["scaleXy"]
 
@@ -219,6 +216,10 @@ def eval(problem, cfgfile, seed=42):
     else:
         pset = gp.PrimitiveSetTyped("Main", [float] * num_variables, float)
 
+    pset = util.add_primitives_to_pset_from_dict(
+        pset, config_file_data["gp"]["primitives"]
+    )
+
     batch_size = config_file_data["gp"]["batch_size"]
     if config_file_data["gp"]["use_constants"]:
         pset.addTerminal(object, float, "a")
@@ -244,25 +245,25 @@ def eval(problem, cfgfile, seed=42):
         callback_func=callback_func,
         print_log=False,
         num_best_inds_str=1,
-        config_file_data=config_file_data,
         save_best_individual=False,
         output_path="./",
         seed=None,
         batch_size=batch_size,
+        **regressor_params,
     )
 
-    train_data = Dataset("dataset", X_train_scaled, y_train_scaled)
-    test_data = Dataset("dataset", X_test_scaled, y_test)
+    # train_data = Dataset("dataset", X_train_scaled, y_train_scaled)
+    # test_data = Dataset("dataset", X_test_scaled, y_test)
 
     if num_variables > 1:
-        train_data.X = [train_data.X[:, i] for i in range(num_variables)]
-        test_data.X = [test_data.X[:, i] for i in range(num_variables)]
+        X_train = [X_train_scaled[:, i] for i in range(num_variables)]
+        X_test = [X_test_scaled[:, i] for i in range(num_variables)]
     else:
-        train_data.X = [train_data.X]
-        test_data.X = [test_data.X]
+        X_train = [X_train_scaled]
+        X_test = [X_test_scaled]
 
     tic = time.time()
-    gpsr.fit(train_data)
+    gpsr.fit(X_train, y_train_scaled)
     toc = time.time()
 
     if hasattr(gpsr.best, "consts"):
@@ -274,7 +275,7 @@ def eval(problem, cfgfile, seed=42):
     )
     print("Individuals per sec = ", individuals_per_sec)
 
-    u_best = gpsr.predict(test_data)
+    u_best = gpsr.predict(X_test)
     # print(u_best)
     # print(y_test)
 
@@ -292,7 +293,7 @@ def eval(problem, cfgfile, seed=42):
     print("MSE on the test set = ", MSE)
     print("R^2 on the test set = ", r2_test)
 
-    pred_train = gpsr.predict(train_data)
+    pred_train = gpsr.predict(X_train)
 
     if scaleXy:
         pred_train = scaler_y.inverse_transform(pred_train.reshape(-1, 1)).flatten()

bench/results/1027_ESL.csv

@@ -1,11 +1,11 @@
 problem;trial;r2_train;r2_test;seed
-1027_ESL;1;0.8689338256921862;0.8499074519865497;29802
-1027_ESL;2;0.867758019976077;0.8551105370726506;22118
-1027_ESL;3;0.8674199139497978;0.8649631203850897;860
-1027_ESL;4;0.8633000304601383;0.8785615546171678;15795
-1027_ESL;5;0.8698221273060668;0.8600147256664417;21575
-1027_ESL;6;0.8696495006874415;0.8514352495870234;5390
+1027_ESL;1;0.8748909209247202;0.8555242762139903;29802
+1027_ESL;2;0.8727993056735073;0.8687545863990538;22118
+1027_ESL;3;0.8655788225418036;0.8922231312905544;860
+1027_ESL;4;0.867303061983018;0.8916465151833094;15795
+1027_ESL;5;0.8736669432607488;0.8273969888641126;21575
+1027_ESL;6;0.8730603777627965;0.8521808175811904;5390
 1027_ESL;7;0.8736255583112233;0.8612788857378055;11964
-1027_ESL;8;0.8693744204394314;0.8642829832699518;6265
-1027_ESL;9;0.8677081484872072;0.8706866048077533;23654
-1027_ESL;10;0.8826240353573471;0.8321416684717158;11284
+1027_ESL;8;0.8694653006180479;0.8643633497900758;6265
+1027_ESL;9;0.871058107389737;0.858477048277418;23654
+1027_ESL;10;0.8800564058940745;0.8278137293745264;11284

bench/results/1028_SWD.csv

@@ -1,11 +1,11 @@
 problem;trial;r2_train;r2_test;seed
-1028_SWD;1;0.41974736363097087;0.38591151944299507;29802
-1028_SWD;2;0.4517804313326744;0.2690526640849469;22118
-1028_SWD;3;0.40896369588755643;0.40529205916548894;860
-1028_SWD;4;0.42619313209794185;0.344849508584405;15795
-1028_SWD;5;0.43293630350349965;0.293534364945674;21575
-1028_SWD;6;0.42481715418024246;0.34221390442067867;5390
-1028_SWD;7;0.43529973436124514;0.3155602583077638;11964
-1028_SWD;8;0.45109071433080694;0.24163364087292183;6265
-1028_SWD;9;0.411954827512946;0.4035198144230855;23654
-1028_SWD;10;0.41881380364353205;0.35402482965759763;11284
+1028_SWD;1;0.4373125219729864;0.3591733546209136;29802
+1028_SWD;2;0.45067849050801434;0.27734018436872376;22118
+1028_SWD;3;0.432540658536116;0.39640604600169704;860
+1028_SWD;4;0.4383346360723498;0.35539252873090454;15795
+1028_SWD;5;0.4351837209211017;0.34356249269993755;21575
+1028_SWD;6;0.45077777466457414;0.331246467426907;5390
+1028_SWD;7;0.45774565018987534;0.31090479799041437;11964
+1028_SWD;8;0.4623140825381532;0.2911353301464765;6265
+1028_SWD;9;0.41985780125866;0.3945376646322808;23654
+1028_SWD;10;0.4362368631680913;0.33558903136597407;11284

bench/results/1029_LEV.csv

@@ -1,11 +1,11 @@
 problem;trial;r2_train;r2_test;seed
-1029_LEV;1;0.5879780782214657;0.5204583881052793;29802
-1029_LEV;2;0.5789393164986216;0.5676559936258676;22118
-1029_LEV;3;0.5654880596620051;0.6014544173446551;860
-1029_LEV;4;0.6009750767452274;0.5121487884369427;15795
-1029_LEV;5;0.5820556772161638;0.5630880040408148;21575
-1029_LEV;6;0.579662627460287;0.5580064249328927;5390
-1029_LEV;7;0.5628308404387818;0.6082523494360423;11964
-1029_LEV;8;0.5919320061434583;0.5239553108909693;6265
-1029_LEV;9;0.5869421867148492;0.5407588303901149;23654
-1029_LEV;10;0.5610457958242968;0.5759109213194484;11284
+1029_LEV;1;0.5844889767108543;0.5241174789850113;29802
+1029_LEV;2;0.5755651903150614;0.549427209698127;22118
+1029_LEV;3;0.566429372563344;0.6028440243460665;860
+1029_LEV;4;0.6004406005484593;0.508368018104046;15795
+1029_LEV;5;0.5862117839676109;0.5581402773171422;21575
+1029_LEV;6;0.5816449064797491;0.5569798462643671;5390
+1029_LEV;7;0.5606708615537527;0.6073771024576097;11964
+1029_LEV;8;0.5942136638927331;0.5204119748868217;6265
+1029_LEV;9;0.5866779188098024;0.5442624434496006;23654
+1029_LEV;10;0.5647757142106716;0.6027015429520087;11284

bench/results/1030_ERA.csv

@@ -1,11 +1,11 @@
 problem;trial;r2_train;r2_test;seed
-1030_ERA;1;0.355946366178447;0.45130876083441573;29802
-1030_ERA;2;0.37290093828465165;0.4120055153836605;22118
-1030_ERA;3;0.3705570168279426;0.4195764121745853;860
-1030_ERA;4;0.38834742453119286;0.38122326189069244;15795
-1030_ERA;5;0.3890787660989249;0.3897748494190886;21575
-1030_ERA;6;0.39790727005128923;0.3652092964276171;5390
-1030_ERA;7;0.39686209532437067;0.34974236928922453;11964
-1030_ERA;8;0.40678859411790114;0.31814387585475756;6265
-1030_ERA;9;0.3923063405726871;0.3400834590518873;23654
-1030_ERA;10;0.405026041012461;0.3124839043779347;11284
+1030_ERA;1;0.3682264585639765;0.45000370188951255;29802
+1030_ERA;2;0.3703982945293156;0.4484664334429165;22118
+1030_ERA;3;0.3694747321551356;0.43183742625673993;860
+1030_ERA;4;0.3890483637162687;0.3854357692512782;15795
+1030_ERA;5;0.38444234563787705;0.39243117425859775;21575
+1030_ERA;6;0.39800028913849717;0.3775160066188339;5390
+1030_ERA;7;0.39553784297250216;0.370361659063548;11964
+1030_ERA;8;0.40500650637303737;0.3288319726806165;6265
+1030_ERA;9;0.3989662123825213;0.3501745992492785;23654
+1030_ERA;10;0.3986027031981597;0.3161933285379288;11284

bench/results/1089_USCrime.csv

@@ -1,11 +1,11 @@
 problem;trial;r2_train;r2_test;seed
-1089_USCrime;1;0.9484139509443629;0.8493853523348904;29802
-1089_USCrime;2;0.9572058470347506;0.81935543633297;22118
-1089_USCrime;3;0.9520102598873852;0.4540434382838121;860
-1089_USCrime;4;0.9513053045283997;0.74447123868373;15795
-1089_USCrime;5;0.9418779030982185;0.7130163733303321;21575
-1089_USCrime;6;0.943966044212423;0.8377965052540635;5390
-1089_USCrime;7;0.9583551955947374;0.7145147539996739;11964
-1089_USCrime;8;0.9563572028892029;0.7684135499103156;6265
-1089_USCrime;9;0.9507182479552773;0.66950062207029;23654
-1089_USCrime;10;0.9449242331234132;0.8216779253190278;11284
+1089_USCrime;1;0.969037608964617;0.7483367280948074;29802
+1089_USCrime;2;0.9686601906897321;0.7314333167731497;22118
+1089_USCrime;3;0.9628279995212059;0.496644706237765;860
+1089_USCrime;4;0.9661516464623952;0.7308406230327349;15795
+1089_USCrime;5;0.9279961242910364;0.9028921075202149;21575
+1089_USCrime;6;0.9620435535455483;0.8035834139223468;5390
+1089_USCrime;7;0.9721744001998275;0.4316886473558669;11964
+1089_USCrime;8;0.9667952188063021;0.776848816174635;6265
+1089_USCrime;9;0.9679690797919094;0.7503110908067083;23654
+1089_USCrime;10;0.9573742889963757;0.7093714233399555;11284

bench/results/1096_FacultySalaries.csv

@@ -1,11 +1,11 @@
 problem;trial;r2_train;r2_test;seed
-1096_FacultySalaries;1;0.9575028902765571;0.8184481267043824;29802
-1096_FacultySalaries;2;0.9568076672378055;0.917434133842035;22118
-1096_FacultySalaries;3;0.9669940054640627;0.8934372879712575;860
-1096_FacultySalaries;4;0.9996757503744214;0.1871532006638934;15795
-1096_FacultySalaries;5;0.9515616447901358;0.8945706203228898;21575
-1096_FacultySalaries;6;0.9981737422612984;-0.6885712121152572;5390
-1096_FacultySalaries;7;0.9795965157536484;0.8997591808890962;11964
-1096_FacultySalaries;8;0.977426037817008;0.8391699685985927;6265
-1096_FacultySalaries;9;0.9684482341572983;0.9618854615626562;23654
-1096_FacultySalaries;10;0.963343554614357;0.8986187469236837;11284
+1096_FacultySalaries;1;0.9757811801086493;0.8468690082448429;29802
+1096_FacultySalaries;2;0.9747072184400877;0.5559189268086008;22118
+1096_FacultySalaries;3;0.9602495975861516;0.6315686809951979;860
+1096_FacultySalaries;4;0.9996757503744214;0.18715319947834508;15795
+1096_FacultySalaries;5;0.9733547803551156;0.9447964356602097;21575
+1096_FacultySalaries;6;0.998173742238267;-0.6885706555756188;5390
+1096_FacultySalaries;7;0.9787889343295242;0.9232090362546187;11964
+1096_FacultySalaries;8;0.9648100352407598;0.8257600154636463;6265
+1096_FacultySalaries;9;0.9792443170894077;0.960647215075818;23654
+1096_FacultySalaries;10;0.9813331788413356;0.8844476997285342;11284

bench/results/192_vineyard.csv

@@ -1,11 +1,11 @@
 problem;trial;r2_train;r2_test;seed
-192_vineyard;1;0.8758445698235056;0.2236397944668479;29802
-192_vineyard;2;0.858616424851334;0.13271060525956901;22118
-192_vineyard;3;0.8537137523938062;0.4780805511251951;860
-192_vineyard;4;0.8692894908232239;0.3758608681462412;15795
-192_vineyard;5;0.87608336889845;0.4455659090090176;21575
-192_vineyard;6;0.8776088928103271;0.3460864519849878;5390
-192_vineyard;7;0.8384373653693135;0.7195907291058308;11964
-192_vineyard;8;0.8223940400933747;0.08607142817359648;6265
-192_vineyard;9;0.8425098503334608;0.6264516080030693;23654
-192_vineyard;10;0.8929706868907198;0.38449967469030966;11284
+192_vineyard;1;0.8708853934183917;-0.4248209450726521;29802
+192_vineyard;2;0.8569606516417794;-0.42892415246500093;22118
+192_vineyard;3;0.8829022683683329;0.09498433432956144;860
+192_vineyard;4;0.8956247954424682;0.15671556194470315;15795
+192_vineyard;5;0.8912368002572013;0.22225606881786264;21575
+192_vineyard;6;0.8829681580907572;0.2039008023908344;5390
+192_vineyard;7;0.8457254813152156;0.47914507962376474;11964
+192_vineyard;8;0.8352999677222162;0.5832638534915283;6265
+192_vineyard;9;0.8619800115440981;0.5736448841534942;23654
+192_vineyard;10;0.8912822500084658;0.5882196088719382;11284

bench/results/197_cpu_act.csv

@@ -1,11 +1,11 @@
 problem;trial;r2_train;r2_test;seed
-197_cpu_act;1;0.9387934899514239;0.9451106322160039;29802
-197_cpu_act;2;0.9333639615224261;0.9290571513295527;22118
-197_cpu_act;3;0.9530034270392485;0.9503027779498951;860
-197_cpu_act;4;0.9470401775214541;0.9451628075750368;15795
-197_cpu_act;5;0.9427855190899835;0.9461699935502532;21575
-197_cpu_act;6;0.9471825707278558;0.9506166699567961;5390
-197_cpu_act;7;0.9564062751447067;0.9520205277468109;11964
-197_cpu_act;8;0.9540432896569563;0.9512028699925679;6265
-197_cpu_act;9;0.9382877197734281;0.9361704125648529;23654
-197_cpu_act;10;0.9374270754243184;0.9250902535585799;11284
+197_cpu_act;1;0.9457665877454394;0.9436749078434319;29802
+197_cpu_act;2;0.9421764711644052;0.9358882728244855;22118
+197_cpu_act;3;0.946313954464179;0.9471130286100041;860
+197_cpu_act;4;0.8496781432845921;0.8582846660255161;15795
+197_cpu_act;5;0.9517077250675727;0.9530442805581179;21575
+197_cpu_act;6;0.9477005153417889;0.9502016383212369;5390
+197_cpu_act;7;0.9486233817092022;0.938278739172296;11964
+197_cpu_act;8;0.9597354596296003;0.9555905049338825;6265
+197_cpu_act;9;0.9648960107234625;0.9634820840877398;23654
+197_cpu_act;10;0.9577340006506451;0.95184836883823;11284

bench/results/210_cloud.csv

@@ -1,11 +1,11 @@
 problem;trial;r2_train;r2_test;seed
-210_cloud;1;0.9247636146137266;0.6455604410596569;29802
-210_cloud;2;0.9343808533567021;0.8770082489331003;22118
-210_cloud;3;0.9311210811645867;0.7127490256169635;860
-210_cloud;4;0.934936620286743;0.9362826105251004;15795
-210_cloud;5;0.9482920134801041;0.9082722075947439;21575
-210_cloud;6;0.942302095803146;0.7227604098373297;5390
-210_cloud;7;0.9505642703201787;0.8504613720142944;11964
-210_cloud;8;0.9360740181207674;0.6395164814517422;6265
-210_cloud;9;0.9298362563595642;0.8894142340721412;23654
-210_cloud;10;0.9517720645617664;0.43475344549546946;11284
+210_cloud;1;0.9362762610302084;0.8691901588735473;29802
+210_cloud;2;0.932913153743925;0.8880995514125988;22118
+210_cloud;3;0.9342923431663721;0.7491585711117132;860
+210_cloud;4;0.9439602033234796;0.8988071127272331;15795
+210_cloud;5;0.9530786572643501;0.8442892691784296;21575
+210_cloud;6;0.9451596735432771;0.8376976230836759;5390
+210_cloud;7;0.9474825821384996;0.8673292587973871;11964
+210_cloud;8;0.9364317165249529;0.5617672429308037;6265
+210_cloud;9;0.9265135160166141;0.9456328880138059;23654
+210_cloud;10;0.9565081594889666;0.39315858636610057;11284

bench/results/225_puma8NH.csv

@@ -1,11 +1,11 @@
 problem;trial;r2_train;r2_test;seed
-225_puma8NH;1;0.6705324281799827;0.6469760131302198;29802
-225_puma8NH;2;0.6529416250592445;0.6648667830014574;22118
-225_puma8NH;3;0.668872418199597;0.6691440462049962;860
-225_puma8NH;4;0.6698248185101057;0.669578706012151;15795
-225_puma8NH;5;0.6806162807788343;0.6962678620000038;21575
-225_puma8NH;6;0.6557342826799404;0.6791402248205318;5390
-225_puma8NH;7;0.6705031948803878;0.668675527501577;11964
-225_puma8NH;8;0.6696418552266217;0.6668672891980192;6265
-225_puma8NH;9;0.6809417218834979;0.6641877090939536;23654
-225_puma8NH;10;0.6778308230365331;0.6596977025598376;11284
+225_puma8NH;1;0.682850204435814;0.6789211747963135;29802
+225_puma8NH;2;0.6754836444737324;0.6930220241212716;22118
+225_puma8NH;3;0.6761584240959573;0.6747595339118583;860
+225_puma8NH;4;0.6685405891987423;0.6638774570350612;15795
+225_puma8NH;5;0.6778781214915794;0.692070342886109;21575
+225_puma8NH;6;0.6753263424574281;0.6922572411975378;5390
+225_puma8NH;7;0.6209830017302066;0.6273467897767163;11964
+225_puma8NH;8;0.6777014266663415;0.6799644712287625;6265
+225_puma8NH;9;0.6822693621746627;0.666476328548008;23654
+225_puma8NH;10;0.6814646567138007;0.6671700746406766;11284