Skip to content

Commit 28a8682

Browse files
suzhoumsuzhoum
committed
update
1 parent c38295a commit 28a8682

File tree

6 files changed

+150
-59
lines changed

6 files changed

+150
-59
lines changed

‎sample_configs/cloud_configs/resources/bench_all.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import random
2+
n_experiments = 5
3+
seeds = []
4+
for i in range(n_experiments):
5+
seeds.append(random.randint(0, 100))
6+
7+
seeds = [22, 92, 54, 86, 41]
8+
9+
config_paths = [
10+
"sample_configs/paper_text_tabular_local_configs.yaml",
11+
"sample_configs/paper_text_local_configs.yaml",
12+
"sample_configs/paper_image_local_configs.yaml",
13+
# "sample_configs/multimodal_cloud_text_configs.yaml",
14+
# "sample_configs/multimodal_cloud_text_fs_configs.yaml",
15+
# "sample_configs/multimodal_cloud_text_tabular_configs.yaml",
16+
# "sample_configs/multimodal_cloud_text_tabular_image_configs.yaml",
17+
# "sample_configs/multimodal_cloud_text_tabular_image_standard_configs.yaml"
18+
]
19+
frameworks = [
20+
# "AutoGluon_best_master",
21+
# "ablation_base",
22+
# "ablation_greedy_soup",
23+
# "ablation_gradient_clip",
24+
"ablation_warmup_steps",
25+
"ablation_cosine_decay",
26+
"ablation_weight_decay",
27+
# "ablation_lr_decay",
28+
# "autokeras_master",
29+
# "torch_compile_best",
30+
# "AutoGluon_best_master",
31+
# "AutoGluon_high_master",
32+
# "AutoGluon_medium_master",
33+
# "AutoGluon_high_vitlarge",
34+
# "AutoGluon_medium_vitlarge",
35+
# "AutoGluon_best_vitlarge",
36+
# "AutoGluon_best_caformer",
37+
# "AutoGluon_best_beit",
38+
# "AutoGluon_best_swinv2"
39+
# "AutoGluon_high_0_8",
40+
# "AutoGluon_medium_0_8",
41+
# "AutoGluon_best_0_8",
42+
]
43+
constraints = [
44+
"g4_12x"
45+
]
46+
fs = [
47+
1,
48+
5,
49+
10
50+
]
51+
# module = "autokeras"
52+
module = "multimodal"
53+
54+
import yaml
55+
import os
56+
import subprocess
57+
58+
config_root = "./temp_configs"
59+
os.makedirs(config_root, exist_ok=True)
60+
61+
for seed in seeds:
62+
print("Seed: ", seed)
63+
for constraint in constraints:
64+
os.makedirs(f"{config_root}/{constraint}", exist_ok=True)
65+
for framework in frameworks:
66+
# for shot in fs:
67+
config_dir = f"{config_root}/{constraint}/{framework}"
68+
os.makedirs(config_dir, exist_ok=True)
69+
70+
for config_path in config_paths:
71+
with open(config_path, "r") as f:
72+
configs = yaml.safe_load(f)
73+
if constraint == "g4_12x":
74+
configs["cdk_context"]["PREFIX"] = f"{configs['cdk_context']['PREFIX']}-multi"
75+
configs["constraint"] = constraint
76+
configs["framework"] = framework
77+
configs["module"] = module
78+
configs["seed"] = seed
79+
# configs["custom_dataloader"]["shot"] = shot
80+
configs["benchmark_name"] = f"{configs['benchmark_name']}-{seed}"
81+
new_config_path = os.path.join(config_dir, os.path.basename(config_path))
82+
with open(new_config_path, "w") as new_f:
83+
yaml.dump(configs, new_f)
84+
print("Running config: ", new_config_path)
85+
command = ["agbench", "run", new_config_path]
86+
subprocess.run(command)
87+

‎sample_configs/cloud_configs/resources/multimodal_frameworks.yaml‎

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,61 +13,78 @@ AutoGluon_best_master:
1313
version: master
1414
params: # MultimodalPredictor.fit(params)
1515
presets: best_quality
16-
# hyperparameters:
17-
# optimization.gradient_clip_val: 0.1
1816

19-
ablation_greedy_soup:
17+
ablation_base:
2018
repo: https://github.com/autogluon/autogluon.git
2119
version: master
2220
params: # MultimodalPredictor.fit(params)
2321
presets: best_quality
2422
hyperparameters:
2523
optimization.top_k_average_method: best
26-
# optimization.gradient_clip_val: 0.1
24+
optimization.gradient_clip_val: 0
25+
optimization.warmup_steps: 0
26+
optimization.lr_schedule: constant
27+
optimization.weight_decay: 0
28+
optimization.lr_decay: 0
29+
30+
31+
ablation_greedy_soup:
32+
repo: https://github.com/autogluon/autogluon.git
33+
version: master
34+
params: # MultimodalPredictor.fit(params)
35+
presets: best_quality
36+
hyperparameters:
37+
optimization.gradient_clip_val: 0
38+
optimization.warmup_steps: 0
39+
optimization.lr_schedule: constant
40+
optimization.weight_decay: 0
41+
optimization.lr_decay: 0
2742

2843
ablation_gradient_clip:
2944
repo: https://github.com/autogluon/autogluon.git
3045
version: master
3146
params: # MultimodalPredictor.fit(params)
3247
presets: best_quality
3348
hyperparameters:
34-
optimization.gradient_clip_val: 10000
49+
optimization.warmup_steps: 0
50+
optimization.lr_schedule: constant
51+
optimization.weight_decay: 0
52+
optimization.lr_decay: 0
3553

3654
ablation_warmup_steps:
3755
repo: https://github.com/autogluon/autogluon.git
3856
version: master
3957
params: # MultimodalPredictor.fit(params)
4058
presets: best_quality
4159
hyperparameters:
42-
optimization.warmup_steps: 0.0
43-
# optimization.gradient_clip_val: 0.1
60+
optimization.lr_schedule: constant
61+
optimization.weight_decay: 0
62+
optimization.lr_decay: 0
4463

4564
ablation_cosine_decay:
4665
repo: https://github.com/autogluon/autogluon.git
4766
version: master
4867
params: # MultimodalPredictor.fit(params)
4968
presets: best_quality
5069
hyperparameters:
51-
optimization.lr_schedule: polynomial_decay
52-
# optimization.gradient_clip_val: 0.1
70+
optimization.weight_decay: 0
71+
optimization.lr_decay: 0
72+
5373

5474
ablation_weight_decay:
5575
repo: https://github.com/autogluon/autogluon.git
5676
version: master
5777
params: # MultimodalPredictor.fit(params)
5878
presets: best_quality
5979
hyperparameters:
60-
optimization.weight_decay: 0.0
61-
# optimization.gradient_clip_val: 0.1
80+
optimization.lr_decay: 0
81+
6282

6383
ablation_lr_decay:
6484
repo: https://github.com/autogluon/autogluon.git
6585
version: master
6686
params: # MultimodalPredictor.fit(params)
6787
presets: best_quality
68-
hyperparameters:
69-
optimization.lr_decay: 0.0
70-
optimization.gradient_clip_val: 0.1
7188

7289

7390
autokeras_master:

‎src/autogluon/bench/custom_configs/dataloaders/__init__.py‎

Whitespace-only changes.

‎src/autogluon/bench/custom_configs/resources/__init__.py‎

Whitespace-only changes.

‎src/autogluon/bench/custom_configs/resources/multimodal_frameworks.yaml‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,53 +14,77 @@ AutoGluon_best_master:
1414
params: # MultimodalPredictor.fit(params)
1515
presets: best_quality
1616

17+
ablation_base:
18+
repo: https://github.com/autogluon/autogluon.git
19+
version: master
20+
params: # MultimodalPredictor.fit(params)
21+
presets: best_quality
22+
hyperparameters:
23+
optimization.top_k_average_method: best
24+
optimization.gradient_clip_val: 0
25+
optimization.warmup_steps: 0
26+
optimization.lr_schedule: constant
27+
optimization.weight_decay: 0
28+
optimization.lr_decay: 0
29+
30+
1731
ablation_greedy_soup:
1832
repo: https://github.com/autogluon/autogluon.git
1933
version: master
2034
params: # MultimodalPredictor.fit(params)
2135
presets: best_quality
2236
hyperparameters:
23-
optimization.top_k_average_method: best
37+
optimization.gradient_clip_val: 0
38+
optimization.warmup_steps: 0
39+
optimization.lr_schedule: constant
40+
optimization.weight_decay: 0
41+
optimization.lr_decay: 0
2442

2543
ablation_gradient_clip:
2644
repo: https://github.com/autogluon/autogluon.git
2745
version: master
2846
params: # MultimodalPredictor.fit(params)
2947
presets: best_quality
3048
hyperparameters:
31-
optimization.gradient_clip_algorithm: value
49+
optimization.warmup_steps: 0
50+
optimization.lr_schedule: constant
51+
optimization.weight_decay: 0
52+
optimization.lr_decay: 0
3253

3354
ablation_warmup_steps:
3455
repo: https://github.com/autogluon/autogluon.git
3556
version: master
3657
params: # MultimodalPredictor.fit(params)
3758
presets: best_quality
3859
hyperparameters:
39-
optimization.warmup_steps: 0.0
60+
optimization.lr_schedule: constant
61+
optimization.weight_decay: 0
62+
optimization.lr_decay: 0
4063

4164
ablation_cosine_decay:
4265
repo: https://github.com/autogluon/autogluon.git
4366
version: master
4467
params: # MultimodalPredictor.fit(params)
4568
presets: best_quality
4669
hyperparameters:
47-
optimization.lr_schedule: polynomial_decay
70+
optimization.weight_decay: 0
71+
optimization.lr_decay: 0
72+
4873

4974
ablation_weight_decay:
5075
repo: https://github.com/autogluon/autogluon.git
5176
version: master
5277
params: # MultimodalPredictor.fit(params)
5378
presets: best_quality
5479
hyperparameters:
55-
optimization.weight_decay: 0.0
80+
optimization.lr_decay: 0
81+
5682

5783
ablation_lr_decay:
5884
repo: https://github.com/autogluon/autogluon.git
5985
version: master
6086
params: # MultimodalPredictor.fit(params)
6187
presets: best_quality
62-
hyperparameters:
63-
optimization.lr_decay: 0.0
6488

6589

6690
autokeras_master:

‎src/autogluon/bench/eval/scripts/run_generate_clean_openml.py‎

Lines changed: 1 addition & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,6 @@
88
import typer
99
from typing_extensions import Annotated
1010

11-
from autogluon.bench.eval.evaluation.constants import (
12-
DATASET,
13-
FOLD,
14-
FRAMEWORK,
15-
METRIC,
16-
METRIC_ERROR,
17-
PROBLEM_TYPE,
18-
TIME_INFER_S,
19-
TIME_TRAIN_S,
20-
)
2111
from autogluon.bench.eval.evaluation.constants import (
2212
DATASET,
2313
FOLD,
@@ -52,7 +42,6 @@
5242
def clean_amlb_results(
5343
benchmark_name: str = typer.Argument(
5444
None, help="Benchmark name populated by benchmark run, in format <benchmark_name>_<timestamp>"
55-
None, help="Benchmark name populated by benchmark run, in format <benchmark_name>_<timestamp>"
5645
),
5746
results_dir: str = typer.Option("data/results/", help="Root directory of raw and prepared results."),
5847
results_dir_input: str = typer.Option(
@@ -126,7 +115,6 @@ def clean_and_save_results(
126115
run_name_in_output_path: bool = True,
127116
save: bool = True,
128117
save_minimal: bool = True,
129-
constraints: List[str] | None = None,
130118
out_path_prefix: str = "openml_ag_",
131119
out_path_suffix: str = "",
132120
framework_suffix_column: str = "constraint",
@@ -151,34 +139,9 @@ def clean_and_save_results(
151139
results_list.append(results)
152140
results_raw = pd.concat(results_list, ignore_index=True, sort=True)
153141

154-
if "framework_parent" in results_raw.columns:
155-
results_raw[FRAMEWORK] = results_raw["framework_parent"] + "_" + run_name + "_" + results_raw[FRAMEWORK]
156-
else:
157-
results_raw[FRAMEWORK] = results_raw[FRAMEWORK] + "_" + run_name
158-
159-
minimal_columns = [
160-
DATASET,
161-
FOLD,
162-
FRAMEWORK,
163-
"constraint",
164-
METRIC,
165-
METRIC_ERROR,
166-
TIME_TRAIN_S,
167-
TIME_INFER_S,
168-
PROBLEM_TYPE,
169-
"tid",
170-
]
171-
172-
results_raw_columns = list(results_raw.columns)
173-
results_raw_columns = [c for c in results_raw_columns if c in minimal_columns] + [
174-
c for c in results_raw_columns if c not in minimal_columns
175-
]
176-
results_raw = results_raw[results_raw_columns]
177-
178142
if save:
179143
if run_name_in_output_path:
180144
save_path = os.path.join(results_dir_output, f"{out_path_prefix}{run_name}{out_path_suffix}")
181-
save_path = os.path.join(results_dir_output, f"{out_path_prefix}{run_name}{out_path_suffix}")
182145
else:
183146
save_path = os.path.join(results_dir_output, f"{out_path_prefix}{out_path_suffix}")
184147
save_path_file = f"{save_path}.csv"
@@ -188,7 +151,7 @@ def clean_and_save_results(
188151
save_path_file_pq = f"{save_path}.parquet"
189152
save_pd.save(path=save_path_file_pq, df=results_raw)
190153
if save_minimal:
191-
results_raw_minimal = results_raw[minimal_columns]
154+
results_raw_minimal = results_raw[MINIMAL_COLUMNS]
192155

193156
save_path_file_minimum = f"{save_path}_min.csv"
194157
save_pd.save(path=save_path_file_minimum, df=results_raw_minimal)

0 commit comments

Comments
 (0)