forked from simonalford42/Agentless
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathagentless_agent.sh
More file actions
170 lines (154 loc) · 8.39 KB
/
Copy pathagentless_agent.sh
File metadata and controls
170 lines (154 loc) · 8.39 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
#%% Setup
TARGET_ID=$1
SAMPLES=1
OUTPUT_DIR="${TARGET_ID}_n${SAMPLES}"
# MODEL="gpt-4o-mini-2024-07-18"
MODEL='gemini-2.0-flash-lite'
# MODEL='gemini-2.5-flash-preview-4-17'
BACKEND='google' # 'openai', 'deepmind', etc.
# add cd to path
export PYTHONPATH=$PYTHONPATH:$(pwd)
# do the same for codearena repo so we can access monkeypatched swebench
cd ../../
export PYTHONPATH=$PYTHONPATH:$(pwd)
cd baselines/Agentless
# exit immediately if any of the commands fail
set -e
# # make sure OPENAI_API_KEY is set
# if [ -z "$OPENAI_API_KEY" ]; then
# echo "OPENAI_API_KEY is not set"
# exit 1
# fi
#%% File-level localization
python agentless/fl/localize.py --file_level \
--output_folder results/$OUTPUT_DIR/file_level \
--num_threads 1 \
--skip_existing \
--target_id $TARGET_ID \
--model $MODEL \
--backend $BACKEND \
--dataset codearena_local
echo "Finished running file-level localization"
#%% Irrelevant file filtering
python agentless/fl/localize.py --file_level \
--irrelevant \
--output_folder results/$OUTPUT_DIR/file_level_irrelevant \
--num_threads 1 \
--skip_existing \
--target_id $TARGET_ID \
--model $MODEL \
--backend $BACKEND \
--dataset codearena_local
echo "Finished running irrelevant file filtering"
#%% Retrieval-based localization
python agentless/fl/retrieve.py --index_type simple \
--filter_type given_files \
--filter_file results/$OUTPUT_DIR/file_level_irrelevant/loc_outputs.jsonl \
--output_folder results/$OUTPUT_DIR/retrievel_embedding \
--persist_dir embedding/swe-bench_simple \
--num_threads 1 \
--target_id $TARGET_ID \
--chunk_size 1024 \
--dataset codearena_local
echo "Finished running retrieval-based localization"
#%% Combine retrieval and model results
python agentless/fl/combine.py --retrieval_loc_file results/$OUTPUT_DIR/retrievel_embedding/retrieve_locs.jsonl \
--model_loc_file results/$OUTPUT_DIR/file_level/loc_outputs.jsonl \
--top_n 3 \
--output_folder results/$OUTPUT_DIR/file_level_combined
echo "Finished running retrieval and model results combination"
#%% Related elements localization
python agentless/fl/localize.py --related_level \
--output_folder results/$OUTPUT_DIR/related_elements \
--top_n 3 \
--compress_assign \
--compress \
--start_file results/$OUTPUT_DIR/file_level_combined/combined_locs.jsonl \
--num_threads 1 \
--skip_existing \
--target_id $TARGET_ID \
--model $MODEL \
--backend $BACKEND \
--dataset codearena_local
echo "Finished running related elements localization"
#%% Fine-grained line-level localization
python agentless/fl/localize.py --fine_grain_line_level \
--output_folder results/$OUTPUT_DIR/edit_location_samples \
--top_n 3 \
--compress \
--temperature 0.8 \
--num_samples 4 \
--start_file results/$OUTPUT_DIR/related_elements/loc_outputs.jsonl \
--num_threads 1 \
--skip_existing \
--target_id $TARGET_ID \
--model $MODEL \
--backend $BACKEND \
--dataset codearena_local
echo "Finished running fine-grained line-level localization"
#%% Merge edit locations
python agentless/fl/localize.py --merge \
--output_folder results/$OUTPUT_DIR/edit_location_individual \
--top_n 3 \
--num_samples 4 \
--start_file results/$OUTPUT_DIR/edit_location_samples/loc_outputs.jsonl \
--target_id $TARGET_ID \
--model $MODEL \
--backend $BACKEND \
--dataset codearena_local
echo "Finished running edit locations merge"
#%% Repair for each set of edit locations
for i in {0..3}; do
python agentless/repair/repair.py --loc_file results/$OUTPUT_DIR/edit_location_individual/loc_merged_${i}-${i}_outputs.jsonl \
--output_folder results/$OUTPUT_DIR/repair_sample_$((i+1)) \
--loc_interval \
--top_n=3 \
--context_window=10 \
--max_samples $SAMPLES \
--cot \
--diff_format \
--gen_and_process \
--num_threads 2 \
--target_id $TARGET_ID \
--model $MODEL \
--backend $BACKEND \
--dataset codearena_local
done
echo "Finished running repair for all edit locations"
#%% Generate regression tests
python agentless/test/run_regression_tests.py --run_id generate_regression_tests \
--output_file results/$OUTPUT_DIR/passing_tests.jsonl \
--instance_ids $TARGET_ID \
--dataset codearena_local
echo "Finished running regression test generation"
#%% Select regression tests
python agentless/test/select_regression_tests.py --passing_tests results/$OUTPUT_DIR/passing_tests.jsonl \
--output_folder results/$OUTPUT_DIR/select_regression \
--target_id $TARGET_ID \
--instance_ids $TARGET_ID \
--model $MODEL \
--backend $BACKEND \
--dataset codearena_local
echo "Finished running regression test selection"
#%% Run regression tests on all patches
for i in {1..4}; do
folder=results/$OUTPUT_DIR/repair_sample_$i
for ((num=0; num<$SAMPLES; num++)); do
run_id_prefix=$(basename $folder)
python agentless/test/run_regression_tests.py --regression_tests results/$OUTPUT_DIR/select_regression/output.jsonl \
--predictions_path="${folder}/output_${num}_processed.jsonl" \
--run_id="${run_id_prefix}_regression_${num}" \
--num_workers 10 \
--instance_ids $TARGET_ID \
--dataset codearena_local
done
done
echo "Finished running regression tests on all patches"
#%% Rerank and select final patches
python agentless/repair/rerank.py --patch_folder results/$OUTPUT_DIR/repair_sample_1/,results/$OUTPUT_DIR/repair_sample_2/,results/$OUTPUT_DIR/repair_sample_3/,results/$OUTPUT_DIR/repair_sample_4/ \
--num_samples=$((4 * SAMPLES)) \
--deduplicate \
--regression \
--target $TARGET_ID \
--output_file results/$OUTPUT_DIR/all_preds.jsonl
echo "Finished everything!"