Skip to content

Commit ac5bf2f

Browse files
JacobBarthelmehJacobBarthelmeh
committed
add initial SFTP benchmark collection scripts
1 parent da85e49 commit ac5bf2f

File tree

3 files changed

+337
-0
lines changed

3 files changed

+337
-0
lines changed

‎.github/workflows/sftp-benchmark.yml‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
name: Benchmark Test
2+
3+
on:
4+
push:
5+
branches: [ '*' ]
6+
pull_request:
7+
branches: [ '*' ]
8+
9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.ref }}
11+
cancel-in-progress: true
12+
13+
jobs:
14+
build_wolfssl:
15+
strategy:
16+
fail-fast: false
17+
matrix:
18+
os: [ ubuntu-latest ]
19+
wolfssl: [ master ]
20+
name: Build wolfssl
21+
runs-on: ${{ matrix.os }}
22+
timeout-minutes: 4
23+
steps:
24+
- name: Checking cache for wolfssl
25+
uses: actions/cache@v4
26+
id: cache-wolfssl
27+
with:
28+
path: build-dir/
29+
key: wolfssh-sshd-wolfssl-${{ matrix.wolfssl }}-${{ matrix.os }}
30+
lookup-only: true
31+
32+
- name: Checkout, build, and install wolfssl
33+
if: steps.cache-wolfssl.outputs.cache-hit != 'true'
34+
uses: wolfSSL/actions-build-autotools-project@v1
35+
with:
36+
repository: wolfssl/wolfssl
37+
ref: ${{ matrix.wolfssl }}
38+
path: wolfssl
39+
configure: --enable-all
40+
check: false
41+
install: true
42+
43+
build_wolfssh:
44+
needs:
45+
- build_wolfssl
46+
- create_matrix
47+
strategy:
48+
fail-fast: false
49+
matrix:
50+
os: [ ubuntu-latest ]
51+
wolfssl: ${{ fromJson(needs.create_matrix.outputs['versions']) }}
52+
name: Build and test wolfsshd
53+
runs-on: ${{ matrix.os }}
54+
timeout-minutes: 10
55+
steps:
56+
- name: Checking cache for wolfssl
57+
uses: actions/cache@v4
58+
with:
59+
path: build-dir/
60+
key: wolfssh-sshd-wolfssl-${{ matrix.wolfssl }}-${{ matrix.os }}
61+
fail-on-cache-miss: true
62+
63+
- uses: actions/checkout@v4
64+
with:
65+
path: wolfssh/
66+
67+
- name: autogen
68+
working-directory: ./wolfssh/
69+
run: ./autogen.sh
70+
71+
- name: configure
72+
working-directory: ./wolfssh/
73+
run : |
74+
./configure --enable-all LDFLAGS="-L${{ github.workspace }}/build-dir/lib" CPPFLAGS="-I${{ github.workspace }}/build-dir/include -DWOLFSSH_NO_FPKI -DWOLFSSH_NO_SFTP_TIMEOUT -DWOLFSSH_MAX_SFTP_RW=4000000 -DMAX_PATH_SZ=120"
75+
76+
- name: make
77+
working-directory: ./wolfssh/
78+
run: make
79+
80+
- name: Get Saved OpenSSH Upload Results
81+
uses: actions/downlad-artifact@v4
82+
with:
83+
path: ./wolfssh/
84+
artifact_id: 'openssh-upload'
85+
86+
87+
- name: Put test key in authorized keys file
88+
run: |
89+
touch ~/.ssh/authorized_keys_test
90+
cat ./keys/hansel-*.pub > authorized_keys_test
91+
chmod 600 ./keys/hansel-key-*.pem
92+
sudo systemctl restart sshd
93+
sudo service sshd restart
94+
95+
- name: Run SFTP client benchmark
96+
working-directory: ./wolfssh/
97+
run: |
98+
./scripts/get-sftp-benchmark.sh
99+
100+
- name: Store Upload Speed PNG
101+
uses: actions/upload-artifact@v4
102+
with:
103+
name: upload-results.png
104+
path: wolfssh/upload-results.png
105+
106+
- name: Store Download Speed PNG
107+
uses: actions/upload-artifact@v4
108+
with:
109+
name: download-results.png
110+
path: wolfssh/download-results.png
111+
112+
- name: Comment on PR about performance
113+
env:
114+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
115+
run: |
116+
PR_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")
117+
gh pr comment $PR_NUMBER --body "Attached is the performance results" \
118+
--attach download-results.png \
119+
--attach upload-results.png
120+
121+
artifacts:
122+
'openssh-upload': { 'path': 'wolfssh/openssh-average-upload.csv' }
123+
'openssh-download': { 'path': 'wolfssh/openssh-average-download.csv' }

‎examples/sftpclient/sftpclient.c‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,8 @@ static void err_msg(const char* s)
132132
#else
133133
#include <sys/time.h>
134134

135+
double current_time_ms(int);
136+
135137
/* return number of seconds*/
136138
word32 current_time(int reset)
137139
{
@@ -142,6 +144,18 @@ static void err_msg(const char* s)
142144
gettimeofday(&tv, 0);
143145
return (word32)tv.tv_sec;
144146
}
147+
148+
/* return number of micro seconds */
149+
double current_time_ms(int reset)
150+
{
151+
struct timeval tv;
152+
153+
(void)reset;
154+
155+
gettimeofday(&tv, 0);
156+
return (word64)(tv.tv_sec*1000000) + tv.tv_usec;
157+
}
158+
145159
#endif /* USE_WINDOWS_API */
146160
#endif /* !WOLFSSH_NO_TIMESTAMP */
147161

@@ -1093,6 +1107,12 @@ static int doAutopilot(int cmd, char* local, char* remote)
10931107
char fullpath[128] = ".";
10941108
WS_SFTPNAME* name = NULL;
10951109
byte remoteAbsPath = 0;
1110+
#if !defined(WOLFSSH_NO_TIMESTAMP) && !defined(USE_WINDOWS_API)
1111+
double currentTime;
1112+
double longBytes = 0;
1113+
FILE* f;
1114+
#endif
1115+
10961116

10971117
/* check if is absolute path before making it one */
10981118
if (remote != NULL && WSTRLEN(remote) > 2 && remote[1] == ':' &&
@@ -1124,6 +1144,18 @@ static int doAutopilot(int cmd, char* local, char* remote)
11241144
remote);
11251145
}
11261146

1147+
#if !defined(WOLFSSH_NO_TIMESTAMP) && !defined(USE_WINDOWS_API)
1148+
ret = WFOPEN(NULL, &f, fullpath, "rb");
1149+
if (ret != 0 || f == WBADFILE) return WS_BAD_FILE_E;
1150+
if (WFSEEK(NULL, f, 0, WSEEK_END) != 0) {
1151+
WFCLOSE(NULL, f);
1152+
return WS_BAD_FILE_E;
1153+
}
1154+
longBytes = (word32)WFTELL(NULL, f);
1155+
WREWIND(NULL, f);
1156+
currentTime = current_time_ms(0);
1157+
#endif
1158+
11271159
do {
11281160
if (cmd == AUTOPILOT_PUT) {
11291161
ret = wolfSSH_SFTP_Put(ssh, local, fullpath, 0, NULL);
@@ -1145,6 +1177,15 @@ static int doAutopilot(int cmd, char* local, char* remote)
11451177
fullpath, local);
11461178
}
11471179
}
1180+
#if !defined(WOLFSSH_NO_TIMESTAMP) && !defined(USE_WINDOWS_API)
1181+
else {
1182+
currentTime = current_time_ms(0) - currentTime;
1183+
double result;
1184+
result = (double)longBytes / 1000000;
1185+
result = result / ((double)currentTime / 1000000);
1186+
printf("Transfered %s at %.2fMB/s\n", fullpath, result);
1187+
}
1188+
#endif
11481189

11491190
wolfSSH_SFTPNAME_list_free(name);
11501191
return ret;

‎scripts/get-sftp-benchmark.sh‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
#!/bin/bash
2+
3+
KEY="keys/hansel-key-ecc.pem"
4+
TEST_FILE="/home/jak/Documents/wolfssh-fork/test"
5+
FILE_SIZES=("5000" "10000" "50000" "100000" "150000" "200000" "250000" "300000" "350000" "400000" "500000" "1000000")
6+
TRANSFER_MBS=""
7+
NUMBER_RUNS=10
8+
LOG_FILE="$PWD/log.csv"
9+
COMPARE_TO=""
10+
AVERAGE_FILE=""
11+
12+
if [ -z $1 ]; then
13+
echo "Assuming default server port of 22 (pass port number as first"
14+
echo "argument if wanting to connect to a different port)"
15+
PORT=22
16+
else
17+
PORT="$1"
18+
fi
19+
20+
do_openssh_put_test() {
21+
cp $TEST_FILE $TEST_FILE-out
22+
sftp_command="sftp -P$PORT -i $KEY [email protected]"
23+
output_file="sftp_log.txt"
24+
25+
# Start the script command to capture the sftp session
26+
script -qc "$sftp_command << EOF
27+
put $TEST_FILE $TEST_FILE-out
28+
bye
29+
EOF" /dev/null 2>&1 | tee $output_file | while read line; do
30+
if [[ "$line" == *'MB/s'* ]]; then
31+
#TRANSFER_MBS=$(echo "$line" | awk '{print $(NF-2)}' | sed 's/MB\/s//')
32+
TRANSFER_MBS="$(echo "$line" | awk '{print $(NF-2)}' | sed 's/MB\/s//')"
33+
printf " $TRANSFER_MBS" >> $LOG_FILE
34+
fi
35+
done
36+
}
37+
38+
do_openssh_get_test() {
39+
cp $TEST_FILE $TEST_FILE-out
40+
sftp_command="sftp -P $PORT -i $KEY [email protected]"
41+
output_file="sftp_log.txt"
42+
43+
# Start the script command to capture the sftp session
44+
script -qc "$sftp_command << EOF
45+
get $TEST_FILE $TEST_FILE-out
46+
bye
47+
EOF" /dev/null 2>&1 | tee $output_file | while read line; do
48+
if [[ "$line" == *'MB/s'* ]]; then
49+
#TRANSFER_MBS=$(echo "$line" | awk '{print $(NF-2)}' | sed 's/MB\/s//')
50+
TRANSFER_MBS="$(echo "$line" | awk '{print $(NF-2)}' | sed 's/MB\/s//')"
51+
printf " $TRANSFER_MBS" >> $LOG_FILE
52+
fi
53+
done
54+
}
55+
56+
do_wolfssh_put_test() {
57+
cp $TEST_FILE $TEST_FILE-out
58+
RESULT=$(./examples/sftpclient/wolfsftp -g -l $TEST_FILE -r $TEST_FILE-out -i $PWD/keys/hansel-key-ecc.der -j $PWD/keys/hansel-key-ecc.pub -u jak -p $PORT)
59+
TRANSFER_MBS="$(echo "$RESULT" | awk '{print $(NF-0)}' | sed 's/MB\/s//')"
60+
printf " $TRANSFER_MBS" >> $LOG_FILE
61+
}
62+
63+
do_wolfssh_get_test() {
64+
cp $TEST_FILE $TEST_FILE-out
65+
RESULT=$(./examples/sftpclient/wolfsftp -G -l $TEST_FILE-out -r $TEST_FILE -i $PWD/keys/hansel-key-ecc.der -j $PWD/keys/hansel-key-ecc.pub -u jak -p $PORT)
66+
TRANSFER_MBS="$(echo "$RESULT" | awk '{print $(NF-0)}' | sed 's/MB\/s//')"
67+
printf " $TRANSFER_MBS" >> $LOG_FILE
68+
}
69+
70+
# Create a log with averages
71+
do_create_average() {
72+
awk -F', ' '{sum[$1]+=$2; count[$1]++} END {for (i in sum) print i, sum[i]/count[i]}' "$LOG_FILE" | sort -n > "$AVERAGE_FILE"
73+
sed -i 's/ /, /' $AVERAGE_FILE
74+
}
75+
76+
77+
do_create_plot() {
78+
gnuplot -e "set title '$TITLE';set ylabel 'MB/s';set xlabel 'File Size in Bytes';set grid; set format x \"%2.1t{/Symbol \264}10^{%L}\"; set term png;set output '$OUTPUT_FILE';plot '$LOG_FILE' using 1:2, '$AVERAGE_FILE' with lines lc rgb 'red' lw 2, '$COMPARE_TO' with lines lc rgb 'gold' lw 2"
79+
}
80+
81+
echo "Starting tests"
82+
echo "Getting the average over $NUMBER_RUNS runs"
83+
84+
# create openssh average if not found
85+
AVERAGE_FILE="$PWD/openssh-average-upload.csv"
86+
if [ ! -f "$AVERAGE_FILE" ]; then
87+
echo "Collecting openssh average upload"
88+
rm -f $LOG_FILE && touch $LOG_FILE
89+
for run in $(seq 1 $NUMBER_RUNS); do
90+
printf "Run $run: "
91+
for i in "${FILE_SIZES[@]}"; do
92+
tail -c "$i" /dev/urandom > "$TEST_FILE"
93+
printf "$i," >> $LOG_FILE
94+
do_openssh_put_test
95+
printf "\n" >> $LOG_FILE
96+
done
97+
printf "done\n"
98+
done
99+
100+
do_create_average
101+
echo ""
102+
fi
103+
104+
# create wolfssh average upload
105+
echo "Collecting wolfssh average upload"
106+
rm -f $LOG_FILE && touch $LOG_FILE
107+
for run in $(seq 1 $NUMBER_RUNS); do
108+
printf "Run $run: "
109+
for i in "${FILE_SIZES[@]}"; do
110+
tail -c "$i" /dev/urandom > "$TEST_FILE"
111+
printf "$i," >> $LOG_FILE
112+
do_wolfssh_put_test
113+
printf "\n" >> $LOG_FILE
114+
done
115+
printf "done\n"
116+
done
117+
118+
# compile and plot the results of average upload
119+
AVERAGE_FILE="$PWD/wolfssh-average-upload.csv"
120+
do_create_average
121+
122+
TITLE="SFTP Client Upload Speeds [$NUMBER_RUNS runs]"
123+
COMPARE_TO="$PWD/openssh-average-upload.csv"
124+
AVERAGE_FILE="$PWD/wolfssh-average-upload.csv"
125+
OUTPUT_FILE="$PWD/upload-results.png"
126+
do_create_plot
127+
128+
# create openssh average download if not found
129+
AVERAGE_FILE="$PWD/openssh-average-download.csv"
130+
if [ ! -f "$AVERAGE_FILE" ]; then
131+
echo "Collecting openssh average download"
132+
rm -f $LOG_FILE && touch $LOG_FILE
133+
for run in $(seq 1 $NUMBER_RUNS); do
134+
printf "Run $run: "
135+
for i in "${FILE_SIZES[@]}"; do
136+
tail -c "$i" /dev/urandom > "$TEST_FILE"
137+
printf "$i," >> $LOG_FILE
138+
do_openssh_get_test
139+
printf "\n" >> $LOG_FILE
140+
done
141+
printf "done\n"
142+
done
143+
144+
do_create_average
145+
echo ""
146+
fi
147+
148+
# create wolfssh average download
149+
echo "Collecting wolfssh average download"
150+
rm -f $LOG_FILE && touch $LOG_FILE
151+
for run in $(seq 1 $NUMBER_RUNS); do
152+
printf "Run $run: "
153+
for i in "${FILE_SIZES[@]}"; do
154+
tail -c "$i" /dev/urandom > "$TEST_FILE"
155+
printf "$i," >> $LOG_FILE
156+
do_wolfssh_get_test
157+
printf "\n" >> $LOG_FILE
158+
done
159+
printf "done\n"
160+
done
161+
162+
# compile and plot the results of average download speeds
163+
AVERAGE_FILE="$PWD/wolfssh-average-download.csv"
164+
do_create_average
165+
166+
TITLE="SFTP Client Download Speeds [$NUMBER_RUNS runs]"
167+
COMPARE_TO="$PWD/openssh-average-download.csv"
168+
AVERAGE_FILE="$PWD/wolfssh-average-download.csv"
169+
OUTPUT_FILE="$PWD/download-results.png"
170+
do_create_plot
171+
172+
rm -rf $TEST_FILE
173+
rm -rf $TEST_FILE-out

0 commit comments

Comments
 (0)