Skip to content

Commit 6918c02

Browse files
Nothing4YouNothing4You
committed
reduce matrix test jobs by running multiple databases in a single ci job
1 parent c9e349d commit 6918c02

File tree

1 file changed

+126
-45
lines changed

1 file changed

+126
-45
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 126 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -27,31 +27,85 @@ jobs:
2727
- '3.9'
2828
- '3.10'
2929
# - '3.11.0-alpha.4'
30-
db:
31-
- [mysql, '5.7']
32-
- [mysql, '8.0']
33-
- [mariadb, '10.2']
34-
- [mariadb, '10.3']
35-
- [mariadb, '10.4']
36-
- [mariadb, '10.5']
37-
- [mariadb, '10.6']
38-
- [mariadb, '10.7']
3930

4031
fail-fast: false
4132
runs-on: ${{ matrix.os }}
42-
timeout-minutes: 15
33+
timeout-minutes: 45
4334

4435
env:
4536
MYSQL_ROOT_PASSWORD: rootpw
4637

4738
services:
48-
mysql:
49-
image: "${{ join(matrix.db, ':') }}"
39+
mysql57:
40+
image: "mysql:5.7"
5041
ports:
5142
- 3306:3306
5243
volumes:
53-
- "/tmp/run-${{ join(matrix.db, '-') }}/:/socket-mount/"
54-
options: '--name=mysqld'
44+
- "/tmp/run-mysql57/:/socket-mount/"
45+
options: '--name=mysql57'
46+
env:
47+
MYSQL_ROOT_PASSWORD: rootpw
48+
mysql80:
49+
image: "mysql:8.0"
50+
ports:
51+
- 3307:3306
52+
volumes:
53+
- "/tmp/run-mysql80/:/socket-mount/"
54+
options: '--name=mysql80'
55+
env:
56+
MYSQL_ROOT_PASSWORD: rootpw
57+
mariadb102:
58+
image: "mariadb:10.2"
59+
ports:
60+
- 3308:3306
61+
volumes:
62+
- "/tmp/run-mariadb102/:/socket-mount/"
63+
options: '--name=mariadb102'
64+
env:
65+
MYSQL_ROOT_PASSWORD: rootpw
66+
mariadb103:
67+
image: "mariadb:10.3"
68+
ports:
69+
- 3309:3306
70+
volumes:
71+
- "/tmp/run-mariadb103/:/socket-mount/"
72+
options: '--name=mariadb103'
73+
env:
74+
MYSQL_ROOT_PASSWORD: rootpw
75+
mariadb104:
76+
image: "mariadb:10.4"
77+
ports:
78+
- 3310:3306
79+
volumes:
80+
- "/tmp/run-mariadb104/:/socket-mount/"
81+
options: '--name=mariadb104'
82+
env:
83+
MYSQL_ROOT_PASSWORD: rootpw
84+
mariadb105:
85+
image: "mariadb:10.5"
86+
ports:
87+
- 3311:3306
88+
volumes:
89+
- "/tmp/run-mariadb105/:/socket-mount/"
90+
options: '--name=mariadb105'
91+
env:
92+
MYSQL_ROOT_PASSWORD: rootpw
93+
mariadb106:
94+
image: "mariadb:10.6"
95+
ports:
96+
- 3312:3306
97+
volumes:
98+
- "/tmp/run-mariadb106/:/socket-mount/"
99+
options: '--name=mariadb106'
100+
env:
101+
MYSQL_ROOT_PASSWORD: rootpw
102+
mariadb107:
103+
image: "mariadb:10.7"
104+
ports:
105+
- 3313:3306
106+
volumes:
107+
- "/tmp/run-mariadb107/:/socket-mount/"
108+
options: '--name=mariadb107'
55109
env:
56110
MYSQL_ROOT_PASSWORD: rootpw
57111

@@ -95,53 +149,80 @@ jobs:
95149
# unfortunately we need this hacky workaround as GitHub Actions service containers can't reference data from our repo.
96150
- name: Prepare mysql
97151
run: |
98-
# ensure server is started up
99-
while :
152+
declare -A DATABASES=(
153+
[mysql57]=3306
154+
[mysql80]=3307
155+
[mariadb102]=3308
156+
[mariadb103]=3309
157+
[mariadb104]=3310
158+
[mariadb105]=3311
159+
[mariadb106]=3312
160+
[mariadb107]=3313
161+
)
162+
163+
PYTEST_CONNECTION_ARGS=""
164+
165+
for db in "${!DATABASES[@]}"
100166
do
101-
sleep 1
102-
mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e 'select version()' && break
167+
# add connection argument for pytest
168+
PYTEST_CONNECTION_ARGS="$PYTEST_CONNECTION_ARGS --mysql-unix-socket "unix-$db=/tmp/run-$db/mysql.sock" --mysql-address "tcp-$db=127.0.0.1:${DATABASES[$db]}"
169+
170+
# ensure server is started up
171+
while :
172+
do
173+
mysql -h127.0.0.1 "-P${DATABASES[$db]}" -uroot "-p$MYSQL_ROOT_PASSWORD" -e 'select version()' && break
174+
sleep 1
175+
done
103176
done
104177
105-
# inject tls configuration
106-
docker container stop mysqld
107-
docker container cp "${{ github.workspace }}/tests/ssl_resources/ssl" mysqld:/etc/mysql/ssl
108-
docker container cp "${{ github.workspace }}/tests/ssl_resources/tls.cnf" mysqld:/etc/mysql/conf.d/aiomysql-tls.cnf
109-
110-
# use custom socket path
111-
# we need to ensure that the socket path is writable for the user running the DB process in the container
112-
sudo chmod 0777 /tmp/run-${{ join(matrix.db, '-') }}
178+
tee -a "$GITHUB_ENV" "PYTEST_CONNECTION_ARGS=$PYTEST_CONNECTION_ARGS"
113179
114-
# mysql 5.7 container overrides the socket path in /etc/mysql/mysql.conf.d/mysqld.cnf
115-
if [ "${{ join(matrix.db, '-') }}" = "mysql-5.7" ]
116-
then
117-
docker container cp "${{ github.workspace }}/tests/ssl_resources/socket.cnf" mysqld:/etc/mysql/mysql.conf.d/zz-aiomysql-socket.cnf
118-
else
119-
docker container cp "${{ github.workspace }}/tests/ssl_resources/socket.cnf" mysqld:/etc/mysql/conf.d/aiomysql-socket.cnf
120-
fi
121-
122-
docker container start mysqld
123-
124-
# ensure server is started up
125-
while :
180+
for db in "${!DATABASES[@]}"
126181
do
127-
sleep 1
128-
mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e 'select version()' && break
182+
# inject tls configuration
183+
docker container stop "$db"
184+
docker container cp "${{ github.workspace }}/tests/ssl_resources/ssl" "$db:/etc/mysql/ssl"
185+
docker container cp "${{ github.workspace }}/tests/ssl_resources/tls.cnf" "$db:/etc/mysql/conf.d/aiomysql-tls.cnf"
186+
187+
# use custom socket path
188+
# we need to ensure that the socket path is writable for the user running the DB process in the container
189+
sudo chmod 0777 "/tmp/run-$db"
190+
191+
# mysql 5.7 container overrides the socket path in /etc/mysql/mysql.conf.d/mysqld.cnf
192+
if [ "$db" = "mysql57" ]
193+
then
194+
docker container cp "${{ github.workspace }}/tests/ssl_resources/socket.cnf" "$db:/etc/mysql/mysql.conf.d/zz-aiomysql-socket.cnf"
195+
else
196+
docker container cp "${{ github.workspace }}/tests/ssl_resources/socket.cnf" "$db:/etc/mysql/conf.d/aiomysql-socket.cnf"
197+
fi
198+
199+
docker container start "$db"
129200
done
130201
131-
mysql -h127.0.0.1 -uroot "-p$MYSQL_ROOT_PASSWORD" -e "SET GLOBAL local_infile=on"
202+
for db in "${!DATABASES[@]}"
203+
do
204+
# ensure server is started up
205+
while :
206+
do
207+
mysql -h127.0.0.1 "-P${DATABASES[$db]}" -uroot "-p$MYSQL_ROOT_PASSWORD" -e 'select version()' && break
208+
sleep 1
209+
done
210+
211+
mysql -h127.0.0.1 "-P${DATABASES[$db]}" -uroot "-p$MYSQL_ROOT_PASSWORD" -e "SET GLOBAL local_infile=on"
212+
done
132213
133214
- name: Run tests
134215
run: |
135216
# timeout ensures a more or less clean stop by sending a KeyboardInterrupt which will still provide useful logs
136-
timeout --preserve-status --signal=INT --verbose 5m \
137-
pytest --color=yes --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql --cov tests ./tests --mysql-unix-socket "unix-${{ join(matrix.db, '') }}=/tmp/run-${{ join(matrix.db, '-') }}/mysql.sock" --mysql-address "tcp-${{ join(matrix.db, '') }}=127.0.0.1:3306"
217+
timeout --preserve-status --signal=INT --verbose 30m \
218+
pytest --color=yes --capture=no --verbosity 2 --cov-report term --cov-report xml --cov aiomysql --cov tests ./tests $PYTEST_CONNECTION_ARGS
138219
env:
139220
PYTHONUNBUFFERED: 1
140-
timeout-minutes: 6
221+
timeout-minutes: 31
141222

142223
- name: Upload coverage
143224
uses: codecov/[email protected]
144225
with:
145226
file: ./coverage.xml
146-
flags: "${{ matrix.os }}_${{ matrix.py }}_${{ join(matrix.db, '-') }}"
227+
flags: "${{ matrix.os }}_${{ matrix.py }}"
147228
fail_ci_if_error: true

0 commit comments

Comments
 (0)