Skip to content

Commit 1fec615

Browse files
nbyavuznbyavuz
committed
meson: ci: test extensions
1 parent dc1866e commit 1fec615

File tree

3 files changed

+283
-1
lines changed

3 files changed

+283
-1
lines changed

‎.cirrus.yml

+17-1
Original file line numberDiff line numberDiff line change
@@ -358,16 +358,32 @@ task:
358358
task:
359359
<<: *linux_opensuse_template
360360

361-
name: Linux - OpenSuse Tumbleweed (LLVM) - Meson
361+
name: Linux - OpenSuse Tumbleweed (LLVM) - Meson - Test Extensions
362362

363363
configure_script:
364364
- su postgres -c 'meson setup --buildtype debug -Dcassert=true -Dssl=openssl -Duuid=e2fs -Dllvm=enabled build'
365365

366+
###
367+
# build & run tests
366368
build_script: su postgres -c 'ninja -C build'
369+
367370
upload_caches: ccache
368371

369372
tests_world_script:
370373
- su postgres -c 'meson test --no-rebuild -C build --num-processes ${TEST_JOBS}'
374+
###
375+
376+
###
377+
# test extensions
378+
install_script: ninja -C build install
379+
380+
# some of the make commands of test_execution script needs to run with sudo,
381+
# that command makes postgres user can execute sudo commands without password
382+
dont_ask_sudo_pw_script:
383+
- echo "postgres ALL=(ALL:ALL) NOPASSWD: ALL" | tee /etc/sudoers.d/postgres
384+
385+
install_extensions_script: su postgres -c 'python3 src/tools/ci/test_extensions'
386+
###
371387

372388
on_failure:
373389
<<: *on_failure

‎src/tools/ci/docker/linux_opensuse_tumbleweed

+13
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,19 @@ RUN \
4949
lz4 \
5050
zstd \
5151
\
52+
### packages needed by extensions ###
53+
sudo \
54+
\
55+
# pgbouncer
56+
libevent-devel \
57+
pandoc \
58+
\
59+
# postgis
60+
geos-devel \
61+
proj-devel \
62+
protobuf-c \
63+
\
64+
###
5265
&& \
5366
zypper -n clean -a
5467

‎src/tools/ci/test_extensions

+253
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,253 @@
1+
#!/usr/bin/env python3
2+
3+
import enum
4+
import os
5+
import shutil
6+
import subprocess
7+
import sys
8+
9+
work_dir = '/tmp/extension_test/'
10+
extensions = []
11+
git_clone_command = ['git', 'clone', '--depth', '1']
12+
13+
class ExitStatus(enum.IntEnum):
14+
success = 0
15+
missing_steps = 1
16+
failed = 2
17+
ext_exit_status = []
18+
19+
# add pg_qualstats
20+
extensions.append(
21+
{
22+
'name': 'pg_qualstats',
23+
'shared_preload_name': 'pg_qualstats',
24+
'download': [
25+
git_clone_command + ['https://github.com/powa-team/pg_qualstats.git'],
26+
],
27+
'build': [
28+
['sudo', 'make', 'install'],
29+
],
30+
'test': [
31+
['make', 'installcheck'],
32+
],
33+
}
34+
)
35+
36+
37+
# add pg_cron
38+
extensions.append(
39+
{
40+
'name': 'pg_cron',
41+
'shared_preload_name': 'pg_cron',
42+
'download': [
43+
git_clone_command + ['https://github.com/citusdata/pg_cron.git'],
44+
],
45+
'build': [
46+
['make'],
47+
['sudo', 'make', 'install'],
48+
],
49+
'test': [
50+
['make', 'installcheck'],
51+
],
52+
}
53+
)
54+
55+
# add hypopg
56+
extensions.append(
57+
{
58+
'name': 'hypopg',
59+
'download': [
60+
git_clone_command + ['https://github.com/HypoPG/hypopg.git'],
61+
],
62+
'build': [
63+
['sudo', 'make', 'install'],
64+
],
65+
'test': [
66+
['make', 'installcheck'],
67+
],
68+
}
69+
)
70+
71+
# add postgresql-hll
72+
extensions.append(
73+
{
74+
'name': 'postgresql-hll',
75+
'download': [
76+
git_clone_command + ['https://github.com/citusdata/postgresql-hll.git'],
77+
],
78+
'build': [
79+
['make'],
80+
['sudo', 'make', 'install'],
81+
],
82+
'test': [
83+
['make', 'installcheck'],
84+
],
85+
}
86+
)
87+
88+
# add orafce
89+
extensions.append(
90+
{
91+
'name': 'orafce',
92+
'download': [
93+
git_clone_command + ['https://github.com/orafce/orafce.git'],
94+
],
95+
'build': [
96+
['make'],
97+
['make', 'install'],
98+
],
99+
'test': [
100+
['make', 'installcheck'],
101+
],
102+
}
103+
)
104+
105+
# add postgis
106+
extensions.append(
107+
{
108+
'name': 'postgis',
109+
'download': [
110+
git_clone_command + ['https://github.com/postgis/postgis.git'],
111+
],
112+
'build': [
113+
['./autogen.sh'],
114+
['./configure', '--without-raster'],
115+
['make'],
116+
],
117+
'test': [
118+
['make', 'check'],
119+
],
120+
}
121+
)
122+
123+
# add pg_partman
124+
# there is confirmed pg15 bug, waiting for fix
125+
# see https://github.com/pgpartman/pg_partman/issues/464
126+
# extensions.append(
127+
# {
128+
# 'name': 'pg_partman',
129+
# 'shared_preload_name': 'pg_partman_bgw',
130+
# 'download': [
131+
# git_clone_command + ['https://github.com/pgpartman/pg_partman.git'],
132+
# ],
133+
# 'build': [
134+
# ['make'],
135+
# ['sudo', 'make', 'install'],
136+
# ],
137+
# 'test': [
138+
# # there is no test command,
139+
# # see bottom of https://github.com/pgpartman/pg_partman
140+
# ],
141+
# }
142+
# )
143+
144+
# add pgbouncer
145+
extensions.append(
146+
{
147+
'name': 'pgbouncer',
148+
'download': [
149+
git_clone_command + ['https://github.com/pgbouncer/pgbouncer.git'],
150+
],
151+
'build': [
152+
['git', 'submodule', 'init'],
153+
['git', 'submodule', 'update'],
154+
['./autogen.sh'],
155+
['./configure', '--prefix=/usr/local'],
156+
['make'],
157+
['sudo', 'make', 'install'],
158+
],
159+
'test': [
160+
['make', '-C', './test', 'all'],
161+
['make', '-C', './test', 'check'],
162+
],
163+
}
164+
)
165+
166+
def start_postgres_server():
167+
shared_preload_libraries = ''
168+
169+
# get shared_preload_libraries
170+
for extension in extensions:
171+
if 'shared_preload_name' in extension:
172+
shared_preload_libraries = shared_preload_libraries + extension['shared_preload_name'] + ', '
173+
# removing last ', '
174+
shared_preload_libraries = shared_preload_libraries[:-2]
175+
176+
postgres_start_commands = [
177+
['sudo', 'mkdir', '-p', '/usr/local/pgsql/data'],
178+
['sudo', 'chown', 'postgres', '/usr/local/pgsql/data'],
179+
['/usr/local/bin/initdb', '-D', '/usr/local/pgsql/data/'],
180+
['sed', '-i',
181+
's/#shared_preload_libraries = \'\'/shared_preload_libraries = \'' + shared_preload_libraries + '\'/g',
182+
'/usr/local/pgsql/data/postgresql.conf'],
183+
['/usr/local/bin/pg_ctl', '-D', '/usr/local/pgsql/data', '-l', 'logfile', 'start'],
184+
]
185+
186+
# run postgres server
187+
print('Starting postgres server...', flush=True)
188+
for command in postgres_start_commands:
189+
subprocess.run(command, check=True)
190+
print('Done.', flush=True)
191+
192+
# if that function returns True, that means current process is successfully finished
193+
# otherwise, there can be error or missing commands
194+
def run_commands(extension, current_process, cwd):
195+
if current_process in extension and extension[current_process]:
196+
print('\n\n### {} {} ###\n'.format(current_process, extension['name']), flush=True)
197+
198+
for command in extension[current_process]:
199+
result = subprocess.run(command, cwd=cwd)
200+
201+
if result.returncode:
202+
failed_step_message = '`{}` is failed at step `{}`'.format(extension['name'], current_process)
203+
ext_exit_status.append([failed_step_message, ExitStatus.failed])
204+
print('\n##### {} #####\n\n'.format(failed_step_message), flush=True)
205+
return False
206+
else:
207+
missing_step_message = 'No `{}` command found for `{}`'.format(current_process, extension['name'])
208+
ext_exit_status.append([missing_step_message, ExitStatus.missing_steps])
209+
print('\n\n### {} ###\n'.format(missing_step_message), flush=True)
210+
return False
211+
return True
212+
213+
def clear_directory(directory):
214+
if os.path.exists(directory) and os.path.isdir(directory):
215+
print('### Clearing {} ###'.format(directory), flush=True)
216+
shutil.rmtree(directory)
217+
218+
def main():
219+
# clear and create work_dir
220+
clear_directory(work_dir)
221+
os.makedirs(work_dir)
222+
223+
# first download then build to get shared_preload_libraries
224+
for extension in extensions:
225+
print('\n\n##### {} #####\n'.format(extension['name']), flush=True)
226+
227+
extension_dir = work_dir + extension['name']
228+
clear_directory(extension_dir)
229+
230+
process_order = [['download', work_dir], ['build', extension_dir]]
231+
for process, cwd in process_order:
232+
if not run_commands(extension, process, cwd):
233+
extension['has_error'] = True
234+
break
235+
236+
# start postgres server
237+
start_postgres_server()
238+
239+
# run tests
240+
for extension in extensions:
241+
if 'has_error' not in extension or not extension['has_error']:
242+
if run_commands(extension, 'test', work_dir + extension['name']):
243+
successfully_finished_message = '`{}` is finished successfully'.format(extension['name'])
244+
ext_exit_status.append([successfully_finished_message, ExitStatus.success])
245+
print('\n##### {} #####\n\n'.format(successfully_finished_message), flush=True)
246+
247+
# sort by exit status(success, missing step, failed)
248+
ext_exit_status.sort(key=lambda tup: tup[1])
249+
for message, _ in ext_exit_status:
250+
print(message, flush=True)
251+
252+
if __name__ == "__main__":
253+
main()

0 commit comments

Comments
 (0)