Skip to content

Commit 3d373c1

Browse files
nbyavuznbyavuz
committed
meson: ci: test extensions
1 parent d1491d3 commit 3d373c1

File tree

2 files changed

+355
-0
lines changed

2 files changed

+355
-0
lines changed

‎.cirrus.yml

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -446,6 +446,61 @@ task:
446446
on_failure:
447447
<<: *on_failure_ac
448448

449+
- name: Linux - Debian Bullseye - Meson - Test Extensions
450+
env:
451+
INSTALL_DIR: ${CIRRUS_WORKING_DIR}/meson-install
452+
PATH: ${INSTALL_DIR}/bin:${PATH}
453+
454+
allow_failures: true
455+
depends_on:
456+
- Linux - Debian Bullseye - Meson
457+
458+
configure_script: |
459+
su postgres <<-EOF
460+
meson setup \
461+
--buildtype=debug \
462+
--prefix=${INSTALL_DIR} \
463+
-Dcassert=true \
464+
${LINUX_MESON_FEATURES} \
465+
-DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
466+
build
467+
EOF
468+
build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}'
469+
470+
install_script: su postgres -c 'ninja -C build install'
471+
472+
install_required_packages_for_extensions_script: |
473+
apt-get update
474+
DEBIAN_FRONTEND=noninteractive apt-get -y install \
475+
autoconf \
476+
libtool \
477+
libgeos-dev \
478+
libproj-dev \
479+
libprotobuf-c-dev \
480+
protobuf-c-compiler
481+
482+
always:
483+
test_pg_qualstats_script: su postgres -c 'python3 src/tools/ci/test_extensions --only pg_qualstats'
484+
test_pg_cron_script: su postgres -c 'python3 src/tools/ci/test_extensions --only pg_cron'
485+
test_hypopg_script: su postgres -c 'python3 src/tools/ci/test_extensions --only hypopg'
486+
test_orafce_script: su postgres -c 'python3 src/tools/ci/test_extensions --only orafce'
487+
test_postgis_script: |
488+
su postgres <<-EOF
489+
export LD_LIBRARY_PATH=${INSTALL_DIR}/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
490+
python3 src/tools/ci/test_extensions --only postgis
491+
EOF
492+
test_pg_partman_script: su postgres -c 'python3 src/tools/ci/test_extensions --only pg_partman'
493+
test_pgbouncer_script: su postgres -c 'python3 src/tools/ci/test_extensions --only pgbouncer'
494+
495+
on_failure:
496+
testrun_artifacts:
497+
paths:
498+
- "extension_test/**/*.log"
499+
- "extension_test/**/*.diffs"
500+
- "extension_test/**/regress_log_*"
501+
- "*-logfile"
502+
type: text/plain
503+
449504
- matrix:
450505
- name: Linux - Debian Bullseye - Meson
451506
- name: Linux - Debian Sid - Meson

‎src/tools/ci/test_extensions

Lines changed: 300 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,300 @@
1+
#!/usr/bin/env python3
2+
3+
import argparse
4+
import enum
5+
import os
6+
import shutil
7+
import subprocess
8+
import sys
9+
10+
# set default workdir to .../${root_path_of_branch}/extension_test
11+
# with the using relative path to the current file
12+
current_file_path = os.path.dirname(os.path.abspath(__file__))
13+
default_workdir = os.path.join(current_file_path, '../../../extension_test')
14+
15+
parser = argparse.ArgumentParser()
16+
17+
parser.add_argument('--only', type=str, required=False, default=[], nargs='+')
18+
parser.add_argument('--workdir', type=str,
19+
required=False, default=default_workdir)
20+
args = parser.parse_args()
21+
22+
only_run_ext = args.only
23+
workdir = args.workdir
24+
25+
extensions = []
26+
git_clone_command = ['git', 'clone', '--depth', '1']
27+
ext_exit_status = []
28+
29+
30+
class ExitStatus(enum.IntEnum):
31+
success = 0
32+
missing_steps = 1
33+
failed = 2
34+
35+
36+
# add pg_qualstats
37+
extensions.append(
38+
{
39+
'name': 'pg_qualstats',
40+
'shared_preload_name': 'pg_qualstats',
41+
'download': [
42+
git_clone_command
43+
+ ['https://github.com/powa-team/pg_qualstats.git'],
44+
],
45+
'build': [
46+
['make', 'install'],
47+
],
48+
'test': [
49+
['make', 'installcheck'],
50+
],
51+
}
52+
)
53+
54+
55+
# add pg_cron
56+
extensions.append(
57+
{
58+
'name': 'pg_cron',
59+
'shared_preload_name': 'pg_cron',
60+
'download': [
61+
git_clone_command + ['https://github.com/citusdata/pg_cron.git'],
62+
],
63+
'build': [
64+
['make', 'install'],
65+
],
66+
'test': [
67+
['make', 'installcheck'],
68+
],
69+
}
70+
)
71+
72+
# add hypopg
73+
extensions.append(
74+
{
75+
'name': 'hypopg',
76+
'download': [
77+
git_clone_command + ['https://github.com/HypoPG/hypopg.git'],
78+
],
79+
'build': [
80+
['make', 'install'],
81+
],
82+
'test': [
83+
['make', 'installcheck'],
84+
],
85+
}
86+
)
87+
88+
# add orafce
89+
extensions.append(
90+
{
91+
'name': 'orafce',
92+
'download': [
93+
git_clone_command + ['https://github.com/orafce/orafce.git'],
94+
],
95+
'build': [
96+
['make'],
97+
['make', 'install'],
98+
],
99+
'test': [
100+
['make', 'installcheck'],
101+
],
102+
}
103+
)
104+
105+
# add postgis
106+
extensions.append(
107+
{
108+
'name': 'postgis',
109+
'download': [
110+
git_clone_command + ['https://github.com/postgis/postgis.git'],
111+
],
112+
'build': [
113+
['./autogen.sh'],
114+
['./configure', '--without-raster'],
115+
['make'],
116+
],
117+
'test': [
118+
['make', 'check'],
119+
],
120+
}
121+
)
122+
123+
# add pg_partman
124+
extensions.append(
125+
{
126+
'name': 'pg_partman',
127+
'shared_preload_name': 'pg_partman_bgw',
128+
'download': [
129+
git_clone_command
130+
+ ['https://github.com/pgpartman/pg_partman.git'],
131+
],
132+
'build': [
133+
['make'],
134+
['make', 'install'],
135+
],
136+
'test': [
137+
# there is no test command,
138+
# see bottom of https://github.com/pgpartman/pg_partman
139+
],
140+
}
141+
)
142+
143+
# add timescaledb
144+
# doesn't support PG 15 yet
145+
# extensions.append(
146+
# {
147+
# 'name': 'timescaledb',
148+
# 'shared_preload_name': 'timescaledb',
149+
# 'download': [
150+
# ['git', 'clone', 'https://github.com/timescale/timescaledb.git'],
151+
# ],
152+
# 'build': [
153+
# ['./bootstrap'],
154+
# ['make', '-C', 'build'],
155+
# ['make', '-C', 'build', 'install'],
156+
# ],
157+
# 'test': [
158+
# ['make', 'check'],
159+
# ],
160+
# }
161+
# )
162+
163+
# add pgbouncer
164+
extensions.append(
165+
{
166+
'name': 'pgbouncer',
167+
'download': [
168+
git_clone_command
169+
+ ['https://github.com/pgbouncer/pgbouncer.git'],
170+
],
171+
'build': [
172+
['git', 'submodule', 'init'],
173+
['git', 'submodule', 'update'],
174+
['./autogen.sh'],
175+
['./configure', '--prefix=/home/naziryavuz/projects/install-meson'],
176+
['make'],
177+
['make', 'install'],
178+
],
179+
'test': [
180+
['make', '-C', './test', 'all'],
181+
['make', '-C', './test', 'check'],
182+
],
183+
}
184+
)
185+
186+
# filter elements of only-run-ext list
187+
if only_run_ext:
188+
# check if all elements of only-list exists
189+
ext_names = [elem['name'] for elem in extensions]
190+
if not set(only_run_ext).issubset(ext_names):
191+
sys.exit('There are unsupported extensions. Supported extensions are: '
192+
+ ' '.join(ext_names))
193+
194+
extensions = [ext for ext in extensions if ext['name'] in only_run_ext]
195+
196+
197+
def stop_and_start_postgres_server(logfile_prefix='',
198+
shared_preload_string=''):
199+
pgsql_data_path = '/tmp/pgsql/data'
200+
logfile_name = '{}-logfile'.format(logfile_prefix).strip('-')
201+
postgresql_conf = pgsql_data_path + '/postgresql.conf'
202+
to_write_pgconf = ''
203+
204+
postgres_server_commands = [
205+
['rm', '-rf', pgsql_data_path],
206+
['mkdir', '-p', pgsql_data_path],
207+
['initdb', '-D', pgsql_data_path],
208+
['pg_ctl', '-D', pgsql_data_path, '-l', logfile_name, 'start'],
209+
]
210+
211+
if os.path.isdir(pgsql_data_path):
212+
postgres_server_commands.insert(0,
213+
['pg_ctl', 'stop', '-D', pgsql_data_path])
214+
215+
if shared_preload_string:
216+
to_write_pgconf = to_write_pgconf + 'shared_preload_libraries = \
217+
\'{}\''.format(shared_preload_string)
218+
219+
# start and stop postgres server
220+
print('Stopping and starting postgres server...\n\n', flush=True)
221+
for i, command in enumerate(postgres_server_commands):
222+
# appending config contents before starting server
223+
if i == len(postgres_server_commands) - 1:
224+
with open(postgresql_conf, 'a') as pg_conf:
225+
pg_conf.write(to_write_pgconf)
226+
227+
if subprocess.run(command).returncode:
228+
sys.exit('\nCan not stop and start postgres server.')
229+
print('Done.', flush=True)
230+
231+
232+
def run_commands(extension, current_process, cwd):
233+
ext_name = extension['name']
234+
ext_command = extension.get(current_process, [])
235+
236+
if extension.get('exit_status', False):
237+
return
238+
elif ext_command:
239+
print('\n\n### `{}` `{}` ###\n'.format(
240+
current_process, ext_name), flush=True)
241+
242+
for command in ext_command:
243+
print('Command = {}'.format(command))
244+
result = subprocess.run(command, cwd=cwd)
245+
246+
if result.returncode:
247+
extension['message'] = '`{}` failed at step `{}`'.format(
248+
ext_name, current_process)
249+
extension['exit_status'] = ExitStatus.failed
250+
else:
251+
extension['message'] = '`{}` finished successfully'.format(
252+
ext_name)
253+
extension['exit_status'] = ExitStatus.success
254+
else:
255+
extension['message'] = 'No `{}` command found for `{}`'.format(
256+
current_process, ext_name)
257+
extension['exit_status'] = ExitStatus.missing_steps
258+
259+
print('\n\n### {} ###\n'.format(extension['message']), flush=True)
260+
261+
262+
def clear_directory(directory):
263+
if os.path.exists(directory) and os.path.isdir(directory):
264+
print('### Clearing {} ###'.format(directory), flush=True)
265+
shutil.rmtree(directory)
266+
267+
268+
def main():
269+
for extension in extensions:
270+
ext_name = extension['name']
271+
ext_workdir = os.path.join(workdir, ext_name)
272+
273+
clear_directory(ext_workdir)
274+
os.makedirs(ext_workdir)
275+
276+
run_commands(extension, 'download', workdir)
277+
run_commands(extension, 'build', ext_workdir)
278+
279+
stop_and_start_postgres_server(
280+
ext_name, extension.get('shared_preload_name', ''))
281+
282+
run_commands(extension, 'test', ext_workdir)
283+
284+
# sort by exit status(success, missing step, failed)
285+
extensions_sorted = sorted(extensions, key=lambda d: d['exit_status'])
286+
287+
previous_exit_status = extensions_sorted[0]['exit_status']
288+
print('\nScript finished, results are:\n', flush=True)
289+
for extension in extensions_sorted:
290+
if extension['exit_status'] != previous_exit_status:
291+
print('\n', flush=True)
292+
print(extension['message'], flush=True)
293+
previous_exit_status = extension['exit_status']
294+
295+
if previous_exit_status == ExitStatus.failed:
296+
sys.exit('\nThere are failed extensions, exiting')
297+
298+
299+
if __name__ == "__main__":
300+
main()

0 commit comments

Comments
 (0)