Skip to content

Commit f5cf897

Browse files
kumaabkumaab
authored
RANGER-5395: Split docker builds across GitHub runners with ranger-service and ranger-plugin builds (#727)
1 parent 4008d96 commit f5cf897

File tree

2 files changed

+89
-20
lines changed

2 files changed

+89
-20
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 86 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ jobs:
101101
name: target-11
102102
path: target/*
103103

104-
docker-build:
104+
services-docker-build:
105105
needs:
106106
- build-8
107107
runs-on: ubuntu-24.04 # bumping os version for more disk space
@@ -115,8 +115,86 @@ jobs:
115115

116116
- name: Copy artifacts for docker build
117117
run: |
118-
cp ranger-*.tar.gz dev-support/ranger-docker/dist
119-
cp version dev-support/ranger-docker/dist
118+
mv ranger-*-admin.tar.gz dev-support/ranger-docker/dist
119+
mv ranger-*-usersync.tar.gz dev-support/ranger-docker/dist
120+
mv ranger-*-tagsync.tar.gz dev-support/ranger-docker/dist
121+
mv ranger-*-kms.tar.gz dev-support/ranger-docker/dist
122+
mv version dev-support/ranger-docker/dist
123+
rm -f ranger-*.tar.gz # clean up workspace
124+
125+
- name: Free up disk space
126+
run: docker system prune --all --force --volumes
127+
128+
- name: Download DB and Log4J connectors
129+
run: |
130+
cd dev-support/ranger-docker
131+
./download-archives.sh none
132+
133+
- name: Build all ranger-service images
134+
run: |
135+
cd dev-support/ranger-docker
136+
export RANGER_DB_TYPE=postgres
137+
docker compose \
138+
-f docker-compose.ranger.yml \
139+
-f docker-compose.ranger-usersync.yml \
140+
-f docker-compose.ranger-tagsync.yml \
141+
-f docker-compose.ranger-kms.yml build
142+
143+
- name: Bring up ranger-service containers
144+
run: |
145+
cd dev-support/ranger-docker
146+
export RANGER_DB_TYPE=postgres
147+
docker compose \
148+
-f docker-compose.ranger.yml \
149+
-f docker-compose.ranger-usersync.yml \
150+
-f docker-compose.ranger-tagsync.yml \
151+
-f docker-compose.ranger-kms.yml up -d
152+
153+
- name: Check status of containers and remove them
154+
run: |
155+
sleep 60
156+
containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms);
157+
flag=true;
158+
for container in "${containers[@]}"; do
159+
if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then
160+
echo "Container $container is running!";
161+
else
162+
flag=false;
163+
echo "Container $container is NOT running! Status: $(docker inspect -f '{{.State.Status}}' $container 2>/dev/null)";
164+
echo "docker logs $container:";
165+
docker logs $container
166+
fi
167+
done
168+
169+
if [[ $flag == true ]]; then
170+
echo "All required containers are up and running";
171+
docker stop $(docker ps -q) && docker rm $(docker ps -aq);
172+
else
173+
docker stop $(docker ps -q) && docker rm $(docker ps -aq);
174+
exit 1;
175+
fi
176+
177+
plugins-docker-build:
178+
needs:
179+
- build-8
180+
runs-on: ubuntu-24.04 # bumping os version for more disk space
181+
timeout-minutes: 60 # bumping timeout to allow containers to come up
182+
steps:
183+
- uses: actions/checkout@v4
184+
- name: Download build-8 artifacts
185+
uses: actions/download-artifact@v4
186+
with:
187+
name: target-8
188+
189+
- name: Copy artifacts for docker build
190+
run: |
191+
mv ranger-*-admin.tar.gz dev-support/ranger-docker/dist
192+
mv ranger-*-plugin.tar.gz dev-support/ranger-docker/dist
193+
mv version dev-support/ranger-docker/dist
194+
rm -f ranger-*.tar.gz # clean up workspace
195+
196+
- name: Free up disk space
197+
run: docker system prune --all --force --volumes
120198

121199
- name: Cache downloaded archives
122200
uses: actions/cache@v4
@@ -130,36 +208,27 @@ jobs:
130208
run: |
131209
cd dev-support/ranger-docker
132210
./download-archives.sh hadoop hive hbase kafka knox ozone
133-
134-
- name: Clean up Docker space
135-
run: docker system prune --all --force --volumes
136-
137-
- name: Build all ranger-service images
211+
212+
- name: Build all ranger-plugin images
138213
run: |
139214
cd dev-support/ranger-docker
140215
export RANGER_DB_TYPE=postgres
141216
docker compose \
142217
-f docker-compose.ranger.yml \
143-
-f docker-compose.ranger-usersync.yml \
144-
-f docker-compose.ranger-tagsync.yml \
145-
-f docker-compose.ranger-kms.yml \
146218
-f docker-compose.ranger-hadoop.yml \
147219
-f docker-compose.ranger-hbase.yml \
148220
-f docker-compose.ranger-kafka.yml \
149221
-f docker-compose.ranger-hive.yml \
150222
-f docker-compose.ranger-knox.yml \
151223
-f docker-compose.ranger-ozone.yml build
152224
153-
- name: Bring up containers
225+
- name: Bring up ranger-plugin containers
154226
run: |
155227
cd dev-support/ranger-docker
156228
./scripts/ozone/ozone-plugin-docker-setup.sh
157229
export RANGER_DB_TYPE=postgres
158230
docker compose \
159231
-f docker-compose.ranger.yml \
160-
-f docker-compose.ranger-usersync.yml \
161-
-f docker-compose.ranger-tagsync.yml \
162-
-f docker-compose.ranger-kms.yml \
163232
-f docker-compose.ranger-hadoop.yml \
164233
-f docker-compose.ranger-hbase.yml \
165234
-f docker-compose.ranger-kafka.yml \
@@ -170,7 +239,7 @@ jobs:
170239
- name: Check status of containers and remove them
171240
run: |
172241
sleep 60
173-
containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode);
242+
containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode);
174243
flag=true;
175244
for container in "${containers[@]}"; do
176245
if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then
@@ -182,7 +251,7 @@ jobs:
182251
docker logs $container
183252
fi
184253
done
185-
254+
186255
if [[ $flag == true ]]; then
187256
echo "All required containers are up and running";
188257
docker stop $(docker ps -q) && docker rm $(docker ps -aq);

‎dev-support/ranger-docker/docker-compose.ranger-hadoop.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,10 @@ services:
2727
ranger:
2828
condition: service_started
2929
healthcheck:
30-
test: 'su -c "/home/ranger/scripts/ranger-hadoop-healthcheck.sh" healthcheck'
31-
interval: 15s
30+
test: ["CMD-SHELL", "/home/ranger/scripts/ranger-hadoop-healthcheck.sh"]
31+
interval: 20s
3232
timeout: 10s
33-
retries: 30
33+
retries: 40
3434
start_period: 40s
3535
environment:
3636
- HADOOP_VERSION

0 commit comments

Comments
 (0)