Skip to content

Commit a0e36d8

Browse files
Jakub CzapigaPatrick Georgi
Jakub Czapiga
authored and
Patrick Georgi
committed
tests: Add support for tests build failures detection
This patch introduces new target: junit.xml-unit-tests, which builds and runs unit-tests. It also creates build log containing build logs. This feature allows for one to see build failures in Jenkins dashboard. Signed-off-by: Jakub Czapiga <[email protected]> Change-Id: I94184379dcc2ac10f1a47f4a9d205cacbeb640fe Reviewed-on: https://review.coreboot.org/c/coreboot/+/67372 Reviewed-by: Julius Werner <[email protected]> Tested-by: build bot (Jenkins) <[email protected]>
1 parent b525ea7 commit a0e36d8

File tree

4 files changed

+99
-12
lines changed

4 files changed

+99
-12
lines changed

‎payloads/libpayload/tests/Makefile.inc

Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -202,19 +202,61 @@ $(TEST_KCONFIG_AUTOHEADER): $(TEST_DOTCONFIG) $(objk)/conf
202202
$(TEST_KCONFIG_AUTOCONFIG): $(TEST_KCONFIG_AUTOHEADER)
203203
true
204204

205-
.PHONY: $(alltests) $(addprefix clean-,$(alltests))
205+
.PHONY: $(alltests) $(addprefix clean-,$(alltests)) $(addprefix try-,$(alltests))
206+
.PHONY: $(addprefix build-,$(alltests)) $(addprefix run-,$(alltests))
206207
.PHONY: unit-tests build-unit-tests run-unit-tests clean-unit-tests
208+
.PHONY: junit.xml-unit-tests clean-junit.xml-unit-tests
207209

208210
ifeq ($(JUNIT_OUTPUT),y)
209-
$(alltests): export CMOCKA_MESSAGE_OUTPUT=xml
210-
$(alltests): export CMOCKA_XML_FILE=$(testobj)/junit-%g.xml
211+
$(addprefix run-,$(alltests)): export CMOCKA_MESSAGE_OUTPUT=xml
212+
$(addprefix run-,$(alltests)): export CMOCKA_XML_FILE=$(testobj)/junit-libpayload-%g.xml
211213
endif
212214

213-
$(alltests): $$($$(@)-bin)
215+
$(addprefix run-,$(alltests)): run-%: $$(%-bin)
214216
rm -f $(testobj)/junit-libpayload-$(subst /,_,$(patsubst $(testobj)/%/,%,$(dir $^)))\(*\).xml
215217
rm -f $(testobj)/$(subst /,_,$^).failed
216218
-$^ || echo failed > $(testobj)/$(subst /,_,$^).failed
217219

220+
$(addprefix build-,$(alltests)): build-%: $$(%-bin)
221+
222+
$(alltests): run-$$(@)
223+
224+
$(addprefix try-,$(alltests)): try-%: clean-% $(CMOCKA_LIB) $(TEST_KCONFIG_AUTOCONFIG)
225+
mkdir -p $(testobj)/$*
226+
echo "<testcase classname='libpayload_build_unit_test' name='$*'>" >> $(testobj)/$*.tmp; \
227+
$(MAKE) V=$(V) Q=$(Q) COV=$(COV) JUNIT_OUTPUT=y "build-$*" >> $(testobj)/$*.tmp.2 2>&1 \
228+
&& type="system-out" || type="failure"; \
229+
if [ $$type = "failure" ]; then \
230+
echo "<failure type='buildFailed'>" >> $(testobj)/$*.tmp; \
231+
else \
232+
echo "<$$type>" >> $(testobj)/$*.tmp; \
233+
fi; \
234+
echo '<![CDATA[' >> $(testobj)/$*.tmp; \
235+
cat $(testobj)/$*.tmp.2 >> $(testobj)/$*.tmp; \
236+
echo "]]></$$type>" >> $(testobj)/$*.tmp; \
237+
rm -f $(testobj)/$*.tmp.2; \
238+
echo "</testcase>" >> $(testobj)/$*.tmp; \
239+
if [ $$type != 'failure' ]; then \
240+
$(MAKE) V=$(V) Q=$(Q) COV=$(COV) JUNIT_OUTPUT=y "run-$*"; \
241+
fi
242+
243+
244+
TESTS_BUILD_XML_FILE := $(testobj)/junit-libpayload-tests-build.xml
245+
246+
$(TESTS_BUILD_XML_FILE): clean-junit.xml-unit-tests $(addprefix try-,$(alltests))
247+
mkdir -p $(dir $@)
248+
echo '<?xml version="1.0" encoding="utf-8"?><testsuite>' > $@
249+
for tst in $(alltests); do \
250+
cat $(testobj)/$$tst.tmp >> $@; \
251+
done
252+
echo "</testsuite>" >> $@
253+
254+
junit.xml-unit-tests: $(TESTS_BUILD_XML_FILE)
255+
256+
clean-junit.xml-unit-tests:
257+
rm -f $(TESTS_BUILD_XML_FILE)
258+
259+
218260
# Build a code coverage report by collecting all the gcov files into a single
219261
# report. If COV is not set, this might be a user error, and they're trying
220262
# to generate a coverage report without first having built and run the code
@@ -254,7 +296,7 @@ run-unit-tests: $(alltests)
254296
exit 0; \
255297
fi
256298

257-
$(addprefix clean-,$(alltests)): clean-%
299+
$(addprefix clean-,$(alltests)): clean-%:
258300
rm -rf $(testobj)/$*
259301

260302
clean-unit-tests:

‎tests/Makefile.common

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,3 +170,5 @@ $(TEST_KCONFIG_AUTOHEADER): $(TEST_DOTCONFIG) $(objutil)/kconfig/conf
170170

171171
$(TEST_KCONFIG_AUTOCONFIG): $(TEST_KCONFIG_AUTOHEADER)
172172
true
173+
174+
TEST_COMMON_DEPENDENCIES := $(CMOCKA_LIB) $(TEST_KCONFIG_AUTOCONFIG)

‎tests/Makefile.inc

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,22 +60,64 @@ $(foreach test, $(alltests), \
6060
DEPENDENCIES += $(addsuffix .d,$(basename $(all-test-objs)))
6161
-include $(DEPENDENCIES)
6262

63-
.PHONY: $(alltests) $(addprefix clean-,$(alltests))
63+
.PHONY: $(alltests) $(addprefix clean-,$(alltests)) $(addprefix try-,$(alltests))
64+
.PHONY: $(addprefix build-,$(alltests)) $(addprefix run-,$(alltests))
6465
.PHONY: unit-tests build-unit-tests run-unit-tests clean-unit-tests
66+
.PHONY: junit.xml-unit-tests clean-junit.xml-unit-tests
6567

6668
# %g in CMOCKA_XML_FILE will be replaced with "__TEST_NAME__(<test-group-name>)"
6769
# by macro cb_run_group_tests(), which should be used for running tests.
6870
# __TEST_NAME__ contains test name including path e.g. tests_lib_rtc-test
6971
ifeq ($(JUNIT_OUTPUT),y)
70-
$(alltests): export CMOCKA_MESSAGE_OUTPUT=xml
71-
$(alltests): export CMOCKA_XML_FILE=$(testobj)/junit-%g.xml
72+
$(addprefix run-,$(alltests)): export CMOCKA_MESSAGE_OUTPUT=xml
73+
$(addprefix run-,$(alltests)): export CMOCKA_XML_FILE=$(testobj)/junit-%g.xml
7274
endif
7375

74-
$(alltests): $$($$(@)-bin)
76+
$(addprefix run-,$(alltests)): run-%: $$(%-bin)
7577
rm -f $(testobj)/junit-$(subst /,_,$(patsubst $(testobj)/%/,%,$(dir $^)))\(*\).xml
7678
rm -f $(testobj)/$(subst /,_,$^).failed
7779
-$^ || echo failed > $(testobj)/$(subst /,_,$^).failed
7880

81+
$(addprefix build-,$(alltests)): build-%: $$(%-bin)
82+
83+
$(alltests): run-$$(@)
84+
85+
$(addprefix try-,$(alltests)): try-%: clean-% $(TEST_COMMON_DEPENDENCIES)
86+
mkdir -p $(testobj)/$*
87+
echo "<testcase classname='coreboot_build_unit_test' name='$*'>" >> $(testobj)/$*.tmp; \
88+
$(MAKE) V=$(V) Q=$(Q) COV=$(COV) JUNIT_OUTPUT=y "build-$*" >> $(testobj)/$*.tmp.2 2>&1 \
89+
&& type="system-out" || type="failure"; \
90+
if [ $$type = "failure" ]; then \
91+
echo "<failure type='buildFailed'>" >> $(testobj)/$*.tmp; \
92+
else \
93+
echo "<$$type>" >> $(testobj)/$*.tmp; \
94+
fi; \
95+
echo '<![CDATA[' >> $(testobj)/$*.tmp; \
96+
cat $(testobj)/$*.tmp.2 >> $(testobj)/$*.tmp; \
97+
echo "]]></$$type>" >> $(testobj)/$*.tmp; \
98+
rm -f $(testobj)/$*.tmp.2; \
99+
echo "</testcase>" >> $(testobj)/$*.tmp; \
100+
if [ $$type != 'failure' ]; then \
101+
$(MAKE) V=$(V) Q=$(Q) COV=$(COV) JUNIT_OUTPUT=y "run-$*"; \
102+
fi
103+
104+
105+
TESTS_BUILD_XML_FILE := $(testobj)/junit-tests-build.xml
106+
107+
$(TESTS_BUILD_XML_FILE): clean-junit.xml-unit-tests $(addprefix try-,$(alltests))
108+
mkdir -p $(dir $@)
109+
echo '<?xml version="1.0" encoding="utf-8"?><testsuite>' > $@
110+
for tst in $(alltests); do \
111+
cat $(testobj)/$$tst.tmp >> $@; \
112+
done
113+
echo "</testsuite>" >> $@
114+
115+
junit.xml-unit-tests: $(TESTS_BUILD_XML_FILE)
116+
117+
clean-junit.xml-unit-tests:
118+
rm -f $(TESTS_BUILD_XML_FILE)
119+
120+
79121
# Build a code coverage report by collecting all the gcov files into a single
80122
# report. If COV is not set, this might be a user error, and they're trying
81123
# to generate a coverage report without first having built and run the code

‎util/testing/Makefile.inc

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -104,9 +104,10 @@ endif
104104
$(MAKE) xcompile=$(COREBOOT_BUILD_DIR)/xcompile $(COREBOOT_BUILD_DIR)/xcompile
105105
$(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR=src/soc/nvidia/tegra124/lp0 BLD=tegra124_lp0 MFLAGS= MAKEFLAGS=xcompile=$(COREBOOT_BUILD_DIR)/xcompile MAKETARGET=all junit.xml
106106
$(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR=src/soc/nvidia/tegra210/lp0 BLD=tegra120_lp0 MFLAGS= MAKEFLAGS=xcompile=$(COREBOOT_BUILD_DIR)/xcompile MAKETARGET=all junit.xml
107-
$(MAKE) unit-tests JUNIT_OUTPUT=y COV=1
108-
(cd payloads/libpayload; unset COREBOOT_BUILD_DIR; $(MAKE) unit-tests coverage-report JUNIT_OUTPUT=y COV=1)
109-
$(MAKE) coverage-report JUNIT_OUTPUT=y COV=1
107+
+$(MAKE) junit.xml-unit-tests COV=1
108+
+(cd payloads/libpayload; unset COREBOOT_BUILD_DIR; $(MAKE) junit.xml-unit-tests COV=1)
109+
+(cd payloads/libpayload; unset COREBOOT_BUILD_DIR; $(MAKE) coverage-report COV=1)
110+
+$(MAKE) coverage-report JUNIT_OUTPUT=y COV=1
110111
find . -name 'tests.info' -exec cat {} + >$(COREBOOT_BUILD_DIR)/coverage.info
111112

112113
test-basic: test-lint test-tools test-abuild test-payloads test-cleanup

0 commit comments

Comments
 (0)