Skip to content

Commit 0bb21f8

Browse files
bbyalcinkayabbyalcinkaya
authored
Rule Coverage (#99)
* add coverage analysis * run basic feature tests in parallel * format whitespace * rename coverage result
1 parent f48b6eb commit 0bb21f8

File tree

7 files changed

+212
-7
lines changed

7 files changed

+212
-7
lines changed

Diff for: ‎.github/workflows/test-pr.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ jobs:
9797
- name: 'Build'
9898
run: docker exec -t elrond-semantics-ci-${GITHUB_SHA} /bin/bash -c 'make build RELEASE=true'
9999
- name: 'Run Basic Feature Test'
100-
run: docker exec -t elrond-semantics-ci-${GITHUB_SHA} /bin/bash -c 'make TEST_CONCRETE_BACKEND=llvm test-elrond-basic-features'
100+
run: docker exec -t elrond-semantics-ci-${GITHUB_SHA} /bin/bash -c 'make TEST_CONCRETE_BACKEND=llvm test-elrond-basic-features -j6'
101101
- name: 'Tear down Docker'
102102
if: always()
103103
run: |

Diff for: ‎Makefile

+20-4
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
test unittest-python mandos-test mandos-coverage test-elrond-contracts \
66
test-elrond-adder test-elrond-crowdfunding-esdt \
77
test-elrond-multisig test-elrond-basic-features \
8+
rule-coverage clean-coverage \
89

910
# Settings
1011
# --------
@@ -94,6 +95,10 @@ wasm-deps:
9495
HOOK_NAMESPACES := KRYPTO
9596
KOMPILE_OPTS := --hook-namespaces \"$(HOOK_NAMESPACES)\" --emit-json
9697

98+
ifneq (,$(K_COVERAGE))
99+
KOMPILE_OPTS += --coverage
100+
endif
101+
97102
LLVM_KOMPILE_OPTS := -L$(LOCAL_LIB) \
98103
$(PLUGIN_SUBMODULE)/plugin-c/plugin_util.cpp \
99104
$(PLUGIN_SUBMODULE)/plugin-c/crypto.cpp \
@@ -156,7 +161,7 @@ KRUN_OPTS :=
156161

157162
# TODO add test-elrond-lottery-esdt
158163
elrond-contract-deps := test-elrond-adder \
159-
test-elrond-crowdfunding-esdt \
164+
test-elrond-crowdfunding-esdt \
160165
test-elrond-multisig \
161166
test-elrond-basic-features
162167
test-elrond-contracts: $(elrond-contract-deps)
@@ -252,12 +257,17 @@ test-elrond-multisig: $(llvm_kompiled)
252257
## Basic Feature Test
253258

254259
ELROND_BASIC_FEATURES_DIR=$(ELROND_CONTRACT)/feature-tests/basic-features
260+
ELROND_BASIC_FEATURES_WASM=$(ELROND_BASIC_FEATURES_DIR)/output/basic-features.wasm
255261
elrond_basic_features_tests=$(shell cat tests/basic_features.test)
256262

257-
# TODO optimize test runner and enable coverage and logging
258-
test-elrond-basic-features: $(llvm_kompiled)
263+
$(ELROND_BASIC_FEATURES_WASM):
259264
mxpy contract build "$(ELROND_BASIC_FEATURES_DIR)" --wasm-symbols
260-
$(TEST_MANDOS) $(elrond_basic_features_tests) --log-level none
265+
266+
# TODO optimize test runner and enable coverage and logging
267+
test-elrond-basic-features: $(elrond_basic_features_tests:=.mandos)
268+
269+
$(ELROND_BASIC_FEATURES_DIR)/scenarios/%.scen.json.mandos: $(llvm_kompiled) $(ELROND_BASIC_FEATURES_WASM)
270+
$(TEST_MANDOS) $(ELROND_BASIC_FEATURES_DIR)/scenarios/$*.scen.json --log-level none
261271

262272
# Unit Tests
263273
# ----------
@@ -266,3 +276,9 @@ unittest-python: $(PYTHON_UNITTEST_FILES:=.unit)
266276

267277
%.unit: %
268278
python3 $<
279+
280+
rule-coverage:
281+
python3 rule_coverage.py $(llvm_dir)/mandos-kompiled $(ELROND_FILES_KWASM_DIR)
282+
283+
clean-coverage:
284+
rm $(llvm_dir)/mandos-kompiled/*_coverage.txt $(llvm_dir)/mandos-kompiled/coverage.txt

Diff for: ‎README.md

+27-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ $ make elrond-clean-sources
4242
$ make elrond-loaded
4343
```
4444

45-
### Run
45+
## Run
4646

4747
To run Mandos tests, first build the contract:
4848

@@ -80,3 +80,29 @@ $ python3 run-elrond-tests.py --coverage \
8080
deps/mx-sdk-rs/contracts/examples/multisig/scenarios/changeQuorum.scen.json \
8181
deps/mx-sdk-rs/contracts/examples/multisig/scenarios/changeQuorum_tooBig.scen.json \
8282
```
83+
84+
## Rule Coverage
85+
86+
Compile the semantics with `K_COVERAGE=true` to enable the coverage analysis. This will make `krun` generate coverage data after every execution:
87+
88+
```
89+
$ make build K_COVERAGE=true
90+
```
91+
92+
Execute the programs to measure rule coverage for:
93+
94+
```
95+
$ make test
96+
```
97+
98+
Run the coverage analysis. This will list the locations of the rules that are not exercised.
99+
100+
```
101+
$ make -s rule-coverage
102+
deps/wasm-semantics/elrond-config.md:163:10
103+
deps/wasm-semantics/elrond-config.md:322:10
104+
deps/wasm-semantics/elrond-config.md:324:10
105+
...
106+
```
107+
108+
The coverage files generated by `krun` are located under the build directory (`.build/defn/llvm/mandos-kompiled`). To clean up the coverage data generated, run `make clean-coverage`.

Diff for: ‎rule_coverage.py

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
from glob import glob
2+
from os.path import abspath, relpath
3+
from sys import argv
4+
5+
def starts_with_one_of(s, ps):
6+
for p in ps:
7+
if s.startswith(p):
8+
return True
9+
return False
10+
11+
def main(kompiled_dir: str, src_files: list[str]):
12+
src_files = [abspath(f) for f in src_files]
13+
14+
all_rules_path = kompiled_dir + '/allRules.txt'
15+
16+
# rule id -> source location
17+
all_rules = dict()
18+
19+
with open(all_rules_path, 'r') as f:
20+
for line in f.readlines():
21+
parts = line.split()
22+
rule_id = parts[0].strip()
23+
rule_loc = parts[1].strip()
24+
25+
if starts_with_one_of(rule_loc, src_files):
26+
all_rules[rule_id] = rule_loc
27+
28+
covered = set()
29+
30+
for filename in glob(kompiled_dir + "/*coverage.txt"):
31+
with open(filename) as f:
32+
for line in f:
33+
rule_id = line.strip()
34+
if rule_id in all_rules:
35+
covered.add(all_rules[rule_id])
36+
37+
not_covered = set(all_rules.values()).difference(covered)
38+
for loc in sorted(not_covered):
39+
print(relpath(loc))
40+
41+
if len(argv) >= 3:
42+
main(argv[1], argv[2:])
43+
else:
44+
print(f'usage: python3 {argv[0]} <kompiled-dir> <source-file>...')

Diff for: ‎run-elrond-tests.py

-1
Original file line numberDiff line numberDiff line change
@@ -349,7 +349,6 @@ def esdt(esdt_value):
349349

350350

351351
def mandos_to_transfer_tx(tx):
352-
print(tx)
353352
sender = mandos_argument_to_kbytes(tx['from'])
354353
to = mandos_argument_to_kbytes(tx['to'])
355354
value = mandos_int_to_kint(getEgldValue(tx))

Diff for: ‎tests/mandos/transfer.scen.json

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
{
2+
"name": "set state",
3+
"comment": "set up state and contract names",
4+
"steps": [
5+
{
6+
"step": "setState",
7+
"comment": "set accounts and balances",
8+
"accounts": {
9+
"address:sender": {
10+
"nonce": "123",
11+
"balance": "100",
12+
"storage": {},
13+
"code": ""
14+
},
15+
"address:receiver": {
16+
"nonce": "456",
17+
"balance": "10",
18+
"storage": {},
19+
"code": ""
20+
}
21+
}
22+
},
23+
{
24+
"step": "transfer",
25+
"txId": "1",
26+
"comment": "simple EGLD transfer",
27+
"tx": {
28+
"from": "address:sender",
29+
"to": "address:receiver",
30+
"egldValue": "20"
31+
}
32+
},
33+
{
34+
"step": "checkState",
35+
"accounts": {
36+
"address:sender": {
37+
"balance": "80"
38+
},
39+
"address:receiver": {
40+
"balance": "30"
41+
}
42+
}
43+
}
44+
]
45+
}

Diff for: ‎tests/uncovered-rules.txt

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
deps/wasm-semantics/elrond-config.md:163:10
2+
deps/wasm-semantics/elrond-config.md:322:10
3+
deps/wasm-semantics/elrond-config.md:324:10
4+
deps/wasm-semantics/elrond-config.md:335:10
5+
deps/wasm-semantics/elrond-config.md:336:10
6+
deps/wasm-semantics/elrond-config.md:337:10
7+
deps/wasm-semantics/elrond-config.md:338:10
8+
deps/wasm-semantics/elrond-config.md:339:10
9+
deps/wasm-semantics/elrond-config.md:340:10
10+
deps/wasm-semantics/elrond-config.md:341:10
11+
deps/wasm-semantics/elrond-config.md:342:10
12+
deps/wasm-semantics/elrond-config.md:366:10
13+
deps/wasm-semantics/elrond-config.md:369:10
14+
deps/wasm-semantics/elrond-config.md:393:10
15+
deps/wasm-semantics/elrond-config.md:399:10
16+
deps/wasm-semantics/elrond-config.md:438:10
17+
deps/wasm-semantics/elrond-config.md:449:10
18+
deps/wasm-semantics/elrond-config.md:455:10
19+
deps/wasm-semantics/elrond-config.md:465:10
20+
deps/wasm-semantics/elrond-config.md:472:10
21+
deps/wasm-semantics/elrond-config.md:481:10
22+
deps/wasm-semantics/elrond-config.md:485:10
23+
deps/wasm-semantics/elrond-config.md:490:10
24+
deps/wasm-semantics/elrond-config.md:638:10
25+
deps/wasm-semantics/mandos.md:70:10
26+
deps/wasm-semantics/vmhooks/baseOps.md:15:10
27+
deps/wasm-semantics/vmhooks/baseOps.md:216:10
28+
deps/wasm-semantics/vmhooks/baseOps.md:225:10
29+
deps/wasm-semantics/vmhooks/baseOps.md:287:10
30+
deps/wasm-semantics/vmhooks/baseOps.md:291:10
31+
deps/wasm-semantics/vmhooks/baseOps.md:295:10
32+
deps/wasm-semantics/vmhooks/baseOps.md:299:10
33+
deps/wasm-semantics/vmhooks/baseOps.md:323:10
34+
deps/wasm-semantics/vmhooks/baseOps.md:367:10
35+
deps/wasm-semantics/vmhooks/baseOps.md:53:10
36+
deps/wasm-semantics/vmhooks/baseOps.md:76:10
37+
deps/wasm-semantics/vmhooks/baseOps.md:81:10
38+
deps/wasm-semantics/vmhooks/baseOps.md:98:10
39+
deps/wasm-semantics/vmhooks/bigIntOps.md:144:10
40+
deps/wasm-semantics/vmhooks/bigIntOps.md:150:10
41+
deps/wasm-semantics/vmhooks/bigIntOps.md:159:10
42+
deps/wasm-semantics/vmhooks/bigIntOps.md:165:10
43+
deps/wasm-semantics/vmhooks/bigIntOps.md:174:10
44+
deps/wasm-semantics/vmhooks/bigIntOps.md:180:10
45+
deps/wasm-semantics/vmhooks/bigIntOps.md:197:10
46+
deps/wasm-semantics/vmhooks/bigIntOps.md:215:10
47+
deps/wasm-semantics/vmhooks/bigIntOps.md:241:10
48+
deps/wasm-semantics/vmhooks/bigIntOps.md:277:10
49+
deps/wasm-semantics/vmhooks/bigIntOps.md:289:10
50+
deps/wasm-semantics/vmhooks/bigIntOps.md:348:10
51+
deps/wasm-semantics/vmhooks/bigIntOps.md:48:10
52+
deps/wasm-semantics/vmhooks/bigIntOps.md:82:10
53+
deps/wasm-semantics/vmhooks/eei-helpers.md:19:8
54+
deps/wasm-semantics/vmhooks/eei-helpers.md:20:8
55+
deps/wasm-semantics/vmhooks/eei-helpers.md:21:8
56+
deps/wasm-semantics/vmhooks/eei-helpers.md:22:8
57+
deps/wasm-semantics/vmhooks/eei-helpers.md:23:8
58+
deps/wasm-semantics/vmhooks/eei-helpers.md:27:8
59+
deps/wasm-semantics/vmhooks/eei-helpers.md:43:8
60+
deps/wasm-semantics/vmhooks/eei-helpers.md:45:8
61+
deps/wasm-semantics/vmhooks/eei-helpers.md:51:8
62+
deps/wasm-semantics/vmhooks/eei-helpers.md:64:8
63+
deps/wasm-semantics/vmhooks/eei-helpers.md:72:8
64+
deps/wasm-semantics/vmhooks/eei-helpers.md:81:8
65+
deps/wasm-semantics/vmhooks/manBufOps.md:214:10
66+
deps/wasm-semantics/vmhooks/managedConversions.md:64:10
67+
deps/wasm-semantics/vmhooks/smallIntOps.md:43:10
68+
deps/wasm-semantics/vmhooks/smallIntOps.md:56:10
69+
deps/wasm-semantics/vmhooks/smallIntOps.md:69:10
70+
deps/wasm-semantics/vmhooks/smallIntOps.md:81:10
71+
deps/wasm-semantics/wasm-coverage.md:125:10
72+
deps/wasm-semantics/wasm-coverage.md:64:10
73+
deps/wasm-semantics/wasm-coverage.md:72:10
74+
deps/wasm-semantics/wasm-coverage.md:80:10
75+
deps/wasm-semantics/wasm-coverage.md:88:10

0 commit comments

Comments
 (0)