Skip to content

Commit bf2707e

Browse files
anvacarugtreptaehildenb
authored
Add test runner python module (#318)
* add test runner module * kiele-coverage script which generates a frontend JSON report from build artifacts and a running client * pyiele/config.py: Put config variable into config class * kiele: Add coverage command * kiele: Add coverage command * pyiele: compile solidity contract * pyiele/utils.py: add basic assert functionality * pyiele: run Solidity tests; compile folders * fix typo * pyiele: add blackbox tester * pyiele/transactionGeneration: fix generate_cons_args * add kiele commands * pyiele: remove time.sleep() usage * kiele: fix test command * pyiele: run solidity tests * some cleanup * pyiele: fix coverage_timeout * pyiele: add number of passing and failing tests * Makefile: Add pyiele to package * pyiele: Update wallet RPC calls for new interface * pyiele/rpc.py: Handle rpc messages in a more structural way * pyiele/testrunner.py: Fix qa_mineBlocks parameter * move kiele-coverage in pyiele module * remove pyielerunner script * Makefile: Add kieleCoverage.py to build * Makefile: Fix removed files in python library * pyiele: default values from config class * Update Makefile Co-authored-by: Everett Hildenbrandt <[email protected]> * Update pyiele/__main__.py Co-authored-by: Everett Hildenbrandt <[email protected]> * pyiele/blackbox: execute contracts once * kieleCoverage: generate report for multiple contracts at once * pyiele: fix typo * kiele: Set PYTHONPATH for installed location * kiele: factor out listener check * kiele: refactor run_pyiele function * kiele: remove redundant calls to pyiele help * Update kiele Co-authored-by: gtrepta <[email protected]> Co-authored-by: Guy Repta <[email protected]> Co-authored-by: Everett Hildenbrandt <[email protected]>
1 parent 55d1ffb commit bf2707e

13 files changed

+970
-26
lines changed

‎Makefile

+29-15
Original file line numberDiff line numberDiff line change
@@ -405,19 +405,32 @@ coverage:
405405
# Install
406406
# -------
407407

408-
build: \
409-
$(IELE_ASSEMBLE) \
410-
$(IELE_CHECK) \
411-
$(IELE_INTERPRETER) \
412-
$(IELE_NODE) \
413-
$(IELE_RUNNER) \
414-
$(IELE_TEST_CLIENT) \
415-
$(IELE_TEST_VM) \
416-
$(IELE_VM) \
417-
$(IELE_LIB)/kiele-generate-report.py \
418-
$(IELE_LIB)/kore-json.py \
419-
$(IELE_LIB)/static-report.html \
420-
$(IELE_LIB)/version \
408+
pyiele_files := pyiele/__init__.py \
409+
pyiele/__main__.py \
410+
pyiele/blackbox.py \
411+
pyiele/config.py \
412+
pyiele/fetchFunctionData.py \
413+
pyiele/kieleCoverage.py \
414+
pyiele/rlp.py \
415+
pyiele/rpc.py \
416+
pyiele/testrunner.py \
417+
pyiele/transactionGeneration.py \
418+
pyiele/utils.py
419+
420+
build: \
421+
$(IELE_ASSEMBLE) \
422+
$(IELE_CHECK) \
423+
$(IELE_INTERPRETER) \
424+
$(IELE_NODE) \
425+
$(IELE_RUNNER) \
426+
$(IELE_TEST_CLIENT) \
427+
$(IELE_TEST_VM) \
428+
$(IELE_VM) \
429+
$(IELE_LIB)/kiele-generate-report.py \
430+
$(IELE_LIB)/kore-json.py \
431+
$(IELE_LIB)/static-report.html \
432+
$(IELE_LIB)/version \
433+
$(patsubst %, $(IELE_LIB)/%, $(pyiele_files)) \
421434
$(haskell_kompiled)
422435

423436
all_bin_sources := $(shell find $(IELE_BIN) -type f \
@@ -471,8 +484,9 @@ install-vm: $(patsubst $(IELE_LIB)/%, $(DESTDIR)$(INSTALL_LIB)/%, $(IELE_VM))
471484

472485
install-check: $(patsubst %, $(DESTDIR)$(INSTALL_LIB)/%, $(iele_check_libs))
473486

474-
install-kiele: $(patsubst $(IELE_BIN)/%, $(DESTDIR)$(INSTALL_BIN)/%, $(IELE_RUNNER))
475-
install-kiele: $(patsubst %, $(DESTDIR)$(INSTALL_LIB)/%, $(kiele_files))
487+
install-kiele: $(patsubst $(IELE_BIN)/%, $(DESTDIR)$(INSTALL_BIN)/%, $(IELE_RUNNER)) \
488+
$(patsubst %, $(DESTDIR)$(INSTALL_LIB)/%, $(kiele_files)) \
489+
$(patsubst %, $(DESTDIR)$(INSTALL_LIB)/%, $(pyiele_files))
476490

477491
install: $(patsubst %, $(DESTDIR)$(INSTALL_BIN)/%, $(all_bin_sources)) \
478492
$(patsubst %, $(DESTDIR)$(INSTALL_LIB)/%, $(all_lib_sources))

‎kiele

+58-11
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ export PATH="${PATH}:${INSTALL_LIB}/standalone/iele-testing-kompiled"
1414
export PATH="${PATH}:${INSTALL_LIB}/check/well-formedness-kompiled"
1515
export PATH="${PATH}:${INSTALL_LIB}/node/iele-testing-kompiled"
1616
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:/usr/local/lib
17+
export PYTHONPATH=${PYTHONPATH:-}:"$(dirname $0)/../lib/kiele"
1718

1819
check_k_install() {
1920
type -p kore-exec &> /dev/null \
@@ -26,6 +27,12 @@ check_k_install() {
2627

2728
check_token() { [ -v FIREFLY_TOKEN ] || fatal "FIREFLY_TOKEN environment variable isn't set!" ; }
2829

30+
check_listener() {
31+
if ! nc -z "$kiele_host" "$kiele_port"; then
32+
fatal "No listener found on port $kiele_port"
33+
fi
34+
}
35+
2936
make_temp_trap() {
3037
local tmpdesc tmpfile
3138
tmpdesc="$1" ; shift
@@ -47,6 +54,10 @@ run_assemble() {
4754
iele-assemble "$run_file" "$@"
4855
}
4956

57+
run_pyiele() {
58+
python3 -m pyiele --host "$kiele_host" --port "$kiele_port" --output "$output_path" "$@"
59+
}
60+
5061
run_interpret() {
5162
local kast output status
5263
kast=$(make_temp_trap kast)
@@ -111,20 +122,18 @@ run_vm() {
111122
"${cmd[@]}"
112123
}
113124

125+
run_coverage() {
126+
notif "Generating coverage json from and client on port $kiele_port"
127+
check_listener
128+
run_pyiele coverage $@
129+
}
130+
114131
run_generate_report() {
115132
if [[ -f $(dirname $(which kiele))/static-report.html ]]; then
116133
report_template=$(dirname $(which kiele))/static-report.html
117134
else
118135
report_template=$INSTALL_LIB/static-report.html
119136
fi
120-
output_path=""
121-
while [[ $# -gt 0 ]]; do
122-
arg="$1"
123-
case $arg in
124-
-o|--output) output_path="$2" ; shift 2 ;;
125-
*) shift ;;
126-
esac
127-
done
128137
kiele-generate-report.py "$report_template" "$run_file" "$output_path" "$create_archive"
129138
}
130139

@@ -141,6 +150,30 @@ run_upload_report() {
141150
rm report.zip
142151
}
143152

153+
run_compile() {
154+
run_pyiele compile "$@"
155+
}
156+
157+
run_testrunner() {
158+
local run_file cmd
159+
check_listener
160+
cmd=(run_pyiele test)
161+
if [[ $# -gt 0 ]] && ([[ -f "$1" ]] || [[ -d "$1" ]]); then
162+
run_file="$1"; shift
163+
if [[ "$run_file" == *.py ]]; then
164+
cmd=(python3 "$run_file")
165+
else
166+
cmd+=(--file "$run_file")
167+
fi
168+
fi
169+
"${cmd[@]}" "$@"
170+
}
171+
172+
run_blackbox() {
173+
check_listener
174+
run_pyiele blackbox "$@"
175+
}
176+
144177
# Main
145178
# ----
146179

@@ -153,8 +186,12 @@ if [[ "$run_command" == 'help' ]] || [[ "$run_command" == '--help' ]] ; then
153186
$0 krun [--backend (standalone|haskell|node|check)] <KIELE arg>* <pgm> <K arg>*
154187
$0 check <KIELE arg>* <pgm> <K arg>*
155188
$0 vm <KIELE arg>*
156-
$0 generate-report [--create-archive] <Report json path> [-o|--output <Report output path>]
189+
$0 coverage <combined.json path> [-p|--port <client port>] [-o|--output <Report output path>]
190+
$0 generate-report [--create-archive] <Report json path> [-o|--output <Report output path>]
157191
$0 upload <Report json path>
192+
$0 compile <COMPILE arg>*
193+
$0 test <TEST arg>*
194+
$0 blackbox <BLACKBOX arg>*
158195
159196
$0 [help|--help|version|--version]
160197
@@ -163,9 +200,12 @@ if [[ "$run_command" == 'help' ]] || [[ "$run_command" == '--help' ]] ; then
163200
$0 krun : Run given input <pgm> using 'krun'
164201
$0 check : Run the IELE well-formedness checker on <pgm>
165202
$0 vm : Run the KIELE VM server.
203+
$0 coverage : Generates a report file containing contract information and coverage data.
166204
$0 generate-report : Generate the static HTML report.
167205
$0 upload : Upload the HTML report to the fireflyblockchain.com website.
168-
206+
$0 compile : Compile solidity contracts using isolc. If no path is provided, all the found .sol files will be compiled. For more information run 'kiele compile --help'.
207+
$0 test : Runs either a python or a solidity test file. If no path is provided, will run all the tests found under the test folder. For more information run 'kiele test --help'.
208+
$0 blackbox : Attempt to find new test inputs via blackbox random testing. For more information run 'kiele blackbox --help'.
169209
$0 help : Display this help message.
170210
$0 version : Display the versions of KIELE in use.
171211
@@ -175,6 +215,7 @@ if [[ "$run_command" == 'help' ]] || [[ "$run_command" == '--help' ]] ; then
175215
<K arg> is an argument you want to pass to K
176216
<KIELE arg> is one of (--no-unparse|--debug|--host|--port|--mode|--coverage
177217
|--schedule|--module|--definition|--depth)
218+
<combined.json path> is the path to the build artifacts from 'isolc --combined-json'
178219
<Report json path> is the path to the report json file
179220
<Report output path> is the output HTML report file path (default: \$uuid.html)
180221
"
@@ -202,6 +243,7 @@ depth=-1
202243
args=()
203244
create_archive='false'
204245
parser='cat'
246+
output_path='/dev/stdout'
205247
while [[ $# -gt 0 ]]; do
206248
arg="$1"
207249
case $arg in
@@ -216,6 +258,7 @@ while [[ $# -gt 0 ]]; do
216258
--coverage) enablecoverage='true' ; shift ;;
217259
--create-archive) create_archive='true' ; shift ;;
218260
--parser) parser="$2" ; shift 2 ;;
261+
-o|--output) output_path="$2" ; shift 2 ;;
219262
-p|--port) kiele_port="$2" ; shift 2 ;;
220263
-h|--host) kiele_host="$2" ; shift 2 ;;
221264
*) args+=("$1") ; shift ;;
@@ -233,7 +276,7 @@ cmode='Lbl'$mode'{}()'
233276
cschedule='Lbl'$schedule'{}()'
234277
cenablecoverage='\dv{SortBool{}}("'$enablecoverage'")'
235278

236-
if [[ "$run_command" != 'vm' ]]; then
279+
if [[ "$run_command" != 'vm' ]] && [[ "$run_command" != 'coverage' ]] && [[ "$run_command" != 'compile' ]] && [[ "$run_command" != 'test' ]] && [[ "$run_command" != 'blackbox' ]]; then
237280
run_file="$1" ; shift
238281
[[ -f "$run_file" ]] || [[ "$run_file" == '-' ]] || fatal "File does not exist: $run_file"
239282
fi
@@ -244,7 +287,11 @@ case "$run_command" in
244287
krun) run_krun "$@" ;;
245288
check) run_check "$@" ;;
246289
vm) run_vm "$@" ;;
290+
coverage) run_coverage "$@" ;;
247291
generate-report) run_generate_report "$@" ;;
248292
upload) run_upload_report "$@" ;;
293+
compile) run_compile "$@" ;;
294+
blackbox) run_blackbox "$@" ;;
295+
test) run_testrunner "$@" ;;
249296
*) $0 help ; fatal "Unknown command: $run_command" ;;
250297
esac

‎pyiele/__init__.py

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/usr/bin/env python3
2+
3+
from .config import *
4+
from .rlp import *
5+
from .rpc import *
6+
from .fetchFunctionData import *
7+
from .testrunner import *
8+
from .utils import *
9+
from .transactionGeneration import *
10+
from .blackbox import *
11+
from .kieleCoverage import *

0 commit comments

Comments
 (0)