test.sh

#!/bin/bash

LIBJUNITPOINTS=lib/junitpoints.jar
LIBJUNIT=lib/junit.jar
LIBJSONSIMPLE=lib/json-simple.jar
LIBALL=$LIBJUNITPOINTS:$LIBJUNIT:$LIBJSONSIMPLE

callerDir=${PWD}
scriptFile=$(readlink -f $0)
scriptDir=$(dirname $scriptFile)

COMPILER_ARGS="$COMPILER_ARGS -g -parameters"

interfacesDirName="interfaces"
cleanroomDirName="cleanroom"
junitDirName="junit"
sutDirName="student"
overallResultFileName="overall.exit"

function info {
	echo -e "\033[1;34m$1\033[0m"
}

function err {
	echo -e "\033[1;31m$1\033[0m"
}

function cleanExit {
	cd ${callerDir}
	if [ $keep -eq 0 ]; then
		info "\ncleaning up"
		rm -rf test.$$
	else
		info "keeping directory"
		rm -f test.latest
		ln -s test.$$ test.latest
	fi
	if [ $# -gt 0 ]; then
		exit $1
	fi
	exit 0
}

function die {
	err "$1"
	cleanExit -1
}

function checkContinue {
	exitcode=$1; shift
	msg=$1; shift
	file=$1; shift
	echo -e "${file}: ok" ${msg} >> ${overallResultFileName}
	if [ ${exitcode} -eq 0 ]; then
		info "${msg}"
	elif [ ${exitcode} -eq 1 ]; then
		err "failed:"
		cat ${file}
		err "${msg}"
	fi
}

function checkExit {
	exitcode=$1; shift
	msg=$1; shift
	file=$1; shift
	if [ ${exitcode} -ne 0 ]; then
		echo -e "${file}:" ${msg} >> ${overallResultFileName}
		err "failed:"
		cat ${file}
		die "${msg}"
	else
		echo -e "${file}: ok" >> ${overallResultFileName}
	fi
}

function checkAnnotationFormatError {
	file=$1; shift
	egrep -q "^(Exception in thread \"main\" )?java.lang.annotation.AnnotationFormatError" $file
	if [ $? -eq 0 ]; then
		err "test case format wrong:"
		cat $file
		err "\nSummary:\n";
		cat $file | egrep -B1 "^(Exception in thread \"main\" )?java.lang.annotation.AnnotationFormatError" | sed -e 's/.*ERROR - /ERROR - /'
		echo -e "${file}: internal error" >> ${overallResultFileName}
		die "\ninternal error\n";
	fi
}

function checkTestFiles {
	file1=$1; shift
	file2=$1;
	# locate the annotations
	secret1=$(grep '@*SecretClass' $file1)
	secret2=$(grep '@*SecretClass' $file2)
	ex1=$(grep '@*Exercises' $file1)
	ex2=$(grep '@*Exercises' $file2)
	# identify test files
	pubTestFile=
	secTestFile=
	if [ "x${secret1}" != "x" ] && [ "x${secret2}" != "x" ]; then
		err "ERROR - Found @SecretClass in BOTH test files"
		cleanExit
	elif [ "x${secret1}" != "x" ]; then
		pubTestFile=$(basename $file2)
		secTestFile=$(basename $file1)
		if [ "x${ex1}" != "x" ] && [ "x${ex2}" != "x" ]; then
			err "WARNING - @Exercises specified in $file1 (SecretTest) -- ignoring"
		fi
	elif [ "x${secret2}" != "x" ]; then
		pubTestFile=$(basename $file1)
		secTestFile=$(basename $file2)
		if [ "x${ex1}" != "x" ] && [ "x${ex2}" != "x" ]; then
			err "WARNING - @Exercises specified in $file2 (SecretTest) -- ignoring"
		fi
	else
		err "ERROR - Found NO @SecretClass in both test files"
		cleanExit
	fi
}

function scanCleanroom {
	first=1
	if [ -d $cleanroomDirName ]; then
		for entry in "$cleanroomDirName"/*; do
			arg=$(basename $entry)
			if [ ${first} -eq 1 ]; then
				cleanroom=$arg
				first=0
			else
				cleanroom="${cleanroom} $arg"
			fi
		done
	else 
		err "WARNING - No cleanroom folder found"
		cleanExit
	fi
}

function scanInterfaces {
	first=1
	if [ -d $interfacesDirName ]; then
		for entry in "$interfacesDirName"/*; do
			arg=$(basename $entry)
			if [ ${first} -eq 1 ];then
				interfaces=$arg
				first=0
			else
		 		interfaces="${interfaces} $arg"
			fi
		done
	else
		err "WARNING - No interfaces folder found"
	fi
}

function scanJunit {
	# look for junit folder
	if [ -d $junitDirName ]; then
		## get the files
		files=()
		for entry in "$junitDirName"/*; do
			files+=("$entry")
		done
		file_count=$(ls -1 $junitDirName | grep -v ^1 | wc -l)
		if [ "${file_count}" == "1" ]; then
			pubTestFile=$(basename ${files[0]})
		elif [ "${file_count}" == "2" ]; then
			checkTestFiles ${files[0]} ${files[1]}
		else
			err " WARNING - Maximum number of test files are 2 (SecretTest and PublicTest) => abort"
			die
		fi
	else
		err "WARNING - No junit folder found => abort\n"
		die
	fi
}

function scanSystemUnderTest {
	for entry in ./*; do
		if [ -d $entry ]; then
			base_entry=$(basename $entry)
			if [ "$base_entry" != "cleanroom" ] && [ "$base_entry" != "interfaces" ] && [ "$base_entry" != "junit" ] \
			&& [ "$base_entry" != "skeleton" ] && [ "$base_entry" != "expected" ] && [[ "$base_entry" != "test."* ]]; then
				((systemUnderTestDirCnt++))
				if [ ${systemUnderTestDirCnt} -eq 1 ]; then
					systemUnderTestDirs=$base_entry
				else
					systemUnderTestDirs="${systemUnderTestDirs} $base_entry"
				fi	
			fi	
		fi
	done
}

function testIt {
	systemUnderTestDir=$1
	first_source=1
	for file in "$systemUnderTestDir"/*; do
		arg=$(basename $file)
		if [ ${first_source} -eq 1 ]; then
			studentSource=$arg
			first_source=0
		else
			studentSource="${studentSource} $arg"
		fi
	done

	if [ "${cleanroom}" != "${studentSource}" ]; then
		err "WARNING - cleanroom sources do not match sources under $1"
		cleanExit
	fi

	info "preparing test setup"
	info "- create test dir"
	testDir="${callerDir}/test.$$"
	mkdir "$testDir" || die "failed to create test dir $testDir"
	cd "$testDir" > /dev/null 2> /dev/null

	info "- copy/install test infrastructure"
	${scriptDir}/tools/install.sh > /dev/null 2> /dev/null || die "failed"

	info "- write var.mk"
	echo "interfacesDirName=${interfacesDirName}" > var.mk
	echo "cleanroomDirName=${cleanroomDirName}" >> var.mk
	echo "junitDirName=${junitDirName}" >> var.mk
	echo "sutDirName=${sutDirName}" >> var.mk
	echo "PUBLICTESTSOURCE=${pubTestFile}" >> var.mk
	echo "SECRETTESTSOURCE=${secTestFile}" >> var.mk

	if [ "x${interfaces}" != "x" ]; then
		info "copy interfaces"
		mkdir ${testDir}/${interfacesDirName} || die "failed to create test sub-dir ${testDir}/${interfacesDirName}"
		if [ -r ../${interfacesDirName} ]; then
			pushd ../${interfacesDirName} > /dev/null || die "failed to chdir to ${interfacesDirName}"
			cp ${interfaces} ${testDir}/${interfacesDirName}/ || die "failed to copy interfaces"
			popd > /dev/null
		fi
	fi

	info "copy system under test (SUT)"
	mkdir ${testDir}/${sutDirName} || die "failed to create test sub-dir ${testDir}/${sutDirName}"
	pushd ../${systemUnderTestDir} > /dev/null || die "failed to chdir to ${systemUnderTestDir}"
	cp ${studentSource} ${testDir}/${sutDirName}/ || die "failed to copy system under test (SUT)"
	popd > /dev/null

	info "\nstage0 (compile interfaces and SUT only)"
	info "- compiling"
	( make compile-stage0 ) > comp0 2>&1
	ec=$?
	checkExit $ec "\nstudent result: ☠\n" comp0

	info "\ncopy cleanroom"
	mkdir ${testDir}/${cleanroomDirName} || die "failed to create test sub-dir ${testDir}/${cleanroomDirName}"
	pushd ../${cleanroomDirName} > /dev/null || die "failed to chdir to ${cleanroomDirName}"
	cp ${studentSource} ${testDir}/${cleanroomDirName} || die "failed to copy cleanroom"
	popd > /dev/null

	info "copy junit tests"
	mkdir ${testDir}/${junitDirName} || die "failed to create test sub-dir ${testDir}/${junitDirName}"
	pushd ../${junitDirName} > /dev/null || die "failed to chdir to ${junitDirName}"
	cp ${pubTestFile} ${testDir}/${junitDirName}/ || die "failed to copy pubTest"
	if [ "x$secTestFile" != "x" ]; then
		cp ${secTestFile} ${testDir}/${junitDirName}/ || die "failed to copy secTest"
	fi
	popd > /dev/null

	info "\nstage1 (run with public test)"
	info "- compiling"
	( make compile-stage1 ) > comp1.out 2> comp1.err
	ec=$?
	checkAnnotationFormatError comp1.err
	checkExit $ec "\nstudent result: ✘\n" comp1.err

	info "- testing"	
	( make run-stage1 ) > run1 2>&1
	ec=$(cat run1.exit)
	for gradeFile in `find ./reports/ -name run.grade -type f`; do cat $gradeFile >> run1.err; rm $gradeFile; done
	cat run1.err >> run1
	if [ $ec -eq 0 ]; then
		checkContinue 0 "\nstudent result: ✔\n" run1
	elif [ $ec -eq 1 ]; then
		checkContinue 1 "\nstudent result: !\n" run1
	else
		checkExit $ec "\ninternal error\n" run1
	fi

	info "\nstage2 (run twice: with secret test and weaving)"
	info "- compiling"
	( make compile-stage2 ) > comp2 2>&1
	ec=$?
	checkAnnotationFormatError comp2
	checkExit $ec "\ninternal error\n" comp2

	info "- testing"
	( make run-stage2 ) > run2 2>&1
	ec=$(cat run2.exit | grep -v 0 | grep -v 1)
	cat run2.err >> run2
	if [ "$ec" != "" ]; then
		checkExit 1 "\ninternal error\n" run2
	fi
	for error in "\\\",\\\"error\\\":\\\"NoSuchFieldError(Class " "\\\",\\\"error\\\":\\\"ExceptionInInitializerError()" ; do
		grep -q "$error" run2
		if [ $? -eq 0 ] ; then
			checkExit 1 "\ninternal error\n" run2
		fi
	done
	checkContinue 2 "\nstudent result: ?\n" run2

	info "  json:"
	cat run2.err

	info "- merging"
	PUBLICTEST=${pubTestFile%.*} # remove suffix (e.g. ".java"/".scala")
	if [ "x$secTestFile" != "x" ]; then
		SECRETTEST=${secTestFile%.*} # remove suffix (e.g. ".java"/".scala")
		( java -cp $LIBALL:$junitDirName:$interfacesDirName:$sutDirName $replace_error -Dpub=$PUBLICTEST -Dsecret=$SECRETTEST tester.tools.PointsMerger run2.err merged ) > merge 2>&1
	else
		( java -cp $LIBALL:$junitDirName:$interfacesDirName:$sutDirName $replace_error -Dpub=$PUBLICTEST tester.tools.PointsMerger run2.err merged ) > merge 2>&1
	fi
	checkExit $? "\ninternal error\n" merge
	cat merged

	cleanExit
}

keep=0
if [ "x$1" == "x-k" ]; then
	keep=1
	shift
fi

replace_error=""
if [ "x$1" == "x--replace-error" ]; then
	replace_error="-DreplaceError=true"
	echo -e "\033[1;33mINFO: option --replace-error is set.\033[0m"
	shift
fi


info "scanning cleanroom"
cleanroom=""
scanCleanroom

info "scanning interfaces"
interfaces=""
scanInterfaces

info "scanning junit"
pubTestFile=""
secTestFile=""
scanJunit

# info "scanning skeleton"
# nothing to do

if [ "$#" -eq 0 ]; then
	info "scanning system under test (SUT) - collecting automatically"
	systemUnderTestDirCnt=0
	systemUnderTestDirs=""
	scanSystemUnderTest
else
	info "scanning system under test (SUT) - using args from commandline"
	systemUnderTestDirCnt=$#
	systemUnderTestDirs=$@
fi

systemUnderTestDir="undertest"
if [ ${systemUnderTestDirCnt} -eq 1 ]; then
	systemUnderTestDir=${systemUnderTestDirs}
	testIt ${systemUnderTestDir}
elif [ ${systemUnderTestDirCnt} -gt 1 ]; then
	for i in ${systemUnderTestDirs}; do
		info "[recur test with $i]"
		keep=0
		(testIt $i)
		if [ $? != 0 ]; then
			die "testing $i failed"
		fi
		info "\n----------------------------------------\n"
	done
	exit 0
fi