Merge pull request #506 from network-intelligence/dev

Merging dev into trunk for 2.8.0

Author: andrewchi

.gitignore

@@ -18,5 +18,5 @@ unit_tests/pcaps/*.pcap
 .vscode/
 *.mk
 *.tgz
-src/libmerc/asn1/oid.cc
-src/libmerc/asn1/oid.h
+*CMakeCache.txt
+*CMakeFiles/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "src/libmerc/xsimd"]
+	path = src/libmerc/xsimd
+	url = https://github.com/xtensor-stack/xsimd.git

Makefile

@@ -44,7 +44,7 @@ ifneq ($(MERCURY_CFG),)
 	$(INSTALLDATA) $(MERCURY_CFG) /etc/mercury/mercury.cfg
 else
 	@echo $(COLOR_RED) "error: no configuration file specified; run as 'make install MERCURY_CFG=filename'" $(COLOR_OFF)
-	@/bin/false
+	@false
 endif
 endif
 
@@ -121,6 +121,69 @@ libs:
 test:
 	cd src && $(MAKE) test
 
+.PHONY: test-coverage
+test-coverage:
+	@bash -e -c -o pipefail ' \
+		mkdir -p coverage && \
+		make && \
+		make clean-helper > /dev/null && \
+		make --directory=src COVERAGE_ENABLED=1 use_fsanitize=no run_unit_test > /dev/null && \
+		lcov -q --directory . --capture --output-file ./coverage/mercury_unit_tests_1.info && \
+		echo -e $(COLOR_GREEN) "created coverage file for unit tests" $(COLOR_OFF) && \
+		make clean-helper > /dev/null && \
+		\
+		make --directory=src/libmerc COVERAGE_ENABLED=1 use_fsanitize=no libmerc.so > /dev/null && \
+		make --directory=unit_tests COVERAGE_ENABLED=1 use_fsanitize=no libmerc_driver_tls_only > /dev/null && \
+		make --directory=unit_tests run_libmerc_tls_only_tests > /dev/null && \
+		lcov -q --directory . --capture --output-file ./coverage/mercury_libmerc_driver_tls_only.info && \
+		echo -e $(COLOR_GREEN) "created coverage file for libmerc driver tls tests" $(COLOR_OFF) && \
+		make clean-helper > /dev/null && \
+		\
+		make --directory=src/libmerc COVERAGE_ENABLED=1 use_fsanitize=no libmerc.so > /dev/null && \
+		make --directory=unit_tests COVERAGE_ENABLED=1 use_fsanitize=no libmerc_driver_multiprotocol > /dev/null && \
+		make --directory=unit_tests run_libmerc_multiprotocol_tests > /dev/null && \
+		lcov -q --directory . --capture --output-file ./coverage/mercury_libmerc_driver_multiprotocol.info && \
+		echo -e $(COLOR_GREEN) "created coverage file for libmerc driver multiprotocol tests" $(COLOR_OFF) && \
+		make clean-helper > /dev/null && \
+		\
+		make --directory=src COVERAGE_ENABLED=1 use_fsanitize=no mercury > /dev/null && \
+		make --directory=test COVERAGE_ENABLED=1 clean comp analysis cert-check memcheck json-validity-test stats > /dev/null && \
+		lcov -q --directory . --capture --output-file ./coverage/mercury_unit_tests_2.info && \
+		echo -e $(COLOR_GREEN) "created coverage file for other unit tests" $(COLOR_OFF) && \
+		make clean-helper > /dev/null && \
+		\
+		lcov --add-tracefile ./coverage/mercury_unit_tests_1.info \
+		     --add-tracefile ./coverage/mercury_libmerc_driver_tls_only.info \
+		     --add-tracefile ./coverage/mercury_libmerc_driver_multiprotocol.info \
+		     --add-tracefile ./coverage/mercury_unit_tests_2.info \
+		     --output-file ./coverage/mercury_total.info 2>&1 | grep -v "function data mismatch" && \
+		lcov -q --remove ./coverage/mercury_total.info "/usr/include/*" "*/src/libmerc/rapidjson/*" "*/unit_tests/*" -o ./coverage/mercury_filtered_coverage.info && \
+		genhtml --no-function-coverage --output-directory coverage_html_report ./coverage/mercury_filtered_coverage.info && \
+		echo -e $(COLOR_GREEN) "created coverage report" $(COLOR_OFF) \
+	' || { echo $(COLOR_RED) "failed to build coverage report" $(COLOR_OFF); exit 1; }
+
+.PHONY: test-coverage-fuzz
+test-coverage-fuzz:
+	@bash -e -c -o pipefail ' \
+		mkdir -p coverage_fuzz && \
+		make --directory=test COVERAGE_ENABLED=1 fuzz-test &&  \
+		find . -name "*.profraw" | xargs -I {} sh -c "llvm-profdata merge -sparse '{}' -o \$$(dirname '{}')/default.profdata" && \
+		find . -name "*exec" | xargs -I {} sh -c "llvm-cov export -format=lcov --instr-profile \$$(dirname '{}')/default.profdata {} > \$$(dirname '{}')/default.info" && \
+		find ./test/fuzz -name "*.info" | sed "s/\\(\\S\\+\\)/--add-tracefile \\1/g" | xargs lcov --output-file ./coverage_fuzz/mercury_fuzz_test_1.info > /dev/null && \
+		lcov -q --directory ./src --capture --output-file ./coverage_fuzz/mercury_fuzz_test_2.info && \
+		echo -e $(COLOR_GREEN) "created coverage files for fuzz tests" $(COLOR_OFF) && \
+		make clean-helper > /dev/null && \
+		\
+		lcov --add-tracefile ./coverage_fuzz/mercury_fuzz_test_1.info \
+		     --add-tracefile ./coverage_fuzz/mercury_fuzz_test_2.info \
+		     --output-file ./coverage_fuzz/mercury_total_with_fuzz.info 2>&1 | grep -v "function data mismatch" && \
+		lcov -q --remove ./coverage_fuzz/mercury_total_with_fuzz.info \
+		        "/usr/include/*" "*/src/libmerc/rapidjson/*" "*/test/fuzz/*" \
+		        -o ./coverage_fuzz/mercury_filtered_coverage_with_fuzz.info && \
+		genhtml --no-function-coverage --output-directory coverage_html_report_fuzz ./coverage_fuzz/mercury_filtered_coverage_with_fuzz.info && \
+		echo -e $(COLOR_GREEN) "created final fuzz coverage report" $(COLOR_OFF) \
+	' || { echo $(COLOR_RED) "failed to build fuzz coverage report" $(COLOR_OFF); exit 1; }
+
 .PHONY: test_strict
 test_strict:
 	cd src && $(MAKE) test
@@ -158,26 +221,34 @@ clean: sphinx-clean
 	for file in Makefile README.md configure.ac Doxyfile; do if [ -e "$$file~" ]; then rm -f "$$file~" ; fi; done
 ifneq ($(wildcard src/Makefile), src/Makefile)
 	@echo $(COLOR_RED) "error: run ./configure before running make (src/Makefile is missing)" $(COLOR_OFF)
-	@/bin/false
+	@false
 else
-	cd src && $(MAKE) clean
-	cd test && $(MAKE) clean
-	cd unit_tests && $(MAKE) clean
+	$(MAKE) clean-helper
 	rm -rf doc/latex
+	rm -rf coverage coverage_fuzz coverage_html_report coverage_html_report_fuzz
 endif
 
 .PHONY: distclean
 distclean: clean
 ifneq ($(wildcard src/Makefile), src/Makefile)
 	@echo $(COLOR_RED) "error: run ./configure before running make (src/Makefile is missing)" $(COLOR_OFF)
-	@/bin/false
+	@false
 else
 	cd src  && $(MAKE) distclean
 	cd test && $(MAKE) distclean
 	rm -rf autom4te.cache config.log config.status Makefile_helper.mk
 	rm -f lib/*.so
 endif
 
+.PHONY: clean-helper
+clean-helper:
+	find . -name "*.gcda" -delete
+	find . -name "*.gcno" -delete
+	find . -name "*.gcov" -delete
+	cd src && $(MAKE) clean
+	cd test && $(MAKE) clean
+	cd unit_tests && $(MAKE) clean
+
 .PHONY: package-deb
 package-deb: mercury
 ifneq ($(wildcard src/Makefile), src/Makefile)

Makefile_helper.mk.in

@@ -35,9 +35,15 @@ is_macos = @IS_MACOS@
 #
 is_macos_arm = @IS_MACOS_ARM@
 
+#The variable platform is used to represent intel or arm based architecture.
+# platform=intel   for intel based platform
+#         =arm     for arm based platform
+#         =unknown otherwise
+platform = @PLATFORM@
+
 CXX     = @CXX@
 CC      = @CC@
-CFLAGS  = --std=c++17
+CFLAGS  += --std=c++17
 CFLAGS += -O3
 # CFLAGS += -march=x86-64 -mtune=generic
 CFLAGS += -Wall -Wextra -Wno-deprecated $(CDEFS) $(MSV)
@@ -52,6 +58,7 @@ CFLAGS += -Wno-deprecated-declarations
 CFLAGS += -Wno-long-long
 CFLAGS += -Wmissing-noreturn
 CFLAGS += -Wunreachable-code
+CFLAGS += -Wno-psabi
 CFLAGS += -fvisibility=hidden
 CFLAGS += -DNDEBUG
 # CFLAGS += -g

README.md

@@ -38,6 +38,21 @@ make
 ```
 to build the package (and check for the programs and python modules required to test it).  TPACKETv3 is present in Linux kernels newer than 3.2.
 
+Building mercury on macOS Apple Silicon is currently experimental.  Note that on
+macOS, standalone mercury can read pcap as input but not capture on an
+interface, since AF_PACKET is Linux-specific.  The following has been tested
+on an M2 mac with Python 3.13.2 installed via the Homebrew command below.
+```
+brew install python openssl zlib
+mkdir -p ~/.envs
+python3 -m venv ~/.envs/merc
+source ~/.envs/merc/bin/activate
+python3 -m pip install --upgrade pip
+python3 -m pip install --upgrade jsonschema cryptography Cython wheel setuptools
+./configure && make
+cd src/cython && make && make wheel
+```
+
 ### Installation
 In the root directory, edit mercury.cfg with the network interface you want to capture from, then run
 ```
@@ -77,7 +92,7 @@ which will remove the mercury program, resources directory, user, group, and sys
 ### Compile-time options
 To create a debugging version of mercury, use the **make debug-mercury** target in the src/ subdirectory.  Be sure to run **make clean** first.
 
-There are compile-time options that can tune mercury for your hardware.  Each of these options is set via a C/C++ preprocessor directive, which should be passed as an argument to "make" through the OPTFLAGS variable.   Frst run **make clean** to remove the previous build, then run **make "OPTFLAGS=<DIRECTIVE>"**.   This runs make, telling it to pass <DIRECTIVE> to the C/C++ compiler.  The available compile time options are:
+There are compile-time options that can tune mercury for your hardware.  Each of these options is set via a C/C++ preprocessor directive, which should be passed as an argument to "make" through the OPTFLAGS variable.   First run **make clean** to remove the previous build, then run **make "OPTFLAGS=<DIRECTIVE>"**.   This runs make, telling it to pass <DIRECTIVE> to the C/C++ compiler.  The available compile time options are:
 
    * -DDEBUG, which turns on debugging, and
    * -FBUFSIZE=16384, which sets the fwrite/fread buffer to 16,384 bytes (for instance).
@@ -212,7 +227,7 @@ DETAILS
    --reassembly enables reassembly
    This option allows mercury to keep track of tcp or udp segment state and
    and reassemble these segments based on the application in payload
- 
+
    "[-u or --user] u" sets the UID and GID to those of user u, so that
    output file(s) are owned by this user.  If this option is not set, then
    the UID is set to SUDO_UID, so that privileges are dropped to those of
@@ -310,9 +325,9 @@ are grateful to the copyright holders for making their excellent
 software available under licensing terms that allow its
 redistribution.
    * RapidJSON
-      [https://github.com/cisco/mercury/src/rapidjson/license.txt](src/rapidjson/license.txt);
+      [https://github.com/cisco/mercury/blob/main/src/libmerc/rapidjson/license.txt](src/libmerc/rapidjson/license.txt);
       this package is Copyright 2015 THL A29 Limited, a Tencent company,
       and Milo Yip.
-   * lctrie [https://github.com/cisco/mercury/src/lctrie](src/lctrie);
+   * lctrie [https://github.com/cisco/mercury/tree/main/src/libmerc/lctrie](src/libmerc/lctrie);
       this package is copyright 2016-2017 Charles Stewart
       <chuckination_at_gmail_dot_com>

VERSION

@@ -1 +1 @@
-2.7.1
+2.8.0

aclocal.m4

@@ -1,6 +1,6 @@
-# generated automatically by aclocal 1.16.5 -*- Autoconf -*-
+# generated automatically by aclocal 1.16.1 -*- Autoconf -*-
 
-# Copyright (C) 1996-2021 Free Software Foundation, Inc.
+# Copyright (C) 1996-2018 Free Software Foundation, Inc.
 
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,

build_pkg.sh

@@ -95,10 +95,14 @@ elif [ "$BUILDTYPE" == "rpm" ]; then
 
     # note: we could detect libssl version here
 
+    # detect platform ID (e.g., "el8" or "el9")
+    source /etc/os-release
+    ELX=$(echo $PLATFORM_ID | cut -f2 -d:)
+
     fpm -s dir -t rpm -n mercury $FPM_LINUX_OPTIONS \
-        --depends 'libssl.so.10()(64bit)' \
-        --depends 'libz.so.1()(64bit)'    \
-        --rpm-dist el7 \
+        --depends 'openssl' \
+        --depends 'zlib'    \
+        --rpm-dist $ELX \
         --rpm-attr 775,mercury,mercury:/usr/local/var/mercury \
         --description "$DESCRIPTION" \
         --after-remove ./install_mercury/postuninstall_rpm \

configure

@@ -639,6 +639,7 @@ ac_includes_default="\
 ac_header_c_list=
 ac_func_c_list=
 ac_subst_vars='LTLIBOBJS
+LCOV
 LLVM
 CLANGPP
 HAVE_AFL
@@ -656,6 +657,7 @@ USE_FSANITIZE
 USE_NO_GNU_UNIQUE
 IS_MACOS_ARM
 IS_MACOS
+PLATFORM
 host_os
 host_vendor
 host_cpu
@@ -4659,6 +4661,27 @@ IFS=$ac_save_IFS
 case $host_os in *\ *) host_os=`echo "$host_os" | sed 's/ /-/g'`;; esac
 
 
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for platform architecture" >&5
+printf %s "checking for platform architecture... " >&6; }
+case "$host_cpu" in
+    i?86 | x86_64)
+        PLATFORM="intel"
+        { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: Intel-compatible ($host_cpu)" >&5
+printf "%s\n" "Intel-compatible ($host_cpu)" >&6; }
+        ;;
+    arm* | aarch64)
+        PLATFORM="arm"
+        { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: ARM-compatible ($host_cpu)" >&5
+printf "%s\n" "ARM-compatible ($host_cpu)" >&6; }
+        ;;
+    *)
+        PLATFORM="unknown"
+        { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: Unknown architecture: $host_cpu" >&5
+printf "%s\n" "$as_me: WARNING: Unknown architecture: $host_cpu" >&2;}
+        ;;
+esac
+
+
 # Update lib and include paths for Apple Silicon
 case $host_os in
     darwin* )
@@ -5869,6 +5892,49 @@ printf "%s\n" "no" >&6; }
 fi
 
 
+# Extract the first word of "lcov", so it can be a program name with args.
+set dummy lcov; ac_word=$2
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+printf %s "checking for $ac_word... " >&6; }
+if test ${ac_cv_prog_LCOV+y}
+then :
+  printf %s "(cached) " >&6
+else case e in #(
+  e) if test -n "$LCOV"; then
+  ac_cv_prog_LCOV="$LCOV" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then
+    ac_cv_prog_LCOV="yes"
+    printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+fi ;;
+esac
+fi
+LCOV=$ac_cv_prog_LCOV
+if test -n "$LCOV"; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LCOV" >&5
+printf "%s\n" "$LCOV" >&6; }
+else
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+
+
 if test "x$TCPREPLAY" = xyes
 then :
 
@@ -5925,6 +5991,14 @@ else case e in #(
 printf "%s\n" "$as_me: WARNING: llvm not found; make fuzz-test may not work" >&2;} ;;
 esac
 fi
+if test "x$LCOV" = xyes
+then :
+
+else case e in #(
+  e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: lcov not found; make test-coverage and test-coverage-fuzz will not work" >&5
+printf "%s\n" "$as_me: WARNING: lcov not found; make test-coverage and test-coverage-fuzz will not work" >&2;} ;;
+esac
+fi
 ac_config_files="$ac_config_files src/libmerc/Makefile src/cython/Makefile src/Makefile test/Makefile unit_tests/Makefile Makefile_helper.mk install_mercury/mercury.service"
 
 cat >confcache <<\_ACEOF

configure.ac

@@ -6,6 +6,23 @@ AC_USE_SYSTEM_EXTENSIONS
 #AC_PROG_CC
 AC_PROG_CXX
 AC_CANONICAL_HOST
+AC_MSG_CHECKING([for platform architecture])
+case "$host_cpu" in
+    i?86 | x86_64)
+        PLATFORM="intel"
+        AC_MSG_RESULT([Intel-compatible ($host_cpu)])
+        ;;
+    arm* | aarch64)
+        PLATFORM="arm"
+        AC_MSG_RESULT([ARM-compatible ($host_cpu)])
+        ;;
+    *)
+        PLATFORM="unknown"
+        AC_MSG_WARN([Unknown architecture: $host_cpu])
+        ;;
+esac
+AC_SUBST([PLATFORM])
+
 # Update lib and include paths for Apple Silicon
 case $host_os in
     darwin* )
@@ -32,6 +49,7 @@ AC_CHECK_FUNCS([munmap])
 AC_CHECK_FUNCS([socket]) 
 AC_CHECK_FUNCS([strerror]) 
 AC_CHECK_FUNCS([strtol]) 
+AC_CHECK_FUNCS([gettid], [AC_DEFINE([HAVE_GETTID], [1], [Define if gettid exists.])])
 AC_CHECK_HEADERS([arpa/inet.h])
 AC_CHECK_HEADERS([fcntl.h])
 AC_CHECK_HEADERS([stddef.h])
@@ -81,6 +99,7 @@ AC_CHECK_PROG(VALGRIND,valgrind,yes)
 AC_CHECK_PROG(HAVE_AFL,afl-g++,yes)
 AC_CHECK_PROG(CLANGPP, clang++, yes)
 AC_CHECK_PROG(LLVM,llvm-link,yes)
+AC_CHECK_PROG(LCOV, lcov, yes)
 AS_IF([test "x$TCPREPLAY" = xyes],
     [],
     [AC_MSG_WARN([tcpreplay not found; test/Makefile dummy-capture test will not work])])
@@ -102,5 +121,8 @@ AS_IF([test "x$CLANGPP" = xyes],
 AS_IF([test "x$LLVM" = xyes],
     [],
     [AC_MSG_WARN([llvm not found; make fuzz-test may not work])])
+AS_IF([test "x$LCOV" = xyes],
+    [],
+    [AC_MSG_WARN([lcov not found; make test-coverage and test-coverage-fuzz will not work])])
 AC_CONFIG_FILES(src/libmerc/Makefile src/cython/Makefile src/Makefile test/Makefile unit_tests/Makefile Makefile_helper.mk install_mercury/mercury.service)
 AC_OUTPUT