infra: update introspector and add new light version (google#12983)

Adds a light version of FI that should work on all projects. This will
only be used if the existing FI does not work, e.g. due to failures
around LTO.

Also starts using the CLI version of fuzz-introspector as opposed to
calling `main.py`.

---------

Signed-off-by: David Korczynski <[email protected]>

Author: DavidKorczynski

infra/base-images/base-builder/compile

@@ -229,17 +229,47 @@ if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ];
 
   # Install Fuzz-Introspector
   pushd /fuzz-introspector/src
-  python3 -m pip install .
+  python3 -m pip install -e .
   popd
 
   if [ "$FUZZING_LANGUAGE" = "python" ]; then
     python3 /fuzz-introspector/src/main.py light --language=python
+    cp -rf $SRC/inspector/ /tmp/inspector-saved
   elif [ "$FUZZING_LANGUAGE" = "jvm" ]; then
     python3 /fuzz-introspector/src/main.py light --language=jvm
+    cp -rf $SRC/inspector/ /tmp/inspector-saved
   elif [ "$FUZZING_LANGUAGE" = "rust" ]; then
     python3 /fuzz-introspector/src/main.py light --language=rust
+    cp -rf $SRC/inspector/ /tmp/inspector-saved
   else
     python3 /fuzz-introspector/src/main.py light
+
+    # Make a copy of the light. This is needed because we run two versions of
+    # introspector: one based on pure statis analysis and one based on
+    # regular LTO.
+    cp -rf $SRC/inspector/ /tmp/inspector-saved
+
+
+    # Move coverage report.
+    if [ -d "$OUT/textcov_reports" ]
+    then
+      find $OUT/textcov_reports/ -name "*.covreport" -exec cp {} $SRC/inspector/ \;
+      find $OUT/textcov_reports/ -name "*.json" -exec cp {} $SRC/inspector/ \;
+    fi
+
+    # Make fuzz-introspector HTML report using light approach.
+    REPORT_ARGS="--name=$PROJECT_NAME"
+
+    # Only pass coverage_url when COVERAGE_URL is set (in cloud builds)
+    if [[ ! -z "${COVERAGE_URL+x}" ]]; then
+      REPORT_ARGS="$REPORT_ARGS --coverage-url=${COVERAGE_URL}"
+    fi
+
+    # Run pure static analysis fuzz introspector
+    fuzz-introspector full --target-dir=$SRC \
+        --language=${FUZZING_LANGUAGE} \
+        --out-dir=$SRC/inspector \
+        ${REPORT_ARGS}
   fi
 
   rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
@@ -313,28 +343,37 @@ if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ];
   unset CFLAGS
   export G_ANALYTICS_TAG="G-8WTFM1Y62J"
 
+  # If we get to here, it means the e.g. LTO had no problems and succeeded.
+  # TO this end, we wlil restore the original light analysis and used the
+  # LTO processing itself.
+  rm -rf $SRC/inspector
+  cp -rf /tmp/inspector-saved $SRC/inspector
+
+  cd /fuzz-introspector/src
+  python3 -m pip install -e .
+  cd /src/
+
   if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
     echo "GOING jvm route"
 
     set -x
     # Output will be put in /out/
-    python3 /fuzz-introspector/frontends/java/oss-fuzz-main.py
+    python3 -m fuzz_introspector.frontends.oss_fuzz --language jvm --target-dir $SRC --entrypoint fuzzerTestOneInput
+
     # Move files temporarily to fit workflow of other languages.
     mkdir -p $SRC/my-fi-data
-    find $OUT/ -name *.data -exec mv {} $SRC/my-fi-data/ \;
-    find $OUT/ -name *.data.yaml -exec mv {} $SRC/my-fi-data/ \;
+    find ./ -name *.data -exec mv {} $SRC/my-fi-data/ \;
+    find ./ -name *.data.yaml -exec mv {} $SRC/my-fi-data/ \;
   elif [ "$FUZZING_LANGUAGE" = "rust" ]; then
     echo "GOING rust route"
 
     # Run the rust frontend
-    pushd /fuzz-introspector/frontends/rust/rust_function_analyser
-    cargo run -- $SRC
+    python3 -m fuzz_introspector.frontends.oss_fuzz --language rust --target-dir $SRC
 
     # Move files temporarily to fix workflow of other languages.
     mkdir -p $SRC/my-fi-data
     find ./ -name "*.data" -exec mv {} $SRC/my-fi-data/ \;
     find ./ -name "*.data.yaml" -exec mv {} $SRC/my-fi-data/ \;
-    popd
 
     # Restore the sanitizer flag for rust
     export SANITIZER="introspector"
@@ -359,43 +398,43 @@ if [ "$SANITIZER" = "introspector" ] || [ "$RUST_SANITIZER" = "introspector" ];
   REPORT_ARGS="--name=$PROJECT_NAME"
   # Only pass coverage_url when COVERAGE_URL is set (in cloud builds)
   if [[ ! -z "${COVERAGE_URL+x}" ]]; then
-    REPORT_ARGS="$REPORT_ARGS --coverage_url=${COVERAGE_URL}"
+    REPORT_ARGS="$REPORT_ARGS --coverage-url=${COVERAGE_URL}"
   fi
 
   # Do different things depending on languages
   if [ "$FUZZING_LANGUAGE" = "python" ]; then
     echo "GOING python route"
     set -x
-    REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector"
+    REPORT_ARGS="$REPORT_ARGS --target-dir=$SRC/inspector"
     REPORT_ARGS="$REPORT_ARGS --language=python"
-    python3 /fuzz-introspector/src/main.py report $REPORT_ARGS
+    fuzz-introspector report $REPORT_ARGS
     rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
   elif [ "$FUZZING_LANGUAGE" = "jvm" ]; then
     echo "GOING jvm route"
     set -x
     find $OUT/ -name "jacoco.xml" -exec cp {} $SRC/inspector/ \;
-    REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector"
+    REPORT_ARGS="$REPORT_ARGS --target-dir=$SRC/inspector"
     REPORT_ARGS="$REPORT_ARGS --language=jvm"
-    python3 /fuzz-introspector/src/main.py report $REPORT_ARGS
+    fuzz-introspector report $REPORT_ARGS
     rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
   elif [ "$FUZZING_LANGUAGE" = "rust" ]; then
     echo "GOING rust route"
-    REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector"
+    REPORT_ARGS="$REPORT_ARGS --target-dir=$SRC/inspector"
     REPORT_ARGS="$REPORT_ARGS --language=rust"
-    python3 /fuzz-introspector/src/main.py report $REPORT_ARGS
+    fuzz-introspector report $REPORT_ARGS
     rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
   else
     # C/C++
-
+    mkdir -p $SRC/inspector
     # Correlate fuzzer binaries to fuzz-introspector's raw data
-    python3 /fuzz-introspector/src/main.py correlate --binaries_dir=$OUT/
+    fuzz-introspector correlate --binaries-dir=$OUT/
 
     # Generate fuzz-introspector HTML report, this generates
     # the file exe_to_fuzz_introspector_logs.yaml
-    REPORT_ARGS="$REPORT_ARGS --target_dir=$SRC/inspector"
+    REPORT_ARGS="$REPORT_ARGS --target-dir=$SRC/inspector"
     # Use the just-generated correlation file
-    REPORT_ARGS="$REPORT_ARGS --correlation_file=exe_to_fuzz_introspector_logs.yaml"
-    python3 /fuzz-introspector/src/main.py report $REPORT_ARGS
+    REPORT_ARGS="$REPORT_ARGS --correlation-file=exe_to_fuzz_introspector_logs.yaml"
+    fuzz-introspector report $REPORT_ARGS
 
     rsync -avu --delete "$SRC/inspector/" "$OUT/inspector"
   fi

infra/base-images/base-builder/install_swift.sh

@@ -37,10 +37,10 @@ apt-get update && apt install -y $SWIFT_PACKAGES && \
   apt install -y $SWIFT_SYMBOLIZER_PACKAGES --no-install-recommends
 
 
-wget -q https://download.swift.org/swift-5.8.1-release/ubuntu2004/swift-5.8.1-RELEASE/swift-5.8.1-RELEASE-ubuntu20.04.tar.gz
-tar xzf swift-5.8.1-RELEASE-ubuntu20.04.tar.gz
-cp -r swift-5.8.1-RELEASE-ubuntu20.04/usr/* /usr/
-rm -rf swift-5.8.1-RELEASE-ubuntu20.04.tar.gz swift-5.8.1-RELEASE-ubuntu20.04/
+wget -q https://download.swift.org/swift-5.10.1-release/ubuntu2004/swift-5.10.1-RELEASE/swift-5.10.1-RELEASE-ubuntu20.04.tar.gz
+tar xzf swift-5.10.1-RELEASE-ubuntu20.04.tar.gz
+cp -r swift-5.10.1-RELEASE-ubuntu20.04/usr/* /usr/
+rm -rf swift-5.10.1-RELEASE-ubuntu20.04.tar.gz swift-5.10.1-RELEASE-ubuntu20.04/
 # TODO: Move to a seperate work dir
 git clone https://github.com/llvm/llvm-project.git
 cd llvm-project

infra/base-images/base-clang/Dockerfile

@@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y wget sudo && \
 RUN apt-get update && apt-get install -y git && \
     git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \
     cd fuzz-introspector && \
-    git checkout 74917384c5a4e368d900862b4bd3d16ce3fe5dd8 && \
+    git checkout eab8da2f42d07fb63d28b30c1cc12a2bfcd8e648 && \
     git submodule init && \
     git submodule update && \
     apt-get autoremove --purge -y git && \