NUTCH-3085 Augment CI by adding code coverage and code quality reporting

.github/workflows/master-build.yml

@@ -66,13 +66,18 @@ jobs:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v4.2.2
+      - name: Check out Repository
+        uses: actions/checkout@v4.2.2
+        # Disabling shallow clone is recommended for improving sonarcloud reporting
+        with:
+          fetch-depth: 0
       - name: Set up JDK ${{ matrix.java }}
         uses: actions/setup-java@v4.5.0
         with:
           java-version: ${{ matrix.java }}
           distribution: 'temurin'
-      - uses: dorny/paths-filter@v3.0.0
+      - name: Filter 'core' and 'plugins' paths
+        uses: dorny/paths-filter@v3.0.0
         id: filter
         with:
           filters: |
@@ -86,14 +91,41 @@ jobs:
               - 'build.xml'
               - 'ivy/ivy.xml'
       # run if the build configuration or both 'core' and 'plugins' files were changed
-      - name: test all
+      - name: Test all
         if: ${{ steps.filter.outputs.buildconf == 'true' || ( steps.filter.outputs.core  == 'true' && steps.filter.outputs.plugins  == 'true' ) }}
         run: ant clean test -buildfile build.xml
       # run only if 'core' files were changed
-      - name: test core
+      - name: Test core
         if: ${{ steps.filter.outputs.core == 'true' && steps.filter.outputs.plugins == 'false' && steps.filter.outputs.buildconf == 'false' }}
         run: ant clean test-core -buildfile build.xml
       # run only if 'plugins' files were changed
-      - name: test plugins
+      - name: Test plugins
         if: ${{ steps.filter.outputs.plugins == 'true' && steps.filter.outputs.core == 'false' && steps.filter.outputs.buildconf == 'false' }}
         run: ant clean test-plugins -buildfile build.xml
+      - name: Fix code coverage paths
+        if: |
+          steps.filter.outputs.core == 'true' ||
+          steps.filter.outputs.plugins == 'true'
+        run: |
+          find . -name jacoco_report.xml -exec sed -i 's#/home/runner/work/nutch/nutch#/github/workspace#g' {} +
+      - name: Analyze with SonarCloud
+        if: |
+          steps.filter.outputs.core == 'true' ||
+          steps.filter.outputs.plugins == 'true'
+        uses: sonarsource/sonarcloud-github-action@v3.1.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        with:
+          args: >
+            -Dsonar.organization=apache
+            -Dsonar.projectKey=apache-nutch
+            -Dsonar.sources=src/java
+            -Dsonar.test.exclusions=src/test/**
+            -Dsonar.tests=src/test/
+            -Dsonar.verbose=true
+            -Dsonar.java.binaries=build/classes,build/**/classes
+            -Dsonar.java.libraries=build/lib/*.jar   
+            -Dsonar.java.test.binaries=build/test/classes/,build/**/test/
+            -Dsonar.java.test.libraries=build/test/lib/*.jar
+            -Dsonar.java.source=11

.gitignore

@@ -27,6 +27,7 @@ ivy/dependency-check-ant/*
 .gradle*
 ivy/apache-rat-*
 .vscode
+ivy/jacoco-*
 # native Hadoop libraries, see lib/native/README.txt
 lib/native/libhadoop.*
 lib/native/libhadooppipes.*

build.xml

@@ -20,7 +20,8 @@
          xmlns:ivy="antlib:org.apache.ivy.ant"
          xmlns:artifact="antlib:org.apache.maven.artifact.ant"
          xmlns:rat="antlib:org.apache.rat.anttasks"
-         xmlns="antlib:org.apache.tools.ant">
+         xmlns="antlib:org.apache.tools.ant"
+         xmlns:jacoco="antlib:org.jacoco.ant">
 
   <!-- Load all the default properties, and any the user wants    -->
   <!-- to contribute (without having to type -D or edit this file -->
@@ -48,6 +49,12 @@
   <property name="apache-rat.home" value="${ivy.dir}/apache-rat-${apache-rat.version}" />
   <property name="apache-rat.jar" value="${apache-rat.home}/apache-rat-${apache-rat.version}.jar" />
 
+  <property name="jacoco.version" value="0.8.12" />
+  <property name="jacoco.home" value="${ivy.dir}/jacoco-${jacoco.version}" />
+  <property name="jacoco.jar" value="${jacoco.home}/lib/jacocoant.jar" />
+  <property name="jacoco.output.dir" location="${build.dir}/jacoco" />
+  <property name="jacoco.destfile" location="${jacoco.output.dir}/jacoco.exec" />
+
   <condition property="using.jdk.11">
     <matches string="${java.version}" pattern="11.+" casesensitive="false" />
   </condition>
@@ -275,9 +282,6 @@
 
       <link href="${javadoc.link.java}"/>
       <link href="${javadoc.link.hadoop}"/>
-      <!--link href="${javadoc.link.lucene.core}"/>
-      <link href="${javadoc.link.lucene.analyzers-common}"/>
-      <link href="${javadoc.link.solr-solrj}"/-->
 
       <classpath refid="classpath"/>
       <classpath>
@@ -475,8 +479,11 @@
     </antcall>
   </target>
 
-  <target name="test-core" depends="compile-core-test, job" description="--> run core JUnit tests only">
+  <target name="test-core" depends="jacoco-download, compile-core-test, job" description="--> run core JUnit tests only">
 
+    <taskdef uri="antlib:org.jacoco.ant" resource="org/jacoco/ant/antlib.xml">
+      <classpath path="${jacoco.jar}" />
+    </taskdef>
     <delete dir="${test.build.data}"/>
     <mkdir dir="${test.build.data}"/>
     <!--
@@ -498,32 +505,54 @@
     <copy file="${test.src.dir}/filter-all.txt"
         todir="${test.build.classes}"/>
 
-    <junit printsummary="yes" haltonfailure="no" fork="yes" dir="${basedir}"
-      errorProperty="tests.failed" failureProperty="tests.failed" maxmemory="1000m">
-      <sysproperty key="test.build.data" value="${test.build.data}"/>
-      <sysproperty key="test.src.dir" value="${test.src.dir}"/>
-      <sysproperty key="test.include.slow" value="${test.include.slow}"/>
-      <sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
-      <classpath refid="test.classpath"/>
-      <formatter type="${test.junit.output.format}" />
-      <batchtest todir="${test.build.dir}" unless="testcase">
-        <fileset dir="${test.src.dir}"
-                 includes="**/Test*.java" excludes="**/${test.exclude}.java" />
-      </batchtest>
-      <batchtest todir="${test.build.dir}" if="testcase">
-        <fileset dir="${test.src.dir}" includes="**/${testcase}.java"/>
-      </batchtest>
-    </junit>
+    <mkdir dir="${build.dir}/jacoco"/>
+    <jacoco:coverage destfile="${build.dir}/jacoco/jacoco.exec">
+      <junit printsummary="yes" haltonerror="true" haltonfailure="true" 
+        fork="yes" dir="${basedir}" errorProperty="tests.failed" 
+        failureProperty="tests.failed" maxmemory="1000m" forkmode="once" >
+        <sysproperty key="test.build.data" value="${test.build.data}"/>
+        <sysproperty key="test.src.dir" value="${test.src.dir}"/>
+        <sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
+        <classpath refid="test.classpath"/>
+        <formatter type="${test.junit.output.format}" />
+        <batchtest todir="${test.build.dir}" unless="testcase">
+          <fileset dir="${test.src.dir}"
+                   includes="**/Test*.java" excludes="**/${test.exclude}.java" />
+        </batchtest>
+        <batchtest todir="${test.build.dir}" if="testcase">
+          <fileset dir="${test.src.dir}" includes="**/${testcase}.java"/>
+        </batchtest>
+      </junit>
+    </jacoco:coverage>
+
+    <mkdir dir="${jacoco.output.dir}/html"/>
+    <mkdir dir="${jacoco.output.dir}/xml"/>
+    <jacoco:report>
+      <executiondata>
+          <file file="${jacoco.destfile}"/>
+      </executiondata>
+      <structure name="${final.name} Jacoco report">
+          <classfiles>
+              <fileset dir="${build.classes}"/>
+          </classfiles>
+          <sourcefiles encoding="UTF-8">
+              <fileset dir="${src.dir}"/>
+          </sourcefiles>
+      </structure>
+      <html destdir="${jacoco.output.dir}/html"
+        footer="${final.name} Jacoco report" encoding="UTF-8"/>
+      <xml destfile="${jacoco.output.dir}/xml/jacoco_report.xml" encoding="UTF-8"/>
+    </jacoco:report>
 
     <fail if="tests.failed">Tests failed!</fail>
 
   </target>
 
-  <target name="test-plugins" depends="resolve-test, compile, compile-core-test" description="--> run plugin JUnit tests only">
+  <target name="test-plugins" depends="resolve-test, jacoco-download, compile, compile-core-test" description="--> run plugin JUnit tests only">
     <ant dir="src/plugin" target="test" inheritAll="false"/>
   </target>
 
-  <target name="test-plugin" depends="resolve-test, compile, compile-core-test" description="--> run a single plugin's JUnit tests">
+  <target name="test-plugin" depends="resolve-test, jacoco-download, compile, compile-core-test" description="--> run a single plugin's JUnit tests">
     <ant dir="src/plugin" target="test-single" inheritAll="false"/>
   </target>
 
@@ -1049,17 +1078,17 @@
     </rat:report>
   </target>
 
-  <!-- ================================================================== -->
-  <!-- Spotbugs                                                           -->
-  <!-- ================================================================== -->
+  <!-- ========= -->
+  <!-- Spotbugs  -->
+  <!-- ========= -->
   <target name="spotbugs-download" description="--> download spotbugs jar">
     <available file="${spotbugs.jar}" property="spotbugs.jar.found"/>
     <antcall target="spotbugs-download-unchecked"/>
   </target>
 
   <target name="spotbugs-download-unchecked" unless="spotbugs.jar.found"
           description="--> downloads the spotbugs binary (spotbugs-*.tgz).">
-    <get src="https://github.com/spotbugs/spotbugs/releases/download/${spotbugs.version}/spotbugs-${spotbugs.version}.tgz "
+    <get src="https://github.com/spotbugs/spotbugs/releases/download/${spotbugs.version}/spotbugs-${spotbugs.version}.tgz"
          dest="${ivy.dir}/spotbugs-${spotbugs.version}.tgz" usetimestamp="false" />
 
     <untar src="${ivy.dir}/spotbugs-${spotbugs.version}.tgz"
@@ -1116,11 +1145,26 @@
     </fileset>
   </path>
 
+  <!-- ======= -->
+  <!-- Jacoco  -->
+  <!-- ======= -->
+  <target name="jacoco-download" description="--> download jacoco jar">
+    <available file="${jacoco.jar}" property="jacoco.jar.found"/>
+    <antcall target="jacoco-download-unchecked"/>
+  </target>
+
+  <target name="jacoco-download-unchecked" unless="jacoco.jar.found"
+    description="--> downloads the jacoco binary">
+    <get src="https://search.maven.org/remotecontent?filepath=org/jacoco/jacoco/${jacoco.version}/jacoco-${jacoco.version}.zip"
+      dest="${jacoco.home}.zip" usetimestamp="false" />
+    <unzip src="${jacoco.home}.zip" dest="${jacoco.home}" />
+    <delete file="${jacoco.home}.zip" />
+  </target>
 
-  <!-- ================================================================== -->
-  <!-- Eclipse targets                                                    -->
-  <!-- ================================================================== -->
 
+  <!-- ================ -->
+  <!-- Eclipse targets  -->
+  <!-- ================ -->
   <!-- classpath for generating eclipse project -->
   <path id="eclipse.classpath">
     <fileset dir="${build.lib.dir}">

default.properties

@@ -52,8 +52,8 @@ src.dist.version.dir=${dist.dir}/${final.name}-src
 bin.dist.version.dir=${dist.dir}/${final.name}-bin
 
 javac.debug=on
-javac.optimize=on
 javac.deprecation=on
+javac.optimize=on
 javac.version=11
 
 runtime.dir=./runtime

src/plugin/build-plugin.xml

@@ -16,7 +16,8 @@
  limitations under the License.
 -->
 <!-- Imported by plugin build.xml files to define default targets. -->
-<project xmlns:ivy="antlib:org.apache.ivy.ant">
+<project xmlns:ivy="antlib:org.apache.ivy.ant"
+         xmlns:jacoco="antlib:org.jacoco.ant">
 
   <property name="name" value="${ant.project.name}"/>
   <property name="root" value="${basedir}"/>
@@ -44,6 +45,12 @@
   <!-- load nutch defaults last so that they can be overridden above -->
   <property file="${nutch.root}/default.properties" />
 
+  <property name="jacoco.version" value="0.8.12" />
+  <property name="jacoco.home" value="${nutch.root}/ivy/jacoco-${jacoco.version}" />
+  <property name="jacoco.jar" value="${jacoco.home}/lib/jacocoant.jar" />
+  <property name="jacoco.output.dir" location="${nutch.root}/build/jacoco"/>
+  <property name="jacoco.destfile" location="${jacoco.output.dir}/jacoco.exec"/>
+
   <ivy:settings id="ivy.instance" file="${nutch.root}/ivy/ivysettings.xml" />
 
   <path id="plugin.deps"/>
@@ -76,16 +83,20 @@
     <pathelement location="${nutch.root}/build"/>
     <!-- test dependencies specific to the current plugin -->
     <fileset dir="${build.test.lib}">
-      <include name="*.jar" />
+      <include name="*.jar"/>
     </fileset>
     <!-- global test dependencies -->
     <fileset dir="${nutch.root}/build/test/lib">
-      <include name="junit*.jar" />
-      <include name="hamcrest*.jar" />
+      <include name="junit*.jar"/>
+      <include name="hamcrest*.jar"/>
     </fileset>
     <path refid="classpath"/>
   </path>
 
+  <taskdef uri="antlib:org.jacoco.ant" resource="org/jacoco/ant/antlib.xml">
+    <classpath path="${jacoco.jar}"/>
+  </taskdef>
+
   <!-- ====================================================== -->
   <!-- Stuff needed by all targets                            -->
   <!-- ====================================================== -->
@@ -123,7 +134,7 @@
   <!-- ====================================================== -->
   <!-- Compile the Java files                                 -->
   <!-- ====================================================== -->
-  <target name="compile" depends="init,deps-jar, resolve-default">
+  <target name="compile" depends="init, deps-jar, resolve-default">
     <echo message="Compiling plugin: ${name}"/>
     <javac 
      encoding="${build.encoding}" 
@@ -207,22 +218,44 @@
   <!-- ================================================================== -->
   <target name="test" depends="compile-test, deploy" if="test.available">
     <echo message="Testing plugin: ${name}"/>
-
-    <junit printsummary="yes" haltonfailure="no" fork="yes"
-      errorProperty="tests.failed" failureProperty="tests.failed">
-      <sysproperty key="test.data" value="${build.test}/data"/>
-      <sysproperty key="test.input" value="${root}/data"/>
-      <sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/> 
-      <classpath refid="test.classpath"/>
-      <formatter type="${test.junit.output.format}" />
-      <batchtest todir="${build.test}" unless="testcase">
-        <fileset dir="${src.test}"
-                 includes="**/Test*.java" excludes="**/${test.exclude}.java" />
-      </batchtest>
-      <batchtest todir="${build.test}" if="testcase">
-        <fileset dir="${src.test}" includes="**/${testcase}.java"/>
-      </batchtest>
-    </junit>
+    <mkdir dir="${jacoco.output.dir}"/>
+    <jacoco:coverage destfile="${jacoco.destfile}">
+      <junit printsummary="yes" haltonerror="true" haltonfailure="true" 
+        fork="yes" errorProperty="tests.failed" failureProperty="tests.failed"
+        forkmode="once" >
+        <sysproperty key="test.data" value="${build.test}/data"/>
+        <sysproperty key="test.input" value="${root}/data"/>
+        <sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/> 
+        <classpath refid="test.classpath"/>
+        <formatter type="${test.junit.output.format}" />
+        <batchtest todir="${build.test}" unless="testcase">
+          <fileset dir="${src.test}"
+                   includes="**/Test*.java" excludes="**/${test.exclude}.java" />
+        </batchtest>
+        <batchtest todir="${build.test}" if="testcase">
+          <fileset dir="${src.test}" includes="**/${testcase}.java"/>
+        </batchtest>
+      </junit>
+    </jacoco:coverage>
+
+    <mkdir dir="${jacoco.output.dir}/html"/>
+    <mkdir dir="${jacoco.output.dir}/xml"/>
+    <jacoco:report>
+      <executiondata>
+          <file file="${jacoco.destfile}"/>
+      </executiondata>
+      <structure name="${final.name} Jacoco report">
+          <classfiles>
+              <fileset dir="${build.classes}"/>
+          </classfiles>
+          <sourcefiles encoding="UTF-8">
+              <fileset dir="${root}/src/java"/>
+          </sourcefiles>
+      </structure>
+      <html destdir="${jacoco.output.dir}/html"
+        footer="${final.name} Jacoco report" encoding="UTF-8"/>
+      <xml destfile="${jacoco.output.dir}/xml/jacoco_report.xml" encoding="UTF-8"/>
+    </jacoco:report>
 
     <fail if="tests.failed">Tests failed!</fail>
 

src/plugin/creativecommons/build.xml

@@ -22,7 +22,6 @@
   <!-- Deploy Unit test dependencies -->
   <target name="deps-test">
     <ant target="deploy" inheritall="false" dir="../nutch-extensionpoints"/>
-   <!--  <ant target="deploy" inheritall="false" dir="../parse-html"/> -->
   </target>
 
 </project>