Enable test retry for daily builds

.github/repository-settings.md

@@ -75,6 +75,7 @@ for [`dependabot/**/**`](https://github.com/open-telemetry/community/blob/main/d
   - Key is associated with [@trask](https://github.com/trask)'s gmail address
 - `SONATYPE_KEY` - owned by [@trask](https://github.com/trask)
 - `SONATYPE_USER` - owned by [@trask](https://github.com/trask)
+- `FLAKY_TEST_REPORTER_ACCESS_KEY` - owned by [@laurit](https://github.com/laurit)
 
 ### Organization secrets
 

.github/workflows/build-common.yml

@@ -290,6 +290,25 @@ jobs:
         if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }}
         run: cat build-scan.txt
 
+      - name: Get current job url
+        uses: Tiryoh/gha-jobid-action@be260d8673c9211a84cdcf37794ebd654ba81eef # v1.4.0
+        id: jobs
+        if: ${{ !cancelled() }}
+        with:
+          job_name: "common / test${{ matrix.test-partition }} (${{ matrix.test-java-version }}, ${{ matrix.vm }})"
+          per_page: 100 # input matrix size here if it is larger than 30
+
+      - name: Flaky test report
+        if: ${{ !cancelled() }}
+        env:
+          FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }}
+          JOB_URL: ${{ steps.jobs.outputs.html_url }}
+        run: |
+          if [ -s build-scan.txt ]; then
+            export BUILD_SCAN_URL=$(cat build-scan.txt)
+          fi
+          ./gradlew :test-report:reportFlakyTests
+
       - name: Upload deadlock detector artifacts if any
         if: failure()
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0

.github/workflows/build-daily-no-build-cache.yml

@@ -10,19 +10,16 @@ jobs:
   common:
     uses: ./.github/workflows/build-common.yml
     with:
-      max-test-retries: 0
       no-build-cache: true
 
   test-latest-deps:
     uses: ./.github/workflows/reusable-test-latest-deps.yml
     with:
-      max-test-retries: 0
       no-build-cache: true
 
   test-indy:
     uses: ./.github/workflows/reusable-test-indy.yml
     with:
-      max-test-retries: 0
       no-build-cache: true
 
   # muzzle is not included here because it doesn't use gradle cache anyway and so is already covered

.github/workflows/build-daily.yml

@@ -9,18 +9,12 @@ on:
 jobs:
   common:
     uses: ./.github/workflows/build-common.yml
-    with:
-      max-test-retries: 0
 
   test-latest-deps:
     uses: ./.github/workflows/reusable-test-latest-deps.yml
-    with:
-      max-test-retries: 0
 
   test-indy:
     uses: ./.github/workflows/reusable-test-indy.yml
-    with:
-      max-test-retries: 0
 
   muzzle:
     uses: ./.github/workflows/reusable-muzzle.yml

.github/workflows/reusable-test-indy.yml

@@ -86,3 +86,22 @@ jobs:
       - name: Build scan
         if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }}
         run: cat build-scan.txt
+
+      - name: Get current job url
+        uses: Tiryoh/gha-jobid-action@be260d8673c9211a84cdcf37794ebd654ba81eef # v1.4.0
+        id: jobs
+        if: ${{ !cancelled() }}
+        with:
+          job_name: "test-indy / testIndy${{ matrix.test-partition }}"
+          per_page: 100 # input matrix size here if it is larger than 30
+
+      - name: Flaky test report
+        if: ${{ !cancelled() }}
+        env:
+          FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }}
+          JOB_URL: ${{ steps.jobs.outputs.html_url }}
+        run: |
+          if [ -s build-scan.txt ]; then
+            export BUILD_SCAN_URL=$(cat build-scan.txt)
+          fi
+          ./gradlew :test-report:reportFlakyTests

.github/workflows/reusable-test-latest-deps.yml

@@ -85,6 +85,25 @@ jobs:
         if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }}
         run: cat build-scan.txt
 
+      - name: Get current job url
+        uses: Tiryoh/gha-jobid-action@be260d8673c9211a84cdcf37794ebd654ba81eef # v1.4.0
+        id: jobs
+        if: ${{ !cancelled() }}
+        with:
+          job_name: "test-latest-deps / testLatestDeps${{ matrix.test-partition }}"
+          per_page: 100 # input matrix size here if it is larger than 30
+
+      - name: Flaky test report
+        if: ${{ !cancelled() }}
+        env:
+          FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }}
+          JOB_URL: ${{ steps.jobs.outputs.html_url }}
+        run: |
+          if [ -s build-scan.txt ]; then
+            export BUILD_SCAN_URL=$(cat build-scan.txt)
+          fi
+          ./gradlew :test-report:reportFlakyTests
+
       - name: Upload deadlock detector artifacts if any
         if: failure()
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0

settings.gradle.kts

@@ -97,6 +97,7 @@ include(":instrumentation-annotations-support-testing")
 
 // misc
 include(":dependencyManagement")
+include(":test-report")
 include(":testing:agent-exporter")
 include(":testing:agent-for-testing")
 include(":testing:armeria-shaded-for-testing")

test-report/build.gradle.kts

@@ -0,0 +1,27 @@
+plugins {
+  id("otel.java-conventions")
+}
+
+dependencies {
+  implementation("com.google.api-client:google-api-client:2.7.1")
+  implementation("com.google.apis:google-api-services-sheets:v4-rev20250106-2.0.0")
+  implementation("com.google.auth:google-auth-library-oauth2-http:1.30.1")
+}
+
+otelJava {
+  minJavaVersionSupported.set(JavaVersion.VERSION_17)
+}
+
+tasks {
+  val reportFlakyTests by registering(JavaExec::class) {
+    dependsOn(classes)
+
+    mainClass.set("io.opentelemetry.instrumentation.testreport.FlakyTestReporter")
+    classpath(sourceSets["main"].runtimeClasspath)
+
+    systemProperty("scanPath", project.rootDir)
+    systemProperty("googleSheetsAccessKey", System.getenv("FLAKY_TEST_REPORTER_ACCESS_KEY"))
+    systemProperty("buildScanUrl", System.getenv("BUILD_SCAN_URL"))
+    systemProperty("jobUrl", System.getenv("JOB_URL"))
+  }
+}