diff --git a/.github/repository-settings.md b/.github/repository-settings.md index 66be1268779c..00bcbb91dfcb 100644 --- a/.github/repository-settings.md +++ b/.github/repository-settings.md @@ -75,6 +75,7 @@ for [`dependabot/**/**`](https://github.com/open-telemetry/community/blob/main/d - Key is associated with [@trask](https://github.com/trask)'s gmail address - `SONATYPE_KEY` - owned by [@trask](https://github.com/trask) - `SONATYPE_USER` - owned by [@trask](https://github.com/trask) +- `FLAKY_TEST_REPORTER_ACCESS_KEY` - owned by [@laurit](https://github.com/laurit) ### Organization secrets diff --git a/.github/workflows/build-common.yml b/.github/workflows/build-common.yml index e4585ec7f6cd..92ce584e2e72 100644 --- a/.github/workflows/build-common.yml +++ b/.github/workflows/build-common.yml @@ -290,6 +290,36 @@ jobs: if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }} run: cat build-scan.txt + - name: Get current job url + id: jobs + if: ${{ !cancelled() }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + matrix: ${{ toJson(matrix) }} + with: + result-encoding: string + script: | + const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + per_page: 100 + }); + const matrix = JSON.parse(process.env.matrix); + const job_name = `common / test${ matrix['test-partition'] } (${ matrix['test-java-version'] }, ${ matrix.vm })`; + return workflow_run.jobs.find((job) => job.name === job_name).html_url; + + - name: Flaky test report + if: ${{ !cancelled() }} + env: + FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }} + JOB_URL: ${{ steps.jobs.outputs.result }} + run: | + if [ -s build-scan.txt ]; then + export BUILD_SCAN_URL=$(cat build-scan.txt) + fi + ./gradlew :test-report:reportFlakyTests + - name: Upload deadlock detector artifacts if any if: failure() uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 diff --git a/.github/workflows/build-daily-no-build-cache.yml b/.github/workflows/build-daily-no-build-cache.yml index 596b46773834..d8d2852b007a 100644 --- a/.github/workflows/build-daily-no-build-cache.yml +++ b/.github/workflows/build-daily-no-build-cache.yml @@ -10,19 +10,16 @@ jobs: common: uses: ./.github/workflows/build-common.yml with: - max-test-retries: 0 no-build-cache: true test-latest-deps: uses: ./.github/workflows/reusable-test-latest-deps.yml with: - max-test-retries: 0 no-build-cache: true test-indy: uses: ./.github/workflows/reusable-test-indy.yml with: - max-test-retries: 0 no-build-cache: true # muzzle is not included here because it doesn't use gradle cache anyway and so is already covered diff --git a/.github/workflows/build-daily.yml b/.github/workflows/build-daily.yml index 2e6aae3033bc..dc6b3de4d45d 100644 --- a/.github/workflows/build-daily.yml +++ b/.github/workflows/build-daily.yml @@ -9,18 +9,12 @@ on: jobs: common: uses: ./.github/workflows/build-common.yml - with: - max-test-retries: 0 test-latest-deps: uses: ./.github/workflows/reusable-test-latest-deps.yml - with: - max-test-retries: 0 test-indy: uses: ./.github/workflows/reusable-test-indy.yml - with: - max-test-retries: 0 muzzle: uses: ./.github/workflows/reusable-muzzle.yml diff --git a/.github/workflows/reusable-test-indy.yml b/.github/workflows/reusable-test-indy.yml index cacebf8458a8..70d7caaac563 100644 --- a/.github/workflows/reusable-test-indy.yml +++ b/.github/workflows/reusable-test-indy.yml @@ -86,3 +86,33 @@ jobs: - name: Build scan if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }} run: cat build-scan.txt + + - name: Get current job url + id: jobs + if: ${{ !cancelled() }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + matrix: ${{ toJson(matrix) }} + with: + result-encoding: string + script: | + const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + per_page: 100 + }); + const matrix = JSON.parse(process.env.matrix); + const job_name = `test-indy / testIndy${ matrix['test-partition'] }`; + return workflow_run.jobs.find((job) => job.name === job_name).html_url; + + - name: Flaky test report + if: ${{ !cancelled() }} + env: + FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }} + JOB_URL: ${{ steps.jobs.outputs.result }} + run: | + if [ -s build-scan.txt ]; then + export BUILD_SCAN_URL=$(cat build-scan.txt) + fi + ./gradlew :test-report:reportFlakyTests diff --git a/.github/workflows/reusable-test-latest-deps.yml b/.github/workflows/reusable-test-latest-deps.yml index fe3c6d730e99..4bf397165387 100644 --- a/.github/workflows/reusable-test-latest-deps.yml +++ b/.github/workflows/reusable-test-latest-deps.yml @@ -85,6 +85,36 @@ jobs: if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }} run: cat build-scan.txt + - name: Get current job url + id: jobs + if: ${{ !cancelled() }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + matrix: ${{ toJson(matrix) }} + with: + result-encoding: string + script: | + const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + per_page: 100 + }); + const matrix = JSON.parse(process.env.matrix); + const job_name = `test-latest-deps / testLatestDeps${ matrix['test-partition'] }`; + return workflow_run.jobs.find((job) => job.name === job_name).html_url; + + - name: Flaky test report + if: ${{ !cancelled() }} + env: + FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }} + JOB_URL: ${{ steps.jobs.outputs.result }} + run: | + if [ -s build-scan.txt ]; then + export BUILD_SCAN_URL=$(cat build-scan.txt) + fi + ./gradlew :test-report:reportFlakyTests + - name: Upload deadlock detector artifacts if any if: failure() uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 diff --git a/settings.gradle.kts b/settings.gradle.kts index 167c4eae3b07..d4dfab53c1b3 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -54,7 +54,8 @@ develocity { termsOfUseUrl.set("https://gradle.com/help/legal-terms-of-use") termsOfUseAgree.set("yes") - if (!gradle.startParameter.taskNames.contains("listTestsInPartition")) { + if (!gradle.startParameter.taskNames.contains("listTestsInPartition") && + !gradle.startParameter.taskNames.contains(":test-report:reportFlakyTests")) { buildScanPublished { File("build-scan.txt").printWriter().use { writer -> writer.println(buildScanUri) @@ -97,6 +98,7 @@ include(":instrumentation-annotations-support-testing") // misc include(":dependencyManagement") +include(":test-report") include(":testing:agent-exporter") include(":testing:agent-for-testing") include(":testing:armeria-shaded-for-testing") diff --git a/test-report/build.gradle.kts b/test-report/build.gradle.kts new file mode 100644 index 000000000000..0b6e15361220 --- /dev/null +++ b/test-report/build.gradle.kts @@ -0,0 +1,27 @@ +plugins { + id("otel.java-conventions") +} + +dependencies { + implementation("com.google.api-client:google-api-client:2.7.1") + implementation("com.google.apis:google-api-services-sheets:v4-rev20250106-2.0.0") + implementation("com.google.auth:google-auth-library-oauth2-http:1.30.1") +} + +otelJava { + minJavaVersionSupported.set(JavaVersion.VERSION_17) +} + +tasks { + val reportFlakyTests by registering(JavaExec::class) { + dependsOn(classes) + + mainClass.set("io.opentelemetry.instrumentation.testreport.FlakyTestReporter") + classpath(sourceSets["main"].runtimeClasspath) + + systemProperty("scanPath", project.rootDir) + systemProperty("googleSheetsAccessKey", System.getenv("FLAKY_TEST_REPORTER_ACCESS_KEY")) + systemProperty("buildScanUrl", System.getenv("BUILD_SCAN_URL")) + systemProperty("jobUrl", System.getenv("JOB_URL")) + } +} diff --git a/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java b/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java new file mode 100644 index 000000000000..a687853d7426 --- /dev/null +++ b/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java @@ -0,0 +1,282 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.instrumentation.testreport; + +import static java.nio.file.FileVisitResult.CONTINUE; + +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.services.sheets.v4.Sheets; +import com.google.api.services.sheets.v4.SheetsScopes; +import com.google.api.services.sheets.v4.model.ValueRange; +import com.google.auth.http.HttpCredentialsAdapter; +import com.google.auth.oauth2.GoogleCredentials; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import org.w3c.dom.Document; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +@SuppressWarnings("SystemOut") +public class FlakyTestReporter { + // https://docs.google.com/spreadsheets/d/1pfa6Ws980AIFI3kKOeIc51-JGEzakG7hkMl4J9h-Tk0 + private static final String SPREADSHEET_ID = "1pfa6Ws980AIFI3kKOeIc51-JGEzakG7hkMl4J9h-Tk0"; + + private int testCount; + private int skippedCount; + private int failureCount; + private int errorCount; + private final List flakyTests = new ArrayList<>(); + + private static class FlakyTest { + final String testClassName; + final String testName; + final String timestamp; + final String message; + + FlakyTest(String testClassName, String testName, String timestamp, String message) { + this.testClassName = testClassName; + this.testName = testName; + this.timestamp = timestamp; + this.message = message; + } + } + + private void addFlakyTest( + String testClassName, String testName, String timestamp, String message) { + flakyTests.add(new FlakyTest(testClassName, testName, timestamp, message)); + } + + private static Document parse(Path testReport) { + try { + DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + return builder.parse(testReport.toFile()); + } catch (Exception exception) { + throw new IllegalStateException("failed to parse test report " + testReport, exception); + } + } + + private void scanTestFile(Path testReport) { + Document doc = parse(testReport); + doc.getDocumentElement().normalize(); + testCount += Integer.parseInt(doc.getDocumentElement().getAttribute("tests")); + skippedCount += Integer.parseInt(doc.getDocumentElement().getAttribute("skipped")); + int failures = Integer.parseInt(doc.getDocumentElement().getAttribute("failures")); + failureCount += failures; + int errors = Integer.parseInt(doc.getDocumentElement().getAttribute("errors")); + errorCount += errors; + String timestamp = doc.getDocumentElement().getAttribute("timestamp"); + + // there are no flaky tests if there are no failures, skip it + if (failures == 0 && errors == 0) { + return; + } + + class TestCase { + final String className; + final String name; + boolean failed; + boolean succeeded; + String message; + + TestCase(String className, String name) { + this.className = className; + this.name = name; + } + + boolean isFlaky() { + return succeeded && failed; + } + } + + Map testcaseMap = new HashMap<>(); + + NodeList testcaseNodes = doc.getElementsByTagName("testcase"); + for (int i = 0; i < testcaseNodes.getLength(); i++) { + Node testNode = testcaseNodes.item(i); + + String testClassName = testNode.getAttributes().getNamedItem("classname").getNodeValue(); + String testName = testNode.getAttributes().getNamedItem("name").getNodeValue(); + String testKey = testClassName + "." + testName; + TestCase testCase = + testcaseMap.computeIfAbsent(testKey, (s) -> new TestCase(testClassName, testName)); + NodeList childNodes = testNode.getChildNodes(); + boolean failed = false; + for (int j = 0; j < childNodes.getLength(); j++) { + Node childNode = childNodes.item(j); + String nodeName = childNode.getNodeName(); + if ("failure".equals(nodeName) || "error".equals(nodeName)) { + failed = true; + // if test fails multiple times we'll use the first failure message + if (testCase.message == null) { + String message = getAttributeValue(childNode, "message"); + if (message != null) { + // compress failure message on a single line + message = message.replaceAll("\n( )*", " "); + } + testCase.message = message; + } + } + } + if (failed) { + testCase.failed = true; + } else { + testCase.succeeded = true; + } + } + + for (TestCase testCase : testcaseMap.values()) { + if (testCase.isFlaky()) { + addFlakyTest(testCase.className, testCase.name, timestamp, testCase.message); + } + } + } + + private static String getAttributeValue(Node node, String attributeName) { + NamedNodeMap attributes = node.getAttributes(); + if (attributes == null) { + return null; + } + Node value = attributes.getNamedItem(attributeName); + return value != null ? value.getNodeValue() : null; + } + + private void scanTestResults(Path buildDir) throws IOException { + Path testResults = buildDir.resolve("test-results"); + if (!Files.exists(testResults)) { + return; + } + + Files.walkFileTree( + testResults, + new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + String name = file.getFileName().toString(); + if (name.startsWith("TEST-") && name.endsWith(".xml")) { + scanTestFile(file); + } + + return CONTINUE; + } + }); + } + + private static FlakyTestReporter scan(Path path) throws IOException { + FlakyTestReporter reporter = new FlakyTestReporter(); + Files.walkFileTree( + path, + new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + throws IOException { + if (dir.endsWith("build")) { + reporter.scanTestResults(dir); + return FileVisitResult.SKIP_SUBTREE; + } + if (dir.endsWith("src")) { + return FileVisitResult.SKIP_SUBTREE; + } + return CONTINUE; + } + }); + return reporter; + } + + private void print() { + System.err.printf( + "Found %d test, skipped %d, failed %d, errored %d\n", + testCount, skippedCount, failureCount, errorCount); + if (!flakyTests.isEmpty()) { + System.err.printf("Found %d flaky test(s):\n", flakyTests.size()); + for (FlakyTest flakyTest : flakyTests) { + System.err.println( + flakyTest.timestamp + + " " + + flakyTest.testClassName + + " " + + flakyTest.testName + + " " + + flakyTest.message); + } + } + } + + // add flaky tests to a google sheet + private void report(String accessKey, String buildScanUrl, String jobUrl) throws Exception { + if (flakyTests.isEmpty()) { + return; + } + + NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); + GoogleCredentials credentials = + GoogleCredentials.fromStream( + new ByteArrayInputStream(accessKey.getBytes(StandardCharsets.UTF_8))) + .createScoped(Collections.singletonList(SheetsScopes.SPREADSHEETS)); + Sheets service = + new Sheets.Builder( + transport, + GsonFactory.getDefaultInstance(), + new HttpCredentialsAdapter(credentials)) + .setApplicationName("Flaky test reporter") + .build(); + + List> data = new ArrayList<>(); + for (FlakyTest flakyTest : flakyTests) { + List row = new ArrayList<>(); + row.add(flakyTest.timestamp); + row.add(flakyTest.testClassName); + row.add(flakyTest.testName); + row.add(buildScanUrl); + row.add(jobUrl); + row.add(flakyTest.message); + data.add(row); + } + + ValueRange valueRange = new ValueRange(); + valueRange.setValues(data); + service + .spreadsheets() + .values() + .append(SPREADSHEET_ID, "Sheet1!A:F", valueRange) + .setValueInputOption("USER_ENTERED") + .execute(); + } + + public static void main(String... args) throws Exception { + String path = System.getProperty("scanPath"); + if (path == null) { + throw new IllegalStateException("scanPath system property must be set"); + } + File file = new File(path).getAbsoluteFile(); + System.err.println("Scanning for flaky tests in " + file.getPath()); + FlakyTestReporter reporter = FlakyTestReporter.scan(file.toPath()); + reporter.print(); + + String accessKey = System.getProperty("googleSheetsAccessKey"); + String buildScanUrl = System.getProperty("buildScanUrl"); + String jobUrl = System.getProperty("jobUrl"); + if (accessKey != null && !accessKey.isEmpty()) { + reporter.report(accessKey, buildScanUrl, jobUrl); + } + } +}