From dbab955e8c077fa9060dbe32d9af3f8ae934aee7 Mon Sep 17 00:00:00 2001 From: Lauri Tulmin Date: Wed, 22 Jan 2025 15:53:36 +0200 Subject: [PATCH 01/13] Enable test retry for daily builds --- .github/repository-settings.md | 1 + .github/workflows/build-common.yml | 16 + .../workflows/build-daily-no-build-cache.yml | 3 - .github/workflows/build-daily.yml | 6 - .../ratpack/v1_7/FlakyTest.java | 29 ++ settings.gradle.kts | 1 + test-report/build.gradle.kts | 27 ++ .../testreport/FlakyTestReporter.java | 285 ++++++++++++++++++ 8 files changed, 359 insertions(+), 9 deletions(-) create mode 100644 instrumentation/ratpack/ratpack-1.7/library/src/test/java/io/opentelemetry/instrumentation/ratpack/v1_7/FlakyTest.java create mode 100644 test-report/build.gradle.kts create mode 100644 test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java diff --git a/.github/repository-settings.md b/.github/repository-settings.md index 66be1268779c..00bcbb91dfcb 100644 --- a/.github/repository-settings.md +++ b/.github/repository-settings.md @@ -75,6 +75,7 @@ for [`dependabot/**/**`](https://github.com/open-telemetry/community/blob/main/d - Key is associated with [@trask](https://github.com/trask)'s gmail address - `SONATYPE_KEY` - owned by [@trask](https://github.com/trask) - `SONATYPE_USER` - owned by [@trask](https://github.com/trask) +- `FLAKY_TEST_REPORTER_ACCESS_KEY` - owned by [@laurit](https://github.com/laurit) ### Organization secrets diff --git a/.github/workflows/build-common.yml b/.github/workflows/build-common.yml index e4585ec7f6cd..ee6f1a6a0769 100644 --- a/.github/workflows/build-common.yml +++ b/.github/workflows/build-common.yml @@ -290,6 +290,22 @@ jobs: if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }} run: cat build-scan.txt + - name: Get current job url + uses: Tiryoh/gha-jobid-action@be260d8673c9211a84cdcf37794ebd654ba81eef # v1.4.0 + id: jobs + with: + job_name: "test${{ matrix.test-partition }} (${{ matrix.test-java-version }}, ${{ matrix.vm }})" + per_page: 50 # input matrix size here if it is larger than 30 + + - name: Flaky test report + if: ${{ !cancelled() }} + env: + FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }} + JOB_URL: ${{ steps.jobs.outputs.html_url }} + run: | + BUILD_SCAN_URL=$(cat build-scan.txt) + ./gradlew :test-report:reportFlakyTests + - name: Upload deadlock detector artifacts if any if: failure() uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 diff --git a/.github/workflows/build-daily-no-build-cache.yml b/.github/workflows/build-daily-no-build-cache.yml index 596b46773834..d8d2852b007a 100644 --- a/.github/workflows/build-daily-no-build-cache.yml +++ b/.github/workflows/build-daily-no-build-cache.yml @@ -10,19 +10,16 @@ jobs: common: uses: ./.github/workflows/build-common.yml with: - max-test-retries: 0 no-build-cache: true test-latest-deps: uses: ./.github/workflows/reusable-test-latest-deps.yml with: - max-test-retries: 0 no-build-cache: true test-indy: uses: ./.github/workflows/reusable-test-indy.yml with: - max-test-retries: 0 no-build-cache: true # muzzle is not included here because it doesn't use gradle cache anyway and so is already covered diff --git a/.github/workflows/build-daily.yml b/.github/workflows/build-daily.yml index 2e6aae3033bc..dc6b3de4d45d 100644 --- a/.github/workflows/build-daily.yml +++ b/.github/workflows/build-daily.yml @@ -9,18 +9,12 @@ on: jobs: common: uses: ./.github/workflows/build-common.yml - with: - max-test-retries: 0 test-latest-deps: uses: ./.github/workflows/reusable-test-latest-deps.yml - with: - max-test-retries: 0 test-indy: uses: ./.github/workflows/reusable-test-indy.yml - with: - max-test-retries: 0 muzzle: uses: ./.github/workflows/reusable-muzzle.yml diff --git a/instrumentation/ratpack/ratpack-1.7/library/src/test/java/io/opentelemetry/instrumentation/ratpack/v1_7/FlakyTest.java b/instrumentation/ratpack/ratpack-1.7/library/src/test/java/io/opentelemetry/instrumentation/ratpack/v1_7/FlakyTest.java new file mode 100644 index 000000000000..94ea49f32a87 --- /dev/null +++ b/instrumentation/ratpack/ratpack-1.7/library/src/test/java/io/opentelemetry/instrumentation/ratpack/v1_7/FlakyTest.java @@ -0,0 +1,29 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.instrumentation.ratpack.v1_7; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Random; +import org.junit.jupiter.api.Test; + +class FlakyTest { + + @Test + void flakyTest1() { + assertThat(new Random().nextInt(10)).isLessThan(3); + } + + @Test + void flakyTest2() { + assertThat(new Random().nextInt(10)).isLessThan(3); + } + + @Test + void flakyTest3() { + assertThat(new Random().nextInt(10)).isLessThan(3); + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 0e4cda313a8d..810570f2c82e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -97,6 +97,7 @@ include(":instrumentation-annotations-support-testing") // misc include(":dependencyManagement") +include(":test-report") include(":testing:agent-exporter") include(":testing:agent-for-testing") include(":testing:armeria-shaded-for-testing") diff --git a/test-report/build.gradle.kts b/test-report/build.gradle.kts new file mode 100644 index 000000000000..0b6e15361220 --- /dev/null +++ b/test-report/build.gradle.kts @@ -0,0 +1,27 @@ +plugins { + id("otel.java-conventions") +} + +dependencies { + implementation("com.google.api-client:google-api-client:2.7.1") + implementation("com.google.apis:google-api-services-sheets:v4-rev20250106-2.0.0") + implementation("com.google.auth:google-auth-library-oauth2-http:1.30.1") +} + +otelJava { + minJavaVersionSupported.set(JavaVersion.VERSION_17) +} + +tasks { + val reportFlakyTests by registering(JavaExec::class) { + dependsOn(classes) + + mainClass.set("io.opentelemetry.instrumentation.testreport.FlakyTestReporter") + classpath(sourceSets["main"].runtimeClasspath) + + systemProperty("scanPath", project.rootDir) + systemProperty("googleSheetsAccessKey", System.getenv("FLAKY_TEST_REPORTER_ACCESS_KEY")) + systemProperty("buildScanUrl", System.getenv("BUILD_SCAN_URL")) + systemProperty("jobUrl", System.getenv("JOB_URL")) + } +} diff --git a/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java b/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java new file mode 100644 index 000000000000..ae53b444686e --- /dev/null +++ b/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java @@ -0,0 +1,285 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.instrumentation.testreport; + +import static java.nio.file.FileVisitResult.CONTINUE; + +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.services.sheets.v4.Sheets; +import com.google.api.services.sheets.v4.SheetsScopes; +import com.google.api.services.sheets.v4.model.ValueRange; +import com.google.auth.http.HttpCredentialsAdapter; +import com.google.auth.oauth2.GoogleCredentials; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import org.w3c.dom.Document; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +@SuppressWarnings("SystemOut") +public class FlakyTestReporter { + // https://docs.google.com/spreadsheets/d/1pfa6Ws980AIFI3kKOeIc51-JGEzakG7hkMl4J9h-Tk0 + private static final String SPREADSHEET_ID = "1pfa6Ws980AIFI3kKOeIc51-JGEzakG7hkMl4J9h-Tk0"; + + private int testCount; + private int skippedCount; + private int failureCount; + private int errorCount; + private final List flakyTests = new ArrayList<>(); + + private static class FlakyTest { + final String testClassName; + final String testName; + final String timestamp; + final String message; + + FlakyTest(String testClassName, String testName, String timestamp, String message) { + this.testClassName = testClassName; + this.testName = testName; + this.timestamp = timestamp; + this.message = message; + } + } + + private void addFlakyTest( + String testClassName, String testName, String timestamp, String message) { + flakyTests.add(new FlakyTest(testClassName, testName, timestamp, message)); + } + + private static Document parse(Path testReport) { + try { + DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + return builder.parse(testReport.toFile()); + } catch (Exception exception) { + throw new IllegalStateException("failed to parse test report " + testReport, exception); + } + } + + private void scanTestFile(Path testReport) { + Document doc = parse(testReport); + doc.getDocumentElement().normalize(); + testCount += Integer.parseInt(doc.getDocumentElement().getAttribute("tests")); + skippedCount += Integer.parseInt(doc.getDocumentElement().getAttribute("skipped")); + int failures = Integer.parseInt(doc.getDocumentElement().getAttribute("failures")); + failureCount += failures; + int errors = Integer.parseInt(doc.getDocumentElement().getAttribute("errors")); + errorCount += errors; + String timestamp = doc.getDocumentElement().getAttribute("timestamp"); + + // there are no flaky tests if there are no failures, skip it + if (failures == 0 && errors == 0) { + return; + } + + class TestCase { + final String className; + final String name; + boolean failed; + boolean succeeded; + String message; + + TestCase(String className, String name) { + this.className = className; + this.name = name; + } + + boolean isFlaky() { + return succeeded && failed; + } + } + + Map testcaseMap = new HashMap<>(); + + NodeList testcaseNodes = doc.getElementsByTagName("testcase"); + for (int i = 0; i < testcaseNodes.getLength(); i++) { + Node testNode = testcaseNodes.item(i); + + String testClassName = testNode.getAttributes().getNamedItem("classname").getNodeValue(); + String testName = testNode.getAttributes().getNamedItem("name").getNodeValue(); + String testKey = testClassName + "." + testName; + TestCase testCase = + testcaseMap.computeIfAbsent(testKey, (s) -> new TestCase(testClassName, testName)); + NodeList childNodes = testNode.getChildNodes(); + boolean failed = false; + for (int j = 0; j < childNodes.getLength(); j++) { + Node childNode = childNodes.item(j); + String nodeName = childNode.getNodeName(); + if ("failure".equals(nodeName) || "error".equals(nodeName)) { + failed = true; + // if test fails multiple times we'll use the first failure message + if (testCase.message == null) { + String message = getAttributeValue(childNode, "message"); + if (message != null) { + // compress failure message on a single line + message = message.replaceAll("\n( )*", " "); + } + testCase.message = message; + } + } + } + if (failed) { + testCase.failed = true; + } else { + testCase.succeeded = true; + } + } + + for (TestCase testCase : testcaseMap.values()) { + if (testCase.isFlaky()) { + addFlakyTest(testCase.className, testCase.name, timestamp, testCase.message); + } + } + } + + private static String getAttributeValue(Node node, String attributeName) { + NamedNodeMap attributes = node.getAttributes(); + if (attributes == null) { + return null; + } + Node value = attributes.getNamedItem(attributeName); + return value != null ? value.getNodeValue() : null; + } + + private void scanTestResults(Path buildDir) throws IOException { + Path testResults = buildDir.resolve("test-results"); + if (!Files.exists(testResults)) { + return; + } + + Files.walkFileTree( + testResults, + new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + String name = file.getFileName().toString(); + if (name.startsWith("TEST-") && name.endsWith(".xml")) { + scanTestFile(file); + } + + return CONTINUE; + } + }); + } + + private static FlakyTestReporter scan(Path path) throws IOException { + FlakyTestReporter reporter = new FlakyTestReporter(); + Files.walkFileTree( + path, + new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + throws IOException { + if (dir.endsWith("build")) { + reporter.scanTestResults(dir); + return FileVisitResult.SKIP_SUBTREE; + } + if (dir.endsWith("src")) { + return FileVisitResult.SKIP_SUBTREE; + } + return CONTINUE; + } + }); + return reporter; + } + + private void print() { + System.err.printf( + "Found %d test, skipped %d, failed %d, errored %d\n", + testCount, skippedCount, failureCount, errorCount); + if (!flakyTests.isEmpty()) { + System.err.printf("Found %d flaky test(s):\n", flakyTests.size()); + for (FlakyTest flakyTest : flakyTests) { + System.err.println( + flakyTest.timestamp + + " " + + flakyTest.testClassName + + " " + + flakyTest.testName + + " " + + flakyTest.message); + } + } + } + + // add flaky tests to a google sheet + private void report(String accessKey, String buildScanUrl, String jobUrl) throws Exception { + if (flakyTests.isEmpty()) { + return; + } + + NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); + GoogleCredentials credentials = + GoogleCredentials.fromStream( + new ByteArrayInputStream(accessKey.getBytes(StandardCharsets.UTF_8))) + .createScoped(Collections.singletonList(SheetsScopes.SPREADSHEETS)); + Sheets service = + new Sheets.Builder( + transport, + GsonFactory.getDefaultInstance(), + new HttpCredentialsAdapter(credentials)) + .setApplicationName("Flaky test reporter") + .build(); + + List> data = new ArrayList<>(); + for (FlakyTest flakyTest : flakyTests) { + List row = new ArrayList<>(); + row.add(flakyTest.timestamp); + row.add(flakyTest.testClassName); + row.add(flakyTest.testName); + row.add(buildScanUrl); + row.add(jobUrl); + row.add(flakyTest.message); + data.add(row); + } + + ValueRange valueRange = new ValueRange(); + valueRange.setValues(data); + service + .spreadsheets() + .values() + .append(SPREADSHEET_ID, "Sheet1!A:F", valueRange) + .setValueInputOption("USER_ENTERED") + .execute(); + } + + public static void main(String... args) throws Exception { + String path = System.getProperty("scanPath"); + if (path == null) { + throw new IllegalStateException("scanPath system property must be set"); + } + File file = new File(path).getAbsoluteFile(); + System.err.println("Scanning for flaky tests in " + file.getPath()); + FlakyTestReporter reporter = FlakyTestReporter.scan(file.toPath()); + reporter.print(); + + String accessKey = System.getProperty("googleSheetsAccessKey"); + String buildScanUrl = System.getProperty("buildScanUrl"); + String jobUrl = System.getProperty("jobUrl"); + System.err.println("buildScanUrl " + buildScanUrl); + System.err.println("jobUrl " + jobUrl); + System.err.println("secret set " + (accessKey != null)); + if (accessKey != null) { + reporter.report(accessKey, buildScanUrl, jobUrl); + } + } +} From 7bc883001018c58e2df37f87553c648539ef12ef Mon Sep 17 00:00:00 2001 From: Lauri Tulmin Date: Wed, 22 Jan 2025 17:11:01 +0200 Subject: [PATCH 02/13] try to fix --- .github/workflows/build-common.yml | 8 +++++--- .../instrumentation/testreport/FlakyTestReporter.java | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-common.yml b/.github/workflows/build-common.yml index ee6f1a6a0769..e1f8ca2eed4a 100644 --- a/.github/workflows/build-common.yml +++ b/.github/workflows/build-common.yml @@ -294,8 +294,8 @@ jobs: uses: Tiryoh/gha-jobid-action@be260d8673c9211a84cdcf37794ebd654ba81eef # v1.4.0 id: jobs with: - job_name: "test${{ matrix.test-partition }} (${{ matrix.test-java-version }}, ${{ matrix.vm }})" - per_page: 50 # input matrix size here if it is larger than 30 + job_name: "common / test${{ matrix.test-partition }} (${{ matrix.test-java-version }}, ${{ matrix.vm }})" + per_page: 100 # input matrix size here if it is larger than 30 - name: Flaky test report if: ${{ !cancelled() }} @@ -303,7 +303,9 @@ jobs: FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }} JOB_URL: ${{ steps.jobs.outputs.html_url }} run: | - BUILD_SCAN_URL=$(cat build-scan.txt) + if [ -s build-scan.txt ]; then + export BUILD_SCAN_URL=$(cat build-scan.txt) + fi ./gradlew :test-report:reportFlakyTests - name: Upload deadlock detector artifacts if any diff --git a/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java b/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java index ae53b444686e..1dbde181a7a7 100644 --- a/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java +++ b/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java @@ -277,8 +277,8 @@ public static void main(String... args) throws Exception { String jobUrl = System.getProperty("jobUrl"); System.err.println("buildScanUrl " + buildScanUrl); System.err.println("jobUrl " + jobUrl); - System.err.println("secret set " + (accessKey != null)); - if (accessKey != null) { + System.err.println("secret length " + (accessKey != null ? accessKey.length() : -1)); + if (accessKey != null && !accessKey.isEmpty()) { reporter.report(accessKey, buildScanUrl, jobUrl); } } From bd4d68a51c643ae06dae53d548c103ddcf8f3434 Mon Sep 17 00:00:00 2001 From: Lauri Tulmin Date: Wed, 22 Jan 2025 17:28:15 +0200 Subject: [PATCH 03/13] run job url task for failed jobs --- .github/workflows/build-common.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-common.yml b/.github/workflows/build-common.yml index e1f8ca2eed4a..79fa1210b03c 100644 --- a/.github/workflows/build-common.yml +++ b/.github/workflows/build-common.yml @@ -293,6 +293,7 @@ jobs: - name: Get current job url uses: Tiryoh/gha-jobid-action@be260d8673c9211a84cdcf37794ebd654ba81eef # v1.4.0 id: jobs + if: ${{ !cancelled() }} with: job_name: "common / test${{ matrix.test-partition }} (${{ matrix.test-java-version }}, ${{ matrix.vm }})" per_page: 100 # input matrix size here if it is larger than 30 From a1567722d3fa06cb2e4ae9b82fe5be55b0c08ee7 Mon Sep 17 00:00:00 2001 From: Lauri Tulmin Date: Wed, 22 Jan 2025 17:28:39 +0200 Subject: [PATCH 04/13] make test less flaky --- .../instrumentation/ratpack/v1_7/FlakyTest.java | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/instrumentation/ratpack/ratpack-1.7/library/src/test/java/io/opentelemetry/instrumentation/ratpack/v1_7/FlakyTest.java b/instrumentation/ratpack/ratpack-1.7/library/src/test/java/io/opentelemetry/instrumentation/ratpack/v1_7/FlakyTest.java index 94ea49f32a87..563236423f61 100644 --- a/instrumentation/ratpack/ratpack-1.7/library/src/test/java/io/opentelemetry/instrumentation/ratpack/v1_7/FlakyTest.java +++ b/instrumentation/ratpack/ratpack-1.7/library/src/test/java/io/opentelemetry/instrumentation/ratpack/v1_7/FlakyTest.java @@ -13,17 +13,7 @@ class FlakyTest { @Test - void flakyTest1() { - assertThat(new Random().nextInt(10)).isLessThan(3); - } - - @Test - void flakyTest2() { - assertThat(new Random().nextInt(10)).isLessThan(3); - } - - @Test - void flakyTest3() { + void flakyTest() { assertThat(new Random().nextInt(10)).isLessThan(3); } } From b302fa5baa5e1f9be23290c2c5dec7c23c7ce4c9 Mon Sep 17 00:00:00 2001 From: Lauri Tulmin Date: Wed, 22 Jan 2025 18:37:40 +0200 Subject: [PATCH 05/13] remove debugging --- .../instrumentation/testreport/FlakyTestReporter.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java b/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java index 1dbde181a7a7..a687853d7426 100644 --- a/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java +++ b/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java @@ -275,9 +275,6 @@ public static void main(String... args) throws Exception { String accessKey = System.getProperty("googleSheetsAccessKey"); String buildScanUrl = System.getProperty("buildScanUrl"); String jobUrl = System.getProperty("jobUrl"); - System.err.println("buildScanUrl " + buildScanUrl); - System.err.println("jobUrl " + jobUrl); - System.err.println("secret length " + (accessKey != null ? accessKey.length() : -1)); if (accessKey != null && !accessKey.isEmpty()) { reporter.report(accessKey, buildScanUrl, jobUrl); } From 07d4052529c7f7130666bf8681051b594fbd8f76 Mon Sep 17 00:00:00 2001 From: Lauri Tulmin Date: Wed, 22 Jan 2025 18:38:13 +0200 Subject: [PATCH 06/13] add flaky test report to indy and latest deps workflows --- .github/workflows/reusable-test-indy.yml | 19 +++++++++++++++++++ .../workflows/reusable-test-latest-deps.yml | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/.github/workflows/reusable-test-indy.yml b/.github/workflows/reusable-test-indy.yml index cacebf8458a8..95e3d43c5051 100644 --- a/.github/workflows/reusable-test-indy.yml +++ b/.github/workflows/reusable-test-indy.yml @@ -86,3 +86,22 @@ jobs: - name: Build scan if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }} run: cat build-scan.txt + + - name: Get current job url + uses: Tiryoh/gha-jobid-action@be260d8673c9211a84cdcf37794ebd654ba81eef # v1.4.0 + id: jobs + if: ${{ !cancelled() }} + with: + job_name: "test-indy / testIndy${{ matrix.test-partition }}" + per_page: 100 # input matrix size here if it is larger than 30 + + - name: Flaky test report + if: ${{ !cancelled() }} + env: + FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }} + JOB_URL: ${{ steps.jobs.outputs.html_url }} + run: | + if [ -s build-scan.txt ]; then + export BUILD_SCAN_URL=$(cat build-scan.txt) + fi + ./gradlew :test-report:reportFlakyTests diff --git a/.github/workflows/reusable-test-latest-deps.yml b/.github/workflows/reusable-test-latest-deps.yml index fe3c6d730e99..7b62f5690cfe 100644 --- a/.github/workflows/reusable-test-latest-deps.yml +++ b/.github/workflows/reusable-test-latest-deps.yml @@ -85,6 +85,25 @@ jobs: if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }} run: cat build-scan.txt + - name: Get current job url + uses: Tiryoh/gha-jobid-action@be260d8673c9211a84cdcf37794ebd654ba81eef # v1.4.0 + id: jobs + if: ${{ !cancelled() }} + with: + job_name: "test-latest-deps / testLatestDeps${{ matrix.test-partition }}" + per_page: 100 # input matrix size here if it is larger than 30 + + - name: Flaky test report + if: ${{ !cancelled() }} + env: + FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }} + JOB_URL: ${{ steps.jobs.outputs.html_url }} + run: | + if [ -s build-scan.txt ]; then + export BUILD_SCAN_URL=$(cat build-scan.txt) + fi + ./gradlew :test-report:reportFlakyTests + - name: Upload deadlock detector artifacts if any if: failure() uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 From ae0a6f1dd18234d1be8889be55d92c0989337863 Mon Sep 17 00:00:00 2001 From: Lauri Tulmin Date: Wed, 22 Jan 2025 18:38:38 +0200 Subject: [PATCH 07/13] delete flaky test --- .../ratpack/v1_7/FlakyTest.java | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 instrumentation/ratpack/ratpack-1.7/library/src/test/java/io/opentelemetry/instrumentation/ratpack/v1_7/FlakyTest.java diff --git a/instrumentation/ratpack/ratpack-1.7/library/src/test/java/io/opentelemetry/instrumentation/ratpack/v1_7/FlakyTest.java b/instrumentation/ratpack/ratpack-1.7/library/src/test/java/io/opentelemetry/instrumentation/ratpack/v1_7/FlakyTest.java deleted file mode 100644 index 563236423f61..000000000000 --- a/instrumentation/ratpack/ratpack-1.7/library/src/test/java/io/opentelemetry/instrumentation/ratpack/v1_7/FlakyTest.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright The OpenTelemetry Authors - * SPDX-License-Identifier: Apache-2.0 - */ - -package io.opentelemetry.instrumentation.ratpack.v1_7; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.Random; -import org.junit.jupiter.api.Test; - -class FlakyTest { - - @Test - void flakyTest() { - assertThat(new Random().nextInt(10)).isLessThan(3); - } -} From a8d4f26df867a99032434d6d27e51d4e1bf5021e Mon Sep 17 00:00:00 2001 From: Lauri Tulmin Date: Thu, 23 Jan 2025 20:10:21 +0200 Subject: [PATCH 08/13] try a different way for getting the job url --- .github/workflows/build-common.yml | 26 ++++++++++++++++--- .../testreport/FlakyTestReporter.java | 2 ++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-common.yml b/.github/workflows/build-common.yml index 79fa1210b03c..777bdc3603b5 100644 --- a/.github/workflows/build-common.yml +++ b/.github/workflows/build-common.yml @@ -290,19 +290,37 @@ jobs: if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }} run: cat build-scan.txt +# - name: Get current job url +# uses: Tiryoh/gha-jobid-action@be260d8673c9211a84cdcf37794ebd654ba81eef # v1.4.0 +# id: jobs +# if: ${{ !cancelled() }} +# with: +# job_name: "common / test${{ matrix.test-partition }} (${{ matrix.test-java-version }}, ${{ matrix.vm }})" +# per_page: 100 # input matrix size here if it is larger than 30 + - name: Get current job url - uses: Tiryoh/gha-jobid-action@be260d8673c9211a84cdcf37794ebd654ba81eef # v1.4.0 id: jobs if: ${{ !cancelled() }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + matrix: ${{ toJson(matrix) }} with: - job_name: "common / test${{ matrix.test-partition }} (${{ matrix.test-java-version }}, ${{ matrix.vm }})" - per_page: 100 # input matrix size here if it is larger than 30 + script: | + const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + const matrix = JSON.parse(process.env.matrix); + const job_name = `common / test${ matrix.test-partition } (${ matrix.test-java-version }, ${ matrix.vm })`; + return workflow_run.jobs.find((job) => job.name === job_name).html_url; - name: Flaky test report if: ${{ !cancelled() }} env: FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }} - JOB_URL: ${{ steps.jobs.outputs.html_url }} + JOB_URL: ${{ steps.jobs.outputs.result }} + # JOB_URL: ${{ steps.jobs.outputs.html_url }} run: | if [ -s build-scan.txt ]; then export BUILD_SCAN_URL=$(cat build-scan.txt) diff --git a/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java b/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java index a687853d7426..ce1194c68bda 100644 --- a/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java +++ b/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java @@ -275,6 +275,8 @@ public static void main(String... args) throws Exception { String accessKey = System.getProperty("googleSheetsAccessKey"); String buildScanUrl = System.getProperty("buildScanUrl"); String jobUrl = System.getProperty("jobUrl"); + System.err.println("buildScanUrl " + buildScanUrl); + System.err.println("jobUrl " + jobUrl); if (accessKey != null && !accessKey.isEmpty()) { reporter.report(accessKey, buildScanUrl, jobUrl); } From 45128963f39974de1f86b464bbd50464e6f59930 Mon Sep 17 00:00:00 2001 From: Lauri Tulmin Date: Thu, 23 Jan 2025 21:54:43 +0200 Subject: [PATCH 09/13] hope this works --- .github/workflows/build-common.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-common.yml b/.github/workflows/build-common.yml index 777bdc3603b5..0f44003547b4 100644 --- a/.github/workflows/build-common.yml +++ b/.github/workflows/build-common.yml @@ -312,7 +312,7 @@ jobs: run_id: context.runId }); const matrix = JSON.parse(process.env.matrix); - const job_name = `common / test${ matrix.test-partition } (${ matrix.test-java-version }, ${ matrix.vm })`; + const job_name = `common / test${ matrix['test-partition'] } (${ matrix['test-java-version'] }, ${ matrix.vm })`; return workflow_run.jobs.find((job) => job.name === job_name).html_url; - name: Flaky test report From 23f60737157aba525ba783b9e427810d69dfd69a Mon Sep 17 00:00:00 2001 From: Lauri Tulmin Date: Thu, 23 Jan 2025 22:43:50 +0200 Subject: [PATCH 10/13] hope this works --- .github/workflows/build-common.yml | 3 ++- settings.gradle.kts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-common.yml b/.github/workflows/build-common.yml index 0f44003547b4..f1a025b9888f 100644 --- a/.github/workflows/build-common.yml +++ b/.github/workflows/build-common.yml @@ -309,7 +309,8 @@ jobs: const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({ owner: context.repo.owner, repo: context.repo.repo, - run_id: context.runId + run_id: context.runId, + per_page: 100 }); const matrix = JSON.parse(process.env.matrix); const job_name = `common / test${ matrix['test-partition'] } (${ matrix['test-java-version'] }, ${ matrix.vm })`; diff --git a/settings.gradle.kts b/settings.gradle.kts index 810570f2c82e..ecc67790f2b5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -54,7 +54,8 @@ develocity { termsOfUseUrl.set("https://gradle.com/help/legal-terms-of-use") termsOfUseAgree.set("yes") - if (!gradle.startParameter.taskNames.contains("listTestsInPartition")) { + if (!gradle.startParameter.taskNames.contains("listTestsInPartition") + && !gradle.startParameter.taskNames.contains(":test-report:reportFlakyTests")) { buildScanPublished { File("build-scan.txt").printWriter().use { writer -> writer.println(buildScanUri) From b21badfc07567686b4f0b151b39c7f647ae58088 Mon Sep 17 00:00:00 2001 From: Lauri Tulmin Date: Fri, 24 Jan 2025 09:31:07 +0200 Subject: [PATCH 11/13] rework get job url in indy and latest tests --- .github/workflows/build-common.yml | 10 +--------- .github/workflows/reusable-test-indy.yml | 19 +++++++++++++++---- .../workflows/reusable-test-latest-deps.yml | 19 +++++++++++++++---- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-common.yml b/.github/workflows/build-common.yml index f1a025b9888f..92ce584e2e72 100644 --- a/.github/workflows/build-common.yml +++ b/.github/workflows/build-common.yml @@ -290,14 +290,6 @@ jobs: if: ${{ !cancelled() && hashFiles('build-scan.txt') != '' }} run: cat build-scan.txt -# - name: Get current job url -# uses: Tiryoh/gha-jobid-action@be260d8673c9211a84cdcf37794ebd654ba81eef # v1.4.0 -# id: jobs -# if: ${{ !cancelled() }} -# with: -# job_name: "common / test${{ matrix.test-partition }} (${{ matrix.test-java-version }}, ${{ matrix.vm }})" -# per_page: 100 # input matrix size here if it is larger than 30 - - name: Get current job url id: jobs if: ${{ !cancelled() }} @@ -305,6 +297,7 @@ jobs: env: matrix: ${{ toJson(matrix) }} with: + result-encoding: string script: | const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({ owner: context.repo.owner, @@ -321,7 +314,6 @@ jobs: env: FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }} JOB_URL: ${{ steps.jobs.outputs.result }} - # JOB_URL: ${{ steps.jobs.outputs.html_url }} run: | if [ -s build-scan.txt ]; then export BUILD_SCAN_URL=$(cat build-scan.txt) diff --git a/.github/workflows/reusable-test-indy.yml b/.github/workflows/reusable-test-indy.yml index 95e3d43c5051..70d7caaac563 100644 --- a/.github/workflows/reusable-test-indy.yml +++ b/.github/workflows/reusable-test-indy.yml @@ -88,18 +88,29 @@ jobs: run: cat build-scan.txt - name: Get current job url - uses: Tiryoh/gha-jobid-action@be260d8673c9211a84cdcf37794ebd654ba81eef # v1.4.0 id: jobs if: ${{ !cancelled() }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + matrix: ${{ toJson(matrix) }} with: - job_name: "test-indy / testIndy${{ matrix.test-partition }}" - per_page: 100 # input matrix size here if it is larger than 30 + result-encoding: string + script: | + const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + per_page: 100 + }); + const matrix = JSON.parse(process.env.matrix); + const job_name = `test-indy / testIndy${ matrix['test-partition'] }`; + return workflow_run.jobs.find((job) => job.name === job_name).html_url; - name: Flaky test report if: ${{ !cancelled() }} env: FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }} - JOB_URL: ${{ steps.jobs.outputs.html_url }} + JOB_URL: ${{ steps.jobs.outputs.result }} run: | if [ -s build-scan.txt ]; then export BUILD_SCAN_URL=$(cat build-scan.txt) diff --git a/.github/workflows/reusable-test-latest-deps.yml b/.github/workflows/reusable-test-latest-deps.yml index 7b62f5690cfe..4bf397165387 100644 --- a/.github/workflows/reusable-test-latest-deps.yml +++ b/.github/workflows/reusable-test-latest-deps.yml @@ -86,18 +86,29 @@ jobs: run: cat build-scan.txt - name: Get current job url - uses: Tiryoh/gha-jobid-action@be260d8673c9211a84cdcf37794ebd654ba81eef # v1.4.0 id: jobs if: ${{ !cancelled() }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + matrix: ${{ toJson(matrix) }} with: - job_name: "test-latest-deps / testLatestDeps${{ matrix.test-partition }}" - per_page: 100 # input matrix size here if it is larger than 30 + result-encoding: string + script: | + const { data: workflow_run } = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + per_page: 100 + }); + const matrix = JSON.parse(process.env.matrix); + const job_name = `test-latest-deps / testLatestDeps${ matrix['test-partition'] }`; + return workflow_run.jobs.find((job) => job.name === job_name).html_url; - name: Flaky test report if: ${{ !cancelled() }} env: FLAKY_TEST_REPORTER_ACCESS_KEY: ${{ secrets.FLAKY_TEST_REPORTER_ACCESS_KEY }} - JOB_URL: ${{ steps.jobs.outputs.html_url }} + JOB_URL: ${{ steps.jobs.outputs.result }} run: | if [ -s build-scan.txt ]; then export BUILD_SCAN_URL=$(cat build-scan.txt) From f7dac2c9468e63f00976e05608c3bb7ade3a267f Mon Sep 17 00:00:00 2001 From: Lauri Tulmin Date: Fri, 24 Jan 2025 09:56:45 +0200 Subject: [PATCH 12/13] remove debug --- .../instrumentation/testreport/FlakyTestReporter.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java b/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java index ce1194c68bda..a687853d7426 100644 --- a/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java +++ b/test-report/src/main/java/io/opentelemetry/instrumentation/testreport/FlakyTestReporter.java @@ -275,8 +275,6 @@ public static void main(String... args) throws Exception { String accessKey = System.getProperty("googleSheetsAccessKey"); String buildScanUrl = System.getProperty("buildScanUrl"); String jobUrl = System.getProperty("jobUrl"); - System.err.println("buildScanUrl " + buildScanUrl); - System.err.println("jobUrl " + jobUrl); if (accessKey != null && !accessKey.isEmpty()) { reporter.report(accessKey, buildScanUrl, jobUrl); } From 79a03541bca393f76755c2f64bfdea5ed624e4e8 Mon Sep 17 00:00:00 2001 From: Lauri Tulmin Date: Fri, 24 Jan 2025 09:58:46 +0200 Subject: [PATCH 13/13] spotless --- settings.gradle.kts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index ecc67790f2b5..281ca16d52d6 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -54,8 +54,8 @@ develocity { termsOfUseUrl.set("https://gradle.com/help/legal-terms-of-use") termsOfUseAgree.set("yes") - if (!gradle.startParameter.taskNames.contains("listTestsInPartition") - && !gradle.startParameter.taskNames.contains(":test-report:reportFlakyTests")) { + if (!gradle.startParameter.taskNames.contains("listTestsInPartition") && + !gradle.startParameter.taskNames.contains(":test-report:reportFlakyTests")) { buildScanPublished { File("build-scan.txt").printWriter().use { writer -> writer.println(buildScanUri)