diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index f5a58ef6896..e54c6af38bf 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -7,8 +7,7 @@ on: androidEmulatorApiLevel: nodeJsVersion: firebaseToolsVersion: - gradleInfoLog: - type: boolean + githubNotificationsIssue: pull_request: paths: - .github/workflows/dataconnect.yml @@ -25,8 +24,7 @@ env: FDC_ANDROID_EMULATOR_API_LEVEL: ${{ inputs.androidEmulatorApiLevel || '34' }} FDC_NODEJS_VERSION: ${{ inputs.nodeJsVersion || '20' }} FDC_FIREBASE_TOOLS_VERSION: ${{ inputs.firebaseToolsVersion || '13.29.1' }} - FDC_FIREBASE_TOOLS_DIR: /tmp/firebase-tools - FDC_FIREBASE_COMMAND: /tmp/firebase-tools/node_modules/.bin/firebase + FDC_FIREBASE_COMMAND: firebase-dataconnect/ci/build/firebase-tools/node_modules/.bin/firebase concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -51,6 +49,11 @@ jobs: - 5432:5432 steps: + - name: Set environment variables + run: | + echo "ORG_GRADLE_PROJECT_firebaseToolsVersion=$FDC_FIREBASE_TOOLS_VERSION" >>"$GITHUB_ENV" + echo "ORG_GRADLE_PROJECT_debugLoggingEnabled=${{ runner.debug || '0' }}" >>"$GITHUB_ENV" + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: show-progress: false @@ -65,13 +68,7 @@ jobs: node-version: ${{ env.FDC_NODEJS_VERSION }} - name: Install Firebase Tools ("firebase" command-line tool) - run: | - set -euo pipefail - set -v - mkdir -p ${{ env.FDC_FIREBASE_TOOLS_DIR }} - cd ${{ env.FDC_FIREBASE_TOOLS_DIR }} - echo '{}' > package.json - npm install --fund=false --audit=false --save --save-exact firebase-tools@${{ env.FDC_FIREBASE_TOOLS_VERSION }} + run: ./gradlew -p firebase-dataconnect/ci installFirebaseTools - name: Restore Gradle Cache id: restore-gradle-cache @@ -87,39 +84,10 @@ jobs: - name: Print Command-Line Tool Versions continue-on-error: true - run: | - set -euo pipefail - - function run_cmd { - echo "===============================================================================" - echo "Running Command: $*" - ("$@" 2>&1) || echo "WARNING: command failed with non-zero exit code $?: $*" - } - - run_cmd uname -a - run_cmd which java - run_cmd java -version - run_cmd which javac - run_cmd javac -version - run_cmd which node - run_cmd node --version - run_cmd ${{ env.FDC_FIREBASE_COMMAND }} --version - run_cmd ./gradlew --version + run: ./gradlew -p firebase-dataconnect/ci printToolVersions - name: Gradle assembleDebugAndroidTest - run: | - set -euo pipefail - set -v - - # Speed up build times and also avoid configuring firebase-crashlytics-ndk - # which is finicky integrating with the Android NDK. - echo >> gradle.properties - echo "org.gradle.configureondemand=true" >> gradle.properties - - ./gradlew \ - --profile \ - ${{ (inputs.gradleInfoLog && '--info') || '' }} \ - :firebase-dataconnect:assembleDebugAndroidTest + run: ./gradlew -p firebase-dataconnect/ci buildIntegrationTests - name: Save Gradle Cache uses: actions/cache/save@d4323d4df104b026a6aa633fdb11d772146be0bf # 4.2.2 @@ -179,7 +147,7 @@ jobs: echo 'emulator.postgresConnectionUrl=postgresql://postgres:password@127.0.0.1:5432?sslmode=disable' > firebase-dataconnect/dataconnect.local.properties ./gradlew \ - ${{ (inputs.gradleInfoLog && '--info') || '' }} \ + ${{ (runner.debug && '--info') || '' }} \ :firebase-dataconnect:connectors:runDebugDataConnectEmulator \ >firebase.emulator.dataconnect.log 2>&1 & @@ -220,8 +188,7 @@ jobs: force-avd-creation: false emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none disable-animations: true - script: | - set -eux && ./gradlew ${{ (inputs.gradleInfoLog && '--info') || '' }} :firebase-dataconnect:connectedCheck :firebase-dataconnect:connectors:connectedCheck + script: ./gradlew -p firebase-dataconnect/ci runIntegrationTests - name: Upload Log Files uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 @@ -256,6 +223,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: show-progress: false + sparse-checkout: '.github/' - uses: docker://rhysd/actionlint:1.7.7 with: args: -color /github/workspace/.github/workflows/dataconnect.yml @@ -267,19 +235,48 @@ jobs: issues: write runs-on: ubuntu-latest steps: - - name: Post Comment on Issue #6857 - if: github.event_name == 'schedule' + - id: issue-id + name: Determine GitHub Issue For Commenting + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + set -xv + + readonly pr_number + pr_number=$(echo '${{ github.ref }}' | sed -En 's#^refs/pull/([0-9]+)/merge$#\1#p') + if [[ -n $pr_number ]] ; then + readonly issue_from_pr_body + issue_from_pr_body=$(gh issue view "$pr_number" --json body --jq '.body' | sed -En 's#(\s|^)trksmnkncd_notification_issue=([0-9]+)(\s|$)#\2#p') + fi + + if [[ -v $issue_from_pr_body ]] ; then + readonly issue="$issue_from_pr_body" + elif [[ '${{ github.event_name }}' == 'schedule' ]] ; then + readonly issue=6857 + else + readonly issue= + fi + + echo "issue=$issue" >> "$GITHUB_OUTPUT" + + - name: Post Comment on GitHub Issue + if: steps.issue-id.outputs.issue != '' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail cat >message.txt <&2 + exit 2 + } + + local -r subcommand_name="$1" + shift + + case "$subcommand_name" { + install_firebase_tools) subcommand/install_firebase_tools "$@" ;; + print_tool_versions) subcommand/print_tool_versions "$@" ;; + gradle_assemble) subcommand/gradle_assemble "$@" ;; + gradle_connected_check) subcommand/gradle_connected_check "$@" ;; + kvm_setup) subcommand/kvm_setup "$@" ;; + start_firebase_emulators) subcommand/start_firebase_emulators "$@" ;; + start_logcat_capture) subcommand/start_logcat_capture "$@" ;; + verify_connected_check_success) subcommand/verify_connected_check_success "$@" ;; + send_notifications) subcommand/send_notifications "$@" ;; + *) + echo "ERROR: no subcommand with the given name: $subcommand_name" >&2 + exit 2 + } +} + +function subcommand/install_firebase_tools { + zparseopts -F -A args - install_dir: firebase_tools_version: + fail_if_missing_arg -install_dir + fail_if_missing_arg -firebase_tools_version + + local -r install_dir="${args[-install_dir]}" + local -r firebase_tools_version="${args[-firebase_tools_version]}" + + set -xv + mkdir -p "$install_dir" + cd "$install_dir" + echo '{}' > package.json + npm install --fund=false --audit=false --save --save-exact "firebase-tools@${firebase_tools_version}" +} + +function run_command_for_print_tool_versions { + echo "===============================================================================" + echo "Running Command: $*" + ("$@" || echo "WARNING: command failed with non-zero exit code $?: $*") 2>&1 +} + +function subcommand/print_tool_versions { + zparseopts -F -A args - firebase_command: + fail_if_missing_arg -firebase_command + + local -r firebase_command="${args[-firebase_command]}" + + run_command_for_print_tool_versions uname -a + run_command_for_print_tool_versions which java + run_command_for_print_tool_versions java -version + run_command_for_print_tool_versions which javac + run_command_for_print_tool_versions javac -version + run_command_for_print_tool_versions which node + run_command_for_print_tool_versions node --version + run_command_for_print_tool_versions "$firebase_command" --version + run_command_for_print_tool_versions ./gradlew --version +} + +function subcommand/gradle_assemble { + local -r gradle_args=( + ./gradlew + --profile + --configure-on-demand + "$@" + :firebase-dataconnect:assembleDebugAndroidTest + ) + + set -xv + "${gradle_args[@]}" +} + +function subcommand/gradle_connected_check { + local -r gradle_args=( + ./gradlew + --profile + --configure-on-demand + "$@" + :firebase-dataconnect:connectedCheck + :firebase-dataconnect:connectors:connectedCheck + ) + + set -xv + "${gradle_args[@]}" +} + +function subcommand/kvm_setup { + if [[ $# -ne 0 ]] { + echo "ERROR: no arguments expected, but got $#: $*" >&2 + exit 2 + } + + set -xv + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm +} + +function subcommand/start_firebase_emulators { + zparseopts -F -A args - firebase_command: postgres_connection_url: log_file: enable_debug_logs: + fail_if_missing_arg -firebase_command + fail_if_missing_arg -postgres_connection_url + fail_if_missing_arg -log_file + fail_if_missing_arg -enable_debug_logs + + local -r firebase_command="${args[-firebase_command]}" + local -r postgres_connection_url="${args[-postgres_connection_url]}" + local -r log_file="${args[-log_file]}" + local -r enable_debug_logs="${args[-enable_debug_logs]}" + + local firebase_args=("$firebase_command") + + if [[ $enable_debug_logs == '1' ]] { + firebase_args=+("--debug") + } elif [[ $enable_debug_logs != '0' ]] { + echo "ERROR: invalid value for -enable_debug_logs: $enable_debug_logs (must be 0 or 1)" >&2 + exit 2 + } + + firebase_args+=( + emulators:start + --only auth,dataconnect + ) + + set -xv + cd firebase-dataconnect/emulator + export FIREBASE_DATACONNECT_POSTGRESQL_STRING="$postgres_connection_url" + "${firebase_args[@]}" >"$log_file" 2>&1 & +} + +function subcommand/start_logcat_capture { + zparseopts -F -A args - adb_command: log_file: + fail_if_missing_arg -adb_command + fail_if_missing_arg -log_file + + local -r adb_command="${args[-adb_command]}" + local -r log_file="${args[-log_file]}" + + set -xv + "$adb_command" logcat >"$log_file" 2>&1 & +} + +function subcommand/verify_connected_check_success { + zparseopts -F -A args - connected_check_step_outcome: + fail_if_missing_arg -connected_check_step_outcome + + local -r outcome="${args[-connected_check_step_outcome]}" + + echo "The outcome of the \"Gradle connectedCheck\" task was: $outcome" + + if [[ $outcome != "success" ]] { + echo "Failing because the outcome of the \"Gradle connectedCheck\" step was not \"success\": $outcome" + exit 1 + } +} + +function subcommand/send_notifications { + zparseopts -F -A args - \ + github_notification_issue: github_workflow_name: github_repository: \ + job_status: git_commit_hash: run_url: \ + run_id: run_number: run_attempt: + + fail_if_missing_arg -github_notification_issue + fail_if_missing_arg -github_workflow_name + fail_if_missing_arg -github_repository + fail_if_missing_arg -job_status + fail_if_missing_arg -git_commit_hash + fail_if_missing_arg -run_url + fail_if_missing_arg -run_id + fail_if_missing_arg -run_number + fail_if_missing_arg -run_attempt + + local -r github_notification_issue="${args[-github_notification_issue]}" + local -r github_workflow_name="${args[-github_workflow_name]}" + local -r github_repository="${args[-github_repository]}" + local -r job_status="${args[-job_status]}" + local -r git_commit_hash="${args[-git_commit_hash]}" + local -r run_url="${args[-run_url]}" + local -r run_id="${args[-run_id]}" + local -r run_number="${args[-run_number]}" + local -r run_attempt="${args[-run_attempt]}" + + local -r message_file=$(mktemp -t message.jpbbdtrx2f.XXXXXX) + echo "Result of workflow $github_workflow_name at $git_commit_hash: $job_status" >>"$message_file" + echo "$run_url" >>"$message_file" + echo "run_id=$run_id run_number=$run_number run_attempt=$run_attempt" >>"$message_file" + + echo "Posting comment on GitHub Issue: https://github.com/firebase/firebase-android-sdk/issues/$github_notification_issue:" + cat "$message_file" + set -xv + gh issue comment "$github_notification_issue" --body-file "$message_file" -R "$github_repository" +} + +function fail_if_missing_arg { + if [[ $# -ne 1 ]] { + echo "INTERNAL ERROR qsvchnvaq7: invalid number of arguments: $# ($*)" >&2 + exit 1 + } + if [[ ! -v args[$1] ]] { + echo "ERROR: $1 must be specified (got ${(k)args})" >&2 + exit 2 + } +} + +main "$@" diff --git a/firebase-dataconnect/ci/build.gradle.kts b/firebase-dataconnect/ci/build.gradle.kts new file mode 100644 index 00000000000..44ab3929a7e --- /dev/null +++ b/firebase-dataconnect/ci/build.gradle.kts @@ -0,0 +1,260 @@ +import java.io.File +import java.io.IOException +import org.gradle.kotlin.dsl.* + +plugins { alias(libs.plugins.spotless) } + +spotless { + kotlinGradle { + target("*.gradle.kts") + ktfmt("0.41").googleStyle() + } +} + +abstract class InstallFirebaseToolsTask : DefaultTask() { + + @get:Input abstract val version: Property + + @get:OutputDirectory abstract val destDir: DirectoryProperty + + @get:OutputFile abstract val firebaseExecutable: RegularFileProperty + + @get:Inject abstract val execOperations: ExecOperations + + fun initializeProperties(version: Provider, destDir: Provider) { + this.version.set(version) + this.destDir.set(destDir) + this.firebaseExecutable.set(destDir.map { it.dir("node_modules").dir(".bin").file("firebase") }) + } + + @TaskAction + fun execute() { + val destDir: File = destDir.get().asFile + val version: String = version.get() + val firebaseExecutable: File = firebaseExecutable.get().asFile + + val packageJsonFile = File(destDir, "package.json") + logger.lifecycle("Creating $packageJsonFile") + packageJsonFile.writeText("{}") + + execOperations.exec { + workingDir = destDir + setCommandLine( + "npm", + "install", + "--fund=false", + "--audit=false", + "--save", + "--save-exact", + "firebase-tools@$version", + ) + logger.lifecycle("Running command in directory $workingDir: ${commandLine.joinToString(" ")}") + } + + execOperations.exec { + setCommandLine(firebaseExecutable.path, "--version") + logger.lifecycle( + "Running command to verify successful installation: ${commandLine.joinToString(" ")}" + ) + } + } +} + +abstract class PrintToolVersions : DefaultTask() { + + @get:InputFile abstract val gradlewExecutable: RegularFileProperty + + @get:InputFile abstract val firebaseExecutable: RegularFileProperty + + @get:Inject abstract val execOperations: ExecOperations + + fun initializeProperties( + firebaseExecutable: Provider, + gradlewExecutable: Provider + ) { + this.firebaseExecutable.set(firebaseExecutable) + this.gradlewExecutable.set(gradlewExecutable) + } + + @TaskAction + fun execute() { + val firebaseExecutable: File = firebaseExecutable.get().asFile + val gradlewExecutable: File = gradlewExecutable.get().asFile + + runCommandIgnoringExitCode("uname", "-a") + runCommandIgnoringExitCode("which", "java") + runCommandIgnoringExitCode("java", "-version") + runCommandIgnoringExitCode("which", "javac") + runCommandIgnoringExitCode("javac", "-version") + runCommandIgnoringExitCode("which", "node") + runCommandIgnoringExitCode("node", "--version") + runCommandIgnoringExitCode(firebaseExecutable.path, "--version") + runCommandIgnoringExitCode(gradlewExecutable.path, "--version") + } + + private fun runCommandIgnoringExitCode(vararg args: String) { + val argsStr = args.joinToString(" ") + logger.lifecycle("Running command: $argsStr") + + val execResult = + try { + execOperations.exec { + commandLine(*args) + isIgnoreExitValue = true + } + } catch (e: Exception) { + var ioException: Throwable? = e + while (ioException !== null && ioException !is IOException) { + ioException = ioException.cause + } + if (ioException !== null) { + logger.warn("WARNING: unable to start process: $argsStr (${e.message})") + return + } else { + throw e + } + } + + if (execResult.exitValue != 0) { + logger.warn( + "WARNING: command completed with non-zero exit code " + "${execResult.exitValue}: $argsStr" + ) + } + } +} + +abstract class BaseGradleTask(@get:Internal val gradleTasks: List) : DefaultTask() { + + @get:Internal abstract val gradleInfoLogsEnabled: Property + + @get:Input abstract val gradleWorkingDir: Property + + @get:InputFile abstract val gradlewExecutable: RegularFileProperty + + @get:Inject abstract val execOperations: ExecOperations + + fun initializeProperties( + gradleWorkingDir: Provider, + gradlewExecutable: Provider, + gradleInfoLogsEnabled: Provider, + ) { + this.gradleWorkingDir.set(gradleWorkingDir.map { it.asFile }) + this.gradlewExecutable.set(gradlewExecutable) + this.gradleInfoLogsEnabled.set(gradleInfoLogsEnabled) + } + + @TaskAction + fun execute() { + val gradleWorkingDir: File = gradleWorkingDir.get() + val gradlewExecutable: File = gradlewExecutable.get().asFile + val gradleInfoLogsEnabled: Boolean = gradleInfoLogsEnabled.get() + + val gradleArgs = buildList { + add(gradlewExecutable.path) + if (gradleInfoLogsEnabled) { + add("--info") + } + add("--profile") + add("--configure-on-demand") + addAll(gradleTasks) + } + + execOperations.exec { + workingDir = gradleWorkingDir + commandLine(gradleArgs) + logger.lifecycle("Running command in directory $workingDir: ${commandLine.joinToString(" ")}") + } + } +} + +abstract class BuildDataConnectIntegrationTests : BaseGradleTask(assembleTasks) { + companion object { + val assembleTasks = + listOf( + ":firebase-dataconnect:assembleDebugAndroidTest", + ":firebase-dataconnect:connectors:assembleDebugAndroidTest", + ) + } +} + +abstract class RunDataConnectIntegrationTests : BaseGradleTask(connectedCheckTasks) { + companion object { + val connectedCheckTasks = + listOf( + ":firebase-dataconnect:connectedCheck", + ":firebase-dataconnect:connectors:connectedCheck", + ) + } +} + +fun ProviderFactory.requiredGradleProperty(propertyName: String) = + gradleProperty(propertyName) + .orElse( + providers.provider { + throw RequiredPropertyMissing( + "Required project property \"$propertyName\" was not set; " + + "consider setting it " + + "by specifying -P$propertyName= on the Gradle command line " + + "or by setting the environment variable ORG_GRADLE_PROJECT_$propertyName" + ) + } + ) + +class RequiredPropertyMissing(message: String) : Exception(message) + +val ciTaskGroup = "Data Connect CI" + +val installFirebaseToolsTask = + tasks.register("installFirebaseTools") { + group = ciTaskGroup + description = "Install the firebase-tools npm package" + initializeProperties( + version = providers.requiredGradleProperty("firebaseToolsVersion"), + destDir = layout.buildDirectory.dir("firebase-tools"), + ) + } + +val gradleWorkingDirProvider = providers.provider { layout.projectDirectory.dir("../..") } + +val gradlewExecutableProvider = providers.provider { layout.projectDirectory.file("../../gradlew") } + +val gradlewInfoLogsEnabledProvider = + providers.requiredGradleProperty("debugLoggingEnabled").map { + when (it) { + "1" -> true + "0" -> false + else -> + throw IllegalArgumentException( + "invalid value for debugLoggingEnabled property: $it (must be either 0 or 1)" + ) + } + } + +tasks.register("printToolVersions") { + group = ciTaskGroup + description = "Print versions of notable command-line tools" + initializeProperties( + firebaseExecutable = installFirebaseToolsTask.flatMap { it.firebaseExecutable }, + gradlewExecutable = gradlewExecutableProvider, + ) +} + +tasks.register("buildIntegrationTests") { + group = ciTaskGroup + description = "Build the Data Connect integration tests" + initializeProperties( + gradleWorkingDir = gradleWorkingDirProvider, + gradlewExecutable = gradlewExecutableProvider, + gradleInfoLogsEnabled = gradlewInfoLogsEnabledProvider, + ) +} + +tasks.register("runIntegrationTests") { + group = ciTaskGroup + description = "Run the Data Connect integration tests" + initializeProperties( + gradleWorkingDir = gradleWorkingDirProvider, + gradlewExecutable = gradlewExecutableProvider, + gradleInfoLogsEnabled = gradlewInfoLogsEnabledProvider, + ) +} diff --git a/firebase-dataconnect/ci/gradle.properties b/firebase-dataconnect/ci/gradle.properties new file mode 100644 index 00000000000..58f3f0e8c7d --- /dev/null +++ b/firebase-dataconnect/ci/gradle.properties @@ -0,0 +1,20 @@ +# See https://docs.gradle.org/current/userguide/build_environment.html#gradle_properties_reference + +# When set to true, Gradle will reuse task outputs from any previous build when possible. +# See https://docs.gradle.org/current/userguide/build_cache.html#build_cache +org.gradle.caching=true + +# Enables configuration caching. Gradle will try to reuse the build configuration from previous builds. +# See https://docs.gradle.org/current/userguide/configuration_cache.html#config_cache:usage:enable +org.gradle.configuration-cache=false + +# Enables incubating configuration-on-demand, where Gradle will attempt to configure only necessary projects. +org.gradle.configureondemand=false + +# When set to quiet, warn, info, or debug, Gradle will use this log level. The values are not case-sensitive. +# Use "info" since this script uses a lot of info logging for normal output. +org.gradle.logging.level=lifecycle + +# When configured, Gradle will fork up to org.gradle.workers.max JVMs to execute projects in parallel. +# See https://docs.gradle.org/current/userguide/performance.html#parallel_execution +org.gradle.parallel=true diff --git a/firebase-dataconnect/ci/settings.gradle.kts b/firebase-dataconnect/ci/settings.gradle.kts new file mode 100644 index 00000000000..e064a15b6d2 --- /dev/null +++ b/firebase-dataconnect/ci/settings.gradle.kts @@ -0,0 +1,11 @@ +rootProject.name = "dataconnect-ci" + +dependencyResolutionManagement { + repositories { + google() + mavenCentral() + } + + // Reuse libs.version.toml from the main Gradle project. + versionCatalogs { create("libs") { from(files("../../gradle/libs.versions.toml")) } } +}