diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09ee495af..f81a757b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,3 +198,16 @@ jobs: uses: ./.github/workflows/integration-test-windows.yml with: unreal-version: ${{ matrix.unreal }} + + integration-test-android: + needs: [test-android] + name: Android UE ${{ matrix.unreal }} + secrets: inherit + strategy: + fail-fast: false + max-parallel: 2 + matrix: + unreal: ['5.4', '5.5', '5.6', '5.7'] + uses: ./.github/workflows/integration-test-android.yml + with: + unreal-version: ${{ matrix.unreal }} diff --git a/.github/workflows/integration-test-android.yml b/.github/workflows/integration-test-android.yml new file mode 100644 index 000000000..178ea26d9 --- /dev/null +++ b/.github/workflows/integration-test-android.yml @@ -0,0 +1,51 @@ +on: + workflow_call: + inputs: + unreal-version: + required: true + type: string + +jobs: + integration-test: + name: Integration Test + runs-on: ubuntu-latest + + env: + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} + GITHUB_TOKEN: ${{ github.token }} + SAUCE_REGION: us-west-1 + SAUCE_DEVICE_NAME: Samsung_Galaxy_S23_15_real_sjc1 + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Download sample build + uses: actions/download-artifact@v4 + with: + name: UE ${{ inputs.unreal-version }} sample build (Android) + path: sample-build + + - name: Run integration tests + id: run-integration-tests + shell: pwsh + working-directory: integration-test + env: + SENTRY_UNREAL_TEST_DSN: ${{ secrets.SENTRY_UNREAL_TEST_DSN }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_API_TOKEN }} + SENTRY_UNREAL_TEST_APP_PATH: ${{ github.workspace }}/sample-build/SentryPlayground-arm64.apk + UNREAL_VERSION: ${{ inputs.unreal-version }} + run: | + cmake -B build -S . + Invoke-Pester Integration.Android.SauceLabs.Tests.ps1 -CI + + - name: Upload integration test output + if: ${{ always() && steps.run-integration-tests.outcome == 'failure' }} + uses: actions/upload-artifact@v4 + with: + name: UE ${{ inputs.unreal-version }} integration test output (Android) + path: | + integration-test/output/ + retention-days: 14 \ No newline at end of file diff --git a/.github/workflows/integration-test-linux.yml b/.github/workflows/integration-test-linux.yml index 66dfc856a..f81a6f44a 100644 --- a/.github/workflows/integration-test-linux.yml +++ b/.github/workflows/integration-test-linux.yml @@ -26,21 +26,15 @@ jobs: chmod +x ${{ github.workspace }}/sample-build/SentryPlayground.sh chmod +x ${{ github.workspace }}/sample-build/SentryPlayground/Plugins/sentry/Binaries/Linux/crashpad_handler - - name: Install Pester - shell: pwsh - run: | - Install-Module -Name Pester -Force -SkipPublisherCheck - - name: Run integration tests id: run-integration-tests shell: pwsh + working-directory: integration-test env: SENTRY_UNREAL_TEST_DSN: ${{ secrets.SENTRY_UNREAL_TEST_DSN }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_API_TOKEN }} SENTRY_UNREAL_TEST_APP_PATH: ${{ github.workspace }}/sample-build/SentryPlayground.sh run: | - cd integration-test - mkdir build cmake -B build -S . Invoke-Pester Integration.Tests.ps1 -CI diff --git a/.github/workflows/integration-test-windows.yml b/.github/workflows/integration-test-windows.yml index 71a72ff71..a528de0f3 100644 --- a/.github/workflows/integration-test-windows.yml +++ b/.github/workflows/integration-test-windows.yml @@ -30,13 +30,12 @@ jobs: - name: Run integration tests id: run-integration-tests + working-directory: integration-test env: SENTRY_UNREAL_TEST_DSN: ${{ secrets.SENTRY_UNREAL_TEST_DSN }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_API_TOKEN }} SENTRY_UNREAL_TEST_APP_PATH: ${{ github.workspace }}/sample-build/SentryPlayground.exe run: | - cd integration-test - mkdir build cmake -B build -S . Invoke-Pester Integration.Tests.ps1 -CI diff --git a/integration-test/CMakeLists.txt b/integration-test/CMakeLists.txt index ec54093bc..566786492 100644 --- a/integration-test/CMakeLists.txt +++ b/integration-test/CMakeLists.txt @@ -7,7 +7,7 @@ include(FetchContent) FetchContent_Declare( app-runner GIT_REPOSITORY https://github.com/getsentry/app-runner.git - GIT_TAG 503795f0ef0f8340fcc0f0bc5fb5437df8cff9ef + GIT_TAG b1d7d0f97959f1ba7b1d52682d45ee9adf3adf96 ) FetchContent_MakeAvailable(app-runner) diff --git a/integration-test/Integration.Android.Adb.Tests.ps1 b/integration-test/Integration.Android.Adb.Tests.ps1 new file mode 100644 index 000000000..6b65a054c --- /dev/null +++ b/integration-test/Integration.Android.Adb.Tests.ps1 @@ -0,0 +1,250 @@ +# Integration tests for Sentry Unreal SDK on Android via ADB +# Requires: +# - Pre-built APK (x64 for emulator) +# - Android emulator or device connected +# - Environment variables: SENTRY_UNREAL_TEST_DSN, SENTRY_AUTH_TOKEN, SENTRY_UNREAL_TEST_APP_PATH + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +BeforeAll { + # Check if configuration file exists + $configFile = "$PSScriptRoot/TestConfig.local.ps1" + if (-not (Test-Path $configFile)) { + throw "Configuration file '$configFile' not found. Run 'cmake -B build -S .' first" + } + + # Load configuration (provides $global:AppRunnerPath) + . $configFile + + # Import app-runner modules (SentryApiClient, test utilities) + . "$global:AppRunnerPath/import-modules.ps1" + + # Validate environment variables + $script:DSN = $env:SENTRY_UNREAL_TEST_DSN + $script:AuthToken = $env:SENTRY_AUTH_TOKEN + $script:ApkPath = $env:SENTRY_UNREAL_TEST_APP_PATH + + if (-not $script:DSN) { + throw "Environment variable SENTRY_UNREAL_TEST_DSN must be set" + } + + if (-not $script:AuthToken) { + throw "Environment variable SENTRY_AUTH_TOKEN must be set" + } + + if (-not $script:ApkPath) { + throw "Environment variable SENTRY_UNREAL_TEST_APP_PATH must be set" + } + + # Connect to Sentry API + Write-Host "Connecting to Sentry API..." -ForegroundColor Yellow + Connect-SentryApi -DSN $script:DSN -ApiToken $script:AuthToken + + # Validate app path + if (-not (Test-Path $script:ApkPath)) { + throw "Application not found at: $script:ApkPath" + } + + # Create output directory + $script:OutputDir = "$PSScriptRoot/output" + if (-not (Test-Path $script:OutputDir)) { + New-Item -ItemType Directory -Path $script:OutputDir | Out-Null + } + + $script:PackageName = "io.sentry.unreal.sample" + $script:ActivityName = "$script:PackageName/com.epicgames.unreal.GameActivity" + + # Connect to Android device via ADB (auto-discovers available device) + Write-Host "Connecting to Android device..." -ForegroundColor Yellow + Connect-Device -Platform AndroidAdb + + # Install APK to device + Write-Host "Installing APK to Android device..." -ForegroundColor Yellow + Install-DeviceApp -Path $script:ApkPath + + # ========================================== + # RUN 1: Crash test - creates minidump + # ========================================== + # The crash is captured but NOT uploaded yet (Android behavior). + # TODO: Re-enable once Android SDK tag persistence is fixed (`test.crash_id` tag set before crash is not synced to the captured crash on Android) + + # Write-Host "Running crash-capture test (will crash)..." -ForegroundColor Yellow + # $cmdlineCrashArgs = "-e cmdline -crash-capture" + # $global:AndroidCrashResult = Invoke-DeviceApp -ExecutablePath $script:ActivityName -Arguments $cmdlineCrashArgs + + # Write-Host "Crash test exit code: $($global:AndroidCrashResult.ExitCode)" -ForegroundColor Cyan + + # ========================================== + # RUN 2: Message test - uploads crash from Run 1 + captures message + # ========================================== + # Currently we need to run again so that Sentry sends the crash event captured during the previous app session. + # TODO: use -SkipReinstall to preserve the crash state. + + Write-Host "Running message-capture test (will upload crash from previous run)..." -ForegroundColor Yellow + $cmdlineMessageArgs = "-e cmdline -message-capture" + $global:AndroidMessageResult = Invoke-DeviceApp -ExecutablePath $script:ActivityName -Arguments $cmdlineMessageArgs + + Write-Host "Message test exit code: $($global:AndroidMessageResult.ExitCode)" -ForegroundColor Cyan +} + +Describe "Sentry Unreal Android Integration Tests" { + + # ========================================== + # NOTE: Crash Capture Tests are DISABLED due to tag sync issue + # Uncomment when Android SDK tag persistence is fixed + # ========================================== + # Context "Crash Capture Tests" { + # BeforeAll { + # # Crash event is sent during the MESSAGE run (Run 2) + # # But the crash_id comes from the CRASH run (Run 1) + # $CrashResult = $global:AndroidCrashResult + # $CrashEvent = $null + # + # # Parse crash event ID from crash run output + # $eventIds = Get-EventIds -AppOutput $CrashResult.Output -ExpectedCount 1 + # + # if ($eventIds -and $eventIds.Count -gt 0) { + # Write-Host "Crash ID captured: $($eventIds[0])" -ForegroundColor Cyan + # $crashId = $eventIds[0] + # + # # Fetch crash event using the tag (event was sent during message run) + # try { + # $CrashEvent = Get-SentryTestEvent -TagName 'test.crash_id' -TagValue "$crashId" + # Write-Host "Crash event fetched from Sentry successfully" -ForegroundColor Green + # } catch { + # Write-Host "Failed to fetch crash event from Sentry: $_" -ForegroundColor Red + # } + # } else { + # Write-Host "Warning: No crash event ID found in output" -ForegroundColor Yellow + # } + # } + # + # It "Should output event ID before crash" { + # $eventIds = Get-EventIds -AppOutput $CrashResult.Output -ExpectedCount 1 + # $eventIds | Should -Not -BeNullOrEmpty + # $eventIds.Count | Should -Be 1 + # } + # + # It "Should capture crash event in Sentry (uploaded during next run)" { + # $CrashEvent | Should -Not -BeNullOrEmpty + # } + # + # It "Should have correct event type and platform" { + # $CrashEvent.type | Should -Be 'error' + # $CrashEvent.platform | Should -Be 'native' + # } + # + # It "Should have exception information" { + # $CrashEvent.exception | Should -Not -BeNullOrEmpty + # $CrashEvent.exception.values | Should -Not -BeNullOrEmpty + # } + # + # It "Should have stack trace" { + # $exception = $CrashEvent.exception.values[0] + # $exception.stacktrace | Should -Not -BeNullOrEmpty + # $exception.stacktrace.frames | Should -Not -BeNullOrEmpty + # } + # + # It "Should have user context" { + # $CrashEvent.user | Should -Not -BeNullOrEmpty + # $CrashEvent.user.username | Should -Be 'TestUser' + # $CrashEvent.user.email | Should -Be 'user-mail@test.abc' + # $CrashEvent.user.id | Should -Be '12345' + # } + # + # It "Should have test.crash_id tag for correlation" { + # $tags = $CrashEvent.tags + # $crashIdTag = $tags | Where-Object { $_.key -eq 'test.crash_id' } + # $crashIdTag | Should -Not -BeNullOrEmpty + # $crashIdTag.value | Should -Not -BeNullOrEmpty + # } + # + # It "Should have integration test tag" { + # $tags = $CrashEvent.tags + # ($tags | Where-Object { $_.key -eq 'test.suite' }).value | Should -Be 'integration' + # } + # + # It "Should have breadcrumbs from before crash" { + # $CrashEvent.breadcrumbs | Should -Not -BeNullOrEmpty + # $CrashEvent.breadcrumbs.values | Should -Not -BeNullOrEmpty + # } + # } + + Context "Message Capture Tests" { + BeforeAll { + $MessageResult = $global:AndroidMessageResult + $MessageEvent = $null + + # Parse event ID from output + $eventIds = Get-EventIds -AppOutput $MessageResult.Output -ExpectedCount 1 + + if ($eventIds -and $eventIds.Count -gt 0) { + Write-Host "Message event ID captured: $($eventIds[0])" -ForegroundColor Cyan + + # Fetch event from Sentry (with polling) + try { + $MessageEvent = Get-SentryTestEvent -EventId $eventIds[0] + Write-Host "Message event fetched from Sentry successfully" -ForegroundColor Green + } catch { + Write-Host "Failed to fetch message event from Sentry: $_" -ForegroundColor Red + } + } else { + Write-Host "Warning: No message event ID found in output" -ForegroundColor Yellow + } + } + + It "Should output event ID" { + $eventIds = Get-EventIds -AppOutput $MessageResult.Output -ExpectedCount 1 + $eventIds | Should -Not -BeNullOrEmpty + $eventIds.Count | Should -Be 1 + } + + It "Should output TEST_RESULT with success" { + $testResultLine = $MessageResult.Output | Where-Object { $_ -match 'TEST_RESULT:' } + $testResultLine | Should -Not -BeNullOrEmpty + $testResultLine | Should -Match '"success"\s*:\s*true' + } + + It "Should capture message event in Sentry" { + $MessageEvent | Should -Not -BeNullOrEmpty + } + + It "Should have correct platform" { + # Android events are captured from Java layer, so platform is 'java' not 'native' + $MessageEvent.platform | Should -Be 'java' + } + + It "Should have message content" { + $MessageEvent.message | Should -Not -BeNullOrEmpty + $MessageEvent.message.formatted | Should -Match 'Integration test message' + } + + It "Should have user context" { + $MessageEvent.user | Should -Not -BeNullOrEmpty + $MessageEvent.user.username | Should -Be 'TestUser' + } + + It "Should have integration test tag" { + $tags = $MessageEvent.tags + ($tags | Where-Object { $_.key -eq 'test.suite' }).value | Should -Be 'integration' + } + + It "Should have breadcrumbs" { + $MessageEvent.breadcrumbs | Should -Not -BeNullOrEmpty + $MessageEvent.breadcrumbs.values | Should -Not -BeNullOrEmpty + } + } +} + +AfterAll { + # Disconnect from Android device + Write-Host "Disconnecting from Android device..." -ForegroundColor Yellow + Disconnect-Device + + # Disconnect from Sentry API + Write-Host "Disconnecting from Sentry API..." -ForegroundColor Yellow + Disconnect-SentryApi + + Write-Host "Integration tests complete" -ForegroundColor Green +} \ No newline at end of file diff --git a/integration-test/Integration.Android.SauceLabs.Tests.ps1 b/integration-test/Integration.Android.SauceLabs.Tests.ps1 new file mode 100644 index 000000000..ca33f7a7e --- /dev/null +++ b/integration-test/Integration.Android.SauceLabs.Tests.ps1 @@ -0,0 +1,273 @@ +# Integration tests for Sentry Unreal SDK on Android via SauceLabs Real Device Cloud +# Requires: +# - Pre-built APK +# - SauceLabs account credentials +# - Environment variables: SENTRY_UNREAL_TEST_DSN, SENTRY_AUTH_TOKEN, SENTRY_UNREAL_TEST_APP_PATH +# SAUCE_USERNAME, SAUCE_ACCESS_KEY, SAUCE_REGION, SAUCE_DEVICE_NAME +# +# Note: SAUCE_DEVICE_NAME must match a device available in SAUCE_REGION. +# Example: For SAUCE_REGION=us-west-1, use devices with 'sjc1' suffix (San Jose DC1) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +BeforeAll { + # Check if configuration file exists + $configFile = "$PSScriptRoot/TestConfig.local.ps1" + if (-not (Test-Path $configFile)) { + throw "Configuration file '$configFile' not found. Run 'cmake -B build -S .' first" + } + + # Load configuration (provides $global:AppRunnerPath) + . $configFile + + # Import app-runner modules (SentryApiClient, test utilities) + . "$global:AppRunnerPath/import-modules.ps1" + + # Validate environment variables + $script:DSN = $env:SENTRY_UNREAL_TEST_DSN + $script:AuthToken = $env:SENTRY_AUTH_TOKEN + $script:ApkPath = $env:SENTRY_UNREAL_TEST_APP_PATH + $script:SauceUsername = $env:SAUCE_USERNAME + $script:SauceAccessKey = $env:SAUCE_ACCESS_KEY + $script:SauceRegion = $env:SAUCE_REGION + $script:SauceDeviceName = $env:SAUCE_DEVICE_NAME + + if (-not $script:DSN) { + throw "Environment variable SENTRY_UNREAL_TEST_DSN must be set" + } + + if (-not $script:AuthToken) { + throw "Environment variable SENTRY_AUTH_TOKEN must be set" + } + + if (-not $script:ApkPath) { + throw "Environment variable SENTRY_UNREAL_TEST_APP_PATH must be set" + } + + if (-not $script:SauceUsername) { + throw "Environment variable SAUCE_USERNAME must be set" + } + + if (-not $script:SauceAccessKey) { + throw "Environment variable SAUCE_ACCESS_KEY must be set" + } + + if (-not $script:SauceRegion) { + throw "Environment variable SAUCE_REGION must be set" + } + + if (-not $script:SauceDeviceName) { + throw "Environment variable SAUCE_DEVICE_NAME must be set" + } + + # Connect to Sentry API + Write-Host "Connecting to Sentry API..." -ForegroundColor Yellow + Connect-SentryApi -DSN $script:DSN -ApiToken $script:AuthToken + + # Validate app path + if (-not (Test-Path $script:ApkPath)) { + throw "Application not found at: $script:ApkPath" + } + + # Create output directory + $script:OutputDir = "$PSScriptRoot/output" + if (-not (Test-Path $script:OutputDir)) { + New-Item -ItemType Directory -Path $script:OutputDir | Out-Null + } + + $script:PackageName = "io.sentry.unreal.sample" + $script:ActivityName = "$script:PackageName/com.epicgames.unreal.GameActivity" + + # Connect to SauceLabs (reads credentials and configuration from environment variables) + Write-Host "Connecting to SauceLabs..." -ForegroundColor Yellow + Connect-Device -Platform AndroidSauceLabs + + # Install APK to SauceLabs device + Write-Host "Installing APK to SauceLabs device..." -ForegroundColor Yellow + Install-DeviceApp -Path $script:ApkPath + + # ========================================== + # RUN 1: Crash test - creates minidump + # ========================================== + # The crash is captured but NOT uploaded yet (Android behavior). + # TODO: Re-enable once Android SDK tag persistence is fixed (`test.crash_id` tag set before crash is not synced to the captured crash on Android) + + # Write-Host "Running crash-capture test (will crash)..." -ForegroundColor Yellow + # $cmdlineCrashArgs = "-e cmdline -crash-capture" + # $global:SauceLabsCrashResult = Invoke-DeviceApp -ExecutablePath $script:ActivityName -Arguments $cmdlineCrashArgs + + # Write-Host "Crash test exit code: $($global:SauceLabsCrashResult.ExitCode)" -ForegroundColor Cyan + + # ========================================== + # RUN 2: Message test - uploads crash from Run 1 + captures message + # ========================================== + # Currently we need to run again so that Sentry sends the crash event captured during the previous app session. + + Write-Host "Running message-capture test (will upload crash from previous run)..." -ForegroundColor Yellow + $cmdlineMessageArgs = "-e cmdline -message-capture" + $global:SauceLabsMessageResult = Invoke-DeviceApp -ExecutablePath $script:ActivityName -Arguments $cmdlineMessageArgs + + Write-Host "Message test exit code: $($global:SauceLabsMessageResult.ExitCode)" -ForegroundColor Cyan +} + +Describe "Sentry Unreal Android Integration Tests (SauceLabs)" { + + # ========================================== + # NOTE: Crash Capture Tests are DISABLED due to tag sync issue + # Uncomment when Android SDK tag persistence is fixed + # ========================================== + # Context "Crash Capture Tests" { + # BeforeAll { + # # Crash event is sent during the MESSAGE run (Run 2) + # # But the crash_id comes from the CRASH run (Run 1) + # $CrashResult = $global:SauceLabsCrashResult + # $CrashEvent = $null + # + # # Parse crash event ID from crash run output + # $eventIds = Get-EventIds -AppOutput $CrashResult.Output -ExpectedCount 1 + # + # if ($eventIds -and $eventIds.Count -gt 0) { + # Write-Host "Crash ID captured: $($eventIds[0])" -ForegroundColor Cyan + # $crashId = $eventIds[0] + # + # # Fetch crash event using the tag (event was sent during message run) + # try { + # $CrashEvent = Get-SentryTestEvent -TagName 'test.crash_id' -TagValue "$crashId" + # Write-Host "Crash event fetched from Sentry successfully" -ForegroundColor Green + # } catch { + # Write-Host "Failed to fetch crash event from Sentry: $_" -ForegroundColor Red + # } + # } else { + # Write-Host "Warning: No crash event ID found in output" -ForegroundColor Yellow + # } + # } + # + # It "Should output event ID before crash" { + # $eventIds = Get-EventIds -AppOutput $CrashResult.Output -ExpectedCount 1 + # $eventIds | Should -Not -BeNullOrEmpty + # $eventIds.Count | Should -Be 1 + # } + # + # It "Should capture crash event in Sentry (uploaded during next run)" { + # $CrashEvent | Should -Not -BeNullOrEmpty + # } + # + # It "Should have correct event type and platform" { + # $CrashEvent.type | Should -Be 'error' + # $CrashEvent.platform | Should -Be 'native' + # } + # + # It "Should have exception information" { + # $CrashEvent.exception | Should -Not -BeNullOrEmpty + # $CrashEvent.exception.values | Should -Not -BeNullOrEmpty + # } + # + # It "Should have stack trace" { + # $exception = $CrashEvent.exception.values[0] + # $exception.stacktrace | Should -Not -BeNullOrEmpty + # $exception.stacktrace.frames | Should -Not -BeNullOrEmpty + # } + # + # It "Should have user context" { + # $CrashEvent.user | Should -Not -BeNullOrEmpty + # $CrashEvent.user.username | Should -Be 'TestUser' + # $CrashEvent.user.email | Should -Be 'user-mail@test.abc' + # $CrashEvent.user.id | Should -Be '12345' + # } + # + # It "Should have test.crash_id tag for correlation" { + # $tags = $CrashEvent.tags + # $crashIdTag = $tags | Where-Object { $_.key -eq 'test.crash_id' } + # $crashIdTag | Should -Not -BeNullOrEmpty + # $crashIdTag.value | Should -Not -BeNullOrEmpty + # } + # + # It "Should have integration test tag" { + # $tags = $CrashEvent.tags + # ($tags | Where-Object { $_.key -eq 'test.suite' }).value | Should -Be 'integration' + # } + # + # It "Should have breadcrumbs from before crash" { + # $CrashEvent.breadcrumbs | Should -Not -BeNullOrEmpty + # $CrashEvent.breadcrumbs.values | Should -Not -BeNullOrEmpty + # } + # } + + Context "Message Capture Tests" { + BeforeAll { + $MessageResult = $global:SauceLabsMessageResult + $MessageEvent = $null + + # Parse event ID from output + $eventIds = Get-EventIds -AppOutput $MessageResult.Output -ExpectedCount 1 + + if ($eventIds -and $eventIds.Count -gt 0) { + Write-Host "Message event ID captured: $($eventIds[0])" -ForegroundColor Cyan + + # Fetch event from Sentry (with polling) + try { + $MessageEvent = Get-SentryTestEvent -EventId $eventIds[0] + Write-Host "Message event fetched from Sentry successfully" -ForegroundColor Green + } catch { + Write-Host "Failed to fetch message event from Sentry: $_" -ForegroundColor Red + } + } else { + Write-Host "Warning: No message event ID found in output" -ForegroundColor Yellow + } + } + + It "Should output event ID" { + $eventIds = Get-EventIds -AppOutput $MessageResult.Output -ExpectedCount 1 + $eventIds | Should -Not -BeNullOrEmpty + $eventIds.Count | Should -Be 1 + } + + It "Should output TEST_RESULT with success" { + $testResultLine = $MessageResult.Output | Where-Object { $_ -match 'TEST_RESULT:' } + $testResultLine | Should -Not -BeNullOrEmpty + $testResultLine | Should -Match '"success"\s*:\s*true' + } + + It "Should capture message event in Sentry" { + $MessageEvent | Should -Not -BeNullOrEmpty + } + + It "Should have correct platform" { + # Android events are captured from Java layer, so platform is 'java' not 'native' + $MessageEvent.platform | Should -Be 'java' + } + + It "Should have message content" { + $MessageEvent.message | Should -Not -BeNullOrEmpty + $MessageEvent.message.formatted | Should -Match 'Integration test message' + } + + It "Should have user context" { + $MessageEvent.user | Should -Not -BeNullOrEmpty + $MessageEvent.user.username | Should -Be 'TestUser' + } + + It "Should have integration test tag" { + $tags = $MessageEvent.tags + ($tags | Where-Object { $_.key -eq 'test.suite' }).value | Should -Be 'integration' + } + + It "Should have breadcrumbs" { + $MessageEvent.breadcrumbs | Should -Not -BeNullOrEmpty + $MessageEvent.breadcrumbs.values | Should -Not -BeNullOrEmpty + } + } +} + +AfterAll { + # Disconnect from SauceLabs device (cleans up session) + Write-Host "Disconnecting from SauceLabs device..." -ForegroundColor Yellow + Disconnect-Device + + # Disconnect from Sentry API + Write-Host "Disconnecting from Sentry API..." -ForegroundColor Yellow + Disconnect-SentryApi + + Write-Host "Integration tests complete" -ForegroundColor Green +} diff --git a/integration-test/README.md b/integration-test/README.md index 589f88f91..05e8f346b 100644 --- a/integration-test/README.md +++ b/integration-test/README.md @@ -2,8 +2,15 @@ This directory contains integration tests for the Sentry Unreal SDK using Pester (PowerShell testing framework). +Supports testing on: +- **Windows** - Desktop (x64) +- **Linux** - Desktop (x64) +- **Android** - Local device/emulator (via adb) or SauceLabs Real Device Cloud + ## Prerequisites +### Common Requirements + - **PowerShell 7+** (Core edition) - **CMake 3.20+** - **Pester 5+** - Install with: `Install-Module -Name Pester -Force -SkipPublisherCheck` @@ -11,7 +18,30 @@ This directory contains integration tests for the Sentry Unreal SDK using Pester - **Environment variables**: - `SENTRY_UNREAL_TEST_DSN` - Sentry test project DSN - `SENTRY_AUTH_TOKEN` - Sentry API authentication token - - `SENTRY_UNREAL_TEST_APP_PATH` - Path to the SentryPlayground executable + - `SENTRY_UNREAL_TEST_APP_PATH` - Path to the SentryPlayground executable/APK + +### Android-Specific Requirements + +#### Option A: Local Testing (via adb) +- **Android device or emulator** connected and visible via `adb devices` +- **ADB (Android Debug Bridge)** installed and in PATH + +#### Option B: Cloud Testing (via SauceLabs) +- **SauceLabs account** with Real Device Cloud access +- **Additional environment variables**: + - `SAUCE_USERNAME` - SauceLabs username + - `SAUCE_ACCESS_KEY` - SauceLabs access key + - `SAUCE_REGION` - SauceLabs region (e.g., `us-west-1`, `eu-central-1`) + - `SAUCE_DEVICE_NAME` - Device name available in the specified region (must match region datacenter suffix) + +**Note**: The device name must match a device available in your SauceLabs region. Device names include a datacenter suffix that must align with the region: +- `us-west-1` → devices ending in `_sjc1` (San Jose DC1) +- `eu-central-1` → devices ending in `_fra1` (Frankfurt DC1) +- `us-east-4` → devices ending in `_use1` (US East DC1) + +Example valid combinations: +- Region: `us-west-1`, Device: `Samsung_Galaxy_S23_15_real_sjc1` ✓ +- Region: `eu-central-1`, Device: `Samsung_Galaxy_S23_15_real_sjc1` ✗ (mismatch) ## Setup @@ -39,6 +69,7 @@ This will: 3. Download the appropriate artifact: - `UE X.X sample build (Windows)` for Windows testing - `UE X.X sample build (Linux)` for Linux testing + - `UE X.X sample build (Android)` for Android testing 4. Extract to a known location #### Option B: Build Locally @@ -75,11 +106,46 @@ cd integration-test pwsh -Command "Invoke-Pester Integration.Tests.ps1" ``` +### Android (Local via adb) + +```bash +# Ensure device/emulator is connected +adb devices + +# Set environment variables +export SENTRY_UNREAL_TEST_DSN="https://key@org.ingest.sentry.io/project" +export SENTRY_AUTH_TOKEN="sntrys_your_token_here" +export SENTRY_UNREAL_TEST_APP_PATH="./path/to/SentryPlayground.apk" + +# Run tests +cd integration-test +pwsh -Command "Invoke-Pester ./Integration.Android.Adb.Tests.ps1" +``` + +### Android (Cloud via SauceLabs) + +```bash +# Set environment variables +export SENTRY_UNREAL_TEST_DSN="https://key@org.ingest.sentry.io/project" +export SENTRY_AUTH_TOKEN="sntrys_your_token_here" +export SENTRY_UNREAL_TEST_APP_PATH="./path/to/SentryPlayground.apk" +export SAUCE_USERNAME="your-saucelabs-username" +export SAUCE_ACCESS_KEY="your-saucelabs-access-key" +export SAUCE_REGION="us-west-1" +export SAUCE_DEVICE_NAME="Samsung_Galaxy_S23_15_real_sjc1" + +# Run tests +cd integration-test +pwsh -Command "Invoke-Pester ./Integration.Android.SauceLabs.Tests.ps1" +``` + +**Note**: Ensure `SAUCE_DEVICE_NAME` matches a device available in your `SAUCE_REGION`. See the [SauceLabs Platform Configurator](https://app.saucelabs.com/live/web-testing) to find available devices for your region. + ## Test Coverage The integration tests cover: -### Crash Capture Tests +### Crash Capture Tests _(Windows/Linux)_ - Application crashes with non-zero exit code - Event ID is captured from output (set via `test.crash_id` tag) - Crash event appears in Sentry @@ -89,8 +155,10 @@ The integration tests cover: - Integration test tags are set - Breadcrumbs are collected -### Message Capture Tests -- Application exits cleanly (exit code 0) +**Note**: Crash capture tests are currently disabled on Android due to a known issue with tag persistence across app sessions. + +### Message Capture Tests _(All platforms)_ +- Application exits cleanly (exit code 0 on Windows/Linux, Android doesn't report exit codes) - Event ID is captured from output - TEST_RESULT indicates success - Message event appears in Sentry @@ -99,9 +167,13 @@ The integration tests cover: - Integration test tags are set - Breadcrumbs are collected +**Note**: On Android, events are captured from the Java layer, so the platform will be `java` instead of `native`. + ## Output Test outputs are saved to `integration-test/output/`: + +### Windows/Linux - `*-crash-stdout.log` - Crash test standard output - `*-crash-stderr.log` - Crash test standard error - `*-crash-result.json` - Full crash test result @@ -110,6 +182,13 @@ Test outputs are saved to `integration-test/output/`: - `*-message-result.json` - Full message test result - `event-*.json` - Events fetched from Sentry API +### Android +- `*-logcat.txt` - Logcat output from app execution (one file per launch) +- `event-*.json` - Events fetched from Sentry API + ## CI Integration -See `.github/workflows/integration-test-windows.yml` and `.github/workflows/integration-test-linux.yml` for CI usage examples. +See the following workflow files for CI usage examples: +- `.github/workflows/integration-test-windows.yml` - Windows desktop testing +- `.github/workflows/integration-test-linux.yml` - Linux desktop testing +- `.github/workflows/integration-test-android.yml` - Android testing via SauceLabs Real Device Cloud diff --git a/sample/Config/DefaultEngine.ini b/sample/Config/DefaultEngine.ini index bca84ad7e..1e8177a5d 100644 --- a/sample/Config/DefaultEngine.ini +++ b/sample/Config/DefaultEngine.ini @@ -215,6 +215,8 @@ KeyStore=debug.keystore KeyAlias=androiddebugkey KeyStorePassword=android KeyPassword=android +bBuildForArm64=True +bBuildForX8664=True [/Script/IOSRuntimeSettings.IOSRuntimeSettings] BundleIdentifier=io.sentry.unreal.sample diff --git a/sample/Source/SentryPlayground/SentryPlayground.Build.cs b/sample/Source/SentryPlayground/SentryPlayground.Build.cs index f1d2de319..35cb4df6c 100644 --- a/sample/Source/SentryPlayground/SentryPlayground.Build.cs +++ b/sample/Source/SentryPlayground/SentryPlayground.Build.cs @@ -7,14 +7,14 @@ public class SentryPlayground : ModuleRules public SentryPlayground(ReadOnlyTargetRules Target) : base(Target) { PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs; - + PublicDependencyModuleNames.AddRange(new string[] { "Core", "CoreUObject", "Engine", "InputCore", "Sentry" }); PrivateDependencyModuleNames.AddRange(new string[] { }); // Uncomment if you are using Slate UI // PrivateDependencyModuleNames.AddRange(new string[] { "Slate", "SlateCore" }); - + // Uncomment if you are using online features // PrivateDependencyModuleNames.Add("OnlineSubsystem"); diff --git a/sample/Source/SentryPlayground/SentryPlaygroundGameInstance.cpp b/sample/Source/SentryPlayground/SentryPlaygroundGameInstance.cpp index 989b52943..9a525c350 100644 --- a/sample/Source/SentryPlayground/SentryPlaygroundGameInstance.cpp +++ b/sample/Source/SentryPlayground/SentryPlaygroundGameInstance.cpp @@ -18,20 +18,22 @@ void USentryPlaygroundGameInstance::Init() { Super::Init(); - const TCHAR* CommandLine = FCommandLine::Get(); + FString CommandLine = FCommandLine::Get(); + + UE_LOG(LogSentrySample, Display, TEXT("Starting app with commandline: %s\n"), *CommandLine); // Check for expected test parameters to decide between running integration tests // or launching the sample app with UI for manual testing - if (FParse::Param(FCommandLine::Get(), TEXT("crash-capture")) || - FParse::Param(FCommandLine::Get(), TEXT("message-capture"))) + if (FParse::Param(*CommandLine, TEXT("crash-capture")) || + FParse::Param(*CommandLine, TEXT("message-capture"))) { RunIntegrationTest(CommandLine); } } -void USentryPlaygroundGameInstance::RunIntegrationTest(const TCHAR* CommandLine) +void USentryPlaygroundGameInstance::RunIntegrationTest(const FString& CommandLine) { - UE_LOG(LogSentrySample, Display, TEXT("Running integration test for command: %s\n"), CommandLine); + UE_LOG(LogSentrySample, Display, TEXT("Running integration test for command: %s\n"), *CommandLine); USentrySubsystem* SentrySubsystem = GEngine->GetEngineSubsystem(); if (!SentrySubsystem) @@ -40,11 +42,11 @@ void USentryPlaygroundGameInstance::RunIntegrationTest(const TCHAR* CommandLine) return; } - SentrySubsystem->InitializeWithSettings(FConfigureSettingsNativeDelegate::CreateLambda([=](USentrySettings* Settings) + SentrySubsystem->InitializeWithSettings(FConfigureSettingsNativeDelegate::CreateLambda([CommandLine](USentrySettings* Settings) { // Override options set in config file if needed FString Dsn; - if (FParse::Value(CommandLine, TEXT("dsn="), Dsn)) + if (FParse::Value(*CommandLine, TEXT("dsn="), Dsn)) { Settings->Dsn = Dsn; } @@ -64,11 +66,11 @@ void USentryPlaygroundGameInstance::RunIntegrationTest(const TCHAR* CommandLine) SentrySubsystem->AddBreadcrumbWithParams( TEXT("Context configuration finished"), TEXT("Test"), TEXT("info"), TMap(), ESentryLevel::Info); - if (FParse::Param(CommandLine, TEXT("crash-capture"))) + if (FParse::Param(*CommandLine, TEXT("crash-capture"))) { RunCrashTest(); } - else if (FParse::Param(CommandLine, TEXT("message-capture"))) + else if (FParse::Param(*CommandLine, TEXT("message-capture"))) { RunMessageTest(); } diff --git a/sample/Source/SentryPlayground/SentryPlaygroundGameInstance.h b/sample/Source/SentryPlayground/SentryPlaygroundGameInstance.h index 636baeda3..0a5a6b0c8 100644 --- a/sample/Source/SentryPlayground/SentryPlaygroundGameInstance.h +++ b/sample/Source/SentryPlayground/SentryPlaygroundGameInstance.h @@ -20,7 +20,7 @@ class SENTRYPLAYGROUND_API USentryPlaygroundGameInstance : public UGameInstance virtual void Init() override; private: - void RunIntegrationTest(const TCHAR* CommandLine); + void RunIntegrationTest(const FString& CommandLine); void RunCrashTest(); void RunMessageTest();