Updating tests to retry after initial if failed

build/jobs/build.yml

@@ -30,22 +30,26 @@ steps:
       workingDirectory: $(System.DefaultWorkingDirectory)
 
 - ${{ if eq(parameters.unitTest, 'true') }}:
-  - task: DotNetCoreCLI@2
-    displayName: 'dotnet test'
-    inputs:
-      command: test
-      projects: '**/*UnitTests/*.csproj'
-      arguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}}'
+  - template: run-tests-with-retry.yml
+    parameters:
+      displayName: 'Unit Tests with Retry'
+      testProjects: '**/*UnitTests/*.csproj'
+      testArguments: '--configuration $(buildConfiguration) --no-build -f ${{parameters.targetBuildFramework}}'
       testRunTitle: 'Unit Tests'
+      maxAttempts: 2
+      retryDelaySeconds: 10
+      workingDirectory: $(System.DefaultWorkingDirectory)
 
 - ${{ if eq(parameters.codeCoverage, 'true') }}:
-  - task: DotNetCoreCLI@2
-    displayName: 'dotnet test with coverage'
-    inputs:
-      command: test
-      projects: '**/*UnitTests/*.csproj'
-      arguments: '--configuration $(buildConfiguration) --no-build --collect "XPlat Code Coverage" -s "$(build.sourcesDirectory)/CodeCoverage.runsettings" -v normal -f ${{parameters.targetBuildFramework}}'
+  - template: run-tests-with-retry.yml
+    parameters:
+      displayName: 'Unit Tests with Coverage and Retry'
+      testProjects: '**/*UnitTests/*.csproj'
+      testArguments: '--configuration $(buildConfiguration) --no-build --collect "XPlat Code Coverage" -s "$(build.sourcesDirectory)/CodeCoverage.runsettings" -v normal -f ${{parameters.targetBuildFramework}}'
       testRunTitle: 'Unit Tests'
+      maxAttempts: 2
+      retryDelaySeconds: 10
+      workingDirectory: $(System.DefaultWorkingDirectory)
   - task: reportgenerator@5
     displayName: 'aggregate code coverage'
     condition: succeededOrFailed()

build/jobs/e2e-tests.yml

@@ -84,46 +84,46 @@ steps:
         Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_TENANT_ID]$env:AZURESUBSCRIPTION_TENANT_ID"
         Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_SERVICE_CONNECTION_ID]$env:AZURESUBSCRIPTION_SERVICE_CONNECTION_ID"
 
-  - task: DotNetCoreCLI@2
-    displayName: 'E2E ${{ parameters.version }} ${{parameters.appServiceType}}'
-    inputs:
-      command: test
-      arguments: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll" --blame-hang-timeout 7m --filter "FullyQualifiedName~${{parameters.appServiceType}}&Category!=ExportLongRunning"'
-      workingDirectory: "$(System.ArtifactsDirectory)"
+  - template: run-tests-with-retry.yml
+    parameters:
+      displayName: 'E2E ${{ parameters.version }} ${{parameters.appServiceType}} with Retry'
+      testArguments: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll" --blame-hang-timeout 7m --filter "FullyQualifiedName~${{parameters.appServiceType}}&Category!=ExportLongRunning"'
       testRunTitle: '${{ parameters.version }} ${{parameters.appServiceType}}'
-    env:
-      'TestEnvironmentUrl': $(TestEnvironmentUrl)
-      'TestEnvironmentUrl_${{ parameters.version }}': $(TestEnvironmentUrl_${{ parameters.version }})
-      'TestEnvironmentUrl_Sql': $(TestEnvironmentUrl_Sql)
-      'TestEnvironmentUrl_${{ parameters.version }}_Sql': $(TestEnvironmentUrl_${{ parameters.version }}_Sql)
-      'Resource': $(Resource)
-      'AllStorageAccounts': $(AllStorageAccounts)
-      'TestExportStoreUri': $(TestExportStoreUri)
-      'TestIntegrationStoreUri': $(TestIntegrationStoreUri)
-      'tenant-admin-service-principal-name': $(tenant-admin-service-principal-name)
-      'tenant-admin-service-principal-password': $(tenant-admin-service-principal-password)
-      'tenant-admin-user-name': $(tenant-admin-user-name)
-      'tenant-admin-user-password': $(tenant-admin-user-password)
-      'tenant-id': $(tenant-id)
-      'app_globalAdminServicePrincipal_id': $(app_globalAdminServicePrincipal_id)
-      'app_globalAdminServicePrincipal_secret': $(app_globalAdminServicePrincipal_secret)
-      'app_nativeClient_id': $(app_nativeClient_id)
-      'app_nativeClient_secret': $(app_nativeClient_secret)
-      'app_wrongAudienceClient_id': $(app_wrongAudienceClient_id)
-      'app_wrongAudienceClient_secret': $(app_wrongAudienceClient_secret)
-      'app_globalAdminUserApp_id': $(app_globalAdminUserApp_id)
-      'app_globalAdminUserApp_secret': $(app_globalAdminUserApp_secret)
-      'app_globalConverterUserApp_id': $(app_globalConverterUserApp_id)
-      'app_globalConverterUserApp_secret': $(app_globalConverterUserApp_secret)
-      'app_globalExporterUserApp_id': $(app_globalExporterUserApp_id)
-      'app_globalExporterUserApp_secret': $(app_globalExporterUserApp_secret)
-      'app_globalImporterUserApp_id': $(app_globalImporterUserApp_id)
-      'app_globalImporterUserApp_secret': $(app_globalImporterUserApp_secret)
-      'app_globalReaderUserApp_id': $(app_globalReaderUserApp_id)
-      'app_globalReaderUserApp_secret': $(app_globalReaderUserApp_secret)
-      'app_globalWriterUserApp_id': $(app_globalWriterUserApp_id)
-      'app_globalWriterUserApp_secret': $(app_globalWriterUserApp_secret)
-      'AZURESUBSCRIPTION_CLIENT_ID': $(AzurePipelinesCredential_ClientId)
-      'AZURESUBSCRIPTION_TENANT_ID': $(AZURESUBSCRIPTION_TENANT_ID)
-      'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': $(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)
-      'SYSTEM_ACCESSTOKEN': $(System.AccessToken)
+      maxAttempts: 3
+      retryDelaySeconds: 45
+      environmentVariables:
+        TestEnvironmentUrl: $(TestEnvironmentUrl)
+        TestEnvironmentUrl_${{ parameters.version }}: $(TestEnvironmentUrl_${{ parameters.version }})
+        TestEnvironmentUrl_Sql: $(TestEnvironmentUrl_Sql)
+        TestEnvironmentUrl_${{ parameters.version }}_Sql: $(TestEnvironmentUrl_${{ parameters.version }}_Sql)
+        Resource: $(Resource)
+        AllStorageAccounts: $(AllStorageAccounts)
+        TestExportStoreUri: $(TestExportStoreUri)
+        TestIntegrationStoreUri: $(TestIntegrationStoreUri)
+        tenant-admin-service-principal-name: $(tenant-admin-service-principal-name)
+        tenant-admin-service-principal-password: $(tenant-admin-service-principal-password)
+        tenant-admin-user-name: $(tenant-admin-user-name)
+        tenant-admin-user-password: $(tenant-admin-user-password)
+        tenant-id: $(tenant-id)
+        app_globalAdminServicePrincipal_id: $(app_globalAdminServicePrincipal_id)
+        app_globalAdminServicePrincipal_secret: $(app_globalAdminServicePrincipal_secret)
+        app_nativeClient_id: $(app_nativeClient_id)
+        app_nativeClient_secret: $(app_nativeClient_secret)
+        app_wrongAudienceClient_id: $(app_wrongAudienceClient_id)
+        app_wrongAudienceClient_secret: $(app_wrongAudienceClient_secret)
+        app_globalAdminUserApp_id: $(app_globalAdminUserApp_id)
+        app_globalAdminUserApp_secret: $(app_globalAdminUserApp_secret)
+        app_globalConverterUserApp_id: $(app_globalConverterUserApp_id)
+        app_globalConverterUserApp_secret: $(app_globalConverterUserApp_secret)
+        app_globalExporterUserApp_id: $(app_globalExporterUserApp_id)
+        app_globalExporterUserApp_secret: $(app_globalExporterUserApp_secret)
+        app_globalImporterUserApp_id: $(app_globalImporterUserApp_id)
+        app_globalImporterUserApp_secret: $(app_globalImporterUserApp_secret)
+        app_globalReaderUserApp_id: $(app_globalReaderUserApp_id)
+        app_globalReaderUserApp_secret: $(app_globalReaderUserApp_secret)
+        app_globalWriterUserApp_id: $(app_globalWriterUserApp_id)
+        app_globalWriterUserApp_secret: $(app_globalWriterUserApp_secret)
+        AZURESUBSCRIPTION_CLIENT_ID: $(AzurePipelinesCredential_ClientId)
+        AZURESUBSCRIPTION_TENANT_ID: $(AZURESUBSCRIPTION_TENANT_ID)
+        AZURESUBSCRIPTION_SERVICE_CONNECTION_ID: $(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken)

build/jobs/run-tests-with-retry.yml

@@ -0,0 +1,137 @@
+parameters:
+- name: displayName
+  type: string
+- name: testArguments  
+  type: string
+  default: ""
+- name: testProjects
+  type: string
+  default: ""
+- name: workingDirectory
+  type: string
+  default: "$(System.ArtifactsDirectory)"
+- name: testRunTitle
+  type: string
+- name: maxAttempts
+  type: number
+  default: 3
+- name: retryDelaySeconds
+  type: number
+  default: 30
+- name: environmentVariables
+  type: object
+  default: {}
+
+steps:
+- ${{ if ne(parameters.testProjects, '') }}:
+  # For unit tests using projects parameter
+  - task: DotNetCoreCLI@2
+    displayName: '${{ parameters.displayName }} (Attempt 1)'
+    name: 'TestAttempt1_Unit'
+    inputs:
+      command: test
+      projects: '${{ parameters.testProjects }}'
+      arguments: '${{ parameters.testArguments }}'
+      testRunTitle: '${{ parameters.testRunTitle }}'
+      workingDirectory: ${{ parameters.workingDirectory }}
+    env: ${{ parameters.environmentVariables }}
+    continueOnError: true
+
+  - ${{ if gt(parameters.maxAttempts, 1) }}:
+    - task: PowerShell@2
+      displayName: 'Retry Logic for Unit Tests'
+      condition: in(variables['TestAttempt1_Unit.result'], 'Failed', 'PartiallySucceeded')
+      inputs:
+        targetType: 'inline'
+        script: |
+          Write-Host "##[warning]First attempt failed or partially succeeded, waiting ${{ parameters.retryDelaySeconds }} seconds before retry..."
+          Start-Sleep -Seconds ${{ parameters.retryDelaySeconds }}
+
+    - task: DotNetCoreCLI@2
+      displayName: '${{ parameters.displayName }} (Attempt 2)'
+      condition: in(variables['TestAttempt1_Unit.result'], 'Failed', 'PartiallySucceeded')
+      name: 'TestAttempt2_Unit'
+      inputs:
+        command: test
+        projects: '${{ parameters.testProjects }}'
+        arguments: '${{ parameters.testArguments }}'
+        testRunTitle: '${{ parameters.testRunTitle }}'
+        workingDirectory: ${{ parameters.workingDirectory }}
+      env: ${{ parameters.environmentVariables }}
+      continueOnError: ${{ lt(2, parameters.maxAttempts) }}
+
+  - ${{ if gt(parameters.maxAttempts, 2) }}:
+    - task: PowerShell@2
+      displayName: 'Retry Logic for Unit Tests (Final)'
+      condition: in(variables['TestAttempt2_Unit.result'], 'Failed', 'PartiallySucceeded')
+      inputs:
+        targetType: 'inline'
+        script: |
+          Write-Host "##[warning]Second attempt failed or partially succeeded, waiting ${{ parameters.retryDelaySeconds }} seconds before final retry..."
+          Start-Sleep -Seconds ${{ parameters.retryDelaySeconds }}
+
+    - task: DotNetCoreCLI@2
+      displayName: '${{ parameters.displayName }} (Attempt 3)'
+      condition: in(variables['TestAttempt2_Unit.result'], 'Failed', 'PartiallySucceeded')
+      inputs:
+        command: test
+        projects: '${{ parameters.testProjects }}'
+        arguments: '${{ parameters.testArguments }}'
+        testRunTitle: '${{ parameters.testRunTitle }}'
+        workingDirectory: ${{ parameters.workingDirectory }}
+      env: ${{ parameters.environmentVariables }}
+
+- ${{ if eq(parameters.testProjects, '') }}:
+  # For integration/E2E tests using arguments parameter
+  - task: DotNetCoreCLI@2
+    displayName: '${{ parameters.displayName }} (Attempt 1)'
+    name: 'TestAttempt1'
+    inputs:
+      command: test
+      arguments: '${{ parameters.testArguments }}'
+      testRunTitle: '${{ parameters.testRunTitle }}'
+      workingDirectory: ${{ parameters.workingDirectory }}
+    env: ${{ parameters.environmentVariables }}
+    continueOnError: true
+
+  - ${{ if gt(parameters.maxAttempts, 1) }}:
+    - task: PowerShell@2
+      displayName: 'Retry Logic'
+      condition: in(variables['TestAttempt1.result'], 'Failed', 'PartiallySucceeded')
+      inputs:
+        targetType: 'inline'
+        script: |
+          Write-Host "##[warning]First attempt failed or partially succeeded, waiting ${{ parameters.retryDelaySeconds }} seconds before retry..."
+          Start-Sleep -Seconds ${{ parameters.retryDelaySeconds }}
+
+    - task: DotNetCoreCLI@2
+      displayName: '${{ parameters.displayName }} (Attempt 2)'
+      condition: in(variables['TestAttempt1.result'], 'Failed', 'PartiallySucceeded')
+      name: 'TestAttempt2'
+      inputs:
+        command: test
+        arguments: '${{ parameters.testArguments }}'
+        testRunTitle: '${{ parameters.testRunTitle }}'
+        workingDirectory: ${{ parameters.workingDirectory }}
+      env: ${{ parameters.environmentVariables }}
+      continueOnError: ${{ lt(2, parameters.maxAttempts) }}
+
+  - ${{ if gt(parameters.maxAttempts, 2) }}:
+    - task: PowerShell@2
+      displayName: 'Retry Logic (Final)'
+      condition: in(variables['TestAttempt2.result'], 'Failed', 'PartiallySucceeded')
+      inputs:
+        targetType: 'inline'
+        script: |
+          Write-Host "##[warning]Second attempt failed or partially succeeded, waiting ${{ parameters.retryDelaySeconds }} seconds before final retry..."
+          Start-Sleep -Seconds ${{ parameters.retryDelaySeconds }}
+
+    - task: DotNetCoreCLI@2
+      displayName: '${{ parameters.displayName }} (Attempt 3)'
+      condition: in(variables['TestAttempt2.result'], 'Failed', 'PartiallySucceeded')
+      inputs:
+        command: test
+        arguments: '${{ parameters.testArguments }}'
+        testRunTitle: '${{ parameters.testRunTitle }}'
+        workingDirectory: ${{ parameters.workingDirectory }}
+      env: ${{ parameters.environmentVariables }}

build/jobs/run-tests.yml

@@ -56,21 +56,21 @@ jobs:
         Write-Host "##vso[task.setvariable variable=DataStoreResourceId]$($dataStoreResourceId)"
 
 
-  - task: DotNetCoreCLI@2
-    displayName: 'Run Cosmos Integration Tests'
-    inputs:
-      command: test
-      arguments: '"$(Agent.TempDirectory)/IntegrationTests/**/*${{ parameters.version }}.Tests.Integration*.dll" --filter DisplayName!~SqlServer -v normal'
-      workingDirectory: "$(System.ArtifactsDirectory)"
+  - template: run-tests-with-retry.yml
+    parameters:
+      displayName: 'Run Cosmos Integration Tests with Retry'
+      testArguments: '"$(Agent.TempDirectory)/IntegrationTests/**/*${{ parameters.version }}.Tests.Integration*.dll" --filter "DisplayName!~SqlServer" -v normal'
       testRunTitle: '${{ parameters.version }} Integration'
-    env:
-      'CosmosDb__Host': $(CosmosDb--Host)
-      'FhirServer__ResourceManager__DataStoreResourceId': '$(DataStoreResourceId)'
-      'CosmosDb__UseManagedIdentity': true
-      'AZURESUBSCRIPTION_CLIENT_ID': '$(AZURESUBSCRIPTION_CLIENT_ID)'
-      'AZURESUBSCRIPTION_TENANT_ID': '$(AZURESUBSCRIPTION_TENANT_ID)'
-      'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': '$(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)'
-      'SYSTEM_ACCESSTOKEN': $(System.AccessToken)
+      maxAttempts: 3
+      retryDelaySeconds: 30
+      environmentVariables:
+        CosmosDb__Host: $(CosmosDb--Host)
+        FhirServer__ResourceManager__DataStoreResourceId: $(DataStoreResourceId)
+        CosmosDb__UseManagedIdentity: true
+        AZURESUBSCRIPTION_CLIENT_ID: $(AZURESUBSCRIPTION_CLIENT_ID)
+        AZURESUBSCRIPTION_TENANT_ID: $(AZURESUBSCRIPTION_TENANT_ID)
+        AZURESUBSCRIPTION_SERVICE_CONNECTION_ID: $(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
 
 - job: "SqlIntegrationTests"
   pool:
@@ -111,19 +111,19 @@ jobs:
         Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_TENANT_ID]$env:AZURESUBSCRIPTION_TENANT_ID"
         Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_SERVICE_CONNECTION_ID]$env:AZURESUBSCRIPTION_SERVICE_CONNECTION_ID"
 
-  - task: DotNetCoreCLI@2
-    displayName: 'Run Sql Integration Tests'
-    inputs:
-      command: test
-      arguments: '"$(Agent.TempDirectory)/IntegrationTests/**/*${{ parameters.version }}.Tests.Integration*.dll" --filter DisplayName!~Cosmos -v normal'
-      workingDirectory: "$(System.ArtifactsDirectory)"
+  - template: run-tests-with-retry.yml
+    parameters:
+      displayName: 'Run SQL Integration Tests with Retry'
+      testArguments: '"$(Agent.TempDirectory)/IntegrationTests/**/*${{ parameters.version }}.Tests.Integration*.dll" --filter "DisplayName!~Cosmos" -v normal'
       testRunTitle: '${{ parameters.version }} Integration'
-    env:
-      'SqlServer:ConnectionString': 'Server=tcp:${{ parameters.integrationSqlServerName }}.database.windows.net,1433;Initial Catalog=master;Persist Security Info=False;Authentication=ActiveDirectoryWorkloadIdentity;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;User Id=$(AZURESUBSCRIPTION_CLIENT_ID);'
-      'AZURESUBSCRIPTION_CLIENT_ID': '$(AZURESUBSCRIPTION_CLIENT_ID)'
-      'AZURESUBSCRIPTION_TENANT_ID': '$(AZURESUBSCRIPTION_TENANT_ID)'
-      'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': '$(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)'
-      'SYSTEM_ACCESSTOKEN': $(System.AccessToken)
+      maxAttempts: 3
+      retryDelaySeconds: 30
+      environmentVariables:
+        SqlServer:ConnectionString: 'Server=tcp:${{ parameters.integrationSqlServerName }}.database.windows.net,1433;Initial Catalog=master;Persist Security Info=False;Authentication=ActiveDirectoryWorkloadIdentity;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;User Id=$(AZURESUBSCRIPTION_CLIENT_ID);'
+        AZURESUBSCRIPTION_CLIENT_ID: $(AZURESUBSCRIPTION_CLIENT_ID)
+        AZURESUBSCRIPTION_TENANT_ID: $(AZURESUBSCRIPTION_TENANT_ID)
+        AZURESUBSCRIPTION_SERVICE_CONNECTION_ID: $(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
 
 - job: 'cosmosE2eTests'
   dependsOn: []