Add a GC Reliability Framework job to the CI (dotnet#11280)

swgillespie · web-flow · commit 30a09322644a · 2017-05-05T15:57:25.000-07:00
* Add GC Reliability Framework CI job

* Three changes:
  1) To address offline feedback, update the CI trigger phase document
  and the GC testing instructions
  2) Set the timeout for GC RF CI jobs to 24 hours, so they don't time
  out (they will run for 15 hours by default)
  3) Refactor "scenario == 'gc_reliability_framework' to its own method
  so that new GC RF scenarios can easily and cleanly be added if
  desired in the future
diff --git a/Documentation/project-docs/ci-trigger-phrases.md b/Documentation/project-docs/ci-trigger-phrases.md
@@ -11,7 +11,10 @@ To trigger a job, post a comment on your PR with "@dotnet-bot {trigger-phrase}".
 
 - **Windows_NT x64 Release Priority 1 Build & Test:** "test Windows_NT pri1"
 - **Windows_NT x64 Release IL RoundTrip Build & Test:** "test Windows_NT ilrt"
-- **Windows_NT x64 Release Long-Running GC Build & Test:**: "test Windows_NT x64 Release longgc"
+- **Windows_NT x64 Release Long-Running GC Build & Test:**: "test Windows_NT Release longgc"
+- **Windows_NT x64 Release GC Simulator:**: "test Windows_NT Release gcsimulator"
+- **Windows_NT x64 Release Standalone GC:**: "test Windows_NT Release standalone_gc"
+- **Windows_NT x64 Release GC Reliability Framework:**: "test Windows_NT Release gc_reliability_framework"
 - **Windows_NT x64 Release Ready-To-Run Priority 0 Build & Test:** "test Windows_NT Release r2r"
 - **Windows_NT x64 Checked Ready-To-Run Priority 0 Build & Test:** "test Windows_NT Checked r2r"
 - **Windows_NT x64 Release Ready-To-Run Priority 1 Build & Test:** "test Windows_NT Release pri1r2r"
@@ -69,6 +72,8 @@ To trigger a job, post a comment on your PR with "@dotnet-bot {trigger-phrase}".
 - **Windows_NT x64 Checked GCStress=0xc JitStress=2 Build & Test:** "test Windows_NT gcstress0xc_jitstress2"
 - **Windows_NT x64 Checked GCStress=0xc MinOpts Heap Verify 1 Build & Test:** "test Windows_NT gcstress0xc_minopts_heapverify1"
 - **Windows_NT x64 Checked Long-Running GC Build & Test:**: "test Windows_NT x64 Checked longgc"
+- **Windows_NT x64 Checked Standalone GC:**: "test Windows_NT Checked standalone_gc"
+- **Windows_NT x64 Checked GC Reliability Framework:**: "test Windows_NT Checked gc_reliability_framework"
 - **Windows_NT x64 Formatting:**: "test Windows_NT formatting"
 - **Windows_NT x64 Checked CoreFX Baseline Build & Test:** "test Windows_NT corefx_baseline"
 - **Windows_NT x64 Checked CoreFX MinOpts Build & Test:** "test Windows_NT corefx_minopts"
@@ -166,6 +171,11 @@ To trigger a job, post a comment on your PR with "@dotnet-bot {trigger-phrase}".
 - **Ubuntu x64 Release Priority 1 Build & Test:** "test Ubuntu pri1"
 - **Ubuntu x64 Release IL RoundTrip Build & Test:** "test Ubuntu ilrt"
 - **Ubuntu x64 Release Long-Running GC Build & Test:**: "test Ubuntu Release longgc"
+- **Ubuntu x64 Release GC Simulator:**: "test Ubuntu Release gcsimulator"
+- **Ubuntu x64 Release Standalone GC:**: "test Ubuntu Release standalone_gc"
+- **Ubuntu x64 Checked Standalone GC:**: "test Ubuntu Checked standalone_gc"
+- **Ubuntu x64 Release GC Reliability Framework:**: "test Ubuntu Release gc_reliability_framework"
+- **Ubuntu x64 Checked GC Reliability Framework:**: "test Ubuntu Checked gc_reliability_framework"
 - **Ubuntu x64 Release Ready-To-Run Priority 0 Build & Test:** "test Ubuntu Release r2r"
 - **Ubuntu x64 Checked Ready-To-Run Priority 0 Build & Test:** "test Ubuntu Checked r2r"
 - **Ubuntu x64 Release Ready-To-Run Priority 1 Build & Test:** "test Ubuntu Release pri1r2r"
@@ -248,6 +258,11 @@ To trigger a job, post a comment on your PR with "@dotnet-bot {trigger-phrase}".
 - **OSX x64 Release Priority 1 Build & Test:** "test OSX pri1"
 - **OSX x64 Release IL RoundTrip Build & Test:** "test OSX ilrt"
 - **OSX x64 Release Long-Running GC Build & Test:**: "test OSX Release longgc"
+- **OSX x64 Release GC Simulator:**: "test OSX10.12 Release gcsimulator"
+- **OSX x64 Release Standalone GC:**: "test OSX10.12 Release standalone_gc"
+- **OSX x64 Checked Standalone GC:**: "test OSX10.12 Checked standalone_gc"
+- **OSX x64 Release GC Reliability Framework:**: "test OSX10.12 Release gc_reliability_framework"
+- **OSX x64 Checked GC Reliability Framework:**: "test OSX10.12 Checked gc_reliability_framework"
 - **OSX x64 Release Ready-To-Run Priority 0 Build & Test:** "test OSX Release r2r"
 - **OSX x64 Checked Ready-To-Run Priority 0 Build & Test:** "test OSX Checked r2r"
 - **OSX x64 Release Ready-To-Run Priority 1 Build & Test:** "test OSX Release pri1r2r"
diff --git a/Documentation/project-docs/garbage-collector-guidelines.md b/Documentation/project-docs/garbage-collector-guidelines.md
@@ -26,12 +26,43 @@ Required Testing: Validation of the behavior of the affected APIs.
 ## Stress Testing ##
 Stress testing must run for at least **48 hours** against a debug build.
 
-Instructions for running stress are located in the repo at tests\src\GC\Stress\stress_run_readme.txt.
+Stress testing for checked and release builds can be done on pull requests with The .NET CI infrastructure.
+A stress run can be requested using the trigger phrase:
+
+```
+@dotnet_bot test <platform> <flavor> gc_reliability_framework
+```
+
+This will run the stress framework for the default amount of time (15 hours) on the given platform and build flavor.
 
 ## Functional Testing ##
 A functional test run executes the same code as a stress run, but only runs for 30 minutes.
 
 Instructions for running stress are located in the repo at tests\src\GC\Stress\stress_run_readme.txt.
 
+It is recommended that you run at least some of the below PR-triggered CI jobs:
+
+```
+@dotnet_bot test Windows_NT Checked longgc
+@dotnet_bot test OSX10.12 Checked longgc
+@dotnet_bot test Ubuntu Checked longgc
+@dotnet_bot test Windows_NT Checked standalone_gc
+@dotnet_bot test OSX10.12 Checked standalone_gc
+@dotnet_bot test Ubuntu Checked standalone_gc
+```
+
+The "Long GC" tests are a series of GC tests whose running time is too long or memory usage is too high to run with
+the rest of the Priority 0 unit tests. The "Standalone GC" build mode builds and runs the GC in a semi-standalone manner
+(see https://github.com/dotnet/coreclr/projects/3).
+
+You may also wish to run the GC Simulator tests. They may take up to 24 hours to complete and are known to sometimes fail on Ubuntu
+due to poor interactions with the Linux OOM killer. However, they have proven to be quite useful in finding bugs in the past:
+
+```
+@dotnet_bot test Windows_NT Release gcsimulator
+@dotnet_bot test Ubuntu Release gcsimulator
+@dotnet_bot test OSX10.12 Release gcsimulator
+```
+
 ## Performance Testing ##
 Coming soon.
diff --git a/netci.groovy b/netci.groovy
@@ -137,6 +137,7 @@ class Constants {
                'gcsimulator',
                'jitdiff',              
                'standalone_gc',
+               'gc_reliability_framework',
                'illink'] + r2rJitStressScenarios
 
     def static configurationList = ['Debug', 'Checked', 'Release']
@@ -205,6 +206,14 @@ def static isJitDiff(def scenario) {
     return (scenario == 'jitdiff')
 }
 
+def static isGcReliabilityFramework(def scenario) {
+    return (scenario == 'gc_reliability_framework')
+}
+
+def static scenarioNeedsPri1Build(def scenario) {
+    return (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage' || isGcReliabilityFramework(scenario))
+}
+
 def static setTestJobTimeOut(newJob, scenario) {
     if (isGCStressRelatedTesting(scenario)) {
         Utilities.setJobTimeout(newJob, 4320)
@@ -227,6 +236,9 @@ def static setTestJobTimeOut(newJob, scenario) {
     else if (isJitDiff(scenario)) {
         Utilities.setJobTimeout(newJob, 240)
     }
+    else if (isGcReliabilityFramework(scenario)) {
+        Utilities.setJobTimeout(newJob, 1440)
+    }
     // Non-test jobs use the default timeout value.
 }
 
@@ -497,6 +509,11 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def
             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
             Utilities.addPeriodicTrigger(job, '@weekly')
             break
+        case 'gc_reliability_framework':
+            assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
+            assert (configuration == 'Release' || configuration == 'Checked')
+            // Only triggered by phrase.
+            break
         case 'ilrt':
             assert !(os in bidailyCrossList)
             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
@@ -766,6 +783,16 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os,
                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
                             }
                             break
+                        case 'standalone_gc':
+                            if (configuration == 'Release' || configuration == 'Checked') {
+                                Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
+                            }
+                            break
+                        case 'gc_reliability_framework':
+                            if (configuration == 'Release' || configuration == 'Checked') {
+                                Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
+                            }
+                            break
                         case 'minopts':
                         case 'forcerelocs':
                         case 'jitstress1':
@@ -939,6 +966,11 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os,
                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
                             }
                             break
+                        case 'gc_reliability_framework':
+                            if (configuration == 'Release' || configuration == 'Checked') {
+                                Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
+                            }
+                            break
                         case 'minopts':
                         case 'forcerelocs':
                         case 'jitstress1':
@@ -1307,7 +1339,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                     // binaries are sent to a default directory whose name is about
                     // 35 characters long.
 
-                    else if (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage') {
+                    else if (scenarioNeedsPri1Build(scenario)) {
                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
                     }
                     else if (isLongGc(scenario)) {
@@ -1435,6 +1467,12 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
                             }
                         }
+                        else if (isGcReliabilityFramework(scenario)) {
+                            buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
+                            buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
+                            Utilities.addArchival(newJob, "stdout.txt")
+                            Utilities.addArchival(newJob, "Logs/**")
+                        }
                         else if (architecture == 'x64' || architecture == 'x86') {
                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
                         }
@@ -1870,6 +1908,7 @@ combinedScenarios.each { scenario ->
                                     return
                                 }
                                 break
+                            case 'gc_reliability_framework':
                             case 'standalone_gc':
                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
                                     return
@@ -2120,10 +2159,12 @@ combinedScenarios.each { scenario ->
                                     return
                                 }
                                 break
+                            case 'gc_reliability_framework':
                             case 'standalone_gc':
                                 if (configuration != 'Release' && configuration != 'Checked') {
                                     return
                                 }
+                                break
                             case 'coverage':
                                 //We only want Ubuntu Release for coverage
                                 if (os != 'Ubuntu') {
@@ -2164,7 +2205,7 @@ combinedScenarios.each { scenario ->
                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
                     // from the default scenario
                     def testBuildScenario = scenario
-                    if (testBuildScenario == 'coverage' || testBuildScenario == 'pri1r2r'|| testBuildScenario == 'gcstress15_pri1r2r') {
+                    if (scenarioNeedsPri1Build(scenario)) {
                         testBuildScenario = 'pri1'
                     }
                     else if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
@@ -2210,6 +2251,7 @@ combinedScenarios.each { scenario ->
                     def runilasmroundtripStr = ''
                     def gcstressStr = ''
                     def illinkStr = ''
+                    def layoutOnlyStr =''
 
                     if (scenario == 'r2r' ||
                         scenario == 'pri1r2r' ||
@@ -2286,6 +2328,10 @@ combinedScenarios.each { scenario ->
                         }
                     }
 
+                    if (isGcReliabilityFramework(scenario)) {
+                        layoutOnlyStr = '--build-overlay-only'
+                    }
+
                     def folder = getJobFolder(scenario)
                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
                         // Add parameters for the inputs
@@ -2438,11 +2484,29 @@ combinedScenarios.each { scenario ->
                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
                 --limitedDumpGeneration \\
-                ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${illinkStr} ${sequentialString} ${playlistString}""")
+                ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} \\
+                ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} \\
+                ${illinkStr} ${sequentialString} ${playlistString} ${layoutOnlyStr}""")
+
+                                if (isGcReliabilityFramework(scenario)) {
+                                    // runtest.sh doesn't actually execute the reliability framework - do it here.
+                                    if (serverGCString != '') {
+                                        shell("export COMPlus_gcServer=1")
+                                    }
+
+                                    shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
+                                }
                             }
                         }
                     }
 
+                    if (isGcReliabilityFramework(scenario))
+                    {
+                        // Both of these are emitted by the RF
+                        Utilities.addArchival(newJob, "stdout.txt")
+                        Utilities.addArchival(newJob, "Logs/**")
+                    }
+
                     if (scenario == 'coverage') {
                         // Publish coverage reports
                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
diff --git a/tests/scripts/run-gc-reliability-framework.cmd b/tests/scripts/run-gc-reliability-framework.cmd
@@ -0,0 +1,10 @@
+@rem Licensed to the .NET Foundation under one or more agreements.
+@rem The .NET Foundation licenses this file to you under the MIT license.
+@rem See the LICENSE file in the project root for more information.
+
+@echo off
+
+set CORE_ROOT=%CD%\bin\tests\Windows_NT.%1.%2\Tests\Core_Root
+set FRAMEWORK_DIR=%CD%\bin\tests\Windows_NT.%1.%2\GC\Stress\Framework\ReliabilityFramework
+powershell "%CORE_ROOT%\CoreRun.exe %FRAMEWORK_DIR%\ReliabilityFramework.exe %FRAMEWORK_DIR%\testmix_gc.config | tee stdout.txt"
+
diff --git a/tests/scripts/run-gc-reliability-framework.sh b/tests/scripts/run-gc-reliability-framework.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export CORE_ROOT=`pwd`/bin/tests/Windows_NT.$1.$2/Tests/coreoverlay
+FRAMEWORK_DIR=`pwd`/bin/tests/Windows_NT.$1.$2/GC/Stress/Framework/ReliabilityFramework
+$CORE_ROOT/corerun $FRAMEWORK_DIR/ReliabilityFramework.exe $FRAMEWORK_DIR/testmix_gc.config | tee stdout.txt
+
diff --git a/tests/src/GC/Stress/Framework/ReliabilityFramework.csproj b/tests/src/GC/Stress/Framework/ReliabilityFramework.csproj
@@ -10,9 +10,9 @@
     <OutputType>Exe</OutputType>
     <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
-    <CLRTestKind>BuildAndRun</CLRTestKind>
+    <CLRTestKind>BuildOnly</CLRTestKind>
+    <GenerateRunScript>false</GenerateRunScript>
     <CLRTestPriority>1</CLRTestPriority>
-    <CLRTestExecutionArguments>testmix_gc.config /maximumExecutionTime:-1</CLRTestExecutionArguments>
     <DefineConstants>$(DefineConstants);STATIC;PROJECTK_BUILD</DefineConstants>
   </PropertyGroup>
   <!-- Default configurations to help VS understand the configurations -->
