|
| 1 | +#!/usr/bin/env superdoit_stone |
| 2 | +options |
| 3 | +{ |
| 4 | + SuperDoitOptionalOptionWithNoArg long: 'debugSuite'. |
| 5 | + SuperDoitRequiredOptionWithRequiredArg long: 'resultDir'. |
| 6 | + SuperDoitOptionalOptionWithRequiredArg long: 'expectedResults'. |
| 7 | + SuperDoitOptionalOptionWithRequiredArg long: 'externalsStBase'. |
| 8 | + SuperDoitOptionalOptionWithRequiredArg long: 'rowanProjectsHome'. |
| 9 | + SuperDoitOptionalOptionWithRequiredArg long: 'rowanProjectsSandbox'. |
| 10 | +} |
| 11 | +% |
| 12 | +usage |
| 13 | +----- |
| 14 | +USAGE |
| 15 | + $basename [--help | -h] [--debug | -D] [OPTIONS] |
| 16 | + |
| 17 | +DESCRIPTION |
| 18 | + Run unit tests for RowanClientServices. Can be run interactively or from a |
| 19 | + battery test. |
| 20 | + |
| 21 | + The environment variables ROWAN_PROJECTS_HOME and ROWAN_PROJECTS_SANDBOX, |
| 22 | + should be defined. Required projects will be cloned by RowanClientServices |
| 23 | + using ROWAN_PROJECTS_HOME. The tests use ROWAN_PROJECTS_SANDBOX as the location |
| 24 | + where their projects are cloned. Note that an alternate ROWAN_PROJECTS_HOME |
| 25 | + can be specified by the --rowanProjectsHome option and an alternate |
| 26 | + ROWAN_PROJECTS_SANDBOX can be specified by the --rowanProjectsSandbox option. |
| 27 | + |
| 28 | +OPTIONS |
| 29 | + --debugSuite If present, test suite will be run in debug |
| 30 | + mode. If -- expectedResults option is used, |
| 31 | + then only those tests that are expected to |
| 32 | + fail will be debugged. |
| 33 | + --expectedResults=<path-to-expected-testResults-json> |
| 34 | + If specified, the testSuiteSample from the JSON |
| 35 | + file will be compared to the testSuiteSample |
| 36 | + produced by the test. The tests results are |
| 37 | + expected to match the expected results, if they |
| 38 | + do not, a difference report is produced. |
| 39 | + --externalsStBase Location where Rowan, RowanClientServices, |
| 40 | + GsTestStats and other projects that are used to |
| 41 | + build the product are found. The environment variable |
| 42 | + EXTERNALS_ST_BASE is used to isolate the projects |
| 43 | + used to build the product (Rowan, etc.) from |
| 44 | + ROWAN_PROJECTS_HOME, which is used by the test |
| 45 | + framework as the location where projects downloaded |
| 46 | + by the tests are located. If EXTERNALS_ST_BASE is |
| 47 | + not defined, then it defaults to ROWAN_PROJECTS_HOME. |
| 48 | + --resultDir=<path-results-directory> |
| 49 | + Location of the battery test result directory |
| 50 | + where the various test directories and results |
| 51 | + files are to be found. REQUIRED. |
| 52 | + --rowanProjectsHome=<directory-path> |
| 53 | + Location where the projects cloned by the tests will |
| 54 | + be located. Use EXTERNALS_ST_BASE or --externalsStBase |
| 55 | + to isolate the projects included in extent0.rowan.dbf |
| 56 | + from the test projects. ROWAN_PROJECTS_HOME MUST BE |
| 57 | + DEFINED. |
| 58 | + --rowanProjectsSandbox=<directory-path> |
| 59 | + Location used by tests when cloning git repositores. |
| 60 | + By default, $ROWAN_PROJECTS_SANDBOX. MUST BE DEFINED. |
| 61 | + -h, --help display usage message |
| 62 | + -D, --debug bring up topaz debugger in the event of a script error |
| 63 | + |
| 64 | +EXAMPLES |
| 65 | + $basename --help -- -L -I .topazini |
| 66 | + $basename --resultDir=TEST_RESULTS -- -L -I .topazini |
| 67 | + $basename --resultDir=TEST_RESULTS --debugSuite -- -L -I .topazini |
| 68 | + $basename --resultDir=TEST_RESULTS --expectedResults=expectedTestResults.json -- -L -I .topazini |
| 69 | + $basename --resultDir=TEST_RESULTS --expectedResults=TEST_RESULTS/expectedTestResults.json -- -L -I .topazini |
| 70 | +----- |
| 71 | +% |
| 72 | +instvars |
| 73 | +testStatusStream |
| 74 | +% |
| 75 | +specs |
| 76 | +[ |
| 77 | +RwLoadSpecificationV2 { |
| 78 | + #specName : 'GsTestStats', |
| 79 | + #projectName : 'GsTestStats', |
| 80 | + #gitUrl : ' [email protected]:GsTestStats', |
| 81 | + #revision : 'v1', |
| 82 | + #projectSpecFile : 'rowan/project.ston', |
| 83 | + #componentNames : [ |
| 84 | + 'Samples' |
| 85 | + ], |
| 86 | + #customConditionalAttributes : [ |
| 87 | + 'tests' ], |
| 88 | + #comment : 'test results tracking' |
| 89 | +} |
| 90 | +] |
| 91 | +% |
| 92 | +method |
| 93 | +writeAppendStreamFor: aFileReference |
| 94 | + "Answer a binary write stream on aFileReference ... only needed until writeAppendStream method added to FileReference (https://github.com/GemTalk/FileSystemGs/issues/70)" |
| 95 | + |
| 96 | + | fileDescriptor optionsForAppend | |
| 97 | + (aFileReference exists and: [aFileReference isFile not]) |
| 98 | + ifTrue: [^FileRequired signalWith: aFileReference]. |
| 99 | + optionsForAppend := aFileReference fileOpeningOptionsClass writeOnly append. |
| 100 | + aFileReference exists ifFalse: [ aFileReference createFile ]. |
| 101 | + fileDescriptor := aFileReference openOptions: optionsForAppend. |
| 102 | + ^ ZnCharacterWriteStream |
| 103 | + on: (ZnBufferedWriteStream on: fileDescriptor binaryWriteStream) |
| 104 | + encoding: 'utf8' |
| 105 | +% |
| 106 | +method |
| 107 | +removeSuperDoitTransientSymbolDictionary |
| 108 | + "SuperDoit_transientSymbolDictionary interferes with some tests, if present" |
| 109 | + |
| 110 | + (Rowan globalNamed: 'SuperDoitCommandParser') |
| 111 | + ifNotNil: [ :superDoitCommandParserClass | |
| 112 | + | session symbolList | |
| 113 | + session := GsCurrentSession currentSession. |
| 114 | + symbolList := session symbolList. |
| 115 | + symbolList |
| 116 | + remove: (symbolList objectNamed: superDoitCommandParserClass transientSymbolDictionaryName) |
| 117 | + ifAbsent: [] ]. |
| 118 | +% |
| 119 | +method |
| 120 | +testStatusStream |
| 121 | + testStatusStream ifNotNil: [ ^ testStatusStream]. |
| 122 | + self resultDir asFileReference ensureCreateDirectory. |
| 123 | + (self resultDir, '/teststatus.txt') asFileReference ensureCreateFile. |
| 124 | + testStatusStream := self writeAppendStreamFor: (self resultDir, '/teststatus.txt') asFileReference. |
| 125 | + ^ testStatusStream |
| 126 | +% |
| 127 | +method |
| 128 | +ansiGreen |
| 129 | +^ (self globalNamed: 'GsTestStatsCI') ansiGreen |
| 130 | +% |
| 131 | +method |
| 132 | +ansiRed |
| 133 | +^ (self globalNamed: 'GsTestStatsCI') ansiRed |
| 134 | +% |
| 135 | +method |
| 136 | +printPassingBanner: message reporter: reporter |
| 137 | + reporter printBanner: message color: self ansiGreen |
| 138 | +% |
| 139 | +method |
| 140 | +printFailingBanner: message reporter: reporter |
| 141 | + reporter printBanner: message color: self ansiRed |
| 142 | +% |
| 143 | +method |
| 144 | +writeTestDone: detailedTestStatus |
| 145 | + "write $resultsDir/testdone.txt" |
| 146 | + self writeTestDone: detailedTestStatus explanation: '' |
| 147 | +% |
| 148 | +method |
| 149 | +writeTestDone: detailedTestStatus explanation: explanation |
| 150 | + "write $resultsDir/testdone.txt" |
| 151 | + | stream | |
| 152 | + stream := self writeAppendStreamFor: (self resultDir, '/testdone.txt') asFileReference. |
| 153 | + stream |
| 154 | + lf; nextPutAll: 'TEST STATUS:: ', detailedTestStatus; lf. |
| 155 | + explanation isEmpty |
| 156 | + ifFalse: [ |
| 157 | + stream lf; nextPutAll: explanation; lf ]. |
| 158 | + stream close |
| 159 | +% |
| 160 | +method |
| 161 | +writeTestStatus: message |
| 162 | + "does not return" |
| 163 | + self _writeTestStatus: 'TEST STATUS:: ', message. |
| 164 | +% |
| 165 | +method |
| 166 | +writePassingTestStatus: runner |
| 167 | + self _writeTestStatus: 'TEST STATUS:: passes with no errors'. |
| 168 | + self |
| 169 | + writeTestDone: 'passes with ', runner summary. |
| 170 | + ^ self noResult |
| 171 | +% |
| 172 | +method |
| 173 | +_writeTestStatus: testStatus |
| 174 | + self testStatusStream |
| 175 | + nextPutAll: testStatus; lf; |
| 176 | + nextPutAll: 'SECTION 2: UNREPORTED BUGS'; lf; "without this entry, batterytest.pl thinks it's a crash (instead of a failure)" |
| 177 | + close. |
| 178 | + testStatusStream := nil. |
| 179 | +% |
| 180 | +method |
| 181 | +loadProjectSpecs |
| 182 | + "load specurls ... fail the run if there are CompileWarnings" |
| 183 | + | warnings rowanProjectsHome rowanProjectsSandbox externalsStBase | |
| 184 | + self rowanProjectsHome |
| 185 | + ifNotNil: [:value | |
| 186 | + rowanProjectsHome := value. |
| 187 | + System gemEnvironmentVariable: 'ROWAN_PROJECTS_HOME' put: value ] |
| 188 | + ifNil: [ |
| 189 | + (rowanProjectsHome := System gemEnvironmentVariable: 'ROWAN_PROJECTS_HOME') |
| 190 | + ifNil: [ self error: 'ROWAN_PROJECTS_HOME env var or --rowanProjectsHome option must be defined' ] ]. |
| 191 | + self stdout nextPutAll: '-- ROWAN_PROJECTS_HOME = ', (System gemEnvironmentVariable: 'ROWAN_PROJECTS_HOME') asFileReference pathString; lf. |
| 192 | + self externalsStBase |
| 193 | + ifNotNil: [:value | |
| 194 | + externalsStBase := value. |
| 195 | + System gemEnvironmentVariable: 'EXTERNALS_ST_BASE' put: value ] |
| 196 | + ifNil: [ |
| 197 | + (externalsStBase := System gemEnvironmentVariable: 'EXTERNALS_ST_BASE') |
| 198 | + ifNil: [ |
| 199 | + externalsStBase := rowanProjectsHome. |
| 200 | + System gemEnvironmentVariable: 'EXTERNALS_ST_BASE' put: rowanProjectsHome ] ]. |
| 201 | + externalsStBase asFileReference exists |
| 202 | + ifFalse: [ self error: 'Directory referred to by EXTERNALS_ST_BASE (', externalsStBase asFileReference pathString, ') does not exist' ]. |
| 203 | + self stdout nextPutAll: '-- EXTERNALS_ST_BASE = ', (System gemEnvironmentVariable: 'EXTERNALS_ST_BASE') asFileReference pathString; lf. |
| 204 | + self rowanProjectsSandbox |
| 205 | + ifNotNil: [:value | |
| 206 | + rowanProjectsSandbox := value. |
| 207 | + System gemEnvironmentVariable: 'ROWAN_PROJECTS_SANDBOX' put: value ] |
| 208 | + ifNil: [ |
| 209 | + (rowanProjectsSandbox := System gemEnvironmentVariable: 'ROWAN_PROJECTS_SANDBOX') |
| 210 | + ifNil: [ self error: 'ROWAN_PROJECTS_SANDBOX env var or --rowanProjectsSandbox option must be used' ] ]. |
| 211 | + self stdout nextPutAll: '-- ROWAN_PROJECTS_SANDBOX = ', (System gemEnvironmentVariable: 'ROWAN_PROJECTS_SANDBOX') asFileReference pathString; lf. |
| 212 | + rowanProjectsHome asFileReference ensureCreateDirectory. |
| 213 | + rowanProjectsSandbox asFileReference ensureCreateDirectory. |
| 214 | + warnings := {}. |
| 215 | + [ |
| 216 | + | loadSpecs remoteServiceReplicationProject | |
| 217 | + self preDoitSpecLoad: [:loadSpec | |
| 218 | + loadSpec projectsHome: externalsStBase ]. |
| 219 | + remoteServiceReplicationProject := Rowan projectNamed: 'RemoteServiceReplication'. |
| 220 | + loadSpecs := remoteServiceReplicationProject loadedLoadSpecifications. |
| 221 | + loadSpecs do: [:loadSpec | |
| 222 | + loadSpec specName = 'RemoteServiceReplication' |
| 223 | + ifTrue: [ |
| 224 | + loadSpec |
| 225 | + gitUrl: 'file:', externalsStBase, '/', loadSpec projectAlias ]. |
| 226 | + loadSpec |
| 227 | + projectsHome: '$ROWAN_PROJECTS_HOME'; |
| 228 | + addCustomConditionalAttributes: #('tests'); |
| 229 | + yourself ]. |
| 230 | + loadSpecs load. |
| 231 | + System commit. |
| 232 | + ] |
| 233 | + on: CompileWarning |
| 234 | + do: [:ex | |
| 235 | + (ex description includesString: 'not optimized') |
| 236 | + ifFalse: [ warnings add: ex asString printString ]. |
| 237 | + ex resume ]. |
| 238 | + self isSolo ifFalse: [ System commit "as loaded" ]. |
| 239 | + warnings isEmpty ifFalse: [ |
| 240 | + | warningStatusStream message | |
| 241 | + warningStatusStream := WriteStream on: String new. |
| 242 | + self testStatusStream nextPutAll: 'COMPILE WARNINGS:'; lf. |
| 243 | + warnings do: [:warning | self testStatusStream nextPutAll: ' ', warning ]. |
| 244 | + self testStatusStream nextPutAll: 'Compile warnings results in test failure'; lf. |
| 245 | + message := ' fails with compile warnings: ', warnings size printString. |
| 246 | + self writeTestStatus: message. |
| 247 | + self |
| 248 | + writeTestDone: message |
| 249 | + explanation: 'See teststatus.txt for more details'. |
| 250 | + self exit: message withStatus: 1 ]. |
| 251 | +% |
| 252 | +method |
| 253 | +buildTestSuite: testSuiteName |
| 254 | + | systemTestSuite | |
| 255 | + systemTestSuite := TestSuite named: testSuiteName. |
| 256 | + { 'RemoteServiceReplication' } |
| 257 | + do: [:projectName | |
| 258 | + | testSuite | |
| 259 | + testSuite := (Rowan projectNamed: projectName) testSuite. |
| 260 | + systemTestSuite addTests: testSuite tests ]. |
| 261 | + ^ systemTestSuite |
| 262 | +% |
| 263 | +method |
| 264 | +runTestSuite: testSuiteName |
| 265 | + | systemTestSuite | |
| 266 | + self removeSuperDoitTransientSymbolDictionary. |
| 267 | + systemTestSuite := self buildTestSuite: testSuiteName. |
| 268 | + self debugSuite |
| 269 | + ifTrue: [ |
| 270 | + | debugOption | |
| 271 | + "interactive debugging of failures" |
| 272 | + debugOption := self optionsDict at: 'debug'. |
| 273 | + debugOption value |
| 274 | + ifFalse: [ |
| 275 | + "set the --debug option" |
| 276 | + debugOption value: true ]. |
| 277 | + self expectedResults |
| 278 | + ifNil: [ |
| 279 | + "debug test failures" |
| 280 | + ^ (self globalNamed: 'GsTestStatsCITestRunner') debugSuite: systemTestSuite ] |
| 281 | + ifNotNil: [ :expectedResultsPath | |
| 282 | + "debug unexpected test failures" |
| 283 | + | expectedFailures | |
| 284 | + expectedFailures := (self globalNamed: 'GsTestSuiteSample') fromJson: expectedResultsPath. |
| 285 | + ^(self globalNamed: 'GsTestStatsCITestRunner') |
| 286 | + debugUnexpectedFailures: systemTestSuite |
| 287 | + expectedFailures: expectedFailures ] ] |
| 288 | + ifFalse: [ |
| 289 | + "run tests and produce report of test failures" |
| 290 | + ^ (self globalNamed: 'GsTestStatsCITestRunner') runSuite: systemTestSuite ]. |
| 291 | +% |
| 292 | +method |
| 293 | +writeFailingTestResultsSummaryReport: runner reporter: reporter |
| 294 | + | failingStatus message | |
| 295 | + failingStatus := 'test failures'. |
| 296 | + self printFailingBanner: failingStatus reporter: reporter. |
| 297 | + (self globalNamed: 'GsTestStatsCITestReporterStdout') |
| 298 | + reportNotPassing: runner |
| 299 | + on: self testStatusStream. |
| 300 | + self testStatusStream lf; nextPutAll: failingStatus; lf; nextPutAll: runner summary; lf. |
| 301 | + message := 'fails with ', runner summary. |
| 302 | + self writeTestStatus: message. |
| 303 | + self |
| 304 | + writeTestDone: message |
| 305 | + explanation: 'See teststatus.txt for more details'. |
| 306 | + self exit: message withStatus: 1 |
| 307 | +% |
| 308 | +method |
| 309 | +writeFailingExpectedResultsSummaryReport: runner reporter: reporter expectedSample: expectedSample testSuiteSample: testSuiteSample |
| 310 | + | failingStatus message | |
| 311 | + failingStatus := 'fails by not matching expected results'. |
| 312 | + self printFailingBanner: failingStatus reporter: reporter. |
| 313 | + (self globalNamed: 'GsTestStatsCITestReporterStdout') |
| 314 | + reportNotPassing: runner |
| 315 | + on: self testStatusStream. |
| 316 | + self testStatusStream nextPutAll: '----------------------------------------------------'; lf. |
| 317 | + self testStatusStream nextPutAll: failingStatus, ' (test results diff report follows)'; lf. |
| 318 | + self testStatusStream nextPutAll: '----------------------------------------------------'; lf. |
| 319 | + { self stdout . self testStatusStream} |
| 320 | + do: [:outputStream | |
| 321 | + testSuiteSample compareTo: expectedSample on: outputStream. |
| 322 | + outputStream nextPutAll: '----------------------------------------------------'; lf; lf ]. |
| 323 | + message := 'fails with differences in expected results'. |
| 324 | + self writeTestStatus: message. |
| 325 | + self |
| 326 | + writeTestDone: message |
| 327 | + explanation: 'See teststatus.txt for more details'. |
| 328 | + self exit: message withStatus: 1 |
| 329 | +% |
| 330 | +doit |
| 331 | + | runner testSuiteSample passingStatus stdoutReporter | |
| 332 | + self loadProjectSpecs. |
| 333 | + "run tests" |
| 334 | + runner := self runTestSuite: 'RemoteServiceReplication projects Test Suite'. |
| 335 | + "report results to stdout" |
| 336 | + stdoutReporter := (self globalNamed: 'GsTestStatsCITestReporterStdout') reportNotPassing: runner. |
| 337 | + "create testSuiteSample with results of test" |
| 338 | + testSuiteSample := ((self globalNamed: 'GsTestStatsCITestReporterTestSuiteSample') report: runner) testSuiteSample. |
| 339 | + "write testSuiteSample to results directory in JSON" |
| 340 | + (self resultDir asFileReference / 'testResults.json') writeStreamDo: [:fileStream | |
| 341 | + testSuiteSample exportJsonTo: fileStream ]. |
| 342 | + "different reporting requirements for failures when expected results are available" |
| 343 | + self expectedResults |
| 344 | + ifNil: [ |
| 345 | + "no expected test results, pass/failure depends on successful test run" |
| 346 | + passingStatus := 'passes with no error'. |
| 347 | + runner isSuccessful |
| 348 | + ifFalse: [ |
| 349 | + "test failures, does not return" |
| 350 | + self writeFailingTestResultsSummaryReport: runner reporter: stdoutReporter ] ] |
| 351 | + ifNotNil: [:expectedResultsPath | |
| 352 | + "with expected test results, pass/failure depends up matching expected results" |
| 353 | + | expectedSample dummyStream | |
| 354 | + expectedSample := (self globalNamed: 'GsTestSuiteSample') fromJson: expectedResultsPath. |
| 355 | + passingStatus := 'passes matching expected results'. |
| 356 | + dummyStream := WriteStream on: String new. |
| 357 | + (testSuiteSample compareTo: expectedSample on: dummyStream) |
| 358 | + ifFalse: [ |
| 359 | + "differences between test results and expected results, does not return" |
| 360 | + self writeFailingExpectedResultsSummaryReport: runner |
| 361 | + reporter: stdoutReporter |
| 362 | + expectedSample: expectedSample |
| 363 | + testSuiteSample: testSuiteSample ] ]. |
| 364 | + "PASSING tests (not test failures, or test results match expected results)" |
| 365 | + self printPassingBanner: passingStatus reporter: stdoutReporter. |
| 366 | + ^ self writePassingTestStatus: runner |
| 367 | +% |
| 368 | + |
0 commit comments