diff --git a/PowerShell/ScubaGear/Modules/CreateReport/CreateReport.psm1 b/PowerShell/ScubaGear/Modules/CreateReport/CreateReport.psm1 index f572f22ab9..218dc4bded 100644 --- a/PowerShell/ScubaGear/Modules/CreateReport/CreateReport.psm1 +++ b/PowerShell/ScubaGear/Modules/CreateReport/CreateReport.psm1 @@ -1,5 +1,70 @@ Import-Module -Name (Join-Path -Path $PSScriptRoot -ChildPath "..\Utility") +function Get-TestResult { + <# + .Description + Given the Rego output for a specific test, determine the result (e.g. "Pass"/"Fail"). + .Functionality + Internal + #> + [CmdletBinding()] + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [object] + $Test, + + [Parameter(Mandatory=$true)] + [AllowNull()] + [array] + $MissingCommands, + + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [object] + $Control + ) + + $Result = @{} + if ($Control.MalformedDescription) { + $Result.DisplayString = "Error" + $Result.SummaryKey = "Errors" + $Result.Details = "Report issue on GitHub" + } + elseif ($Control.Deleted) { + $Result.DisplayString = "-" + $Result.SummaryKey = "-" + $Result.Details = "-" + } + elseif ($MissingCommands.Count -gt 0) { + $Result.DisplayString = "Error" + $Result.SummaryKey = "Errors" + $MissingString = $MissingCommands -Join ", " + $Result.Details = "This test depends on the following command(s) which did not execute successfully: $($MissingString). See terminal output for more details." + } + elseif ($Test.RequirementMet) { + $Result.DisplayString = "Pass" + $Result.SummaryKey = "Passes" + $Result.Details = $Test.ReportDetails + } + elseif ($Test.Criticality -eq "Should") { + $Result.DisplayString = "Warning" + $Result.SummaryKey = "Warnings" + $Result.Details = $Test.ReportDetails + } + elseif ($Test.Criticality.EndsWith('3rd Party') -or $Test.Criticality.EndsWith('Not-Implemented')) { + $Result.DisplayString = "N/A" + $Result.SummaryKey = "Manual" + $Result.Details = $Test.ReportDetails + } + else { + $Result.DisplayString = "Fail" + $Result.SummaryKey = "Failures" + $Result.Details = $Test.ReportDetails + } + $Result +} + function New-Report { <# .Description @@ -104,6 +169,9 @@ function New-Report { $Test = $TestResults | Where-Object -Property PolicyId -eq $Control.Id if ($null -ne $Test){ + $MissingCommands = $Test.Commandlet | Where-Object {$SettingsExport."$($BaselineName)_successful_commands" -notcontains $_} + $Result = Get-TestResult $Test $MissingCommands $Control + # Check if the config file indicates the control should be omitted $Config = $SettingsExport.scuba_config $Omit = Get-OmissionState $Config $Control.Id @@ -123,61 +191,26 @@ function New-Report { "Result"= "Omitted" "Criticality"= $Test.Criticality "Details"= "Test omitted by user. $($OmitRationale)" + "OmittedEvaluationResult"=$Result.DisplayString + "OmittedEvaluationDetails"=$Result.Details } continue } - $MissingCommands = $Test.Commandlet | Where-Object {$SettingsExport."$($BaselineName)_successful_commands" -notcontains $_} - - if ($MissingCommands.Count -gt 0) { - $Result = "Error" - $ReportSummary.Errors += 1 - $MissingString = $MissingCommands -Join ", " - $Test.ReportDetails = "This test depends on the following command(s) which did not execute successfully: $($MissingString). See terminal output for more details." - } - elseif ($Test.RequirementMet) { - $Result = "Pass" - $ReportSummary.Passes += 1 - } - elseif ($Test.Criticality -eq "Should") { - $Result = "Warning" - $ReportSummary.Warnings += 1 - } - elseif ($Test.Criticality.EndsWith('3rd Party') -or $test.Criticality.EndsWith('Not-Implemented')) { - $Result = "N/A" - $ReportSummary.Manual += 1 - } - else { - $Result = "Fail" - $ReportSummary.Failures += 1 - } - + # This is the typical case, the test result is not missing or omitted + $ReportSummary[$Result.SummaryKey] += 1 $Fragment += [pscustomobject]@{ "Control ID"=$Control.Id "Requirement"=$Control.Value - "Result"= if ($Control.Deleted) { - "-" - } - elseif ($Control.MalformedDescription) { - $ReportSummary.Errors += 1 - "Error" - } - else { - $Result - } + "Result"= $Result.DisplayString "Criticality"=if ($Control.Deleted -or $Control.MalformedDescription) {"-"} else {$Test.Criticality} - "Details"=if ($Control.Deleted) { - "-" - } - elseif ($Control.MalformedDescription){ - "Report issue on GitHub" - } - else { - $Test.ReportDetails - } + "Details"= $Result.Details + "OmittedEvaluationResult"="N/A" + "OmittedEvaluationDetails"="N/A" } } else { + # The test result is missing $ReportSummary.Errors += 1 $Fragment += [pscustomobject]@{ "Control ID"=$Control.Id @@ -185,6 +218,8 @@ function New-Report { "Result"= "Error - Test results missing" "Criticality"= "-" "Details"= "Report issue on GitHub" + "OmittedEvaluationResult"="N/A" + "OmittedEvaluationDetails"="N/A" } Write-Warning -Message "WARNING: No test results found for Control Id $($Control.Id)" } @@ -197,7 +232,16 @@ function New-Report { $GroupAnchor = New-MarkdownAnchor -GroupNumber $BaselineGroup.GroupNumber -GroupName $BaselineGroup.GroupName $GroupReferenceURL = "$($ScubaGitHubUrl)/blob/v$($SettingsExport.module_version)/PowerShell/ScubaGear/baselines/$($BaselineName.ToLower()).md$GroupAnchor" $MarkdownLink = "$Name" - $Fragments += $Fragment | ConvertTo-Html -PreContent "

$Number $MarkdownLink

" -Fragment + # Create a version of the object without the omitted evaluation keys, otherwise they + # would show up as columns on the HTML report. + $FragmentWithoutOmitted = $Fragment | ForEach-Object -Process {[pscustomobject]@{ + "Control ID" = $_."Control ID"; + "Requirement" = $_."Requirement"; + "Result" = $_."Result"; + "Criticality" = $_."Criticality"; + "Details" = $_."Details"; + }} + $Fragments += $FragmentWithoutOmitted | ConvertTo-Html -PreContent "

$Number $MarkdownLink

" -Fragment # Package Assessment Report into Report JSON by Policy Group $ReportJson.Results += [pscustomobject]@{ diff --git a/PowerShell/ScubaGear/Testing/Unit/PowerShell/CreateReport/Get-TestResult.Tests.ps1 b/PowerShell/ScubaGear/Testing/Unit/PowerShell/CreateReport/Get-TestResult.Tests.ps1 new file mode 100644 index 0000000000..6831a564ba --- /dev/null +++ b/PowerShell/ScubaGear/Testing/Unit/PowerShell/CreateReport/Get-TestResult.Tests.ps1 @@ -0,0 +1,205 @@ +Import-Module (Join-Path -Path $PSScriptRoot -ChildPath '../../../../Modules/CreateReport') ` + -Function 'Get-TestResult' -Force + +InModuleScope CreateReport { + Describe -Tag CreateReport -Name 'Get-TestResult' { + Context "When the control is valid" { + BeforeAll { + # PS Script Analyzer doesn't play well with Pester scoping and can't tell that + # these variables *are* used. Using the script scope is the suggested workaround. + # See https://github.com/PowerShell/PSScriptAnalyzer/issues/946. + $script:ExampleReportDetails = "Example details" + $script:NormalControl = [PSCustomObject]@{ + MalformedDescription=$false; + Deleted=$false; + } + } + Context "When the control is implemented" { + It 'Returns pass if the test passed for SHALLs' { + $Test = [PSCustomObject]@{ + RequirementMet=$true; + Criticality="Shall"; + ReportDetails=$ExampleReportDetails; + } + $MissingCommands = $null + $Control = $NormalControl + $Result = Get-TestResult $Test $MissingCommands $Control + $Result.DisplayString | Should -Be "Pass" + $Result.SummaryKey | Should -Be "Passes" + $Result.Details | Should -Be $ExampleReportDetails + } + It 'Returns pass if the test passed for SHOULDs' { + $Test = [PSCustomObject]@{ + RequirementMet=$true; + Criticality="Should"; + ReportDetails=$ExampleReportDetails; + } + $MissingCommands = $null + $Control = $NormalControl + $Result = Get-TestResult $Test $MissingCommands $Control + $Result.DisplayString | Should -Be "Pass" + $Result.SummaryKey | Should -Be "Passes" + $Result.Details | Should -Be $ExampleReportDetails + } + It 'Returns fail for failed SHALLs' { + $Test = [PSCustomObject]@{ + RequirementMet=$false; + Criticality="Shall"; + ReportDetails=$ExampleReportDetails; + } + $MissingCommands = $null + $Control = $NormalControl + $Result = Get-TestResult $Test $MissingCommands $Control + $Result.DisplayString | Should -Be "Fail" + $Result.SummaryKey | Should -Be "Failures" + $Result.Details | Should -Be $ExampleReportDetails + } + It 'Returns warning for failed SHOULDs' { + $Test = [PSCustomObject]@{ + RequirementMet=$false; + Criticality="Should"; + ReportDetails=$ExampleReportDetails; + } + $MissingCommands = $null + $Control = $NormalControl + $Result = Get-TestResult $Test $MissingCommands $Control + $Result.DisplayString | Should -Be "Warning" + $Result.SummaryKey | Should -Be "Warnings" + $Result.Details | Should -Be $ExampleReportDetails + } + } + Context "When the control is not implemented" { + It 'Returns N/A for not implemented SHALLs' { + $Test = [PSCustomObject]@{ + RequirementMet=$false; + Criticality="Shall/Not-Implemented"; + ReportDetails=$ExampleReportDetails; + } + $MissingCommands = $null + $Control = $NormalControl + $Result = Get-TestResult $Test $MissingCommands $Control + $Result.DisplayString | Should -Be "N/A" + $Result.SummaryKey | Should -Be "Manual" + $Result.Details | Should -Be $ExampleReportDetails + } + It 'Returns N/A for not implemented SHOULDs' { + $Test = [PSCustomObject]@{ + RequirementMet=$false; + Criticality="Should/Not-Implemented"; + ReportDetails=$ExampleReportDetails; + } + $MissingCommands = $null + $Control = $NormalControl + $Result = Get-TestResult $Test $MissingCommands $Control + $Result.DisplayString | Should -Be "N/A" + $Result.SummaryKey | Should -Be "Manual" + $Result.Details | Should -Be $ExampleReportDetails + } + It 'Returns N/A for third-party SHALLs' { + $Test = [PSCustomObject]@{ + RequirementMet=$false; + Criticality="Shall/3rd Party"; + ReportDetails=$ExampleReportDetails; + } + $MissingCommands = $null + $Control = $NormalControl + $Result = Get-TestResult $Test $MissingCommands $Control + $Result.DisplayString | Should -Be "N/A" + $Result.SummaryKey | Should -Be "Manual" + $Result.Details | Should -Be $ExampleReportDetails + } + It 'Returns N/A for third-party SHOULDs' { + $Test = [PSCustomObject]@{ + RequirementMet=$false; + Criticality="Should/3rd Party"; + ReportDetails=$ExampleReportDetails; + } + $MissingCommands = $null + $Control = $NormalControl + $Result = Get-TestResult $Test $MissingCommands $Control + $Result.DisplayString | Should -Be "N/A" + $Result.SummaryKey | Should -Be "Manual" + $Result.Details | Should -Be $ExampleReportDetails + } + } + } + + Context "When the control/test has errors" { + BeforeAll { + $script:ScubaGitHubUrl = "https://github.com/cisagov/ScubaGear" + $script:ExampleMissingCommand1 = "Get-Example1" + $script:ExampleMissingCommand2 = "Get-Example2" + $script:ExampleMissingDetails = @("This test depends on the following command(s) which did not execute ", + "successfully: Get-Example1, Get-Example2. See terminal output for more details.") -Join "" + } + It 'When the test has missing commands and passed' { + $Test = [PSCustomObject]@{ + RequirementMet=$true; + Criticality="Should"; + ReportDetails=$ExampleReportDetails; + } + $MissingCommands = @($ExampleMissingCommand1, $ExampleMissingCommand2) + $Control = [PSCustomObject]@{ + MalformedDescription=$false; + Deleted=$false; + } + $Result = Get-TestResult $Test $MissingCommands $Control + $Result.DisplayString | Should -Be "Error" + $Result.SummaryKey | Should -Be "Errors" + $Result.Details | Should -Be $ExampleMissingDetails + } + It 'When the test has missing commands and failed' { + $Test = [PSCustomObject]@{ + RequirementMet=$true; + Criticality="Should"; + ReportDetails=$ExampleReportDetails; + } + $MissingCommands = @($ExampleMissingCommand1, $ExampleMissingCommand2) + $Control = [PSCustomObject]@{ + MalformedDescription=$false; + Deleted=$false; + } + $Result = Get-TestResult $Test $MissingCommands $Control + $Result.DisplayString | Should -Be "Error" + $Result.SummaryKey | Should -Be "Errors" + $Result.Details | Should -Be $ExampleMissingDetails + } + It 'When the control has been deleted' { + $Test = [PSCustomObject]@{ + RequirementMet=$true; + Criticality="Should"; + ReportDetails=$ExampleReportDetails; + } + $MissingCommands = @($ExampleMissingCommand1, $ExampleMissingCommand2) + $Control = [PSCustomObject]@{ + MalformedDescription=$false; + Deleted=$true; + } + $Result = Get-TestResult $Test $MissingCommands $Control + $Result.DisplayString | Should -Be "-" + $Result.SummaryKey | Should -Be "-" + $Result.Details | Should -Be "-" + } + It 'When the control description is malformed' { + $Test = [PSCustomObject]@{ + RequirementMet=$true; + Criticality="Should"; + ReportDetails=$ExampleReportDetails; + } + $MissingCommands = @($ExampleMissingCommand1, $ExampleMissingCommand2) + $Control = [PSCustomObject]@{ + MalformedDescription=$true; + Deleted=$true; + } + $Result = Get-TestResult $Test $MissingCommands $Control + $Result.DisplayString | Should -Be "Error" + $Result.SummaryKey | Should -Be "Errors" + $Result.Details | Should -Be "Report issue on GitHub" + } + } + } + + AfterAll { + Remove-Module CreateReport -ErrorAction SilentlyContinue + } +}