Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Save omitted rego evaulation results #1604

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
+294 −45
Open

Save omitted rego evaulation results #1604

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4f0c839
Initial implementation
adhilto Feb 3, 2025
479bae1
Add unit tests
adhilto Feb 28, 2025
6ee1d06
Force function import for unit test file
adhilto Feb 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 89 additions & 45 deletions PowerShell/ScubaGear/Modules/CreateReport/CreateReport.psm1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,70 @@
Import-Module -Name (Join-Path -Path $PSScriptRoot -ChildPath "..\Utility")

function Get-TestResult {
<#
.Description
Given the Rego output for a specific test, determine the result (e.g. "Pass"/"Fail").
.Functionality
Internal
#>
[CmdletBinding()]
param (
[Parameter(Mandatory=$true)]
[ValidateNotNullOrEmpty()]
[object]
$Test,

[Parameter(Mandatory=$true)]
[AllowNull()]
[array]
$MissingCommands,

[Parameter(Mandatory=$true)]
[ValidateNotNullOrEmpty()]
[object]
$Control
)

$Result = @{}
if ($Control.MalformedDescription) {
$Result.DisplayString = "Error"
$Result.SummaryKey = "Errors"
$Result.Details = "Report issue on <a href=`"$ScubaGitHubUrl/issues`" target=`"_blank`">GitHub</a>"
}
elseif ($Control.Deleted) {
$Result.DisplayString = "-"
$Result.SummaryKey = "-"
$Result.Details = "-"
}
elseif ($MissingCommands.Count -gt 0) {
$Result.DisplayString = "Error"
$Result.SummaryKey = "Errors"
$MissingString = $MissingCommands -Join ", "
$Result.Details = "This test depends on the following command(s) which did not execute successfully: $($MissingString). See terminal output for more details."
}
elseif ($Test.RequirementMet) {
$Result.DisplayString = "Pass"
$Result.SummaryKey = "Passes"
$Result.Details = $Test.ReportDetails
}
elseif ($Test.Criticality -eq "Should") {
$Result.DisplayString = "Warning"
$Result.SummaryKey = "Warnings"
$Result.Details = $Test.ReportDetails
}
elseif ($Test.Criticality.EndsWith('3rd Party') -or $Test.Criticality.EndsWith('Not-Implemented')) {
$Result.DisplayString = "N/A"
$Result.SummaryKey = "Manual"
$Result.Details = $Test.ReportDetails
}
else {
$Result.DisplayString = "Fail"
$Result.SummaryKey = "Failures"
$Result.Details = $Test.ReportDetails
}
$Result
}

function New-Report {
<#
.Description
Expand Down Expand Up @@ -104,6 +169,9 @@ function New-Report {
$Test = $TestResults | Where-Object -Property PolicyId -eq $Control.Id

if ($null -ne $Test){
$MissingCommands = $Test.Commandlet | Where-Object {$SettingsExport."$($BaselineName)_successful_commands" -notcontains $_}
$Result = Get-TestResult $Test $MissingCommands $Control

# Check if the config file indicates the control should be omitted
$Config = $SettingsExport.scuba_config
$Omit = Get-OmissionState $Config $Control.Id
Expand All @@ -123,68 +191,35 @@ function New-Report {
"Result"= "Omitted"
"Criticality"= $Test.Criticality
"Details"= "Test omitted by user. $($OmitRationale)"
"OmittedEvaluationResult"=$Result.DisplayString
"OmittedEvaluationDetails"=$Result.Details
}
continue
}

$MissingCommands = $Test.Commandlet | Where-Object {$SettingsExport."$($BaselineName)_successful_commands" -notcontains $_}

if ($MissingCommands.Count -gt 0) {
$Result = "Error"
$ReportSummary.Errors += 1
$MissingString = $MissingCommands -Join ", "
$Test.ReportDetails = "This test depends on the following command(s) which did not execute successfully: $($MissingString). See terminal output for more details."
}
elseif ($Test.RequirementMet) {
$Result = "Pass"
$ReportSummary.Passes += 1
}
elseif ($Test.Criticality -eq "Should") {
$Result = "Warning"
$ReportSummary.Warnings += 1
}
elseif ($Test.Criticality.EndsWith('3rd Party') -or $test.Criticality.EndsWith('Not-Implemented')) {
$Result = "N/A"
$ReportSummary.Manual += 1
}
else {
$Result = "Fail"
$ReportSummary.Failures += 1
}

# This is the typical case, the test result is not missing or omitted
$ReportSummary[$Result.SummaryKey] += 1
$Fragment += [pscustomobject]@{
"Control ID"=$Control.Id
"Requirement"=$Control.Value
"Result"= if ($Control.Deleted) {
"-"
}
elseif ($Control.MalformedDescription) {
$ReportSummary.Errors += 1
"Error"
}
else {
$Result
}
"Result"= $Result.DisplayString
"Criticality"=if ($Control.Deleted -or $Control.MalformedDescription) {"-"} else {$Test.Criticality}
"Details"=if ($Control.Deleted) {
"-"
}
elseif ($Control.MalformedDescription){
"Report issue on <a href=`"$ScubaGitHubUrl/issues`" target=`"_blank`">GitHub</a>"
}
else {
$Test.ReportDetails
}
"Details"= $Result.Details
"OmittedEvaluationResult"="N/A"
"OmittedEvaluationDetails"="N/A"
}
}
else {
# The test result is missing
$ReportSummary.Errors += 1
$Fragment += [pscustomobject]@{
"Control ID"=$Control.Id
"Requirement"=$Control.Value
"Result"= "Error - Test results missing"
"Criticality"= "-"
"Details"= "Report issue on <a href=`"$ScubaGitHubUrl/issues`" target=`"_blank`">GitHub</a>"
"OmittedEvaluationResult"="N/A"
"OmittedEvaluationDetails"="N/A"
}
Write-Warning -Message "WARNING: No test results found for Control Id $($Control.Id)"
}
Expand All @@ -197,7 +232,16 @@ function New-Report {
$GroupAnchor = New-MarkdownAnchor -GroupNumber $BaselineGroup.GroupNumber -GroupName $BaselineGroup.GroupName
$GroupReferenceURL = "$($ScubaGitHubUrl)/blob/v$($SettingsExport.module_version)/PowerShell/ScubaGear/baselines/$($BaselineName.ToLower()).md$GroupAnchor"
$MarkdownLink = "<a class='control_group' href=`"$($GroupReferenceURL)`" target=`"_blank`">$Name</a>"
$Fragments += $Fragment | ConvertTo-Html -PreContent "<h2>$Number $MarkdownLink</h2>" -Fragment
# Create a version of the object without the omitted evaluation keys, otherwise they
# would show up as columns on the HTML report.
$FragmentWithoutOmitted = $Fragment | ForEach-Object -Process {[pscustomobject]@{
"Control ID" = $_."Control ID";
"Requirement" = $_."Requirement";
"Result" = $_."Result";
"Criticality" = $_."Criticality";
"Details" = $_."Details";
}}
$Fragments += $FragmentWithoutOmitted | ConvertTo-Html -PreContent "<h2>$Number $MarkdownLink</h2>" -Fragment

# Package Assessment Report into Report JSON by Policy Group
$ReportJson.Results += [pscustomobject]@{
Expand Down
205 changes: 205 additions & 0 deletions PowerShell/ScubaGear/Testing/Unit/PowerShell/CreateReport/Get-TestResult.Tests.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
Import-Module (Join-Path -Path $PSScriptRoot -ChildPath '../../../../Modules/CreateReport') `
-Function 'Get-TestResult' -Force

InModuleScope CreateReport {
Describe -Tag CreateReport -Name 'Get-TestResult' {
Context "When the control is valid" {
BeforeAll {
# PS Script Analyzer doesn't play well with Pester scoping and can't tell that
# these variables *are* used. Using the script scope is the suggested workaround.
# See https://github.com/PowerShell/PSScriptAnalyzer/issues/946.
$script:ExampleReportDetails = "Example details"
$script:NormalControl = [PSCustomObject]@{
MalformedDescription=$false;
Deleted=$false;
}
}
Context "When the control is implemented" {
It 'Returns pass if the test passed for SHALLs' {
$Test = [PSCustomObject]@{
RequirementMet=$true;
Criticality="Shall";
ReportDetails=$ExampleReportDetails;
}
$MissingCommands = $null
$Control = $NormalControl
$Result = Get-TestResult $Test $MissingCommands $Control
$Result.DisplayString | Should -Be "Pass"
$Result.SummaryKey | Should -Be "Passes"
$Result.Details | Should -Be $ExampleReportDetails
}
It 'Returns pass if the test passed for SHOULDs' {
$Test = [PSCustomObject]@{
RequirementMet=$true;
Criticality="Should";
ReportDetails=$ExampleReportDetails;
}
$MissingCommands = $null
$Control = $NormalControl
$Result = Get-TestResult $Test $MissingCommands $Control
$Result.DisplayString | Should -Be "Pass"
$Result.SummaryKey | Should -Be "Passes"
$Result.Details | Should -Be $ExampleReportDetails
}
It 'Returns fail for failed SHALLs' {
$Test = [PSCustomObject]@{
RequirementMet=$false;
Criticality="Shall";
ReportDetails=$ExampleReportDetails;
}
$MissingCommands = $null
$Control = $NormalControl
$Result = Get-TestResult $Test $MissingCommands $Control
$Result.DisplayString | Should -Be "Fail"
$Result.SummaryKey | Should -Be "Failures"
$Result.Details | Should -Be $ExampleReportDetails
}
It 'Returns warning for failed SHOULDs' {
$Test = [PSCustomObject]@{
RequirementMet=$false;
Criticality="Should";
ReportDetails=$ExampleReportDetails;
}
$MissingCommands = $null
$Control = $NormalControl
$Result = Get-TestResult $Test $MissingCommands $Control
$Result.DisplayString | Should -Be "Warning"
$Result.SummaryKey | Should -Be "Warnings"
$Result.Details | Should -Be $ExampleReportDetails
}
}
Context "When the control is not implemented" {
It 'Returns N/A for not implemented SHALLs' {
$Test = [PSCustomObject]@{
RequirementMet=$false;
Criticality="Shall/Not-Implemented";
ReportDetails=$ExampleReportDetails;
}
$MissingCommands = $null
$Control = $NormalControl
$Result = Get-TestResult $Test $MissingCommands $Control
$Result.DisplayString | Should -Be "N/A"
$Result.SummaryKey | Should -Be "Manual"
$Result.Details | Should -Be $ExampleReportDetails
}
It 'Returns N/A for not implemented SHOULDs' {
$Test = [PSCustomObject]@{
RequirementMet=$false;
Criticality="Should/Not-Implemented";
ReportDetails=$ExampleReportDetails;
}
$MissingCommands = $null
$Control = $NormalControl
$Result = Get-TestResult $Test $MissingCommands $Control
$Result.DisplayString | Should -Be "N/A"
$Result.SummaryKey | Should -Be "Manual"
$Result.Details | Should -Be $ExampleReportDetails
}
It 'Returns N/A for third-party SHALLs' {
$Test = [PSCustomObject]@{
RequirementMet=$false;
Criticality="Shall/3rd Party";
ReportDetails=$ExampleReportDetails;
}
$MissingCommands = $null
$Control = $NormalControl
$Result = Get-TestResult $Test $MissingCommands $Control
$Result.DisplayString | Should -Be "N/A"
$Result.SummaryKey | Should -Be "Manual"
$Result.Details | Should -Be $ExampleReportDetails
}
It 'Returns N/A for third-party SHOULDs' {
$Test = [PSCustomObject]@{
RequirementMet=$false;
Criticality="Should/3rd Party";
ReportDetails=$ExampleReportDetails;
}
$MissingCommands = $null
$Control = $NormalControl
$Result = Get-TestResult $Test $MissingCommands $Control
$Result.DisplayString | Should -Be "N/A"
$Result.SummaryKey | Should -Be "Manual"
$Result.Details | Should -Be $ExampleReportDetails
}
}
}

Context "When the control/test has errors" {
BeforeAll {
$script:ScubaGitHubUrl = "https://github.com/cisagov/ScubaGear"
$script:ExampleMissingCommand1 = "Get-Example1"
$script:ExampleMissingCommand2 = "Get-Example2"
$script:ExampleMissingDetails = @("This test depends on the following command(s) which did not execute ",
"successfully: Get-Example1, Get-Example2. See terminal output for more details.") -Join ""
}
It 'When the test has missing commands and passed' {
$Test = [PSCustomObject]@{
RequirementMet=$true;
Criticality="Should";
ReportDetails=$ExampleReportDetails;
}
$MissingCommands = @($ExampleMissingCommand1, $ExampleMissingCommand2)
$Control = [PSCustomObject]@{
MalformedDescription=$false;
Deleted=$false;
}
$Result = Get-TestResult $Test $MissingCommands $Control
$Result.DisplayString | Should -Be "Error"
$Result.SummaryKey | Should -Be "Errors"
$Result.Details | Should -Be $ExampleMissingDetails
}
It 'When the test has missing commands and failed' {
$Test = [PSCustomObject]@{
RequirementMet=$true;
Criticality="Should";
ReportDetails=$ExampleReportDetails;
}
$MissingCommands = @($ExampleMissingCommand1, $ExampleMissingCommand2)
$Control = [PSCustomObject]@{
MalformedDescription=$false;
Deleted=$false;
}
$Result = Get-TestResult $Test $MissingCommands $Control
$Result.DisplayString | Should -Be "Error"
$Result.SummaryKey | Should -Be "Errors"
$Result.Details | Should -Be $ExampleMissingDetails
}
It 'When the control has been deleted' {
$Test = [PSCustomObject]@{
RequirementMet=$true;
Criticality="Should";
ReportDetails=$ExampleReportDetails;
}
$MissingCommands = @($ExampleMissingCommand1, $ExampleMissingCommand2)
$Control = [PSCustomObject]@{
MalformedDescription=$false;
Deleted=$true;
}
$Result = Get-TestResult $Test $MissingCommands $Control
$Result.DisplayString | Should -Be "-"
$Result.SummaryKey | Should -Be "-"
$Result.Details | Should -Be "-"
}
It 'When the control description is malformed' {
$Test = [PSCustomObject]@{
RequirementMet=$true;
Criticality="Should";
ReportDetails=$ExampleReportDetails;
}
$MissingCommands = @($ExampleMissingCommand1, $ExampleMissingCommand2)
$Control = [PSCustomObject]@{
MalformedDescription=$true;
Deleted=$true;
}
$Result = Get-TestResult $Test $MissingCommands $Control
$Result.DisplayString | Should -Be "Error"
$Result.SummaryKey | Should -Be "Errors"
$Result.Details | Should -Be "Report issue on <a href=`"$ScubaGitHubUrl/issues`" target=`"_blank`">GitHub</a>"
}
}
}

AfterAll {
Remove-Module CreateReport -ErrorAction SilentlyContinue
}
}
Loading