feat(new): Added Azure.AKS.AuditAdmin (Azure#2994)

BenjaminEngeset · BernieWhite · web-flow · commit f8b30532f806 · 2024-07-20T14:40:09.000+10:00
* feat(new): Added Azure.AKS.AuditAdmin

* Update src/PSRule.Rules.Azure/en/PSRule-rules.psd1

Co-authored-by: Bernie White &lt;bewhite@microsoft.com&gt;

* Update src/PSRule.Rules.Azure/rules/Azure.AKS.Rule.ps1

Co-authored-by: Bernie White &lt;bewhite@microsoft.com&gt;

* Update docs/en/rules/Azure.AKS.AuditAdmin.md

Co-authored-by: Bernie White &lt;bewhite@microsoft.com&gt;

* Delete docs/en/rules/json

---------

Co-authored-by: Bernie White &lt;bewhite@microsoft.com&gt;
diff --git a/docs/CHANGELOG-v1.md b/docs/CHANGELOG-v1.md
@@ -29,6 +29,11 @@ See [upgrade notes][1] for helpful information when upgrading from previous vers
 
 ## Unreleased
 
+- New rules:
+  - Azure Kubernetes Service:
+    - Verify that clusters have kube-audit logging disabled when not required by @BenjaminEngeset.
+      [#2450](https://github.com/Azure/PSRule.Rules.Azure/issues/2450)
+
 ## v1.39.0-B0009 (pre-release)
 
 What's changed since v1.38.0:
diff --git a/docs/en/rules/Azure.AKS.AuditAdmin.md b/docs/en/rules/Azure.AKS.AuditAdmin.md
@@ -0,0 +1,127 @@
+---
+severity: Important
+pillar: Cost Optimization
+category: CO:07 Component costs
+resource: Azure Kubernetes Service
+online version: https://azure.github.io/PSRule.Rules.Azure/en/rules/Azure.AKS.AuditAdmin/
+---
+
+# kube-audit-admin
+
+## SYNOPSIS
+
+Use kube-audit-admin instead of kube-audit to capture administrative actions in AKS clusters.
+
+## DESCRIPTION
+
+Key components in a Kubernetes cluster regularly scan or check for updated Kubernetes resources against the API server.
+These _get_ and _list_ operations typically occur more frequently as a Kubernetes cluster grows.
+
+Auditing within AKS writes log events for each operation that occur against the API server.
+As a result, collecting audit logs for _get_ and _list_ operations of a production AKS cluster can increase cost exponentially.
+
+AKS provides two log categories for collecting audit logs, `kube-audit` and `kube-audit-admin`.
+
+- `kube-audit` - Audit log data for every audit event including _get_, _list_, _create_, _update_, _delete_, _patch_, and _post_.
+- `kube-audit-admin` - Is a subset of the `kube-audit` log category that excludes _get_ and list_ audit events.
+
+In other words, both `kube-audit` and `kube-audit-admin` contain the same data except `kube-audit-admin` does not contain _get_ and _list_ events.
+Changes to the cluster configuration are captured with _create_, _update_, _delete_, _patch_, and _post_ events.
+
+By using `kube-audit-admin`, changes to resources in AKS are audited, however events relating to reading resources and configuration are not.
+This significantly reduces the number of logs and overall cost for collecting and storing AKS audit events.
+
+## RECOMMENDATION
+
+Consider using kube-audit-admin logging instead of kube-audit when detailed logging of every API request is not required.
+This approach helps in managing log volume and associated costs while still capturing essential administrative actions.
+
+## EXAMPLES
+
+### Configure with Azure template
+
+To deploy AKS clusters that pass this rule:
+
+- Deploy a diagnostic settings sub-resource.
+- Enable logging for the `kube-audit-admin` category and disable logging for the `kube-audit` category.
+
+For example:
+
+```json
+{
+  "type": "Microsoft.Insights/diagnosticSettings",
+  "apiVersion": "2021-05-01-preview",
+  "scope": "[format('Microsoft.ContainerService/managedClusters/{0}', parameters('clusterName'))]",
+  "name": "[parameters('name')]",
+  "properties": {
+    "logs": [
+      {
+        "category": "kube-audit-admin",
+        "enabled": true,
+        "retentionPolicy": {
+          "days": 0,
+          "enabled": false
+        }
+      },
+      {
+        "category": "kube-audit",
+        "enabled": false,
+        "retentionPolicy": {
+          "days": 0,
+          "enabled": false
+        }
+      }
+    ],
+    "workspaceId": "[parameters('workspaceId')]",
+    "logAnalyticsDestinationType": "Dedicated"
+  },
+  "dependsOn": [
+    "[resourceId('Microsoft.ContainerService/managedClusters', parameters('clusterName'))]"
+  ]
+}
+```
+
+### Configure with Bicep
+
+To deploy AKS clusters that pass this rule:
+
+- Deploy a diagnostic settings sub-resource.
+- Enable logging for the `kube-audit-admin` category and disable logging for the `kube-audit` category.
+
+For example:
+
+```bicep
+resource diagnosticSetting 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = {
+  name: name
+  scope: aks
+  properties: {
+    logs: [
+      {
+        category: 'kube-audit-admin'
+        enabled: true
+        retentionPolicy: {
+          days: 0
+          enabled: false
+        }
+      }
+      {
+        category: 'kube-audit'
+        enabled: false
+        retentionPolicy: {
+          days: 0
+          enabled: false
+        }
+      }
+    ]
+    workspaceId: workspaceId
+    logAnalyticsDestinationType: 'Dedicated'
+  }
+}
+```
+
+## LINKS
+
+- [CO:07 Component costs](https://learn.microsoft.com/azure/well-architected/cost-optimization/optimize-component-costs)
+- [Monitor AKS](https://learn.microsoft.com/azure/aks/monitor-aks)
+- [AKS control plane/resource logs](https://learn.microsoft.com/azure/aks/monitor-aks#aks-control-planeresource-logs)
+- [Azure deployment reference](https://learn.microsoft.com/azure/templates/microsoft.insights/diagnosticsettings)
diff --git a/src/PSRule.Rules.Azure/en/PSRule-rules.psd1 b/src/PSRule.Rules.Azure/en/PSRule-rules.psd1
@@ -15,6 +15,7 @@
     AKSAvailabilityZone = "The agent pool ({0}) deployed to region ({1}) should use following availability zones [{2}]."
     AKSAuditLogs = "The diagnostic setting ({0}) logs should enable at least one of (kube-audit, kube-audit-admin) and guard."
     AKSPlatformLogs = "The diagnostic setting ({0}) logs should enable ({1})."
+    AKSAuditAdmin = "The diagnostic setting ({0}) should use 'kube-audit-admin' instead of the 'kube-audit' log category."
     AKSEphemeralOSDiskNotConfigured = "The OS disk type 'Managed' should be of type 'Ephemeral'."
     SubnetNSGNotConfigured = "The subnet ({0}) has no NSG associated."
     ServiceUrlNotHttps = "The service URL for '{0}' is not a HTTPS endpoint."
diff --git a/src/PSRule.Rules.Azure/rules/Azure.AKS.Rule.ps1 b/src/PSRule.Rules.Azure/rules/Azure.AKS.Rule.ps1
@@ -37,7 +37,7 @@ Rule 'Azure.AKS.PoolVersion' -Ref 'AZR-000016' -Type 'Microsoft.ContainerService
     }
     foreach ($agentPool in $agentPools) {
         $Assert.HasDefaultValue($agentPool, 'orchestratorVersion', $clusterVersion).
-            Reason($LocalizedData.AKSNodePoolVersion, $agentPool.name, $agentPool.orchestratorVersion);
+        Reason($LocalizedData.AKSNodePoolVersion, $agentPool.name, $agentPool.orchestratorVersion);
     }
 }
 
@@ -50,7 +50,7 @@ Rule 'Azure.AKS.PoolScaleSet' -Ref 'AZR-000017' -Type 'Microsoft.ContainerServic
     }
     foreach ($agentPool in $agentPools) {
         $Assert.HasFieldValue($agentPool, 'type', 'VirtualMachineScaleSets').
-            Reason($LocalizedData.AKSNodePoolType, $agentPool.name);
+        Reason($LocalizedData.AKSNodePoolType, $agentPool.name);
     }
 }
 
@@ -99,11 +99,11 @@ Rule 'Azure.AKS.CNISubnetSize' -Ref 'AZR-000020' -If { IsExport } -With 'Azure.A
         $subnetAddressPrefixSize = [int]$subnet.Properties.addressPrefix.Split('/')[-1];
         
         $Assert.LessOrEqual($subnetAddressPrefixSize, '.', $configurationMinimumSubnetSize).
-            Reason(
-                $LocalizedData.AKSAzureCNI, 
-                $subnet.Name, 
-                $configurationMinimumSubnetSize
-            );
+        Reason(
+            $LocalizedData.AKSAzureCNI, 
+            $subnet.Name, 
+            $configurationMinimumSubnetSize
+        );
     }
 } -Configure @{ AZURE_AKS_CNI_MINIMUM_CLUSTER_SUBNET_SIZE = 23 }
 
@@ -134,7 +134,7 @@ Rule 'Azure.AKS.AvailabilityZone' -Ref 'AZR-000021' -Type 'Microsoft.ContainerSe
         # Availability zones only available on virtual machine scale sets
         if ($Assert.HasFieldValue($agentPool, 'type', 'VirtualMachineScaleSets').Result) {
             $Assert.HasFieldValue($agentPool, 'availabilityZones').
-                Reason($LocalizedData.AKSAvailabilityZone, $agentPool.name, $TargetObject.Location, $joinedZoneString);
+            Reason($LocalizedData.AKSAvailabilityZone, $agentPool.name, $TargetObject.Location, $joinedZoneString);
         }
         else {
             $Assert.Pass();
@@ -150,15 +150,15 @@ Rule 'Azure.AKS.AuditLogs' -Ref 'AZR-000022' -Type 'Microsoft.ContainerService/m
 
     foreach ($setting in $diagnosticLogs) {
         $kubeAuditEnabledLog = @($setting.Properties.logs | Where-Object {
-            $_.category -in 'kube-audit', 'kube-audit-admin' -and $_.enabled
-        });
+                $_.category -in 'kube-audit', 'kube-audit-admin' -and $_.enabled
+            });
 
         $guardEnabledLog = @($setting.Properties.logs | Where-Object {
-            $_.category -eq 'guard' -and $_.enabled
-        });
+                $_.category -eq 'guard' -and $_.enabled
+            });
 
         $auditLogsEnabled = $Assert.Greater($kubeAuditEnabledLog, '.', 0).Result -and
-                            $Assert.Greater($guardEnabledLog, '.', 0).Result;
+        $Assert.Greater($guardEnabledLog, '.', 0).Result;
 
         $Assert.Create($auditLogsEnabled, $LocalizedData.AKSAuditLogs, $setting.name);
     }
@@ -177,7 +177,7 @@ Rule 'Azure.AKS.PlatformLogs' -Ref 'AZR-000023' -Type 'Microsoft.ContainerServic
     $Assert.Greater($diagnosticLogs, '.', 0).Reason($LocalizedData.DiagnosticSettingsNotConfigured, $TargetObject.name);
 
     $availableLogCategories = @{
-        Logs = @(
+        Logs    = @(
             'cluster-autoscaler', 
             'kube-apiserver', 
             'kube-controller-manager', 
@@ -189,12 +189,12 @@ Rule 'Azure.AKS.PlatformLogs' -Ref 'AZR-000023' -Type 'Microsoft.ContainerServic
     }
 
     $configurationLogCategories = @($configurationLogCategoriesList | Where-Object {
-        $_ -in $availableLogCategories.Logs
-    });
+            $_ -in $availableLogCategories.Logs
+        });
 
     $configurationMetricCategories = @($configurationLogCategoriesList | Where-Object {
-        $_ -in $availableLogCategories.Metrics
-    });
+            $_ -in $availableLogCategories.Metrics
+        });
 
     $logCategoriesNeeded = [System.Math]::Min(
         $configurationLogCategories.Length, 
@@ -210,19 +210,19 @@ Rule 'Azure.AKS.PlatformLogs' -Ref 'AZR-000023' -Type 'Microsoft.ContainerServic
 
     foreach ($setting in $diagnosticLogs) {
         $platformLogs = @($setting.Properties.logs | Where-Object {
-            $_.enabled -and
-            $_.category -in $configurationLogCategories -and
-            $_.category -in $availableLogCategories.Logs
-        });
+                $_.enabled -and
+                $_.category -in $configurationLogCategories -and
+                $_.category -in $availableLogCategories.Logs
+            });
 
         $metricLogs = @($setting.Properties.metrics | Where-Object {
-            $_.enabled -and 
-            $_.category -in $configurationMetricCategories -and
-            $_.category -in $availableLogCategories.Metrics
-        });
+                $_.enabled -and 
+                $_.category -in $configurationMetricCategories -and
+                $_.category -in $availableLogCategories.Metrics
+            });
 
         $platformLogsEnabled = $Assert.HasFieldValue($platformLogs, 'Length', $logCategoriesNeeded).Result -and 
-                               $Assert.HasFieldValue($metricLogs, 'Length', $metricCategoriesNeeded).Result
+        $Assert.HasFieldValue($metricLogs, 'Length', $metricCategoriesNeeded).Result
 
         $Assert.Create(
             $platformLogsEnabled, 
@@ -262,8 +262,8 @@ Rule 'Azure.AKS.MinNodeCount' -Ref 'AZR-000024' -Type 'Microsoft.ContainerServic
 Rule 'Azure.AKS.MinUserPoolNodes' -Ref 'AZR-000412' -Type 'Microsoft.ContainerService/managedClusters', 'Microsoft.ContainerService/managedClusters/agentPools' -Tag @{ release = 'GA'; ruleSet = '2024_03'; 'Azure.WAF/pillar' = 'Reliability' } {
     $excludedPools = $Configuration.GetStringValues('AZURE_AKS_CLUSTER_USER_POOL_EXCLUDED_FROM_MINIMUM_NODES');
     $agentPools = @(GetAgentPoolProfiles | Where-Object {
-        $_.mode -eq 'user' -and $_.name -notin $excludedPools -and $_.scaleSetPriority -ne 'Spot'
-    })
+            $_.mode -eq 'user' -and $_.name -notin $excludedPools -and $_.scaleSetPriority -ne 'Spot'
+        })
 
     if ($agentPools.Length -eq 0) {
         return $Assert.Pass();
@@ -305,6 +305,20 @@ Rule 'Azure.AKS.EphemeralOSDisk' -Ref 'AZR-000287' -Level Warning -Type 'Microso
     }
 }
 
+# Synopsis: Use kube-audit-admin instead of kube-audit to capture administrative actions in AKS clusters.
+Rule 'Azure.AKS.AuditAdmin' -Ref 'AZR-000445' -Type 'Microsoft.ContainerService/managedClusters' -Tag @{ release = 'GA'; ruleSet = '2024_09'; 'Azure.WAF/pillar' = 'Cost Optimization'; } {
+    $kubeAuditLogs = @(GetSubResources -ResourceType 'Microsoft.Insights/diagnosticSettings' |
+        Where-Object { $_.properties.logs | Where-Object { $_.category -eq 'kube-audit' -and $_.enabled } } )
+
+    if ($kubeAuditLogs.Count -eq 0) {
+        return $Assert.Pass()
+    }
+
+    foreach ($kubeAuditLog in $kubeAuditLogs) {
+        $Assert.Fail().Reason($LocalizedData.AKSAuditAdmin, $kubeAuditLog.name)
+    }
+}
+
 #region Helper functions
 
 function global:GetAgentPoolProfiles {
@@ -315,36 +329,36 @@ function global:GetAgentPoolProfiles {
         if ($PSRule.TargetType -eq 'Microsoft.ContainerService/managedClusters') {
             $TargetObject.Properties.agentPoolProfiles;
             @(GetSubResources -ResourceType 'Microsoft.ContainerService/managedClusters/agentPools' | ForEach-Object {
-                [PSCustomObject]@{
-                    name = $_.name
-                    type = $_.properties.type
-                    mode = $_.properties.mode
-                    maxPods = $_.properties.maxPods
-                    orchestratorVersion = $_.properties.orchestratorVersion
-                    enableAutoScaling = $_.properties.enableAutoScaling
-                    availabilityZones = $_.properties.availabilityZones
-                    osDiskType = $_.properties.osDiskType
-                    count = [int]$_.properties.count
-                    minCount = [int]$_.properties.minCount
-                    maxCount = [int]$_.properties.maxCount
-                    scaleSetPriority = $_.properties.scaleSetPriority
-                }
-            });
+                    [PSCustomObject]@{
+                        name                = $_.name
+                        type                = $_.properties.type
+                        mode                = $_.properties.mode
+                        maxPods             = $_.properties.maxPods
+                        orchestratorVersion = $_.properties.orchestratorVersion
+                        enableAutoScaling   = $_.properties.enableAutoScaling
+                        availabilityZones   = $_.properties.availabilityZones
+                        osDiskType          = $_.properties.osDiskType
+                        count               = [int]$_.properties.count
+                        minCount            = [int]$_.properties.minCount
+                        maxCount            = [int]$_.properties.maxCount
+                        scaleSetPriority    = $_.properties.scaleSetPriority
+                    }
+                });
         }
         elseif ($PSRule.TargetType -eq 'Microsoft.ContainerService/managedClusters/agentPools') {
             [PSCustomObject]@{
-                name = $TargetObject.name
-                type = $TargetObject.properties.type
-                mode = $TargetObject.properties.mode
-                maxPods = $TargetObject.properties.maxPods
+                name                = $TargetObject.name
+                type                = $TargetObject.properties.type
+                mode                = $TargetObject.properties.mode
+                maxPods             = $TargetObject.properties.maxPods
                 orchestratorVersion = $TargetObject.properties.orchestratorVersion
-                enableAutoScaling = $TargetObject.properties.enableAutoScaling
-                availabilityZones = $TargetObject.properties.availabilityZones
-                osDiskType = $TargetObject.properties.osDiskType
-                count = [int]$TargetObject.properties.count
-                minCount = [int]$TargetObject.properties.minCount
-                maxCount = [int]$TargetObject.properties.maxCount
-                scaleSetPriority = $TargetObject.properties.scaleSetPriority
+                enableAutoScaling   = $TargetObject.properties.enableAutoScaling
+                availabilityZones   = $TargetObject.properties.availabilityZones
+                osDiskType          = $TargetObject.properties.osDiskType
+                count               = [int]$TargetObject.properties.count
+                minCount            = [int]$TargetObject.properties.minCount
+                maxCount            = [int]$TargetObject.properties.maxCount
+                scaleSetPriority    = $TargetObject.properties.scaleSetPriority
             }
         }
     }
diff --git a/tests/PSRule.Rules.Azure.Tests/Azure.AKS.Tests.ps1 b/tests/PSRule.Rules.Azure.Tests/Azure.AKS.Tests.ps1
@@ -526,6 +526,23 @@ Describe 'Azure.AKS' -Tag AKS {
             $ruleResult.Length | Should -Be 1;
             $ruleResult.TargetName | Should -BeIn 'cluster-L';
         }
+
+        It 'Azure.AKS.AuditAdmin' {
+            $filteredResult = $result | Where-Object { $_.RuleName -eq 'Azure.AKS.AuditAdmin' };
+
+            # Fail
+            $ruleResult = @($filteredResult | Where-Object { $_.Outcome -eq 'Fail' });
+            $ruleResult.Length | Should -Be 2;
+            $ruleResult.TargetName | Should -Be 'cluster-I', 'cluster-J';
+
+            $ruleResult[0].Reason | Should -Be "The diagnostic setting (metrics) should use 'kube-audit-admin' instead of the 'kube-audit' log category.";
+            $ruleResult[1].Reason | Should -Be "The diagnostic setting (metrics) should use 'kube-audit-admin' instead of the 'kube-audit' log category.";
+
+            # Pass
+            $ruleResult = @($filteredResult | Where-Object { $_.Outcome -eq 'Pass' });
+            $ruleResult.Length | Should -Be 9;
+            $ruleResult.TargetName | Should -Be 'cluster-A', 'cluster-B', 'cluster-C', 'cluster-D', 'cluster-F', 'cluster-G', 'cluster-H', 'cluster-K', 'cluster-L';
+        }
     }
 
     Context 'Resource name' {
