Skip to content

Commit 6b12228

Browse files
BernieWhiteBernieWhite
authored
Fixes Azure.AKS.AuthorizedIPs for private cluster Azure#2677 (Azure#2679)
1 parent dca5bde commit 6b12228

File tree

7 files changed

+387
-18
lines changed

7 files changed

+387
-18
lines changed

‎docs/CHANGELOG-v1.md

+6
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ See [upgrade notes][1] for helpful information when upgrading from previous vers
3232

3333
## Unreleased
3434

35+
What's changed since v1.33.0:
36+
37+
- Bug fixes:
38+
- Fixed `Azure.AKS.AuthorizedIPs` is not valid for a private cluster by @BernieWhite.
39+
[#2677](https://github.com/Azure/PSRule.Rules.Azure/issues/2677)
40+
3541
## v1.33.0
3642

3743
What's changed since v1.32.1:

‎docs/en/rules/Azure.AKS.AuthorizedIPs.md

+163-8
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
---
2+
reviewed: 2024-02-07
23
severity: Important
34
pillar: Security
4-
category: Design
5+
category: SE:06 Network controls
56
resource: Azure Kubernetes Service
67
online version: https://azure.github.io/PSRule.Rules.Azure/en/rules/Azure.AKS.AuthorizedIPs/
78
---
@@ -20,30 +21,184 @@ Access to the API server is required by various cluster functions as well as all
2021
All activities performed against the cluster require authorization.
2122
To improve cluster security, the API server can be restricted to a limited set of IP address ranges.
2223

23-
Restricting authorized IP addresses for the API server as the following limitations:
24+
Restricting authorized IP addresses for the API server has the following limitations:
2425

2526
- Requires AKS clusters configured with a Standard Load Balancer SKU.
2627
- This feature is not compatible with clusters that use Public IP per Node.
28+
- This feature is not compatible with AKS private clusters.
2729

28-
When configuring this feature you must specify the IP address ranges that will be authorized.
30+
When configuring this feature, you must specify the IP address ranges that will be authorized.
2931
To allow only the outbound public IP of the Standard SKU load balancer, use `0.0.0.0/32`.
3032

33+
You should add these ranges to the allow list:
34+
35+
- Include output IP addresses for cluster nodes
36+
- Any range where administration will connect to the API server, including CI/CD systems, monitoring, and management systems.
37+
3138
## RECOMMENDATION
3239

3340
Consider restricting network traffic to the API server endpoints to trusted IP addresses.
34-
Include output IP addresses for cluster nodes and any range where administration will occur from.
3541

3642
## EXAMPLES
3743

44+
### Configure with Azure template
45+
46+
To deploy clusters that pass this rule:
47+
48+
- Set the `properties.apiServerAccessProfile.authorizedIPRanges` property to a list of authorized IP ranges.
49+
50+
For example:
51+
52+
```json
53+
{
54+
"type": "Microsoft.ContainerService/managedClusters",
55+
"apiVersion": "2023-11-01",
56+
"name": "[parameters('name')]",
57+
"location": "[parameters('location')]",
58+
"identity": {
59+
"type": "UserAssigned",
60+
"userAssignedIdentities": {
61+
"[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('identityName')))]": {}
62+
}
63+
},
64+
"properties": {
65+
"kubernetesVersion": "[parameters('kubernetesVersion')]",
66+
"disableLocalAccounts": true,
67+
"enableRBAC": true,
68+
"dnsPrefix": "[parameters('dnsPrefix')]",
69+
"agentPoolProfiles": "[variables('allPools')]",
70+
"aadProfile": {
71+
"managed": true,
72+
"enableAzureRBAC": true,
73+
"adminGroupObjectIDs": "[parameters('clusterAdmins')]",
74+
"tenantID": "[subscription().tenantId]"
75+
},
76+
"networkProfile": {
77+
"networkPlugin": "azure",
78+
"networkPolicy": "azure",
79+
"loadBalancerSku": "standard",
80+
"serviceCidr": "[variables('serviceCidr')]",
81+
"dnsServiceIP": "[variables('dnsServiceIP')]"
82+
},
83+
"apiServerAccessProfile": {
84+
"authorizedIPRanges": [
85+
"0.0.0.0/32"
86+
]
87+
},
88+
"autoUpgradeProfile": {
89+
"upgradeChannel": "stable"
90+
},
91+
"oidcIssuerProfile": {
92+
"enabled": true
93+
},
94+
"addonProfiles": {
95+
"azurepolicy": {
96+
"enabled": true
97+
},
98+
"omsagent": {
99+
"enabled": true,
100+
"config": {
101+
"logAnalyticsWorkspaceResourceID": "[parameters('workspaceId')]"
102+
}
103+
},
104+
"azureKeyvaultSecretsProvider": {
105+
"enabled": true,
106+
"config": {
107+
"enableSecretRotation": "true"
108+
}
109+
}
110+
}
111+
},
112+
"dependsOn": [
113+
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('identityName'))]"
114+
]
115+
}
116+
```
117+
118+
### Configure with Bicep
119+
120+
To deploy resource that pass this rule:
121+
122+
- Set the `properties.apiServerAccessProfile.authorizedIPRanges` property to a list of authorized IP ranges.
123+
124+
For example:
125+
126+
```bicep
127+
resource cluster 'Microsoft.ContainerService/managedClusters@2023-11-01' = {
128+
location: location
129+
name: name
130+
identity: {
131+
type: 'UserAssigned'
132+
userAssignedIdentities: {
133+
'${identity.id}': {}
134+
}
135+
}
136+
properties: {
137+
kubernetesVersion: kubernetesVersion
138+
disableLocalAccounts: true
139+
enableRBAC: true
140+
dnsPrefix: dnsPrefix
141+
agentPoolProfiles: allPools
142+
aadProfile: {
143+
managed: true
144+
enableAzureRBAC: true
145+
adminGroupObjectIDs: clusterAdmins
146+
tenantID: subscription().tenantId
147+
}
148+
networkProfile: {
149+
networkPlugin: 'azure'
150+
networkPolicy: 'azure'
151+
loadBalancerSku: 'standard'
152+
serviceCidr: serviceCidr
153+
dnsServiceIP: dnsServiceIP
154+
}
155+
apiServerAccessProfile: {
156+
authorizedIPRanges: [
157+
'0.0.0.0/32'
158+
]
159+
}
160+
autoUpgradeProfile: {
161+
upgradeChannel: 'stable'
162+
}
163+
oidcIssuerProfile: {
164+
enabled: true
165+
}
166+
addonProfiles: {
167+
azurepolicy: {
168+
enabled: true
169+
}
170+
omsagent: {
171+
enabled: true
172+
config: {
173+
logAnalyticsWorkspaceResourceID: workspaceId
174+
}
175+
}
176+
azureKeyvaultSecretsProvider: {
177+
enabled: true
178+
config: {
179+
enableSecretRotation: 'true'
180+
}
181+
}
182+
}
183+
}
184+
}
185+
```
186+
38187
### Configure with Azure CLI
39188

40189
```bash
41190
az aks update -n '<name>' -g '<resource_group>' --api-server-authorized-ip-ranges '0.0.0.0/32'
42191
```
43192

193+
### Configure with Azure PowerShell
194+
195+
```powershell
196+
Set-AzAksCluster -Name '<name>' -ResourceGroupName '<resource_group>' -ApiServerAccessAuthorizedIpRange '0.0.0.0/32'
197+
```
198+
44199
## LINKS
45200

46-
- [Network security](https://learn.microsoft.com/azure/architecture/framework/security/design-network)
47-
- [Secure access to the API server using authorized IP address ranges in Azure Kubernetes Service (AKS)](https://docs.microsoft.com/azure/aks/api-server-authorized-ip-ranges)
48-
- [Best practices for cluster security and upgrades in Azure Kubernetes Service (AKS)](https://docs.microsoft.com/azure/aks/operator-best-practices-cluster-security#secure-access-to-the-api-server-and-cluster-nodes)
49-
- [Azure deployment reference](https://docs.microsoft.com/azure/templates/microsoft.containerservice/managedclusters)
201+
- [SE:06 Network controls](https://learn.microsoft.com/azure/well-architected/security/networking)
202+
- [Secure access to the API server using authorized IP address ranges in Azure Kubernetes Service (AKS)](https://learn.microsoft.com/azure/aks/api-server-authorized-ip-ranges)
203+
- [Best practices for cluster security and upgrades in Azure Kubernetes Service (AKS)](https://learn.microsoft.com/azure/aks/operator-best-practices-cluster-security#secure-access-to-the-api-server-and-cluster-nodes)
204+
- [Azure deployment reference](https://learn.microsoft.com/azure/templates/microsoft.containerservice/managedclusters)

‎docs/examples-aks.bicep

+98-2
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ resource identity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31'
130130
}
131131

132132
// An example AKS cluster
133-
resource cluster 'Microsoft.ContainerService/managedClusters@2023-07-01' = {
133+
resource cluster 'Microsoft.ContainerService/managedClusters@2023-11-01' = {
134134
location: location
135135
name: name
136136
identity: {
@@ -158,6 +158,11 @@ resource cluster 'Microsoft.ContainerService/managedClusters@2023-07-01' = {
158158
serviceCidr: serviceCidr
159159
dnsServiceIP: dnsServiceIP
160160
}
161+
apiServerAccessProfile: {
162+
authorizedIPRanges: [
163+
'0.0.0.0/32'
164+
]
165+
}
161166
autoUpgradeProfile: {
162167
upgradeChannel: 'stable'
163168
}
@@ -185,7 +190,7 @@ resource cluster 'Microsoft.ContainerService/managedClusters@2023-07-01' = {
185190
}
186191

187192
// An example AKS cluster with pools defined.
188-
resource clusterWithPools 'Microsoft.ContainerService/managedClusters@2023-07-01' = {
193+
resource clusterWithPools 'Microsoft.ContainerService/managedClusters@2023-11-01' = {
189194
location: location
190195
name: name
191196
identity: {
@@ -240,6 +245,97 @@ resource clusterWithPools 'Microsoft.ContainerService/managedClusters@2023-07-01
240245
serviceCidr: serviceCidr
241246
dnsServiceIP: dnsServiceIP
242247
}
248+
apiServerAccessProfile: {
249+
authorizedIPRanges: [
250+
'0.0.0.0/32'
251+
]
252+
}
253+
autoUpgradeProfile: {
254+
upgradeChannel: 'stable'
255+
}
256+
oidcIssuerProfile: {
257+
enabled: true
258+
}
259+
addonProfiles: {
260+
azurepolicy: {
261+
enabled: true
262+
}
263+
omsagent: {
264+
enabled: true
265+
config: {
266+
logAnalyticsWorkspaceResourceID: workspaceId
267+
}
268+
}
269+
azureKeyvaultSecretsProvider: {
270+
enabled: true
271+
config: {
272+
enableSecretRotation: 'true'
273+
}
274+
}
275+
}
276+
}
277+
}
278+
279+
// An example private AKS cluster with pools defined.
280+
resource privateCluster 'Microsoft.ContainerService/managedClusters@2023-11-01' = {
281+
location: location
282+
name: name
283+
identity: {
284+
type: 'UserAssigned'
285+
userAssignedIdentities: {
286+
'${identity.id}': {}
287+
}
288+
}
289+
properties: {
290+
kubernetesVersion: kubernetesVersion
291+
disableLocalAccounts: true
292+
enableRBAC: true
293+
dnsPrefix: dnsPrefix
294+
agentPoolProfiles: [
295+
{
296+
name: 'system'
297+
osDiskSizeGB: 0
298+
minCount: 3
299+
maxCount: 5
300+
enableAutoScaling: true
301+
maxPods: 50
302+
vmSize: 'Standard_D4s_v5'
303+
type: 'VirtualMachineScaleSets'
304+
vnetSubnetID: clusterSubnetId
305+
mode: 'System'
306+
osDiskType: 'Ephemeral'
307+
}
308+
{
309+
name: 'user'
310+
osDiskSizeGB: 0
311+
minCount: 3
312+
maxCount: 20
313+
enableAutoScaling: true
314+
maxPods: 50
315+
vmSize: 'Standard_D4s_v5'
316+
type: 'VirtualMachineScaleSets'
317+
vnetSubnetID: clusterSubnetId
318+
mode: 'User'
319+
osDiskType: 'Ephemeral'
320+
}
321+
]
322+
aadProfile: {
323+
managed: true
324+
enableAzureRBAC: true
325+
adminGroupObjectIDs: clusterAdmins
326+
tenantID: subscription().tenantId
327+
}
328+
networkProfile: {
329+
networkPlugin: 'azure'
330+
networkPolicy: 'azure'
331+
loadBalancerSku: 'standard'
332+
serviceCidr: serviceCidr
333+
dnsServiceIP: dnsServiceIP
334+
}
335+
apiServerAccessProfile: {
336+
enablePrivateCluster: true
337+
enablePrivateClusterPublicFQDN: false
338+
}
243339
autoUpgradeProfile: {
244340
upgradeChannel: 'stable'
245341
}

0 commit comments

Comments
 (0)