Skip to content

Commit a109f23

Browse files
mriccellmriccell
authored
Merge pull request #146 from galiacheng/private-vnet-support
"Bring your own vnet" for Application Gateway on AKS offer
2 parents dd1d178 + e0cc918 commit a109f23

27 files changed

+562
-209
lines changed

‎.github/workflows/testWlsAksWithDependencyCreation.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ on:
2121
types: [aks-integration-test-with-dependency-creation,integration-test-all]
2222

2323
env:
24-
refArmttk: 7dc2c2a7822c2825ea3524ac2af72e561847fece
25-
refJavaee: 13fe6ec487024eb61355d661ab5700ae90cb0a8f
24+
refArmttk: 500a3a612ef51d4421c114183f97b425c9bfaab8
25+
refJavaee: f25ab89a2a8848da39b84e5d6c927f4c4cb47200
2626
azCliVersion: 2.30.0
2727
azureCredentials: ${{ secrets.AZURE_CREDENTIALS }}
2828
location: eastus

‎.github/workflows/testWlsAksWithoutDependencyCreation.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ on:
3636
types: [aks-integration-test-without-dependency-creation,integration-test-all]
3737

3838
env:
39-
refArmttk: 7dc2c2a7822c2825ea3524ac2af72e561847fece
40-
refJavaee: 13fe6ec487024eb61355d661ab5700ae90cb0a8f
39+
refArmttk: 500a3a612ef51d4421c114183f97b425c9bfaab8
40+
refJavaee: f25ab89a2a8848da39b84e5d6c927f4c4cb47200
4141
azCliVersion: 2.30.0
4242
azureCredentials: ${{ secrets.AZURE_CREDENTIALS }}
4343
location: eastus

‎weblogic-azure-aks/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
<groupId>com.oracle.weblogic.azure</groupId>
1919
<artifactId>wls-on-aks-azure-marketplace</artifactId>
20-
<version>1.0.37</version>
20+
<version>1.0.38</version>
2121

2222
<parent>
2323
<groupId>com.microsoft.azure.iaas</groupId>

‎weblogic-azure-aks/src/main/arm/createUiDefinition.json

Lines changed: 53 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1029,7 +1029,7 @@
10291029
"text": "Enabling a HTTPS (Secure) port for the Administration Console requires you to obtain a valid TLS/SSL certificate. The offer will look for the certificate and other configuration items in the Azure Key Vault specified here.",
10301030
"link": {
10311031
"label": "Learn more",
1032-
"uri": "https://aka.ms/arm-oraclelinux-wls-cluster-app-gateway-key-vault"
1032+
"uri": "https://aka.ms/wls-aks-ssl-key-vault"
10331033
}
10341034
}
10351035
},
@@ -1394,6 +1394,50 @@
13941394
"required": false
13951395
}
13961396
},
1397+
{
1398+
"name": "vnetForApplicationGateway",
1399+
"type": "Microsoft.Network.VirtualNetworkCombo",
1400+
"label": {
1401+
"virtualNetwork": "Virtual network",
1402+
"subnets": "Subnets"
1403+
},
1404+
"toolTip": {
1405+
"virtualNetwork": "Select a virtual network in which to place the Application Gateway.",
1406+
"subnets": "The subnet must be dedicated only for use by the Application Gateway."
1407+
},
1408+
"defaultValue": {
1409+
"name": "[concat('wlsaks-vnet',take(guid(), 8))]",
1410+
"addressPrefixSize": "/24"
1411+
},
1412+
"constraints": {
1413+
"minAddressPrefixSize": "/24"
1414+
},
1415+
"options": {
1416+
"hideExisting": false
1417+
},
1418+
"subnets": {
1419+
"gatewaySubnet": {
1420+
"label": "Subnet",
1421+
"defaultValue": {
1422+
"name": "wls-aks-gateway-subnet",
1423+
"addressPrefixSize": "/24"
1424+
},
1425+
"constraints": {
1426+
"minAddressPrefixSize": "/24",
1427+
"minAddressCount": 38,
1428+
"requireContiguousAddresses": false
1429+
}
1430+
}
1431+
},
1432+
"visible": "[steps('section_appGateway').appgwIngress.enableAppGateway]"
1433+
},
1434+
{
1435+
"name": "appgwUsePrivateIP",
1436+
"type": "Microsoft.Common.CheckBox",
1437+
"label": "Configure frontend IP with private IP address",
1438+
"toolTip": "If checked, expose WebLogic Server with private IP address.",
1439+
"visible": "[steps('section_appGateway').appgwIngress.enableAppGateway]"
1440+
},
13971441
{
13981442
"name": "keyVaultText00",
13991443
"type": "Microsoft.Common.TextBlock",
@@ -1426,7 +1470,7 @@
14261470
"text": "    ⁃ Generate a self-signed front-end certificate: Generate a self-signed front-end certificate and apply it during deployment.",
14271471
"link": {
14281472
"label": "Learn more",
1429-
"uri": "https://aka.ms/arm-oraclelinux-wls-cluster-app-gateway-key-vault"
1473+
"uri": "https://aka.ms/wls-aks-application-gateway-ssl-key-vault"
14301474
}
14311475
}
14321476
},
@@ -1576,17 +1620,17 @@
15761620
"visible": "[steps('section_appGateway').appgwIngress.enableAppGateway]",
15771621
"options": {
15781622
"icon": "Info",
1579-
"text": "You must input an Active Directory Service Principal that is encoded with base64 to create the Application Gateway Ingress Controller. See this <a href=https://aka.ms/wls-aks-agic-sp-doc target='_blank'>document</a> for more information.</br>You can generate one with command <b>az ad sp create-for-rbac --role Contributor --sdk-auth | base64 -w0</b>. On macOS omit the <b>-w0</b>."
1623+
"text": "You must input an Active Directory Service Principal that is encoded with base64 to create the Application Gateway Ingress Controller. See this <a href=https://aka.ms/wls-aks-agic-sp-doc target='_blank'>document</a> for more information.</br>You can generate one with command <b>az ad sp create-for-rbac --sdk-auth --role Contributor --scopes /subscriptions/&lt;AZURE_SUBSCRIPTION_ID&gt; | base64 -w0</b>. On macOS omit the <b>-w0</b>."
15801624
}
15811625
},
15821626
{
15831627
"name": "servicePrincipal",
15841628
"type": "Microsoft.Common.PasswordBox",
15851629
"label": {
15861630
"password": "Service Principal",
1587-
"confirmPassword": "Confirm password"
1631+
"confirmPassword": "Confirm service principal"
15881632
},
1589-
"toolTip": "Base64 encoded JSON blob of the service principal. You can generate one with command 'az ad sp create-for-rbac --role Contributor --sdk-auth | base64 -w0' On macOS omit the -w0.",
1633+
"toolTip": "Base64 encoded JSON blob of the service principal. You can generate one with command 'az ad sp create-for-rbac --sdk-auth --role Contributor --scopes /subscriptions/&lt;AZURE_SUBSCRIPTION_ID&gt; | base64 -w0' On macOS omit the -w0.",
15901634
"constraints": {
15911635
"required": true
15921636
},
@@ -2066,6 +2110,7 @@
20662110
"appGatewaySSLCertPassword": "[steps('section_appGateway').appgwIngress.appGatewaySSLCertPassword]",
20672111
"appgwForAdminServer": "[steps('section_appGateway').appgwIngress.appgwForAdminServer]",
20682112
"appgwForRemoteConsole": "[steps('section_appGateway').appgwIngress.appgwForAdminRemote]",
2113+
"appgwUsePrivateIP": "[steps('section_appGateway').appgwIngress.appgwUsePrivateIP]",
20692114
"appPackageUrls": "[steps('section_aks').jeeAppInfo.appPackageUrl]",
20702115
"appReplicas": "[int(steps('section_aks').jeeAppInfo.appReplicas)]",
20712116
"createACR": "[bool(steps('section_aks').imageInfo.oracleCreateACR)]",
@@ -2105,6 +2150,7 @@
21052150
"keyVaultSSLCertDataSecretName": "[steps('section_appGateway').appgwIngress.keyVaultSSLCertDataSecretName]",
21062151
"keyVaultSSLCertPasswordSecretName": "[steps('section_appGateway').appgwIngress.keyVaultSSLCertPasswordSecretName]",
21072152
"managedServerPrefix": "[basics('basicsOptional').managedServerPrefix]",
2153+
"newOrExistingVnetForApplicationGateway": "[steps('section_appGateway').appgwIngress.vnetForApplicationGateway.newOrExisting]",
21082154
"ocrSSOPSW": "[steps('section_aks').imageInfo.ocrSSOPassword]",
21092155
"ocrSSOUser": "[steps('section_aks').imageInfo.ocrSSOUserName]",
21102156
"servicePrincipal": "[steps('section_appGateway').appgwIngress.servicePrincipal]",
@@ -2132,6 +2178,8 @@
21322178
"userProvidedAcr": "[last(split(steps('section_aks').imageInfo.userProvidedAcrSelector.id, '/'))]",
21332179
"userProvidedImagePath": "[steps('section_aks').imageInfo.userProvidedImagePath]",
21342180
"validateApplications": "[bool(steps('section_aks').jeeAppInfo.validateApplications)]",
2181+
"vnetForApplicationGateway": "[steps('section_appGateway').appgwIngress.vnetForApplicationGateway]",
2182+
"vnetRGNameForApplicationGateway": "[steps('section_appGateway').appgwIngress.vnetForApplicationGateway.resourceGroup]",
21352183
"wdtRuntimePassword": "[basics('basicsRequired').wdtRuntimePassword]",
21362184
"wlsClusterSize": "[basics('basicsOptional').wlsClusterSize]",
21372185
"wlsDomainName": "[basics('basicsOptional').wlsDomainName]",

‎weblogic-azure-aks/src/main/arm/scripts/appgw-helm-config.yaml.template

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ appgw:
1515
subscriptionId: @SUB_ID@
1616
resourceGroup: @APPGW_RG_NAME@
1717
name: @APPGW_NAME@
18-
usePrivateIP: false
18+
usePrivateIP: @USE_PRIVATE_IP@
1919

2020
# Setting appgw.shared to "true" will create an AzureIngressProhibitedTarget CRD.
2121
# This prohibits AGIC from applying config for any host/path.

‎weblogic-azure-aks/src/main/arm/scripts/common.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@
33
# This script runs on Azure Container Instance with Alpine Linux that Azure Deployment script creates.
44

55
export checkPodStatusInterval=20 # interval of checking pod status.
6-
export checkPodStatusMaxAttemps=30 # max attempt to check pod status.
6+
export checkPodStatusMaxAttemps=50 # max attempt to check pod status.
77
export checkPVStateInterval=5 # interval of checking pvc status.
88
export checkPVStateMaxAttempt=10 # max attempt to check pvc status.
9-
export checkSVCStateMaxAttempt=10
9+
export checkSVCStateMaxAttempt=50
1010
export checkSVCInterval=30 #seconds
1111

1212
export constAdminT3AddressEnvName="T3_TUNNELING_ADMIN_ADDRESS"

‎weblogic-azure-aks/src/main/arm/scripts/createAppGatewayIngress.sh

Lines changed: 45 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -350,31 +350,52 @@ function network_peers_aks_appgw() {
350350

351351
aksNetWorkId=$(az resource list -g ${aksMCRGName} --resource-type Microsoft.Network/virtualNetworks -o tsv --query '[*].id')
352352
aksNetworkName=$(az resource list -g ${aksMCRGName} --resource-type Microsoft.Network/virtualNetworks -o tsv --query '[*].name')
353-
az network vnet peering create \
354-
--name aks-appgw-peer \
355-
--remote-vnet ${aksNetWorkId} \
356-
--resource-group ${curRGName} \
357-
--vnet-name ${vnetName} \
358-
--allow-vnet-access
359-
utility_validate_status "Create network peers for $aksNetWorkId and ${vnetName}."
360-
361-
appgwNetworkId=$(az resource list -g ${curRGName} --name ${vnetName} -o tsv --query '[*].id')
362-
az network vnet peering create \
363-
--name aks-appgw-peer \
364-
--remote-vnet ${appgwNetworkId} \
365-
--resource-group ${aksMCRGName} \
366-
--vnet-name ${aksNetworkName} \
367-
--allow-vnet-access
368-
369-
utility_validate_status "Create network peers for $aksNetWorkId and ${vnetName}."
353+
appGatewaySubnetId=$(az network application-gateway show -g ${curRGName} --name ${appgwName} -o tsv --query "gatewayIpConfigurations[0].subnet.id")
354+
appGatewayVnetResourceGroup=$(az network application-gateway show -g ${curRGName} --name ${appgwName} -o tsv --query "gatewayIpConfigurations[0].subnet.resourceGroup")
355+
appGatewaySubnetName=$(az resource show --ids ${appGatewaySubnetId} --query "name" -o tsv)
356+
appgwNetworkId=$(echo $appGatewaySubnetId | sed s/"\/subnets\/${appGatewaySubnetName}"//)
357+
appgwVnetName=$(az resource show --ids ${appgwNetworkId} --query "name" -o tsv)
358+
359+
local toPeer=true
360+
# if the AKS and App Gateway have the same VNET, need not peer.
361+
if [ "${aksNetWorkId}" == "${appgwNetworkId}" ]; then
362+
echo_stdout "AKS and Application Gateway are in the same virtual network: ${appgwNetworkId}."
363+
toPeer=false
364+
fi
365+
366+
# check if the Vnets have been peered.
367+
local ret=$(az network vnet peering list \
368+
--resource-group ${appGatewayVnetResourceGroup} \
369+
--vnet-name ${appgwVnetName} -o json \
370+
| jq ".[] | select(.remoteVirtualNetwork.id==\"${aksNetWorkId}\")")
371+
if [ -n "$ret" ]; then
372+
echo_stdout "VNET of AKS ${aksNetWorkId} and Application Gateway ${appgwNetworkId} is peering."
373+
toPeer=false
374+
fi
375+
376+
if [ "${toPeer}" == "true" ]; then
377+
az network vnet peering create \
378+
--name aks-appgw-peer \
379+
--remote-vnet ${aksNetWorkId} \
380+
--resource-group ${appGatewayVnetResourceGroup} \
381+
--vnet-name ${appgwVnetName} \
382+
--allow-vnet-access
383+
utility_validate_status "Create network peers for $aksNetWorkId and ${appgwNetworkId}."
384+
385+
az network vnet peering create \
386+
--name aks-appgw-peer \
387+
--remote-vnet ${appgwNetworkId} \
388+
--resource-group ${aksMCRGName} \
389+
--vnet-name ${aksNetworkName} \
390+
--allow-vnet-access
391+
392+
utility_validate_status "Complete creating network peers for $aksNetWorkId and ${appgwNetworkId}."
393+
fi
370394

371395
# For Kbectl network plugin: https://azure.github.io/application-gateway-kubernetes-ingress/how-tos/networking/#with-kubenet
372396
# find route table used by aks cluster
373397
routeTableId=$(az network route-table list -g $aksMCRGName --query "[].id | [0]" -o tsv)
374398

375-
# get the application gateway's subnet
376-
appGatewaySubnetId=$(az network application-gateway show -n $appgwName -g $curRGName -o tsv --query "gatewayIpConfigurations[0].subnet.id")
377-
378399
# associate the route table to Application Gateway's subnet
379400
az network vnet subnet update \
380401
--ids $appGatewaySubnetId \
@@ -411,6 +432,7 @@ function install_azure_ingress() {
411432
sed -i -e "s:@APPGW_NAME@:${appgwName}:g" ${customAppgwHelmConfig}
412433
sed -i -e "s:@WATCH_NAMESPACE@:${wlsDomainNS}:g" ${customAppgwHelmConfig}
413434
sed -i -e "s:@SP_ENCODING_CREDENTIALS@:${spBase64String}:g" ${customAppgwHelmConfig}
435+
sed -i -e "s:@USE_PRIVATE_IP@:${appgwUsePrivateIP,,}:g" ${customAppgwHelmConfig}
414436

415437
helm install ingress-azure \
416438
-f ${customAppgwHelmConfig} \
@@ -542,7 +564,7 @@ function appgw_ingress_svc_for_admin_server() {
542564
generate_appgw_admin_config_file
543565
kubectl apply -f ${adminAppgwIngressYamlPath}
544566
utility_validate_status "Create appgw ingress svc."
545-
utility_waitfor_lb_svc_completed \
567+
utility_waitfor_ingress_completed \
546568
${adminIngressName} \
547569
${wlsDomainNS} \
548570
${checkSVCStateMaxAttempt} \
@@ -556,7 +578,7 @@ function appgw_ingress_svc_for_remote_console() {
556578

557579
kubectl apply -f ${adminRemoteAppgwIngressYamlPath}
558580
utility_validate_status "Create appgw ingress svc."
559-
utility_waitfor_lb_svc_completed \
581+
utility_waitfor_ingress_completed \
560582
${adminRemoteIngressName} \
561583
${wlsDomainNS} \
562584
${checkSVCStateMaxAttempt} \
@@ -631,7 +653,7 @@ wlsDomainUID=$3
631653
subID=$4
632654
curRGName=$5
633655
appgwName=$6
634-
vnetName=$7
656+
appgwUsePrivateIP=$7
635657
appgwForAdminServer=$8
636658
enableCustomDNSAlias=$9
637659
dnsRGName=${10}

‎weblogic-azure-aks/src/main/arm/scripts/inline-scripts/queryPrivateIPForAppGateway.sh

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Copyright (c) 2022, Oracle Corporation and/or its affiliates.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
3+
# This script runs on Azure Container Instance with Alpine Linux that Azure Deployment script creates.
4+
#
5+
# env inputs:
6+
# SUBNET_ID
7+
# KNOWN_IP
8+
9+
function query_ip() {
10+
echo_stdout "Subnet Id: ${SUBNET_ID}"
11+
12+
# select a available private IP
13+
# azure reserves the first 3 private IPs.
14+
local ret=$(az network vnet check-ip-address \
15+
--ids ${SUBNET_ID} \
16+
--ip-address ${KNOWN_IP})
17+
local available=$(echo ${ret} | jq -r .available)
18+
if [[ "${available,,}" == "true" ]]; then
19+
outputPrivateIP=${KNOWN_IP}
20+
else
21+
local privateIPAddress=$(echo ${ret} | jq -r .availableIpAddresses[0])
22+
if [[ -z "${privateIPAddress}" ]] || [[ "${privateIPAddress}"=="null" ]]; then
23+
echo_stderr "ERROR: make sure there is available IP for application gateway in your subnet."
24+
fi
25+
26+
outputPrivateIP=${privateIPAddress}
27+
fi
28+
}
29+
30+
function output_result() {
31+
echo "Available Private IP: ${outputPrivateIP}"
32+
result=$(jq -n -c \
33+
--arg privateIP "$outputPrivateIP" \
34+
'{privateIP: $privateIP}')
35+
echo "result is: $result"
36+
echo $result >$AZ_SCRIPTS_OUTPUT_PATH
37+
}
38+
39+
# main script
40+
outputPrivateIP="10.0.0.1"
41+
42+
query_ip
43+
44+
output_result

0 commit comments

Comments
 (0)