From 586a2b6e1178eb801194de54d46d40ca194ae68a Mon Sep 17 00:00:00 2001 From: Andrei Ilas Date: Fri, 23 May 2025 18:44:26 +0300 Subject: [PATCH] add support for compute-cluster --- docs/src/guide/cluster.md | 1 + docs/src/guide/network_subnets.md | 6 + .../vars-cluster-addons.auto.tfvars | 10 +- .../cluster/vars-cluster-basic.auto.tfvars | 4 +- .../cluster/vars-cluster-enhanced.auto.tfvars | 7 +- ...ars-extensions-service-account.auto.tfvars | 18 +- examples/istio-mc/README.md | 2 +- examples/istio-mc/terraform.tfvars.example | 2 +- examples/istio-mc/variables.tf | 2 +- examples/network/vars-network.auto.tfvars | 7 +- .../rms/oke-cluster-only/variables-cluster.tf | 2 +- examples/rms/oke-workers-only/variables.tf | 2 +- .../workers/vars-workers-instance.auto.tfvars | 2 +- .../vars-workers-instancepool.auto.tfvars | 16 +- module-cluster.tf | 2 + module-extensions.tf | 28 +-- module-iam.tf | 10 +- module-network.tf | 68 ++++--- module-workers.tf | 7 +- modules/bastion/cloudinit.tf | 2 +- modules/bastion/variables.tf | 2 +- modules/cluster/cluster.tf | 9 +- modules/cluster/variables.tf | 1 + modules/extensions/cilium.tf | 24 +-- modules/extensions/service_account.tf | 24 +-- modules/iam/policy.tf | 14 ++ modules/iam/variables.tf | 3 +- modules/iam/versions.tf | 4 +- modules/network/locals.tf | 11 +- modules/network/nsg-controlplane.tf | 8 + modules/network/nsg-loadbalancers-int.tf | 5 + modules/network/nsg-loadbalancers-pub.tf | 5 + modules/network/nsg-operator.tf | 22 ++- modules/network/nsg-pods.tf | 25 ++- modules/network/nsg-workers.tf | 28 ++- modules/network/rules.tf | 116 +++++------ modules/network/subnets.tf | 91 +++++++-- modules/network/variables.tf | 8 + modules/operator/cloudinit.tf | 2 +- modules/operator/variables.tf | 2 +- modules/operator/versions.tf | 4 +- modules/workers/computecluster.tf | 181 ++++++++++++++++++ modules/workers/instance.tf | 10 +- modules/workers/instanceconfig.tf | 10 +- modules/workers/instancepools.tf | 4 +- modules/workers/locals.tf | 40 ++-- modules/workers/nodepools.tf | 4 +- modules/workers/variables.tf | 10 + modules/workers/versions.tf | 2 +- variables-cluster.tf | 6 + variables-extensions.tf | 8 +- variables-network.tf | 53 +++-- variables-workers.tf | 10 + 53 files changed, 700 insertions(+), 244 deletions(-) create mode 100644 modules/workers/computecluster.tf diff --git a/docs/src/guide/cluster.md b/docs/src/guide/cluster.md index 8a350d08..aaf018b9 100644 --- a/docs/src/guide/cluster.md +++ b/docs/src/guide/cluster.md @@ -12,6 +12,7 @@ The OKE parameters concern mainly the following: * number of node pools and their respective size of the cluster * services and pods cidr blocks * whether to use encryption +* whether you want to enable [dual-stack](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/conteng_ipv4-and-ipv6.htm): IPv4 & IPv6 ```admonish notice If you need to change the default services and pods' CIDRs, note the following: diff --git a/docs/src/guide/network_subnets.md b/docs/src/guide/network_subnets.md index d57e9c4a..84ee6058 100644 --- a/docs/src/guide/network_subnets.md +++ b/docs/src/guide/network_subnets.md @@ -27,6 +27,12 @@ Subnets are created for core components managed within the module, namely: {{#include ../../../examples/network/vars-network-subnets-create-cidr.auto.tfvars:4:}} ``` +## Create new subnets with IPv4 and IPv6 (CIDR notation) + +```javascript +{{#include ../../../examples/network/vars-network-subnets-create-cidr-ipv4-and-ipv6.tfvars:4:}} +``` + ## Use existing subnets ```javascript diff --git a/examples/cluster-addons/vars-cluster-addons.auto.tfvars b/examples/cluster-addons/vars-cluster-addons.auto.tfvars index 48bd9f7e..447e0ff5 100644 --- a/examples/cluster-addons/vars-cluster-addons.auto.tfvars +++ b/examples/cluster-addons/vars-cluster-addons.auto.tfvars @@ -4,12 +4,12 @@ cluster_addons = { "CertManager" = { remove_addon_resources_on_delete = true - override_existing = true # Default is false if not specified + override_existing = true # Default is false if not specified # The list of supported configurations for the cluster addons is here: https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringclusteraddons-configurationarguments.htm#contengconfiguringclusteraddons-configurationarguments_CertificateManager configurations = [ { - key = "numOfReplicas" - value = "1" + key = "numOfReplicas" + value = "1" } ] } @@ -20,7 +20,7 @@ cluster_addons = { # Prevent Flannel pods from being scheduled using a non-existing label as nodeSelector "Flannel" = { remove_addon_resources_on_delete = true - override_existing = true # Override the existing configuration with this one, if Flannel addon in already enabled + override_existing = true # Override the existing configuration with this one, if Flannel addon in already enabled configurations = [ { key = "nodeSelectors" @@ -31,7 +31,7 @@ cluster_addons = { # Prevent Kube-Proxy pods from being scheduled using a non-existing label as nodeSelector "KubeProxy" = { remove_addon_resources_on_delete = true - override_existing = true # Override the existing configuration with this one, if KubeProxy addon in already enabled + override_existing = true # Override the existing configuration with this one, if KubeProxy addon in already enabled configurations = [ { key = "nodeSelectors" diff --git a/examples/cluster/vars-cluster-basic.auto.tfvars b/examples/cluster/vars-cluster-basic.auto.tfvars index 22503303..bfdba0ea 100644 --- a/examples/cluster/vars-cluster-basic.auto.tfvars +++ b/examples/cluster/vars-cluster-basic.auto.tfvars @@ -1,5 +1,5 @@ -# Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. +# Copyright (c) 2017, 2025 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl cluster_name = "oke-example" -kubernetes_version = "v1.31.1" +kubernetes_version = "v1.32.1" diff --git a/examples/cluster/vars-cluster-enhanced.auto.tfvars b/examples/cluster/vars-cluster-enhanced.auto.tfvars index 88acac27..5361f558 100644 --- a/examples/cluster/vars-cluster-enhanced.auto.tfvars +++ b/examples/cluster/vars-cluster-enhanced.auto.tfvars @@ -1,4 +1,4 @@ -# Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. +# Copyright (c) 2017, 2025 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl create_cluster = true // *true/false @@ -7,9 +7,10 @@ cluster_kms_key_id = null cluster_name = "oke" cluster_type = "enhanced" // *basic/enhanced cni_type = "flannel" // *flannel/npn -assign_public_ip_to_control_plane = true // true/*false +assign_public_ip_to_control_plane = true // true/*false image_signing_keys = [] -kubernetes_version = "v1.31.1" +kubernetes_version = "v1.32.1" pods_cidr = "10.244.0.0/16" services_cidr = "10.96.0.0/16" use_signed_images = false // true/*false +enable_ipv6 = false //true/*false diff --git a/examples/extensions/vars-extensions-service-account.auto.tfvars b/examples/extensions/vars-extensions-service-account.auto.tfvars index c55ce5a9..0038ab1f 100644 --- a/examples/extensions/vars-extensions-service-account.auto.tfvars +++ b/examples/extensions/vars-extensions-service-account.auto.tfvars @@ -2,26 +2,26 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl create_service_account = true -service_accounts = { +service_accounts = { # Example to create a cluster role binding using a cluster role. example_cluster_role_binding = { - sa_name = "sa1" - sa_namespace = "kube-system" - sa_cluster_role = "cluster-admin" + sa_name = "sa1" + sa_namespace = "kube-system" + sa_cluster_role = "cluster-admin" sa_cluster_role_binding = "sa1-crb" } # Example to create a role binding using a cluster role. example_role_binding = { - sa_name = "sa2" - sa_namespace = "default" + sa_name = "sa2" + sa_namespace = "default" sa_cluster_role = "cluster-admin" sa_role_binding = "sa1-rb" } # Example to create a role binding using a role, the role needs to exist within the namespace. example_role_binding = { - sa_name = "sa3" - sa_namespace = "kube-system" - sa_role = "system:controller:token-cleaner" + sa_name = "sa3" + sa_namespace = "kube-system" + sa_role = "system:controller:token-cleaner" sa_role_binding = "sa3-rb" } } \ No newline at end of file diff --git a/examples/istio-mc/README.md b/examples/istio-mc/README.md index 8b4fd915..0714147a 100644 --- a/examples/istio-mc/README.md +++ b/examples/istio-mc/README.md @@ -51,7 +51,7 @@ clusters = { 5. Configure additional parameters if necessary: ``` -kubernetes_version = "v1.28.2" +kubernetes_version = "v1.32.1" cluster_type = "basic" diff --git a/examples/istio-mc/terraform.tfvars.example b/examples/istio-mc/terraform.tfvars.example index 363bbc36..583c5fe2 100755 --- a/examples/istio-mc/terraform.tfvars.example +++ b/examples/istio-mc/terraform.tfvars.example @@ -23,7 +23,7 @@ clusters = { c2 = { region = "melbourne", vcn = "10.2.0.0/16", pods = "10.202.0.0/16", services = "10.102.0.0/16", enabled = true } } -kubernetes_version = "v1.28.2" +kubernetes_version = "v1.32.1" cluster_type = "basic" diff --git a/examples/istio-mc/variables.tf b/examples/istio-mc/variables.tf index 0ed7d8cb..4be58889 100644 --- a/examples/istio-mc/variables.tf +++ b/examples/istio-mc/variables.tf @@ -61,7 +61,7 @@ variable "clusters" { } variable "kubernetes_version" { - default = "v1.30.1" + default = "v1.32.1" description = "The version of Kubernetes to use." type = string } diff --git a/examples/network/vars-network.auto.tfvars b/examples/network/vars-network.auto.tfvars index 2d5b7bbd..bc9c68d5 100644 --- a/examples/network/vars-network.auto.tfvars +++ b/examples/network/vars-network.auto.tfvars @@ -12,6 +12,7 @@ vcn_id = null # Ignored if create_vcn = true vcn_cidrs = ["10.0.0.0/16"] # Ignored if create_vcn = false vcn_dns_label = "oke" # Ignored if create_vcn = false vcn_name = "oke" # Ignored if create_vcn = false +enable_ipv6 = false # true/*false # Subnets subnets = { @@ -74,7 +75,8 @@ drg_display_name = "drg" drg_id = null # Routing -ig_route_table_id = null # Optional ID of existing internet gateway route table +ig_route_table_id = null # Optional ID of existing internet gateway route table +internet_gateway_id = null # Optional ID of existing internet gateway internet_gateway_route_rules = [ # { # destination = "192.168.0.0/16" # Route Rule Destination CIDR @@ -84,6 +86,9 @@ internet_gateway_route_rules = [ # }, ] +igw_ngw_mixed_route_id = null # Optional ID of existing mixed route table NAT GW for IPv4 and Internet GW for IPv6 + +nat_gateway_id = null # Optional ID of existing NAT gateway nat_gateway_public_ip_id = "none" nat_route_table_id = null # Optional ID of existing NAT gateway route table nat_gateway_route_rules = [ diff --git a/examples/rms/oke-cluster-only/variables-cluster.tf b/examples/rms/oke-cluster-only/variables-cluster.tf index 3cb55bba..085e63b2 100644 --- a/examples/rms/oke-cluster-only/variables-cluster.tf +++ b/examples/rms/oke-cluster-only/variables-cluster.tf @@ -25,7 +25,7 @@ variable "services_cidr" { default = "10.96.0.0/16" type = string } -variable "kubernetes_version" { default = "v1.26.2" } +variable "kubernetes_version" { default = "v1.32.1" } variable "cluster_kms_vault_id" { default = null diff --git a/examples/rms/oke-workers-only/variables.tf b/examples/rms/oke-workers-only/variables.tf index 39ee0c2d..7996e26b 100644 --- a/examples/rms/oke-workers-only/variables.tf +++ b/examples/rms/oke-workers-only/variables.tf @@ -32,7 +32,7 @@ variable "cluster_id" { } variable "cni_type" { default = "Flannel" } variable "kubernetes_version" { - default = "v1.26.2" + default = "v1.32.1" type = string } diff --git a/examples/workers/vars-workers-instance.auto.tfvars b/examples/workers/vars-workers-instance.auto.tfvars index a800d225..13cf2a50 100644 --- a/examples/workers/vars-workers-instance.auto.tfvars +++ b/examples/workers/vars-workers-instance.auto.tfvars @@ -18,6 +18,6 @@ worker_pools = { description = "Self-managed Instance With Bursting", mode = "instance", size = 1, - burst = "BASELINE_1_8", # Valid values BASELINE_1_8,BASELINE_1_2 + burst = "BASELINE_1_8", # Valid values BASELINE_1_8,BASELINE_1_2 }, } diff --git a/examples/workers/vars-workers-instancepool.auto.tfvars b/examples/workers/vars-workers-instancepool.auto.tfvars index 3615ce8e..a6489570 100644 --- a/examples/workers/vars-workers-instancepool.auto.tfvars +++ b/examples/workers/vars-workers-instancepool.auto.tfvars @@ -18,19 +18,19 @@ worker_pools = { description = "Self-managed Instance Pool With Bursting", mode = "instance-pool", size = 1, - burst = "BASELINE_1_8", # Valid values BASELINE_1_8,BASELINE_1_2 + burst = "BASELINE_1_8", # Valid values BASELINE_1_8,BASELINE_1_2 }, oke-vm-instance-pool-with-block-volume = { - description = "Self-managed Instance Pool with block volume", - mode = "instance-pool", - size = 1, - disable_block_volume = false, + description = "Self-managed Instance Pool with block volume", + mode = "instance-pool", + size = 1, + disable_block_volume = false, block_volume_size_in_gbs = 60, }, oke-vm-instance-pool-without-block-volume = { - description = "Self-managed Instance Pool without block volume", - mode = "instance-pool", - size = 1, + description = "Self-managed Instance Pool without block volume", + mode = "instance-pool", + size = 1, disable_block_volume = true, }, } diff --git a/module-cluster.tf b/module-cluster.tf index 8d68f99c..4f7433ff 100644 --- a/module-cluster.tf +++ b/module-cluster.tf @@ -60,6 +60,7 @@ module "cluster" { vcn_id = local.vcn_id cni_type = var.cni_type control_plane_is_public = var.control_plane_is_public + enable_ipv6 = var.enable_ipv6 assign_public_ip_to_control_plane = var.assign_public_ip_to_control_plane control_plane_nsg_ids = compact(flatten([var.control_plane_nsg_ids, try(module.network.control_plane_nsg_id, null)])) control_plane_subnet_id = try(module.network.control_plane_subnet_id, "") # safe destroy; validated in submodule @@ -70,6 +71,7 @@ module "cluster" { : try(module.network.int_lb_subnet_id, "") ) + # Cluster cluster_kms_key_id = var.cluster_kms_key_id cluster_name = local.cluster_name diff --git a/module-extensions.tf b/module-extensions.tf index afb7f30f..c9abc1af 100644 --- a/module-extensions.tf +++ b/module-extensions.tf @@ -2,26 +2,26 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { - cluster_private_endpoint = ( var.create_cluster ? - coalesce(split(":", lookup(one(module.cluster[*].endpoints), "private_endpoint", ""))...) : - ( length(local.existing_cluster_endpoints) > 0 ? - coalesce(split(":", lookup(local.existing_cluster_endpoints, "private_endpoint", ""))...): + cluster_private_endpoint = (var.create_cluster ? + coalesce(split(":", lookup(one(module.cluster[*].endpoints), "private_endpoint", ""))...) : + (length(local.existing_cluster_endpoints) > 0 ? + coalesce(split(":", lookup(local.existing_cluster_endpoints, "private_endpoint", ""))...) : null ) ) } module "extensions" { - source = "./modules/extensions" - depends_on = [ module.network ] - count = alltrue([var.create_cluster, local.operator_enabled]) ? 1 : 0 - region = var.region - state_id = local.state_id + source = "./modules/extensions" + depends_on = [module.network] + count = alltrue([var.create_cluster, local.operator_enabled]) ? 1 : 0 + region = var.region + state_id = local.state_id # Cluster - kubernetes_version = var.kubernetes_version - expected_node_count = local.worker_count_expected - worker_pools = one(module.workers[*].worker_pools) + kubernetes_version = var.kubernetes_version + expected_node_count = local.worker_count_expected + worker_pools = one(module.workers[*].worker_pools) cluster_private_endpoint = local.cluster_private_endpoint # Bastion/operator connection @@ -120,6 +120,6 @@ module "extensions" { mpi_operator_version = var.mpi_operator_version # Service Account - create_service_account = var.create_service_account - service_accounts = var.service_accounts + create_service_account = var.create_service_account + service_accounts = var.service_accounts } diff --git a/module-iam.tf b/module-iam.tf index a998b49e..8d8640fd 100644 --- a/module-iam.tf +++ b/module-iam.tf @@ -80,6 +80,9 @@ module "iam_cluster_prerequisites" { autoscaler_compartments = [] worker_compartments = [] + enable_ipv6 = false + network_compartment_id = var.network_compartment_id + providers = { oci.home = oci.home } @@ -113,6 +116,9 @@ module "iam" { autoscaler_compartments = local.autoscaler_compartments worker_compartments = local.worker_compartments + enable_ipv6 = var.enable_ipv6 + network_compartment_id = var.network_compartment_id + providers = { oci.home = oci.home } @@ -125,7 +131,7 @@ output "availability_domains" { output "dynamic_group_ids" { description = "Cluster IAM dynamic group IDs" - value = concat( + value = concat( coalesce(module.iam_cluster_prerequisites.dynamic_group_ids, []), coalesce(module.iam.dynamic_group_ids, []) ) @@ -133,7 +139,7 @@ output "dynamic_group_ids" { output "policy_statements" { description = "Cluster IAM policy statements" - value = concat( + value = concat( coalesce(module.iam_cluster_prerequisites.policy_statements, []), coalesce(module.iam.policy_statements, []) ) diff --git a/module-network.tf b/module-network.tf index 5bef8335..dcb243ed 100644 --- a/module-network.tf +++ b/module-network.tf @@ -14,12 +14,36 @@ locals { vcn_lookup = coalesce(one(data.oci_core_vcn.oke[*].cidr_blocks), []) vcn_lookup_cidr_blocks = flatten(local.vcn_lookup) vcn_cidrs = var.create_vcn ? var.vcn_cidrs : local.vcn_lookup_cidr_blocks - + vcn_ipv6_cidr = var.create_vcn ? one(try(coalescelist(module.vcn[0].vcn_all_attributes["ipv6cidr_blocks"], module.vcn[0].vcn_all_attributes["byoipv6cidr_blocks"], module.vcn[0].vcn_all_attributes["ipv6private_cidr_blocks"]), [])) : one(try(coalescelist(data.oci_core_vcn.oke[0].ipv6cidr_blocks, data.oci_core_vcn.oke[0].byoipv6cidr_blocks, data.oci_core_vcn.oke[0].ipv6private_cidr_blocks), [])) # Created route table if enabled, else var.ig_route_table_id ig_route_table_id = var.create_vcn ? try(one(module.vcn[*].ig_route_id), var.ig_route_table_id) : var.ig_route_table_id # Created route table if enabled, else var.nat_route_table_id nat_route_table_id = var.create_vcn ? try(one(module.vcn[*].nat_route_id), var.ig_route_table_id) : var.nat_route_table_id + + create_internet_gateway = alltrue([ + var.vcn_create_internet_gateway != "never", # always disable + anytrue([ # enable for configurations that generally utilize it + var.vcn_create_internet_gateway == "always", # always enable + var.create_bastion && var.bastion_is_public, # enable for public bastion + var.control_plane_is_public, # enable for cluster w/ public endpoint + var.load_balancers != "internal", # enable for cluster w/ public load balancers + ]) + ]) + + create_nat_gateway = alltrue([ + var.vcn_create_nat_gateway != "never", # always disable + anytrue([ # enable for configurations that generally utilize it + var.vcn_create_nat_gateway == "always", # always enable + !var.worker_is_public, # enable for private workers + var.create_operator, # enable for operator + !var.control_plane_is_public, # enable for cluster w/ private endpoint + contains(["internal", "both"], var.load_balancers), # enable for cluster w/ private load balancers + ]) + ]) + + internet_gateway_id = var.create_vcn ? try(one(module.vcn[*].internet_gateway_id), var.internet_gateway_id) : var.internet_gateway_id + nat_gateway_id = var.create_vcn ? try(one(module.vcn[*].nat_gateway_id), var.nat_gateway_id) : var.nat_gateway_id } module "vcn" { @@ -45,26 +69,9 @@ module "vcn" { attached_drg_id = var.drg_id != null ? var.drg_id : (tobool(var.create_drg) ? module.drg[0].drg_id : null) - create_internet_gateway = alltrue([ - var.vcn_create_internet_gateway != "never", # always disable - anytrue([ # enable for configurations that generally utilize it - var.vcn_create_internet_gateway == "always", # always enable - var.create_bastion && var.bastion_is_public, # enable for public bastion - var.control_plane_is_public, # enable for cluster w/ public endpoint - var.load_balancers != "internal", # enable for cluster w/ public load balancers - ]) - ]) + create_internet_gateway = local.create_internet_gateway - create_nat_gateway = alltrue([ - var.vcn_create_nat_gateway != "never", # always disable - anytrue([ # enable for configurations that generally utilize it - var.vcn_create_nat_gateway == "always", # always enable - !var.worker_is_public, # enable for private workers - var.create_operator, # enable for operator - !var.control_plane_is_public, # enable for cluster w/ private endpoint - contains(["internal", "both"], var.load_balancers), # enable for cluster w/ private load balancers - ]) - ]) + create_nat_gateway = local.create_nat_gateway create_service_gateway = var.vcn_create_service_gateway != "never" internet_gateway_route_rules = var.internet_gateway_route_rules @@ -72,9 +79,11 @@ module "vcn" { lockdown_default_seclist = var.lockdown_default_seclist nat_gateway_public_ip_id = var.nat_gateway_public_ip_id nat_gateway_route_rules = var.nat_gateway_route_rules - vcn_cidrs = local.vcn_cidrs - vcn_dns_label = var.assign_dns ? coalesce(var.vcn_dns_label, local.state_id) : null - vcn_name = coalesce(var.vcn_name, "oke-${local.state_id}") + + enable_ipv6 = var.enable_ipv6 + vcn_cidrs = local.vcn_cidrs + vcn_dns_label = var.assign_dns ? coalesce(var.vcn_dns_label, local.state_id) : null + vcn_name = coalesce(var.vcn_name, "oke-${local.state_id}") } module "drg" { @@ -84,8 +93,8 @@ module "drg" { compartment_id = coalesce(var.network_compartment_id, local.compartment_id) drg_compartment_id = var.drg_compartment_id - drg_id = one([var.drg_id]) # existing DRG ID or null - drg_display_name = coalesce(var.drg_display_name, "oke-${local.state_id}") + drg_id = one([var.drg_id]) # existing DRG ID or null + drg_display_name = coalesce(var.drg_display_name, "oke-${local.state_id}") drg_vcn_attachments = tobool(var.create_drg) || var.drg_id != null ? { for k, v in module.vcn : k => { # gets the vcn_id values dynamically from the vcn module vcn_id : v.vcn_id @@ -129,15 +138,22 @@ module "network" { control_plane_is_public = var.control_plane_is_public create_cluster = var.create_cluster create_bastion = var.create_bastion + create_internet_gateway = local.create_internet_gateway + create_nat_gateway = local.create_nat_gateway + enable_ipv6 = var.enable_ipv6 nsgs = var.nsgs create_operator = local.operator_enabled drg_attachments = var.drg_attachments enable_waf = var.enable_waf ig_route_table_id = local.ig_route_table_id + igw_ngw_mixed_route_id = var.igw_ngw_mixed_route_id + internet_gateway_id = local.internet_gateway_id load_balancers = var.load_balancers + nat_gateway_id = local.nat_gateway_id nat_route_table_id = local.nat_route_table_id subnets = var.subnets vcn_cidrs = local.vcn_cidrs + vcn_ipv6_cidr = local.vcn_ipv6_cidr vcn_id = local.vcn_id worker_is_public = var.worker_is_public } @@ -254,4 +270,4 @@ output "network_security_rules" { output "lpg_all_attributes" { description = "all attributes of created lpg" value = try(one(module.vcn[*].lpg_all_attributes), null) -} +} \ No newline at end of file diff --git a/module-workers.tf b/module-workers.tf index 55bc0d57..33fed173 100644 --- a/module-workers.tf +++ b/module-workers.tf @@ -38,6 +38,9 @@ module "workers" { cluster_type = var.cluster_type kubernetes_version = var.kubernetes_version + # Compute clusters + compute_clusters = var.worker_compute_clusters + # Worker pools worker_pool_mode = var.worker_pool_mode worker_pool_size = var.worker_pool_size @@ -59,7 +62,7 @@ module "workers" { indexed_images = local.indexed_images kubeproxy_mode = var.kubeproxy_mode max_pods_per_node = var.max_pods_per_node - node_labels = alltrue([var.cluster_type == "basic", var.cilium_install == true]) ? merge(var.worker_node_labels, {"oci.oraclecloud.com/custom-k8s-networking" = true}) : var.worker_node_labels + node_labels = alltrue([var.cluster_type == "basic", var.cilium_install == true]) ? merge(var.worker_node_labels, { "oci.oraclecloud.com/custom-k8s-networking" = true }) : var.worker_node_labels node_metadata = var.worker_node_metadata agent_config = var.agent_config platform_config = var.platform_config @@ -103,4 +106,4 @@ output "worker_pool_ids" { output "worker_pool_ips" { description = "Created worker instance private IPs by pool for available modes ('node-pool', 'instance')." value = local.worker_count_expected > 0 ? try(one(module.workers[*].worker_pool_ips), null) : null -} +} \ No newline at end of file diff --git a/modules/bastion/cloudinit.tf b/modules/bastion/cloudinit.tf index 1660dc71..dfbe457b 100644 --- a/modules/bastion/cloudinit.tf +++ b/modules/bastion/cloudinit.tf @@ -11,7 +11,7 @@ data "cloudinit_config" "bastion" { # https://cloudinit.readthedocs.io/en/latest/reference/examples.html#run-commands-on-first-boot content = <<-EOT runcmd: - - ${format("dnf config-manager --disable ol%v_addons --disable ol%v_appstream", var.bastion_image_os_version, var.bastion_image_os_version) } + - ${format("dnf config-manager --disable ol%v_addons --disable ol%v_appstream", var.bastion_image_os_version, var.bastion_image_os_version)} EOT } diff --git a/modules/bastion/variables.tf b/modules/bastion/variables.tf index 9ebadecc..79eadcae 100644 --- a/modules/bastion/variables.tf +++ b/modules/bastion/variables.tf @@ -9,7 +9,7 @@ variable "state_id" { type = string } variable "await_cloudinit" { type = string } variable "assign_dns" { type = bool } variable "availability_domain" { type = string } -variable "bastion_image_os_version" {type = string} +variable "bastion_image_os_version" { type = string } variable "image_id" { type = string } variable "is_public" { type = bool } variable "nsg_ids" { type = list(string) } diff --git a/modules/cluster/cluster.tf b/modules/cluster/cluster.tf index 79c3536b..f39e7ffd 100644 --- a/modules/cluster/cluster.tf +++ b/modules/cluster/cluster.tf @@ -61,8 +61,8 @@ resource "oci_containerengine_cluster" "k8s_cluster" { dynamic "required_claims" { for_each = lookup(var.oidc_token_authentication_config, "required_claims", []) content { - key = lookup(required_claims.value, "key") - value = lookup(required_claims.value, "value") + key = lookup(required_claims.value, "key") + value = lookup(required_claims.value, "value") } } configuration_file = lookup(var.oidc_token_authentication_config, "configuration_file", null) @@ -73,6 +73,8 @@ resource "oci_containerengine_cluster" "k8s_cluster" { is_open_id_connect_discovery_enabled = var.oidc_discovery_enabled } + + kubernetes_network_config { pods_cidr = var.pods_cidr services_cidr = var.services_cidr @@ -89,6 +91,7 @@ resource "oci_containerengine_cluster" "k8s_cluster" { } service_lb_subnet_ids = compact([var.service_lb_subnet_id]) + ip_families = var.enable_ipv6 ? ["IPv4", "IPv6"] : ["IPv4"] } timeouts { @@ -96,7 +99,7 @@ resource "oci_containerengine_cluster" "k8s_cluster" { } lifecycle { - ignore_changes = [defined_tags, freeform_tags, cluster_pod_network_options] + ignore_changes = [defined_tags, freeform_tags, cluster_pod_network_options, options["kubernetes_network_config"]] precondition { condition = var.service_lb_subnet_id != null diff --git a/modules/cluster/variables.tf b/modules/cluster/variables.tf index 7135f740..732c31b1 100644 --- a/modules/cluster/variables.tf +++ b/modules/cluster/variables.tf @@ -10,6 +10,7 @@ variable "cluster_kms_key_id" { type = string } variable "cluster_name" { type = string } variable "cluster_type" { type = string } variable "cni_type" { type = string } +variable "enable_ipv6" { type = bool } variable "control_plane_is_public" { type = bool } variable "control_plane_nsg_ids" { type = set(string) } variable "assign_public_ip_to_control_plane" { type = bool } diff --git a/modules/extensions/cilium.tf b/modules/extensions/cilium.tf index 335c5c3a..c53c0a94 100644 --- a/modules/extensions/cilium.tf +++ b/modules/extensions/cilium.tf @@ -11,22 +11,22 @@ locals { cilium_helm_crds = one(data.helm_template.cilium[*].crds) cilium_helm_values_override = one(data.helm_template.cilium[*].values) - - cilium_helm_repository = "https://helm.cilium.io" + + cilium_helm_repository = "https://helm.cilium.io" cilium_vxlan_cni = { - install = true - exclusive = true # !var.multus_install + install = true + exclusive = true # !var.multus_install } cilium_helm_values = { - annotateK8sNode = true - cluster = { + annotateK8sNode = true + cluster = { name = "oke-${var.state_id}" id = 1 } - clustermesh = { - useAPIServer = false + clustermesh = { + useAPIServer = false apiserver = { kvstoremesh = { enabled = false @@ -42,7 +42,7 @@ locals { pmtuDiscovery = { enabled = true } rollOutCiliumPods = true tunnelProtocol = local.cilium_tunnel - + hubble = { metrics = { dashboards = { enabled = var.prometheus_install } @@ -53,7 +53,7 @@ locals { } k8s = { - requireIPv4PodCIDR = true # wait for Kubernetes to provide the PodCIDR (ipam kubernetes) + requireIPv4PodCIDR = true # wait for Kubernetes to provide the PodCIDR (ipam kubernetes) } # Prometheus metrics @@ -66,7 +66,7 @@ locals { } } - # TODO Support Flannel w/ generic-veth & tunnel disabled + # TODO Support Flannel w/ generic-veth & tunnel disabled cilium_tunnel = "vxlan" # var.cni_type == "flannel" ? "disabled" : "vxlan" cilium_flannel_cni = { @@ -76,7 +76,7 @@ locals { customConf = var.cni_type == "flannel" exclusive = !var.multus_install } - + cilium_net_attach_def_conf = { cniVersion = "0.3.1" name = "cilium" diff --git a/modules/extensions/service_account.tf b/modules/extensions/service_account.tf index 050d2068..26d584c2 100644 --- a/modules/extensions/service_account.tf +++ b/modules/extensions/service_account.tf @@ -3,13 +3,13 @@ locals { sa_with_cluster_role_bindings = { - for k, v in var.service_accounts: k => v + for k, v in var.service_accounts : k => v if lookup(v, "sa_cluster_role_binding", null) != null } sa_with_role_bindings = { - for k, v in var.service_accounts: k => v + for k, v in var.service_accounts : k => v if lookup(v, "sa_role_binding", null) != null - } + } } resource "null_resource" "service_account_crb" { @@ -49,7 +49,7 @@ resource "null_resource" "service_account_crb" { } provisioner "remote-exec" { - when = destroy + when = destroy on_failure = continue inline = [ "kubectl delete clusterrolebinding ${self.triggers.service_account_cluster_role_binding}", @@ -72,11 +72,11 @@ resource "null_resource" "service_account_rb" { for_each = var.create_service_account ? local.sa_with_role_bindings : {} triggers = { - service_account_name = each.value.sa_name - service_account_namespace = each.value.sa_namespace - service_account_cluster_role = each.value.sa_cluster_role - service_account_role = lookup(each.value, "sa_role", "") - service_account_role_binding = each.value.sa_role_binding + service_account_name = each.value.sa_name + service_account_namespace = each.value.sa_namespace + service_account_cluster_role = each.value.sa_cluster_role + service_account_role = lookup(each.value, "sa_role", "") + service_account_role_binding = each.value.sa_role_binding # Parameters ignored as triggers in the life_cycle block. Required to establish connections. bastion_host = var.bastion_host @@ -102,13 +102,13 @@ resource "null_resource" "service_account_rb" { "kubectl get ns ${self.triggers.service_account_namespace} || kubectl create ns ${self.triggers.service_account_namespace}", "kubectl create sa -n ${self.triggers.service_account_namespace} ${self.triggers.service_account_name}", self.triggers.service_account_role != "" ? - "kubectl create rolebinding -n ${self.triggers.service_account_namespace} ${self.triggers.service_account_role_binding} --role=${self.triggers.service_account_role} --serviceaccount=${self.triggers.service_account_namespace}:${self.triggers.service_account_name}" : - "kubectl create rolebinding -n ${self.triggers.service_account_namespace} ${self.triggers.service_account_role_binding} --clusterrole=${self.triggers.service_account_cluster_role} --serviceaccount=${self.triggers.service_account_namespace}:${self.triggers.service_account_name}" + "kubectl create rolebinding -n ${self.triggers.service_account_namespace} ${self.triggers.service_account_role_binding} --role=${self.triggers.service_account_role} --serviceaccount=${self.triggers.service_account_namespace}:${self.triggers.service_account_name}" : + "kubectl create rolebinding -n ${self.triggers.service_account_namespace} ${self.triggers.service_account_role_binding} --clusterrole=${self.triggers.service_account_cluster_role} --serviceaccount=${self.triggers.service_account_namespace}:${self.triggers.service_account_name}" ] } provisioner "remote-exec" { - when = destroy + when = destroy on_failure = continue inline = [ "kubectl delete rolebinding -n ${self.triggers.service_account_namespace} ${self.triggers.service_account_role_binding}", diff --git a/modules/iam/policy.tf b/modules/iam/policy.tf index 66463eea..cc8a906c 100644 --- a/modules/iam/policy.tf +++ b/modules/iam/policy.tf @@ -30,3 +30,17 @@ resource "oci_identity_policy" "cluster" { ignore_changes = [defined_tags, freeform_tags] } } + +resource "oci_identity_policy" "cluster_ipv6" { + provider = oci.home + count = var.enable_ipv6 && var.create_iam_resources ? 1 : 0 + compartment_id = var.network_compartment_id + description = format("Policies for OKE Terraform state %v", var.state_id) + name = var.policy_name + statements = [format("Allow any-user to use ipv6s in compartment %s where all { request.principal.id = '%s' }", var.network_compartment_id, var.cluster_id )] + defined_tags = local.defined_tags + freeform_tags = local.freeform_tags + lifecycle { + ignore_changes = [defined_tags, freeform_tags] + } +} \ No newline at end of file diff --git a/modules/iam/variables.tf b/modules/iam/variables.tf index 499bc6fc..7510bad7 100644 --- a/modules/iam/variables.tf +++ b/modules/iam/variables.tf @@ -4,10 +4,11 @@ # Common variable "cluster_id" { type = string } variable "compartment_id" { type = string } +variable "network_compartment_id" { type = string } variable "state_id" { type = string } variable "tenancy_id" { type = string } variable "worker_compartments" { type = list(string) } - +variable "enable_ipv6" { type = bool } # Tags variable "create_iam_defined_tags" { type = bool } variable "create_iam_tag_namespace" { type = bool } diff --git a/modules/iam/versions.tf b/modules/iam/versions.tf index 27bbb86f..6ee4453b 100644 --- a/modules/iam/versions.tf +++ b/modules/iam/versions.tf @@ -9,8 +9,8 @@ terraform { required_providers { oci = { configuration_aliases = [oci.home] - source = "oracle/oci" - version = ">= 4.119.0" + source = "oracle/oci" + version = ">= 4.119.0" } } } \ No newline at end of file diff --git a/modules/network/locals.tf b/modules/network/locals.tf index 8e373d59..dde73ed9 100644 --- a/modules/network/locals.tf +++ b/modules/network/locals.tf @@ -17,12 +17,15 @@ locals { # Protocols # See https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml - all_protocols = "all" - icmp_protocol = 1 - tcp_protocol = 6 - udp_protocol = 17 + all_protocols = "all" + icmp_protocol = 1 + icmpv6_protocol = 58 + tcp_protocol = 6 + udp_protocol = 17 + anywhere = "0.0.0.0/0" + anywhere_ipv6 = "::/0" rule_type_nsg = "NETWORK_SECURITY_GROUP" rule_type_cidr = "CIDR_BLOCK" rule_type_service = "SERVICE_CIDR_BLOCK" diff --git a/modules/network/nsg-controlplane.tf b/modules/network/nsg-controlplane.tf index 556cdbb9..bfa2b5f9 100644 --- a/modules/network/nsg-controlplane.tf +++ b/modules/network/nsg-controlplane.tf @@ -47,6 +47,14 @@ locals { protocol = local.icmp_protocol, source = local.worker_nsg_id, source_type = local.rule_type_nsg, }, }, + var.enable_ipv6 ? { + "Allow ICMPv6 egress for path discovery to worker nodes" : { + protocol = local.icmpv6_protocol, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, + }, + "Allow ICMPv6 ingress for path discovery from worker nodes" : { + protocol = local.icmpv6_protocol, source = local.worker_nsg_id, source_type = local.rule_type_nsg, + }, + } : {}, local.operator_nsg_enabled ? { "Allow TCP ingress to kube-apiserver from operator instance" : { protocol = local.tcp_protocol, port = local.apiserver_port, source = local.operator_nsg_id, source_type = local.rule_type_nsg, diff --git a/modules/network/nsg-loadbalancers-int.tf b/modules/network/nsg-loadbalancers-int.tf index 8fcf8f7b..957b0d08 100644 --- a/modules/network/nsg-loadbalancers-int.tf +++ b/modules/network/nsg-loadbalancers-int.tf @@ -29,6 +29,11 @@ locals { protocol = local.tcp_protocol, port = local.health_check_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, }, + var.enable_ipv6 ? { + "Allow ICMPv6 egress from internal load balancers to worker nodes for path discovery" : { + protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, + }, + } : {}, var.enable_waf ? local.waf_rules : {}, var.allow_rules_internal_lb, ) : {} diff --git a/modules/network/nsg-loadbalancers-pub.tf b/modules/network/nsg-loadbalancers-pub.tf index 01e3915b..1a5ae8ab 100644 --- a/modules/network/nsg-loadbalancers-pub.tf +++ b/modules/network/nsg-loadbalancers-pub.tf @@ -29,6 +29,11 @@ locals { protocol = local.icmp_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, }, + var.enable_ipv6 ? { + "Allow ICMPv6 egress from public load balancers to worker nodes for path discovery" : { + protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, + }, + } : {}, var.enable_waf ? local.waf_rules : {}, var.allow_rules_public_lb, ) : {} diff --git a/modules/network/nsg-operator.tf b/modules/network/nsg-operator.tf index 377ef4bf..d7045638 100644 --- a/modules/network/nsg-operator.tf +++ b/modules/network/nsg-operator.tf @@ -27,14 +27,20 @@ locals { }, }, - local.bastion_nsg_enabled ? { - "Allow ICMP ingress to operator from bastion for path discovery" : { - protocol = local.icmp_protocol, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, - } - "Allow SSH ingress to operator from bastion" : { - protocol = local.tcp_protocol, port = local.ssh_port, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, - } - } : {}, + local.bastion_nsg_enabled ? merge( + var.enable_ipv6 ? { + "Allow ICMPv6 ingress to operator from bastion for path discovery" : { + protocol = local.icmpv6_protocol, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, + } + } : {}, + { + "Allow ICMP ingress to operator from bastion for path discovery" : { + protocol = local.icmp_protocol, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, + } + "Allow SSH ingress to operator from bastion" : { + protocol = local.tcp_protocol, port = local.ssh_port, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, + } + }) : {}, ) : {} } diff --git a/modules/network/nsg-pods.tf b/modules/network/nsg-pods.tf index 2624c62f..9d5f0ed0 100644 --- a/modules/network/nsg-pods.tf +++ b/modules/network/nsg-pods.tf @@ -48,11 +48,28 @@ locals { protocol = local.icmp_protocol, port = local.all_ports, source = local.anywhere, source_type = local.rule_type_cidr, } }, - var.allow_pod_internet_access ? { - "Allow ALL egress from pods to internet" = { - protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, - } + + var.enable_ipv6 ? { + "Allow ICMPv6 ingress to pods for path discovery" : { + protocol = local.icmpv6_protocol, port = local.all_ports, source = local.anywhere_ipv6, source_type = local.rule_type_cidr, + }, + "Allow ICMPv6 egress from pods for path discovery" : { + protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, + }, } : {}, + + var.allow_pod_internet_access ? + merge( + var.enable_ipv6 ? { + "Allow ALL IPv6 egress from pods to internet" = { + protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, + } + } : {}, + { + "Allow ALL egress from pods to internet" = { + protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, + } + }) : {}, var.allow_rules_pods ) : {} } diff --git a/modules/network/nsg-workers.tf b/modules/network/nsg-workers.tf index 04190a65..43dd2fa8 100644 --- a/modules/network/nsg-workers.tf +++ b/modules/network/nsg-workers.tf @@ -47,6 +47,15 @@ locals { }, }, + var.enable_ipv6 ? { + "Allow ICMPv6 ingress to workers for path discovery" : { + protocol = local.icmpv6_protocol, port = local.all_ports, source = local.anywhere_ipv6, source_type = local.rule_type_cidr, + }, + "Allow ICMPv6 egress from workers for path discovery" : { + protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, + }, + } : {}, + local.pod_nsg_enabled ? { "Allow ALL egress from workers to pods" : { protocol = local.all_protocols, port = local.all_ports, destination = local.pod_nsg_id, destination_type = local.rule_type_nsg, @@ -56,11 +65,18 @@ locals { }, } : {}, - var.allow_worker_internet_access ? { - "Allow ALL egress from workers to internet" : { - protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, - }, - } : {}, + var.allow_worker_internet_access ? + merge( + var.enable_ipv6 ? { + "Allow ALL IPv6 egress from workers to internet" = { + protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, + } + } : {}, + { + "Allow ALL egress from workers to internet" : { + protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, + }, + }) : {}, local.int_lb_nsg_enabled ? { "Allow TCP ingress to workers from internal load balancers" : { @@ -120,7 +136,7 @@ locals { }, } : {}, var.allow_rules_workers - ) : {} + ) : {} } resource "oci_core_network_security_group" "workers" { diff --git a/modules/network/rules.tf b/modules/network/rules.tf index 4b289f34..5dd4a9fd 100644 --- a/modules/network/rules.tf +++ b/modules/network/rules.tf @@ -24,22 +24,22 @@ locals { network_security_group_id = lookup(y, "nsg_id") direction = contains(keys(y), "source") ? "INGRESS" : "EGRESS" protocol = lookup(y, "protocol") - source = ( + source = ( alltrue([ upper(lookup(y, "source_type", "")) == local.rule_type_nsg, - length(regexall("ocid\\d+\\.networksecuritygroup", lower(lookup(y, "source", "")))) == 0]) ? - lookup(local.all_nsg_ids, lower(lookup(y, "source", "")), null) : - lookup(y, "source", null) + length(regexall("ocid\\d+\\.networksecuritygroup", lower(lookup(y, "source", "")))) == 0]) ? + lookup(local.all_nsg_ids, lower(lookup(y, "source", "")), null) : + lookup(y, "source", null) ) - source_type = lookup(y, "source_type", null) - destination = ( + source_type = lookup(y, "source_type", null) + destination = ( alltrue([ upper(lookup(y, "destination_type", "")) == local.rule_type_nsg, - length(regexall("ocid\\d+\\.networksecuritygroup", lower(lookup(y, "destination", "")))) == 0]) ? - lookup(local.all_nsg_ids, lower(lookup(y, "destination", "")), null) : - lookup(y, "destination", null) + length(regexall("ocid\\d+\\.networksecuritygroup", lower(lookup(y, "destination", "")))) == 0]) ? + lookup(local.all_nsg_ids, lower(lookup(y, "destination", "")), null) : + lookup(y, "destination", null) ) - destination_type = lookup(y, "destination_type", null) + destination_type = lookup(y, "destination_type", null) }) } # Dynamic map of all NSG IDs for enabled NSGs @@ -72,33 +72,33 @@ resource "oci_core_network_security_group_security_rule" "oke" { tonumber(lookup(each.value, "port", 0)) != local.all_ports ? [each.value] : [] ) content { - dynamic destination_port_range { - for_each = ( - (contains(keys(tcp_options.value), "destination_port_min" ) && - contains(keys(tcp_options.value), "destination_port_max" )) || - (contains(keys(tcp_options.value), "source_port_min" ) && - contains(keys(tcp_options.value), "source_port_max" )) - ) ? []: [tcp_options.value] - content { - min = tonumber(lookup(destination_port_range.value, "port_min", lookup(destination_port_range.value, "port", 0))) - max = tonumber(lookup(destination_port_range.value, "port_max", lookup(destination_port_range.value, "port", 0))) - } + dynamic "destination_port_range" { + for_each = ( + (contains(keys(tcp_options.value), "destination_port_min") && + contains(keys(tcp_options.value), "destination_port_max")) || + (contains(keys(tcp_options.value), "source_port_min") && + contains(keys(tcp_options.value), "source_port_max")) + ) ? [] : [tcp_options.value] + content { + min = tonumber(lookup(destination_port_range.value, "port_min", lookup(destination_port_range.value, "port", 0))) + max = tonumber(lookup(destination_port_range.value, "port_max", lookup(destination_port_range.value, "port", 0))) + } } dynamic "destination_port_range" { - for_each = (contains(keys(tcp_options.value), "destination_port_min") && - contains(keys(tcp_options.value), "destination_port_max") ) ? [tcp_options.value]: [] - content { - min = tonumber(lookup(destination_port_range.value, "destination_port_min", 0)) - max = tonumber(lookup(destination_port_range.value, "destination_port_max", 0)) - } + for_each = (contains(keys(tcp_options.value), "destination_port_min") && + contains(keys(tcp_options.value), "destination_port_max")) ? [tcp_options.value] : [] + content { + min = tonumber(lookup(destination_port_range.value, "destination_port_min", 0)) + max = tonumber(lookup(destination_port_range.value, "destination_port_max", 0)) + } } dynamic "source_port_range" { - for_each = (contains(keys(tcp_options.value), "source_port_min") && - contains(keys(tcp_options.value), "source_port_max") ) ? [tcp_options.value]: [] - content { - min = tonumber(lookup(source_port_range.value, "source_port_min", 0)) - max = tonumber(lookup(source_port_range.value, "source_port_max", 0)) - } + for_each = (contains(keys(tcp_options.value), "source_port_min") && + contains(keys(tcp_options.value), "source_port_max")) ? [tcp_options.value] : [] + content { + min = tonumber(lookup(source_port_range.value, "source_port_min", 0)) + max = tonumber(lookup(source_port_range.value, "source_port_max", 0)) + } } } } @@ -109,32 +109,32 @@ resource "oci_core_network_security_group_security_rule" "oke" { ) content { dynamic "destination_port_range" { - for_each = ( - (contains(keys(udp_options.value), "destination_port_min" ) && - contains(keys(udp_options.value), "destination_port_max" )) || - (contains(keys(udp_options.value), "source_port_min" ) && - contains(keys(udp_options.value), "source_port_max" )) - ) ? []: [udp_options.value] + for_each = ( + (contains(keys(udp_options.value), "destination_port_min") && + contains(keys(udp_options.value), "destination_port_max")) || + (contains(keys(udp_options.value), "source_port_min") && + contains(keys(udp_options.value), "source_port_max")) + ) ? [] : [udp_options.value] content { min = tonumber(lookup(destination_port_range.value, "port_min", lookup(destination_port_range.value, "port", 0))) max = tonumber(lookup(destination_port_range.value, "port_max", lookup(destination_port_range.value, "port", 0))) } } dynamic "destination_port_range" { - for_each = (contains(keys(udp_options.value), "destination_port_min") && - contains(keys(udp_options.value), "destination_port_max") ) ? [udp_options.value]: [] - content { - min = tonumber(lookup(destination_port_range.value, "destination_port_min", 0)) - max = tonumber(lookup(destination_port_range.value, "destination_port_max", 0)) - } + for_each = (contains(keys(udp_options.value), "destination_port_min") && + contains(keys(udp_options.value), "destination_port_max")) ? [udp_options.value] : [] + content { + min = tonumber(lookup(destination_port_range.value, "destination_port_min", 0)) + max = tonumber(lookup(destination_port_range.value, "destination_port_max", 0)) + } } dynamic "source_port_range" { - for_each = (contains(keys(udp_options.value), "source_port_min") && - contains(keys(udp_options.value), "source_port_max") ) ? [udp_options.value]: [] - content { - min = tonumber(lookup(source_port_range.value, "source_port_min", 0)) - max = tonumber(lookup(source_port_range.value, "source_port_max", 0)) - } + for_each = (contains(keys(udp_options.value), "source_port_min") && + contains(keys(udp_options.value), "source_port_max")) ? [udp_options.value] : [] + content { + min = tonumber(lookup(source_port_range.value, "source_port_min", 0)) + max = tonumber(lookup(source_port_range.value, "source_port_max", 0)) + } } } } @@ -147,12 +147,20 @@ resource "oci_core_network_security_group_security_rule" "oke" { } } + dynamic "icmp_options" { + for_each = tostring(each.value.protocol) == tostring(local.icmpv6_protocol) ? [1] : [] + content { + type = 2 + code = 0 + } + } + lifecycle { precondition { - condition = tostring(each.value.protocol) == tostring(local.icmp_protocol) || contains(keys(each.value), "port") || ( + condition = contains([tostring(local.icmp_protocol), tostring(local.icmpv6_protocol)], tostring(each.value.protocol)) || contains(keys(each.value), "port") || ( contains(keys(each.value), "port_min") && contains(keys(each.value), "port_max")) || ( contains(keys(each.value), "source_port_min") && contains(keys(each.value), "source_port_max") || ( - contains(keys(each.value), "destination_port_min") && contains(keys(each.value), "destination_port_max") + contains(keys(each.value), "destination_port_min") && contains(keys(each.value), "destination_port_max") ) ) error_message = "TCP/UDP rule must contain a port or port range: '${each.key}'" @@ -160,7 +168,7 @@ resource "oci_core_network_security_group_security_rule" "oke" { precondition { condition = ( - tostring(each.value.protocol) == tostring(local.icmp_protocol) + contains([tostring(local.icmp_protocol), tostring(local.icmpv6_protocol)], tostring(each.value.protocol)) || can(tonumber(each.value.port)) || (can(tonumber(each.value.port_min)) && can(tonumber(each.value.port_max))) || (can(tonumber(each.value.source_port_min)) && can(tonumber(each.value.source_port_max))) diff --git a/modules/network/subnets.tf b/modules/network/subnets.tf index 5f75c983..4ce64e90 100644 --- a/modules/network/subnets.tf +++ b/modules/network/subnets.tf @@ -48,13 +48,37 @@ locals { local.subnet_cidrs_netnum_newbits_ranges, ) + # IPv6 Default CIDRs + default_ipv6_cidrs = { + bastion = { ipv6_cidr = "8, 0" } + operator = { ipv6_cidr = "8, 1" } + cp = { ipv6_cidr = "8, 2" } + int_lb = { ipv6_cidr = "8, 3" } + pub_lb = { ipv6_cidr = "8, 4" } + workers = { ipv6_cidr = "8, 5" } + pods = { ipv6_cidr = "8, 6" } + fss = { ipv6_cidr = "8, 7" } + } + + # Add default ipv6 cidrs to var.subnets if missing + subnets_with_ipv6_cidr_defaults = { for k, v in var.subnets : + k => merge(v, lookup(v, "ipv6_cidr", null) == null ? lookup(local.default_ipv6_cidrs, k, { "ipv6_cidr" : "::/0" }) : {}) + } + + # Generate IPv6 CIDRs + subnets_ipv6_cidr = var.enable_ipv6 == true ? { + for k, v in local.subnets_with_ipv6_cidr_defaults : k => merge(v, { + "ipv6_cidr" = length(regexall("^\\d+,[ ]?\\d+$", lookup(v, "ipv6_cidr"))) > 0 ? cidrsubnet(var.vcn_ipv6_cidr, tonumber(split(",", lookup(v, "ipv6_cidr"))[0]), tonumber(trim(split(",", lookup(v, "ipv6_cidr"))[1], " "))) : lookup(v, "ipv6_cidr") + }) if try(v.create, "auto") != "never" + } : { for k, v in var.subnets : k => merge(v, { "ipv6_cidr" : null }) if try(v.create, "auto") != "never" } + # Map of subnets for standard components with additional configuration derived # TODO enumerate worker pools for public/private overrides, conditional subnets for both subnet_info = { bastion = { create = var.create_bastion, is_public = var.bastion_is_public } - cp = { create = var.create_cluster, is_public = var.control_plane_is_public } - workers = { create = var.create_cluster, is_public = var.worker_is_public } - pods = { create = var.create_cluster && var.cni_type == "npn" } + cp = { create = var.create_cluster, is_public = var.enable_ipv6 == true ? true : var.control_plane_is_public } + workers = { create = var.create_cluster, is_public = var.enable_ipv6 == true ? true : var.worker_is_public } + pods = { create = var.create_cluster && var.cni_type == "npn", is_public = var.enable_ipv6 == true ? true : false } operator = { create = var.create_operator } fss = { create = contains(keys(var.subnets), "fss") } int_lb = { @@ -97,6 +121,13 @@ locals { subnet_output = { for k, v in var.subnets : k => lookup(v, "id", null) != null ? v.id : lookup(lookup(oci_core_subnet.oke, k, {}), "id", null) } + + create_mixed_igw_ngw_route_table = alltrue([ + var.enable_ipv6 == true, + var.create_internet_gateway == true, + var.create_nat_gateway == true, + var.igw_ngw_mixed_route_id == null + ]) } resource "null_resource" "validate_subnets" { @@ -120,30 +151,64 @@ resource "null_resource" "validate_subnets" { } } +resource "oci_core_route_table" "igw_ngw_mixed_route_id" { + count = local.create_mixed_igw_ngw_route_table ? 1 : 0 + + compartment_id = var.compartment_id + display_name = format("%v-%v", "igw-ngw-mixed-rt", var.state_id) + vcn_id = var.vcn_id + defined_tags = var.defined_tags + freeform_tags = var.freeform_tags + route_rules { + #Required + network_entity_id = var.nat_gateway_id + + description = "Default route for IPv4." + destination = local.anywhere + destination_type = local.rule_type_cidr + } + + route_rules { + #Required + network_entity_id = var.internet_gateway_id + + description = "Default route for IPv6." + destination = local.anywhere_ipv6 + destination_type = local.rule_type_cidr + } + + lifecycle { + ignore_changes = [ + freeform_tags, defined_tags, + ] + } +} + resource "oci_core_subnet" "oke" { for_each = local.subnets_to_create - compartment_id = var.compartment_id - vcn_id = var.vcn_id - cidr_block = lookup(local.subnet_cidrs_all, each.key) - display_name = ( lookup(var.subnets, each.key, null) != null ? - ( lookup(var.subnets[each.key], "display_name", null) != null ? - var.subnets[each.key]["display_name"] : - format("%v-%v", each.key, var.state_id) + compartment_id = var.compartment_id + vcn_id = var.vcn_id + cidr_block = lookup(local.subnet_cidrs_all, each.key) + display_name = (lookup(var.subnets, each.key, null) != null ? + (lookup(var.subnets[each.key], "display_name", null) != null ? + var.subnets[each.key]["display_name"] : + format("%v-%v", each.key, var.state_id) ) : format("%v-%v", each.key, var.state_id) ) dns_label = lookup(local.subnet_dns_labels, each.key, null) prohibit_public_ip_on_vnic = !tobool(lookup(each.value, "is_public", false)) - route_table_id = !tobool(lookup(each.value, "is_public", false)) ? var.nat_route_table_id : var.ig_route_table_id + route_table_id = var.enable_ipv6 && var.cni_type == "npn" && each.key == "pods" ? coalesce(one(oci_core_route_table.igw_ngw_mixed_route_id[*].id), var.igw_ngw_mixed_route_id) : !tobool(lookup(each.value, "is_public", false)) ? var.nat_route_table_id : var.ig_route_table_id security_list_ids = compact([lookup(lookup(oci_core_security_list.oke, each.key, {}), "id", null)]) defined_tags = var.defined_tags freeform_tags = var.freeform_tags + ipv6cidr_block = var.enable_ipv6 ? lookup(local.subnets_ipv6_cidr[each.key], "ipv6_cidr", null) : null lifecycle { ignore_changes = [ freeform_tags, defined_tags, - cidr_block, dns_label, security_list_ids, vcn_id, route_table_id, + cidr_block, dns_label, security_list_ids, vcn_id, ] } } @@ -218,4 +283,4 @@ output "fss_subnet_id" { } output "fss_subnet_cidr" { value = contains(keys(local.subnet_output), "fss") ? lookup(local.subnet_cidrs_all, "fss", null) : null -} +} \ No newline at end of file diff --git a/modules/network/variables.tf b/modules/network/variables.tf index b2da8656..bc8f3d08 100644 --- a/modules/network/variables.tf +++ b/modules/network/variables.tf @@ -30,13 +30,20 @@ variable "control_plane_allowed_cidrs" { type = set(string) } variable "control_plane_is_public" { type = bool } variable "create_cluster" { type = bool } variable "create_bastion" { type = bool } +variable "create_internet_gateway" { type = bool } +variable "create_nat_gateway" { type = bool } variable "create_operator" { type = bool } variable "drg_attachments" { type = any } +variable "enable_ipv6" { type = bool } variable "enable_waf" { type = bool } variable "ig_route_table_id" { type = string } +variable "igw_ngw_mixed_route_id" { type = string } +variable "internet_gateway_id" { type = string } variable "load_balancers" { type = string } +variable "nat_gateway_id" { type = string } variable "nat_route_table_id" { type = string } variable "vcn_cidrs" { type = list(string) } +variable "vcn_ipv6_cidr" { type = string } variable "vcn_id" { type = string } variable "worker_is_public" { type = bool } @@ -49,6 +56,7 @@ variable "subnets" { cidr = optional(string) display_name = optional(string) dns_label = optional(string) + ipv6_cidr = optional(string) })) } diff --git a/modules/operator/cloudinit.tf b/modules/operator/cloudinit.tf index eeebc74c..6515994d 100644 --- a/modules/operator/cloudinit.tf +++ b/modules/operator/cloudinit.tf @@ -35,7 +35,7 @@ data "cloudinit_config" "operator" { "golang", var.install_helm ? "helm" : null, var.install_istioctl ? "istio-istioctl" : null, - var.install_kubectl_from_repo ? "kubectl": null, + var.install_kubectl_from_repo ? "kubectl" : null, ]) yum_repos = { "${local.developer_EPEL}" = { diff --git a/modules/operator/variables.tf b/modules/operator/variables.tf index 1642fb12..604b5e88 100644 --- a/modules/operator/variables.tf +++ b/modules/operator/variables.tf @@ -21,7 +21,7 @@ variable "install_helm" { type = bool } variable "install_helm_from_repo" { type = bool } variable "install_istioctl" { type = bool } variable "install_k9s" { type = bool } -variable "install_kubectl_from_repo" { +variable "install_kubectl_from_repo" { type = bool default = true } diff --git a/modules/operator/versions.tf b/modules/operator/versions.tf index d5ccd4f1..ae390452 100644 --- a/modules/operator/versions.tf +++ b/modules/operator/versions.tf @@ -16,8 +16,8 @@ terraform { } oci = { - source = "oracle/oci" - version = ">= 4.119.0" + source = "oracle/oci" + version = ">= 4.119.0" } } } diff --git a/modules/workers/computecluster.tf b/modules/workers/computecluster.tf new file mode 100644 index 00000000..a9782cad --- /dev/null +++ b/modules/workers/computecluster.tf @@ -0,0 +1,181 @@ +# Copyright (c) 2022, 2025 Oracle Corporation and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl + +# Create the shared compute clusters defined in workers_compute_clusters + +resource "oci_core_compute_cluster" "shared" { + # Create an OCI Compute Cluster resource for each enabled entry of the worker_pools map with that mode. + for_each = var.compute_clusters + compartment_id = lookup(each.value, "compartment_id", var.compartment_id) + display_name = each.key + defined_tags = merge( + var.defined_tags, + lookup(each.value, "defined_tags", {}) + ) + freeform_tags = merge( + var.freeform_tags, + lookup(each.value, "freeform_tags", {}) + ) + + availability_domain = lookup(var.ad_numbers_to_names, lookup(each.value, "placement_ad", 1)) + + lifecycle { + ignore_changes = [ + display_name, defined_tags, freeform_tags, + ] + } +} + +# Dynamic resource block for Compute Cluster groups defined in worker_pools +resource "oci_core_compute_cluster" "workers" { + # Create an OCI Compute Cluster resource for each enabled entry of the worker_pools map with that mode. + for_each = { for k, v in local.enabled_compute_clusters : k => v if length(lookup(v, "instance_ids", [])) > 0 && lookup(v, "compute_cluster", null) == null } + compartment_id = each.value.compartment_id + display_name = each.key + defined_tags = each.value.defined_tags + freeform_tags = each.value.freeform_tags + availability_domain = lookup(each.value, "placement_ad", null) != null ? lookup(var.ad_numbers_to_names, lookup(each.value, "placement_ad")) : element(each.value.availability_domains, 0) + + lifecycle { + ignore_changes = [ + display_name, defined_tags, freeform_tags, + ] + } +} + +resource "oci_core_instance" "compute_cluster_workers" { + for_each = local.compute_cluster_instance_map + + availability_domain = (lookup(oci_core_compute_cluster.shared, lookup(each.value, "compute_cluster", ""), null) != null ? + oci_core_compute_cluster.shared[lookup(each.value, "compute_cluster", "")].availability_domain : + lookup(each.value, "placement_ad", null) != null ? lookup(var.ad_numbers_to_names, lookup(each.value, "placement_ad")) : element(each.value.availability_domains, 0) + ) + fault_domain = try(each.value.placement_fds[0], null) + compartment_id = each.value.compartment_id + display_name = format("%s-%s", element(split("###", each.key), 0), element(split("###", each.key), 1)) + preserve_boot_volume = false + shape = each.value.shape + + defined_tags = each.value.defined_tags + freeform_tags = each.value.freeform_tags + extended_metadata = each.value.extended_metadata + capacity_reservation_id = each.value.capacity_reservation_id + compute_cluster_id = (lookup(oci_core_compute_cluster.shared, lookup(each.value, "compute_cluster", ""), null) != null ? + oci_core_compute_cluster.shared[lookup(each.value, "compute_cluster", "")].id : + (lookup(oci_core_compute_cluster.workers, element(split("###", each.key), 0), null) != null ? + oci_core_compute_cluster.workers[element(split("###", each.key), 0)].id : + lookup(each.value, "compute_cluster", "") + ) + ) + + dynamic "platform_config" { + for_each = each.value.platform_config != null ? [1] : [] + content { + type = lookup( + # Attempt lookup against data source for the associated 'type' of configured worker shape + lookup(local.platform_config_by_shape, each.value.shape, {}), "type", + # Fall back to 'type' on pool with custom platform_config, or INTEL_VM default + lookup(each.value.platform_config, "type", "INTEL_VM") + ) + # Remaining parameters as configured, validated by instance/instance config resource + are_virtual_instructions_enabled = lookup(each.value.platform_config, "are_virtual_instructions_enabled", null) + is_access_control_service_enabled = lookup(each.value.platform_config, "is_access_control_service_enabled", null) + is_input_output_memory_management_unit_enabled = lookup(each.value.platform_config, "is_input_output_memory_management_unit_enabled", null) + is_measured_boot_enabled = lookup(each.value.platform_config, "is_measured_boot_enabled", null) + is_memory_encryption_enabled = lookup(each.value.platform_config, "is_memory_encryption_enabled", null) + is_secure_boot_enabled = lookup(each.value.platform_config, "is_secure_boot_enabled", null) + is_symmetric_multi_threading_enabled = lookup(each.value.platform_config, "is_symmetric_multi_threading_enabled", null) + is_trusted_platform_module_enabled = lookup(each.value.platform_config, "is_trusted_platform_module_enabled", null) + numa_nodes_per_socket = lookup(each.value.platform_config, "numa_nodes_per_socket", null) + percentage_of_cores_enabled = lookup(each.value.platform_config, "percentage_of_cores_enabled", null) + } + } + + agent_config { + are_all_plugins_disabled = each.value.agent_config.are_all_plugins_disabled + is_management_disabled = each.value.agent_config.is_management_disabled + is_monitoring_disabled = each.value.agent_config.is_monitoring_disabled + dynamic "plugins_config" { + for_each = merge( + { + "Compute HPC RDMA Authentication" : "ENABLED", + "Compute HPC RDMA Auto-Configuration" : "ENABLED" + }, + each.value.agent_config.plugins_config + ) + content { + name = plugins_config.key + desired_state = plugins_config.value + } + } + } + + create_vnic_details { + assign_private_dns_record = var.assign_dns + assign_public_ip = each.value.assign_public_ip + nsg_ids = each.value.nsg_ids + subnet_id = each.value.subnet_id + defined_tags = each.value.defined_tags + freeform_tags = each.value.freeform_tags + } + + instance_options { + are_legacy_imds_endpoints_disabled = false + } + + metadata = merge( + { + apiserver_host = var.apiserver_private_host + cluster_ca_cert = var.cluster_ca_cert + oke-k8version = var.kubernetes_version + oke-kubeproxy-proxy-mode = var.kubeproxy_mode + oke-tenancy-id = var.tenancy_id + oke-initial-node-labels = join(",", [for k, v in each.value.node_labels : format("%v=%v", k, v)]) + secondary_vnics = jsonencode(lookup(each.value, "secondary_vnics", {})) + ssh_authorized_keys = var.ssh_public_key + user_data = lookup(lookup(data.cloudinit_config.workers, element(split("###", each.key), 0), {}), "rendered", "") + }, + + # Add labels required for NPN CNI. + var.cni_type == "npn" ? { + oke-native-pod-networking = true + oke-max-pods = var.max_pods_per_node + pod-subnets = coalesce(var.pod_subnet_id, var.worker_subnet_id, "none") + pod-nsgids = join(",", each.value.pod_nsg_ids) + } : {}, + + # Only provide cluster DNS service address if set explicitly; determined automatically in practice. + coalesce(var.cluster_dns, "none") == "none" ? {} : { kubedns_svc_ip = var.cluster_dns }, + + # Extra user-defined fields merged last + var.node_metadata, # global + lookup(each.value, "node_metadata", {}), # pool-specific + ) + + source_details { + boot_volume_size_in_gbs = each.value.boot_volume_size + boot_volume_vpus_per_gb = each.value.boot_volume_vpus_per_gb + source_id = each.value.image_id + source_type = "image" + } + + lifecycle { + precondition { + condition = coalesce(each.value.image_id, "none") != "none" + error_message = <<-EOT + Missing image_id; check provided value if image_type is 'custom', or image_os/image_os_version if image_type is 'oke' or 'platform'. + pool: ${element(split("###", each.key), 0)} + image_type: ${coalesce(each.value.image_type, "none")} + image_id: ${coalesce(each.value.image_id, "none")} + EOT + } + + ignore_changes = [ + agent_config, # TODO Not updateable; remove when supported + defined_tags, freeform_tags, display_name, + metadata["cluster_ca_cert"], metadata["user_data"], + create_vnic_details[0].defined_tags, + create_vnic_details[0].freeform_tags, + ] + } +} diff --git a/modules/workers/instance.tf b/modules/workers/instance.tf index e530f241..36bd308b 100644 --- a/modules/workers/instance.tf +++ b/modules/workers/instance.tf @@ -83,11 +83,15 @@ resource "oci_core_instance" "workers" { secondary_vnics = jsonencode(lookup(each.value, "secondary_vnics", {})) ssh_authorized_keys = var.ssh_public_key user_data = lookup(lookup(data.cloudinit_config.workers, lookup(each.value, "key", ""), {}), "rendered", "") - oke-native-pod-networking = var.cni_type == "npn" ? true : false + }, + + # Add labels required for NPN CNI. + var.cni_type == "npn" ? { + oke-native-pod-networking = true oke-max-pods = var.max_pods_per_node pod-subnets = coalesce(var.pod_subnet_id, var.worker_subnet_id, "none") - pod-nsgids = var.cni_type == "npn" ? join(",", each.value.pod_nsg_ids) : null - }, + pod-nsgids = join(",", each.value.pod_nsg_ids) + } : {}, # Only provide cluster DNS service address if set explicitly; determined automatically in practice. coalesce(var.cluster_dns, "none") == "none" ? {} : { kubedns_svc_ip = var.cluster_dns }, diff --git a/modules/workers/instanceconfig.tf b/modules/workers/instanceconfig.tf index 2115a31c..807a46b0 100644 --- a/modules/workers/instanceconfig.tf +++ b/modules/workers/instanceconfig.tf @@ -61,11 +61,15 @@ resource "oci_core_instance_configuration" "workers" { secondary_vnics = jsonencode(lookup(each.value, "secondary_vnics", {})) ssh_authorized_keys = var.ssh_public_key user_data = lookup(lookup(data.cloudinit_config.workers, each.key, {}), "rendered", "") - oke-native-pod-networking = var.cni_type == "npn" ? true : false + }, + + # Add labels required for NPN CNI. + var.cni_type == "npn" ? { + oke-native-pod-networking = true oke-max-pods = var.max_pods_per_node pod-subnets = coalesce(var.pod_subnet_id, var.worker_subnet_id, "none") - pod-nsgids = var.cni_type == "npn" ? join(",", each.value.pod_nsg_ids) : null - }, + pod-nsgids = join(",", each.value.pod_nsg_ids) + } : {}, # Only provide cluster DNS service address if set explicitly; determined automatically in practice. coalesce(var.cluster_dns, "none") == "none" ? {} : { kubedns_svc_ip = var.cluster_dns }, diff --git a/modules/workers/instancepools.tf b/modules/workers/instancepools.tf index a1d655e4..08a5bdc9 100644 --- a/modules/workers/instancepools.tf +++ b/modules/workers/instancepools.tf @@ -4,7 +4,7 @@ # Dynamic resource block for Instance Pool groups defined in worker_pools resource "oci_core_instance_pool" "tfscaled_workers" { # Create an OCI Instance Pool resource for each enabled entry of the worker_pools map with that mode. - for_each = { for key, value in local.enabled_instance_pools: key => value if tobool(lookup(value, "ignore_initial_pool_size", false)) == false } + for_each = { for key, value in local.enabled_instance_pools : key => value if tobool(lookup(value, "ignore_initial_pool_size", false)) == false } compartment_id = each.value.compartment_id display_name = each.key size = each.value.size @@ -59,7 +59,7 @@ resource "oci_core_instance_pool" "tfscaled_workers" { resource "oci_core_instance_pool" "autoscaled_workers" { # Create an OCI Instance Pool resource for each enabled entry of the worker_pools map with that mode. - for_each = { for key, value in local.enabled_instance_pools: key => value if tobool(lookup(value, "ignore_initial_pool_size", false)) == true } + for_each = { for key, value in local.enabled_instance_pools : key => value if tobool(lookup(value, "ignore_initial_pool_size", false)) == true } compartment_id = each.value.compartment_id display_name = each.key size = each.value.size diff --git a/modules/workers/locals.tf b/modules/workers/locals.tf index 6e6234f8..09d1872d 100644 --- a/modules/workers/locals.tf +++ b/modules/workers/locals.tf @@ -104,20 +104,20 @@ locals { # Use provided image_id for 'custom' type, or first match for all shape + OS criteria image_id = ( - pool.image_type == "custom" ? - pool.image_id : - element(split("###", element(reverse(sort([for entry in tolist(setintersection([ - pool.image_type == "oke" ? - setintersection( - lookup(var.image_ids, "oke", null), - lookup(var.image_ids, trimprefix(lower(pool.kubernetes_version), "v"), null) - ) : - lookup(var.image_ids, "platform", null), - lookup(var.image_ids, pool.image_type, null), - length(regexall("GPU", pool.shape)) > 0 ? var.image_ids.gpu : var.image_ids.nongpu, - length(regexall("A[12]\\.", pool.shape)) > 0 ? var.image_ids.aarch64 : var.image_ids.x86_64, - lookup(var.image_ids, format("%v %v", pool.os, split(".", pool.os_version)[0]), null), - ]...)): "${var.indexed_images[entry].sort_key}###${entry}"])), 0)), 1) + pool.image_type == "custom" ? + pool.image_id : + element(split("###", element(reverse(sort([for entry in tolist(setintersection([ + pool.image_type == "oke" ? + setintersection( + lookup(var.image_ids, "oke", null), + lookup(var.image_ids, trimprefix(lower(pool.kubernetes_version), "v"), null) + ) : + lookup(var.image_ids, "platform", null), + lookup(var.image_ids, pool.image_type, null), + length(regexall("GPU", pool.shape)) > 0 ? var.image_ids.gpu : var.image_ids.nongpu, + length(regexall("A[12]\\.", pool.shape)) > 0 ? var.image_ids.aarch64 : var.image_ids.x86_64, + lookup(var.image_ids, format("%v %v", pool.os, split(".", pool.os_version)[0]), null), + ]...)) : "${var.indexed_images[entry].sort_key}###${entry}"])), 0)), 1) ) # Standard tags as defined if enabled for use @@ -224,6 +224,16 @@ locals { for k, v in local.enabled_worker_pools : k => v if lookup(v, "mode", "") == "cluster-network" } + # Enabled worker_pool map entries for compute clusters + enabled_compute_clusters = { + for k, v in local.enabled_worker_pools : k => v if lookup(v, "mode", "") == "compute-cluster" + } + + # Prepare a map workers node enabled for compute_clusters { "pool_id###worker_id" => pool_values } + compute_cluster_instance_ids_map = { for k, v in local.enabled_compute_clusters : k => toset(lookup(v, "instance_ids", [])) } + compute_cluster_instance_ids = toset(concat(flatten([for k, v in local.compute_cluster_instance_ids_map : [for id in v : format("%s###%s", k, id)]]))) + compute_cluster_instance_map = { for id in local.compute_cluster_instance_ids : id => lookup(local.enabled_compute_clusters, element(split("###", id), 0), {}) } + # Sanitized worker_pools output; some conditionally-used defaults would be misleading worker_pools_final = { for pool_name, pool in local.enabled_worker_pools : pool_name => { for a, b in pool : a => b @@ -270,4 +280,4 @@ locals { # Yields { = { = }} for modes: 'node-pool', 'instance' worker_pool_ips = merge(local.worker_instance_ips, local.worker_nodepool_ips) -} +} \ No newline at end of file diff --git a/modules/workers/nodepools.tf b/modules/workers/nodepools.tf index b4e4a6a0..69bbd886 100644 --- a/modules/workers/nodepools.tf +++ b/modules/workers/nodepools.tf @@ -4,7 +4,7 @@ # Dynamic resource block for Node Pool groups defined in worker_pools resource "oci_containerengine_node_pool" "tfscaled_workers" { # Create an OKE node pool resource for each enabled entry of the worker_pools map with that mode. - for_each = { for key, value in local.enabled_node_pools: key => value if tobool(lookup(value, "ignore_initial_pool_size", false)) == false } + for_each = { for key, value in local.enabled_node_pools : key => value if tobool(lookup(value, "ignore_initial_pool_size", false)) == false } cluster_id = var.cluster_id compartment_id = each.value.compartment_id defined_tags = each.value.defined_tags @@ -158,7 +158,7 @@ resource "oci_containerengine_node_pool" "tfscaled_workers" { resource "oci_containerengine_node_pool" "autoscaled_workers" { # Create an OKE node pool resource for each enabled entry of the worker_pools map with that mode. - for_each = { for key, value in local.enabled_node_pools: key => value if tobool(lookup(value, "ignore_initial_pool_size", false)) == true } + for_each = { for key, value in local.enabled_node_pools : key => value if tobool(lookup(value, "ignore_initial_pool_size", false)) == true } cluster_id = var.cluster_id compartment_id = each.value.compartment_id defined_tags = each.value.defined_tags diff --git a/modules/workers/variables.tf b/modules/workers/variables.tf index 04735137..f14e4f04 100644 --- a/modules/workers/variables.tf +++ b/modules/workers/variables.tf @@ -318,3 +318,13 @@ variable "agent_config" { plugins_config = map(string), }) } + +# +# Workers: compute-cluster +# + +variable "compute_clusters" { + default = {} + description = "Whether to create compute clusters shared by nodes across multiple worker pools enabled for 'compute-cluster'." + type = map(any) +} \ No newline at end of file diff --git a/modules/workers/versions.tf b/modules/workers/versions.tf index c1447783..9d2d2785 100644 --- a/modules/workers/versions.tf +++ b/modules/workers/versions.tf @@ -12,7 +12,7 @@ terraform { oci = { source = "oracle/oci" - version = ">= 4.119.0" + version = ">= 6.37.0" } } } diff --git a/variables-cluster.tf b/variables-cluster.tf index bac0a031..652734a0 100644 --- a/variables-cluster.tf +++ b/variables-cluster.tf @@ -51,6 +51,12 @@ variable "cni_type" { } } +variable "enable_ipv6" { + default = false + description = "Whether to create a dual-stack (IPv4/IPv6) cluster." + type = bool +} + variable "pods_cidr" { default = "10.244.0.0/16" description = "The CIDR range used for IP addresses by the pods. A /16 CIDR is generally sufficient. This CIDR should not overlap with any subnet range in the VCN (it can also be outside the VCN CIDR range). Ignored when cni_type = 'npn'." diff --git a/variables-extensions.tf b/variables-extensions.tf index f2e2520b..c4cb094f 100644 --- a/variables-extensions.tf +++ b/variables-extensions.tf @@ -376,11 +376,11 @@ variable "create_service_account" { } variable "service_accounts" { - default = { + default = { kubeconfigsa = { - sa_name = "kubeconfigsa" - sa_namespace = "kube-system" - sa_cluster_role = "cluster-admin" + sa_name = "kubeconfigsa" + sa_namespace = "kube-system" + sa_cluster_role = "cluster-admin" sa_cluster_role_binding = "kubeconfigsa-crb" } } diff --git a/variables-network.tf b/variables-network.tf index 2e6a6cb3..43a671ec 100644 --- a/variables-network.tf +++ b/variables-network.tf @@ -49,18 +49,48 @@ variable "vcn_create_service_gateway" { } } +variable "vcn_enable_ipv6_gua" { + default = true + description = "Whether to enable IPv6 GUA when IPv6 is enabled." + type = bool +} + +variable "vcn_ipv6_ula_cidrs" { + default = [] + description = "IPv6 ULA CIDR blocks to be used for the VCN." + type = list(string) +} + +variable "internet_gateway_id" { + default = null + description = "Optional ID of existing Internet gateway in VCN." + type = string +} + variable "ig_route_table_id" { default = null - description = "Optional ID of existing internet gateway in VCN." + description = "Optional ID of existing internet gateway route table in VCN." type = string } -variable "nat_route_table_id" { +variable "nat_gateway_id" { default = null description = "Optional ID of existing NAT gateway in VCN." type = string } +variable "nat_route_table_id" { + default = null + description = "Optional ID of existing NAT gateway route table in VCN." + type = string +} + +variable "igw_ngw_mixed_route_id" { + default = null + description = "Optional ID of the existing route table in VCN using IGW for IPv6 and NGW for IPv4." + type = string +} + variable "create_drg" { default = false description = "Whether to create a Dynamic Routing Gateway and attach it to the VCN." @@ -129,13 +159,13 @@ variable "nat_gateway_public_ip_id" { variable "subnets" { default = { - bastion = { newbits = 13 } - operator = { newbits = 13 } - cp = { newbits = 13 } - int_lb = { newbits = 11 } - pub_lb = { newbits = 11 } - workers = { newbits = 4 } - pods = { newbits = 2 } + bastion = { newbits = 13, ipv6_cidr = "8, 0" } + operator = { newbits = 13, ipv6_cidr = "8, 1" } + cp = { newbits = 13, ipv6_cidr = "8, 2" } + int_lb = { newbits = 11, ipv6_cidr = "8, 3" } + pub_lb = { newbits = 11, ipv6_cidr = "8, 4" } + workers = { newbits = 4, ipv6_cidr = "8, 5" } + pods = { newbits = 2, ipv6_cidr = "8, 6" } } description = "Configuration for standard subnets. The 'create' parameter of each entry defaults to 'auto', creating subnets when other enabled components are expected to utilize them, and may be configured with 'never' or 'always' to force disabled/enabled." type = map(object({ @@ -146,6 +176,7 @@ variable "subnets" { cidr = optional(string) display_name = optional(string) dns_label = optional(string) + ipv6_cidr = optional(string) })) validation { condition = alltrue([ @@ -155,10 +186,10 @@ variable "subnets" { } validation { condition = alltrue([ - for v in flatten([for k, v in var.subnets : keys(v)]) : contains(["create", "id", "cidr", "netnum", "newbits", "display_name", "dns_label"], v) + for v in flatten([for k, v in var.subnets : keys(v)]) : contains(["create", "id", "cidr", "netnum", "newbits", "display_name", "dns_label", "ipv6_cidr"], v) ]) error_message = format("Invalid subnet configuration keys: %s", jsonencode(distinct([ - for v in flatten([for k, v in var.subnets : keys(v)]) : v if !contains(["create", "id", "cidr", "netnum", "newbits", "display_name", "dns_label"], v) + for v in flatten([for k, v in var.subnets : keys(v)]) : v if !contains(["create", "id", "cidr", "netnum", "newbits", "display_name", "dns_label", "ipv6_cidr"], v) ]))) } } diff --git a/variables-workers.tf b/variables-workers.tf index 41625c8e..e9cb1f09 100644 --- a/variables-workers.tf +++ b/variables-workers.tf @@ -55,6 +55,16 @@ variable "worker_pool_size" { type = number } +# +# Workers: Compute clusters +# + +variable "worker_compute_clusters" { + default = {} + description = "Whether to create compute clusters shared by nodes across multiple worker pools enabled for 'compute-cluster'." + type = map(any) +} + # # Workers: network #