diff --git a/README.md b/README.md index 15c5dcba6..942452ba2 100644 --- a/README.md +++ b/README.md @@ -167,20 +167,16 @@ This chart deploys the GlueOps Platform | host_network.kube_pometheus_stack.prometheusOperator.tls.internal_port | int | `45040` | | | host_network.nginx_public.controller.host_port.ports.http | int | `45030` | | | host_network.nginx_public.controller.host_port.ports.https | int | `45031` | | -| kube_etcd.enabled | string | `"placeholder_enable_kubeadm_cluster"` | | -| kube_etcd.serviceMonitor.caFile | string | `"/etc/prometheus/secrets/etcd-client/ca.crt"` | | -| kube_etcd.serviceMonitor.certFile | string | `"/etc/prometheus/secrets/etcd-client/apiserver-etcd-client.crt"` | | -| kube_etcd.serviceMonitor.keyFile | string | `"/etc/prometheus/secrets/etcd-client/apiserver-etcd-client.key"` | | +| kubeadm.enabled | string | `"placeholder_enable_kubeadm_cluster"` | | +| kubeadm.kube_etcd.serviceMonitor.caFile | string | `"/etc/prometheus/secrets/etcd-client/ca.crt"` | | +| kubeadm.kube_etcd.serviceMonitor.certFile | string | `"/etc/prometheus/secrets/etcd-client/apiserver-etcd-client.crt"` | | +| kubeadm.kube_etcd.serviceMonitor.keyFile | string | `"/etc/prometheus/secrets/etcd-client/apiserver-etcd-client.key"` | | | loki.aws_accessKey | string | `"placeholder_loki_aws_access_key"` | Part of `loki_s3_iam_credentials` output from terraform-module-cloud-multy-prerequisites: https://github.com/GlueOps/terraform-module-cloud-multy-prerequisites | | loki.aws_region | string | `"placeholder_aws_region"` | Should be the same `primary_region` you used in: https://github.com/GlueOps/terraform-module-cloud-multy-prerequisites | | loki.aws_secretKey | string | `"placeholder_loki_aws_secret_key"` | Part of `loki_s3_iam_credentials` output from terraform-module-cloud-multy-prerequisites: https://github.com/GlueOps/terraform-module-cloud-multy-prerequisites | | loki.bucket | string | `"glueops-tenant-placeholder_tenant_key-placeholder_cluster_environment-loki-primary"` | Format: glueops-tenant-placeholder_tenant_key-placeholder_cluster_environment-loki-primary, Credentials found at `loki_credentials` of json output of terraform-module-cloud-multy-prerequisites | | nginx.controller_replica_count | int | `2` | number of replicas for ingress controller | -| node_ports.enabled | string | `"placeholder_enable_kubeadm_cluster"` | | -| node_ports.nginx.ports.http | int | `30020` | | -| node_ports.nginx.ports.https | int | `30021` | | -| node_ports.oauth2_proxy.ports.http | int | `30010` | | -| node_ports.oauth2_proxy.ports.https | int | `30011` | | +| nginx.internal.enabled | bool | `false` | | | prometheus.volume_claim_storage_request | string | `"50"` | Volume of storage requested for each Prometheus PVC, in Gi | | pull_request_bot.watch_for_apps_delay_seconds | string | `"10"` | number of seconds to wait before checking ArgoCD for new applications | | tls_cert_restore.aws_accessKey | string | `"placeholder_tls_cert_restore_aws_access_key"` | Part of `loki_log_exporter` output from terraform-module-cloud-multy-prerequisites: https://github.com/GlueOps/terraform-module-cloud-multy-prerequisites | diff --git a/templates/application-kube-prometheus-stack.yaml b/templates/application-kube-prometheus-stack.yaml index 4b3877884..c290da155 100644 --- a/templates/application-kube-prometheus-stack.yaml +++ b/templates/application-kube-prometheus-stack.yaml @@ -81,12 +81,12 @@ spec: registry: {{ .Values.base_registries.registry_k8s_io }} {{- toYaml .Values.glueops_node_and_tolerations | nindent 14 }} {{- toYaml .Values.glueops_node_and_tolerations | nindent 10 }} - {{- if .Values.kube_etcd.enabled }} + {{- if .Values.kubeadm.enabled }} kubeEtcd: serviceMonitor: - caFile: {{ .Values.kube_etcd.serviceMonitor.caFile }} - certFile: {{ .Values.kube_etcd.serviceMonitor.certFile }} - keyFile: {{ .Values.kube_etcd.serviceMonitor.keyFile }} + caFile: {{ .Values.kubeadm.kube_etcd.serviceMonitor.caFile }} + certFile: {{ .Values.kubeadm.kube_etcd.serviceMonitor.certFile }} + keyFile: {{ .Values.kubeadm.kube_etcd.serviceMonitor.keyFile }} scheme: https service: port: 2379 @@ -118,9 +118,9 @@ spec: requests: storage: {{ .Values.prometheus.volume_claim_storage_request}}Gi hostNetwork: {{ .Values.host_network.enabled }} - {{- if .Values.kube_etcd.enabled }} + {{- if .Values.kubeadm.enabled }} secrets: - - etcd-client + - etcd-client-certs {{- end }} grafana: {{- toYaml .Values.glueops_node_and_tolerations | nindent 10 }} diff --git a/templates/application-nginx-public.yaml b/templates/application-nginx-public.yaml index ef876294d..4b2716c77 100644 --- a/templates/application-nginx-public.yaml +++ b/templates/application-nginx-public.yaml @@ -105,32 +105,26 @@ spec: {{- if .Values.enable_chisel_proxy_protocol }} chisel-operator.io/proxy-protocol: "true" {{- end }} - - type: {{- if .Values.node_ports.enabled }} "NodePort" {{- else }} "LoadBalancer" {{- end }} - {{- if .Values.node_ports.enabled }} - nodePorts: - http: {{ .Values.node_ports.nginx.ports.http }} - https: {{ .Values.node_ports.nginx.ports.https }} + type: "LoadBalancer" + {{- if .Values.kubeadm.enabled }} + externalIPs: [{{ .Values.public_loadbalancer_ips.public | join ", " }}] {{- end }} - externalTrafficPolicy: "Local" + internal: + enabled: {{ .Values.nginx.internal.enabled }} + type: "ClusterIP" + {{- if .Values.kubeadm.enabled }} + externalIPs: [{{ .Values.public_loadbalancer_ips.private | join ", " }}] + {{- end }} + externalTrafficPolicy: "Local" + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: nlb # https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/2366 # https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/2366#issuecomment-1788923154 updateStrategy: rollingUpdate: maxSurge: 1 maxUnavailable: 0 - # Add a pause to make time for the pod to be registered in the AWS NLB target group before proceeding with the next - # https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/1834#issuecomment-781530724 - # https://alexklibisz.com/2021/07/20/speed-limits-for-rolling-restarts-in-kubernetes#round-3-set-minreadyseconds-maxunavailable-to-0-and-maxsurge-to-1 - minReadySeconds: 180 - # Add sleep on preStop to allow for graceful shutdown with AWS NLB - # https://github.com/kubernetes/ingress-nginx/issues/6928 - # https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/2366#issuecomment-1118312709 - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "sleep 240; /wait-shutdown"] metrics: enabled: true serviceMonitor: @@ -144,6 +138,14 @@ spec: default: false controllerValue: "k8s.io/public-ingress-nginx" affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: use-as-public-lb + operator: In + values: + - "true" podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 diff --git a/templates/glueops-platform-ingress/application-nginx-glueops-platform-oauth2.yaml b/templates/glueops-platform-ingress/application-nginx-glueops-platform-oauth2.yaml index 17f288091..0d40b4622 100644 --- a/templates/glueops-platform-ingress/application-nginx-glueops-platform-oauth2.yaml +++ b/templates/glueops-platform-ingress/application-nginx-glueops-platform-oauth2.yaml @@ -106,32 +106,25 @@ spec: {{- if .Values.enable_chisel_proxy_protocol }} chisel-operator.io/proxy-protocol: "true" {{- end }} - - type: {{- if .Values.node_ports.enabled }} "NodePort" {{- else }} "LoadBalancer" {{- end }} - {{- if .Values.node_ports.enabled }} - nodePorts: - http: {{ .Values.node_ports.oauth2_proxy.ports.http }} - https: {{ .Values.node_ports.oauth2_proxy.ports.https }} + type: "LoadBalancer" + {{- if .Values.kubeadm.enabled }} + externalIPs: [{{ .Values.private_loadbalancer_ips.public | join ", " }}] {{- end }} - externalTrafficPolicy: "Local" - # https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/2366 - # https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/2366#issuecomment-1788923154 + internal: + enabled: {{ .Values.nginx.internal.enabled }} + type: "ClusterIP" + {{- if .Values.kubeadm.enabled }} + externalIPs: [{{ .Values.private_loadbalancer_ips.private | join ", " }}] + {{- end }} + externalTrafficPolicy: "Local" + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: nlb + updateStrategy: rollingUpdate: maxSurge: 1 maxUnavailable: 0 - # Add a pause to make time for the pod to be registered in the AWS NLB target group before proceeding with the next - # https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/1834#issuecomment-781530724 - # https://alexklibisz.com/2021/07/20/speed-limits-for-rolling-restarts-in-kubernetes#round-3-set-minreadyseconds-maxunavailable-to-0-and-maxsurge-to-1 - minReadySeconds: 180 - # Add sleep on preStop to allow for graceful shutdown with AWS NLB - # https://github.com/kubernetes/ingress-nginx/issues/6928 - # https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/2366#issuecomment-1118312709 - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "sleep 240; /wait-shutdown"] metrics: enabled: true serviceMonitor: @@ -145,6 +138,14 @@ spec: default: false controllerValue: "k8s.io/glueops-platform-ingress-nginx" affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: use-as-lb + operator: In + values: + - "true" podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 diff --git a/values.yaml b/values.yaml index 074a8020d..e7d570a86 100644 --- a/values.yaml +++ b/values.yaml @@ -37,17 +37,6 @@ host_network: metricServer: port: 45056 -node_ports: - enabled: placeholder_enable_kubeadm_cluster - # the chosen http/https values are the same with LoadBalancer values - oauth2_proxy: - ports: - http: 30010 - https: 30011 - nginx: - ports: - http: 30020 - https: 30021 # -- The Route53 subdomain for the services on your cluster. It will be used as the suffix url for argocd, grafana, vault, and any other services that come out of the box in the glueops platform. Note: you need to create this before using this repo as this repo does not provision DNS Zones for you. # This is the domain you created through: https://github.com/GlueOps/terraform-module-cloud-multy-prerequisites @@ -151,6 +140,8 @@ vault: nginx: # -- number of replicas for ingress controller controller_replica_count: 2 + internal: + enabled: false pull_request_bot: # -- number of seconds to wait before checking ArgoCD for new applications @@ -396,10 +387,11 @@ container_images: enable_chisel_proxy_protocol: false -kube_etcd: +kubeadm: enabled: placeholder_enable_kubeadm_cluster - serviceMonitor: - caFile: /etc/prometheus/secrets/etcd-client/ca.crt - certFile: /etc/prometheus/secrets/etcd-client/apiserver-etcd-client.crt - keyFile: /etc/prometheus/secrets/etcd-client/apiserver-etcd-client.key + kube_etcd: + serviceMonitor: + caFile: /etc/prometheus/secrets/etcd-client/ca.crt + certFile: /etc/prometheus/secrets/etcd-client/apiserver-etcd-client.crt + keyFile: /etc/prometheus/secrets/etcd-client/apiserver-etcd-client.key