diff --git a/fides-minimal/templates/_helpers.tpl b/fides-minimal/templates/_helpers.tpl index 7689f66..8a31254 100644 --- a/fides-minimal/templates/_helpers.tpl +++ b/fides-minimal/templates/_helpers.tpl @@ -18,7 +18,7 @@ If release name contains chart name it will be used as a full name. {{- if contains $baseName .Release.Name }} {{- $baseName = .Release.Name }} {{- else }} - {{- printf "%s-%s" .Release.Name $baseName }} + {{- $baseName = printf "%s-%s" .Release.Name $baseName }} {{- end }} {{- end }} {{- $baseName | trunc 63 | trimSuffix "-"}} @@ -150,11 +150,29 @@ Create the name of the config map to store the fides.toml file. List of CORS origins, concatenated, deduplicated, and formatted. */}} {{- define "fides.corsOrigins" -}} -{{ $cors := list (printf "https://%s" .Values.privacyCenter.publicHostname | quote ) (printf "https://%s" .Values.fides.publicHostname | quote) }} +{{- $cors := list }} + +{{- if .Values.privacyCenter.publicHostname }} + {{- $cors = append $cors (printf "https://%s" .Values.privacyCenter.publicHostname | quote) }} +{{- end }} +{{- if .Values.fides.publicHostname }} + {{- $cors = append $cors (printf "https://%s" .Values.fides.publicHostname | quote) }} +{{- end }} + +{{- if eq .Values.fides.service.type "LoadBalancer" }} + {{- if .Values.fides.publicHostname }} + {{- $cors = append $cors (printf "http://%s" .Values.fides.publicHostname | quote) }} + {{- end }} + {{- if and .Values.privacyCenter.enabled .Values.privacyCenter.publicHostname }} + {{- $cors = append $cors (printf "http://%s" .Values.privacyCenter.publicHostname | quote) }} + {{- end }} +{{- end }} + {{- range (.Values.fides.configuration.additionalCORSOrigins | compact) }} - {{- $cors = . | quote | append $cors }} + {{- $cors = append $cors (. | quote) }} {{- end }} -{{ $cors = $cors | uniq }} + +{{- $cors = $cors | compact | uniq }} {{ printf "[%s]" (join "," $cors) }} {{- end }} @@ -167,7 +185,7 @@ The set of environment variables for Fides and workers {{- $redisDeployment := .Values.redis }} {{- $pgDeployment := .Values.postgresql }} {{- with .Values.fides.configuration }} -{{- .additionalEnvVars | toYaml }} +{{- include "fides.processedEnvVars" $ }} - name: FIDES__DATABASE__SERVER valueFrom: secretKeyRef: @@ -210,3 +228,39 @@ The set of environment variables for Fides and workers key: REDIS_PASSWORD {{- end }} {{- end }} + +{{/* +Detect if fidesplus is being used based on the repository name +*/}} +{{- define "fides.isFidesplus" -}} +{{- if contains "fidesplus" (.Values.fides.image.repository | lower) -}} +true +{{- else -}} +false +{{- end -}} +{{- end }} + +{{/* +Get processed environment variables with additional settings +*/}} +{{- define "fides.processedEnvVars" -}} +{{- $envVars := .Values.fides.configuration.additionalEnvVars | default list }} +{{- $hiddenEnvVar := dict "name" "FIDES__EXECUTION__MONITOR_CELERY_TASKS_ENABLED" "value" "true" }} +{{- $envVars = append $envVars $hiddenEnvVar }} +{{- $envVars | toYaml }} +{{- end }} + +{{/* +Validates that all worker types have unique names. Fails if duplicate names are found. +*/}} +{{- define "fides.worker.validateUniqueNames" -}} +{{- $workers := .Values.fides.workerConfiguration.workers | default list }} +{{- $names := dict }} +{{- range $workers }} +{{- if hasKey $names .name }} +{{- fail (printf "Duplicate worker name found: '%s'. Worker names must be unique" .name) }} +{{- else }} +{{- $_ := set $names .name "used" }} +{{- end }} +{{- end }} +{{- end }} diff --git a/fides-minimal/templates/fides/fides-config.yaml b/fides-minimal/templates/fides/fides-config.yaml index c962318..395ab71 100644 --- a/fides-minimal/templates/fides/fides-config.yaml +++ b/fides-minimal/templates/fides/fides-config.yaml @@ -14,7 +14,7 @@ data: [celery] event_queue_prefix = "fides_worker" task_default_queue = "fides" - task_always_eager = {{ not $.worker }} + task_always_eager = false [security] cors_origins = {{ include "fides.corsOrigins" . | trim }} diff --git a/fides-minimal/templates/fides/worker-config.yaml b/fides-minimal/templates/fides/worker-config.yaml index 3521740..cf774fa 100644 --- a/fides-minimal/templates/fides/worker-config.yaml +++ b/fides-minimal/templates/fides/worker-config.yaml @@ -1,3 +1,4 @@ +{{- $_ := set $ "worker" ( ge (len .Values.fides.workerConfiguration.workers) 1) }} {{- if $.worker }} apiVersion: v1 kind: ConfigMap @@ -15,7 +16,7 @@ data: [celery] event_queue_prefix = "fides_worker" task_default_queue = "fides" - task_always_eager = true + task_always_eager = false redis_socket_keepalive = true [security] diff --git a/fides-minimal/templates/fides/worker-deployment.yaml b/fides-minimal/templates/fides/worker-deployment.yaml index ed9dc38..7eb3209 100644 --- a/fides-minimal/templates/fides/worker-deployment.yaml +++ b/fides-minimal/templates/fides/worker-deployment.yaml @@ -1,54 +1,110 @@ -{{- $_ := set $ "worker" ( ge (.Values.fides.workers.count | int) 1) }} +{{- include "fides.worker.validateUniqueNames" . }} +{{- $userWorkers := .Values.fides.workerConfiguration.workers | default list }} +{{- $isFidesplus := include "fides.isFidesplus" . }} + +{{/* Build default workers list */}} +{{- $defaultWorkers := list }} +{{/* Both Fides and Fidesplus get DSR and other workers */}} +{{- $defaultWorkers = list + (dict "name" "dsr" "count" 1 "queues" (list "fides.dsr") "resources" (dict "limits" (dict "memory" "512Mi") "requests" (dict "cpu" "100m" "memory" "256Mi"))) + (dict "name" "other" "count" 1 "excludeQueues" (list "fides.dsr" "fides.privacy_preferences") "resources" (dict "limits" (dict "memory" "512Mi") "requests" (dict "cpu" "100m" "memory" "256Mi"))) +}} +{{/* Fidesplus additionally gets classification, helios, and consent workers */}} +{{- if eq $isFidesplus "true" }} +{{- $defaultWorkers = concat $defaultWorkers (list + (dict "name" "classification" "count" 1 "queues" (list "fidesplus.discovery_monitors_classification") "resources" (dict "limits" (dict "memory" "512Mi") "requests" (dict "cpu" "100m" "memory" "256Mi"))) + (dict "name" "helios" "count" 1 "queues" (list "fidesplus.discovery_monitors_promotion" "fidesplus.discovery_monitors_detection") "resources" (dict "limits" (dict "memory" "512Mi") "requests" (dict "cpu" "100m" "memory" "256Mi"))) + (dict "name" "consent" "count" 1 "queues" (list "fides.privacy_preferences" "fides.consent_webhooks") "resources" (dict "limits" (dict "memory" "512Mi") "requests" (dict "cpu" "100m" "memory" "256Mi"))) +) }} +{{- end }} + +{{/* Merge user workers with defaults */}} +{{- $mergedWorkers := list }} +{{- $userWorkerNames := dict }} +{{- range $userWorkers }} +{{- $_ := set $userWorkerNames .name true }} +{{- $mergedWorkers = append $mergedWorkers . }} +{{- end }} +{{- range $defaultWorkers }} +{{- if not (hasKey $userWorkerNames .name) }} +{{- $mergedWorkers = append $mergedWorkers . }} +{{- end }} +{{- end }} + +{{/* Check if we have any active workers */}} +{{- $hasActiveWorkers := false }} +{{- range $mergedWorkers }} +{{- if gt (.count | int) 0 }} +{{- $hasActiveWorkers = true }} +{{- end }} +{{- end }} +{{- $_ := set $ "worker" $hasActiveWorkers }} + {{- if $.worker }} +{{- range $mergedWorkers }} +{{- $workerCount := .count | int }} +{{- if gt $workerCount 0 }} {{- $volume := "config" }} {{- $configPath := "/etc/fides/config" }} apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "fides.worker.fullname" . }} + name: {{ printf "%s-%s" (include "fides.worker.fullname" $) .name}} labels: - {{- include "fides.labels" . | nindent 4 }} + {{- include "fides.labels" $ | nindent 4 }} + fid.es/worker: {{ .name }} spec: - replicas: {{ .Values.fides.workers.count | int }} + replicas: {{ $workerCount | int }} selector: matchLabels: - {{- include "fides.worker.selectorLabels" . | nindent 6 }} + {{- include "fides.worker.selectorLabels" $ | nindent 6 }} + fid.es/worker: {{ .name }} strategy: - {{- include "fides.deploymentStrategy" . | nindent 4 }} + {{- include "fides.deploymentStrategy" $ | nindent 4 }} template: metadata: - {{- with .Values.podAnnotations }} + {{- with $.Values.podAnnotations }} annotations: {{- toYaml . | nindent 8 }} {{- end }} labels: - {{- include "fides.worker.selectorLabels" . | nindent 8 }} + {{- include "fides.worker.selectorLabels" $ | nindent 8 }} + fid.es/worker: {{ .name }} spec: - {{- with .Values.imagePullSecrets }} + {{- with $.Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "fides.serviceAccountName" . }} + serviceAccountName: {{ include "fides.serviceAccountName" $ }} securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml $.Values.podSecurityContext | nindent 8 }} containers: - name: fides securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: {{ printf "%s:%s" .Values.fides.image.repository ( include "fides.dockerTag" .)}} - imagePullPolicy: {{ .Values.fides.image.pullPolicy }} + {{- toYaml $.Values.securityContext | nindent 12 }} + image: {{ printf "%s:%s" $.Values.fides.image.repository (default (include "fides.dockerTag" $) .imageTagOverride) }} + imagePullPolicy: {{ $.Values.fides.image.pullPolicy }} command: ["fides"] - args: ["worker"] + {{- if and (hasKey . "queues") (hasKey . "excludeQueues") }} + {{- fail (printf "Worker '%s' cannot have both --queues and --exclude-queues passed" .name) }} + {{- end }} + args: + - worker + {{- if hasKey . "queues" }} + - {{ printf "--queues=%s" (join "," .queues) }} + {{- else if hasKey . "excludeQueues" }} + - {{ printf "--exclude-queues=%s" (join "," .excludeQueues) }} + {{- end }} env: - name: FIDES__CONFIG_PATH value: {{ printf "%s/fides.toml" $configPath }} - {{- include "fides.env" . | nindent 12 }} + {{- include "fides.env" $ | nindent 12 }} envFrom: - secretRef: - name: {{ include "fides.fidesSecuritySecretName" . }} - {{- if .Values.fides.configuration.additionalEnvVarsSecret }} + name: {{ include "fides.fidesSecuritySecretName" $ }} + {{- if $.Values.fides.configuration.additionalEnvVarsSecret }} - secretRef: - name: {{ .Values.fides.configuration.additionalEnvVarsSecret }} + name: {{ $.Values.fides.configuration.additionalEnvVarsSecret }} {{- end }} livenessProbe: exec: @@ -57,28 +113,31 @@ spec: "-c", "celery --quiet --no-color --app fides.api.tasks inspect ping --destination celery@$HOSTNAME --json" ] - initialDelaySeconds: {{ .Values.fides.startupTimeSeconds | default 30 }} + initialDelaySeconds: {{ $.Values.fides.startupTimeSeconds | default 30 }} periodSeconds: 60 - timeoutSeconds: {{ .Values.fides.healthCheckTimeoutSeconds | default 5 }} + timeoutSeconds: {{ $.Values.fides.healthCheckTimeoutSeconds | default 5 }} volumeMounts: - name: {{ $volume }} mountPath: {{ $configPath }} resources: - {{- toYaml .Values.resources | nindent 12 }} + {{- toYaml .resources | nindent 12 }} volumes: - name: {{ $volume }} configMap: - name: {{ include "fides.worker.tomlConfigMapName" . }} - {{- with .Values.nodeSelector }} + name: {{ include "fides.worker.tomlConfigMapName" $ }} + {{- with $.Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} + {{- with $.Values.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with $.Values.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} +--- +{{- end }} +{{- end }} {{- end }} diff --git a/fides-minimal/values.yaml b/fides-minimal/values.yaml index 6cdc689..3b9da43 100644 --- a/fides-minimal/values.yaml +++ b/fides-minimal/values.yaml @@ -31,6 +31,8 @@ fides: value: "false" - name: FIDES__REDIS__SSL_CERT_REQS # Accepted values include: none, optional and require. value: "none" + - name: FIDES__EXECUTION__USE_DSR_3_0 + value: "true" # Additional environment variables may be declared here. # fides.configuration.additionalEnvVarsSecret is an optional parameter representing the name of an existing secret containing environment variables to pass into the Fides containers. additionalEnvVarsSecret: "" @@ -38,8 +40,11 @@ fides: # FIDES__SECURITY__APP_ENCRYPTION_KEY, FIDES__SECURITY__OAUTH_ROOT_CLIENT_ID, FIDES__SECURITY__OAUTH_ROOT_CLIENT_SECRET, FIDES__SECURITY__DRP_JWT_SECRET fidesSecuritySecretName: "" # fides.configuration.additionalCORSOrigins is an optional parameter to configure allowed CORS origins in addition to the Fides and Privacy Center URLs. + # Note: When using LoadBalancer service type, the chart automatically includes both HTTP and HTTPS variants of your hostnames additionalCORSOrigins: [] # fides.publicHostname is used to set the allowed CORS origins for Fides, e.g. fides.example.com + # For LoadBalancer services: You can leave this empty initially and set it later once you know the LoadBalancer endpoint, + # or specify a CNAME/DNS name that points to your LoadBalancer publicHostname: "" fullnameOverride: "" count: 1 @@ -53,10 +58,34 @@ fides: startupTimeSeconds: 30 # fides.healthCheckTimeoutSeconds configures the timeoutSeconds of the liveness and readiness probes. healthCheckTimeoutSeconds: 5 - workers: - # fides.workers.count determines how many workers the deployment will use to process DSRs. - # To disable workers, set count to 0. This should be set to at least 1 in production environments. - count: 0 + # fides.workerConfiguration configures the Celery workers that process background tasks. + # + # DEFAULT WORKER CONFIGURATION: + # Both Fides and Fidesplus deployments automatically get: + # - 1 DSR worker + # - 1 other worker + # + # Fidesplus deployments additionally get: + # - 1 classification worker + # - 1 helios worker + # - 1 consent worker + # + # To override defaults, explicitly define workers below. To disable a worker, set count: 0. + # For more information, see: https://www.ethyca.com/docs/dev-docs/get-started/advanced#running-workers + workerConfiguration: + workers: [] + # Example worker override: + # - name: other + # count: 1 + # excludeQueues: + # - fides.dsr + # - fides.privacy_preferences + # resources: + # limits: + # memory: 512Mi + # requests: + # cpu: 100m + # memory: 256Mi # privacyCenter is the end-user facing application where data subjects can submit privacy requests. privacyCenter: @@ -82,6 +111,8 @@ privacyCenter: additionalEnvVars: [] nameOverride: "" # privacyCenter.publicHostname is used to set the allowed CORS origins for Fides, e.g. privacy.example.com + # For LoadBalancer services: You can leave this empty initially and set it later once you know the LoadBalancer endpoint, + # or specify a CNAME/DNS name that points to your LoadBalancer publicHostname: "" fullnameOverride: "" count: 1 diff --git a/fides/templates/_helpers.tpl b/fides/templates/_helpers.tpl index f085fb2..540f917 100644 --- a/fides/templates/_helpers.tpl +++ b/fides/templates/_helpers.tpl @@ -18,7 +18,7 @@ If release name contains chart name it will be used as a full name. {{- if contains $baseName .Release.Name }} {{- $baseName = .Release.Name }} {{- else }} - {{- printf "%s-%s" .Release.Name $baseName }} + {{- $baseName = printf "%s-%s" .Release.Name $baseName }} {{- end }} {{- end }} {{- $baseName | trunc 63 | trimSuffix "-"}} @@ -150,11 +150,29 @@ Create the name of the config map to store the fides.toml file. List of CORS origins, concatenated, deduplicated, and formatted. */}} {{- define "fides.corsOrigins" -}} -{{ $cors := list (printf "https://%s" .Values.privacyCenter.publicHostname | quote ) (printf "https://%s" .Values.fides.publicHostname | quote) }} +{{- $cors := list }} + +{{- if .Values.privacyCenter.publicHostname }} + {{- $cors = append $cors (printf "https://%s" .Values.privacyCenter.publicHostname | quote) }} +{{- end }} +{{- if .Values.fides.publicHostname }} + {{- $cors = append $cors (printf "https://%s" .Values.fides.publicHostname | quote) }} +{{- end }} + +{{- if eq .Values.fides.service.type "LoadBalancer" }} + {{- if .Values.fides.publicHostname }} + {{- $cors = append $cors (printf "http://%s" .Values.fides.publicHostname | quote) }} + {{- end }} + {{- if and .Values.privacyCenter.enabled .Values.privacyCenter.publicHostname }} + {{- $cors = append $cors (printf "http://%s" .Values.privacyCenter.publicHostname | quote) }} + {{- end }} +{{- end }} + {{- range (.Values.fides.configuration.additionalCORSOrigins | compact) }} - {{- $cors = . | quote | append $cors }} + {{- $cors = append $cors (. | quote) }} {{- end }} -{{ $cors = $cors | uniq }} + +{{- $cors = $cors | compact | uniq }} {{ printf "[%s]" (join "," $cors) }} {{- end }} @@ -167,7 +185,7 @@ The set of environment variables for Fides and workers {{- $redisDeployment := .Values.redis }} {{- $pgDeployment := .Values.postgresql }} {{- with .Values.fides.configuration }} -{{- .additionalEnvVars | toYaml }} +{{- include "fides.processedEnvVars" $ }} {{- $dbConfig := lookup "v1" "Secret" $namespace .dbSecretName }} {{- $redisConfig := lookup "v1" "Secret" $namespace .redisSecretName }} - name: FIDES__DATABASE__SERVER @@ -286,3 +304,39 @@ Redis CA path key: {{ default "value" $config.secretKey }} {{- end }} {{- end }} + +{{/* +Detect if fidesplus is being used based on the repository name +*/}} +{{- define "fides.isFidesplus" -}} +{{- if contains "fidesplus" (.Values.fides.image.repository | lower) -}} +true +{{- else -}} +false +{{- end -}} +{{- end }} + +{{/* +Get processed environment variables with additional settings +*/}} +{{- define "fides.processedEnvVars" -}} +{{- $envVars := .Values.fides.configuration.additionalEnvVars | default list }} +{{- $hiddenEnvVar := dict "name" "FIDES__EXECUTION__MONITOR_CELERY_TASKS_ENABLED" "value" "true" }} +{{- $envVars = append $envVars $hiddenEnvVar }} +{{- $envVars | toYaml }} +{{- end }} + +{{/* +Validates that all worker types have unique names. Fails if duplicate names are found. +*/}} +{{- define "fides.worker.validateUniqueNames" -}} +{{- $workers := .Values.fides.workerConfiguration.workers | default list }} +{{- $names := dict }} +{{- range $workers }} +{{- if hasKey $names .name }} +{{- fail (printf "Duplicate worker name found: '%s'. Worker names must be unique" .name) }} +{{- else }} +{{- $_ := set $names .name "used" }} +{{- end }} +{{- end }} +{{- end }} diff --git a/fides/templates/fides/fides-config.yaml b/fides/templates/fides/fides-config.yaml index c962318..395ab71 100644 --- a/fides/templates/fides/fides-config.yaml +++ b/fides/templates/fides/fides-config.yaml @@ -14,7 +14,7 @@ data: [celery] event_queue_prefix = "fides_worker" task_default_queue = "fides" - task_always_eager = {{ not $.worker }} + task_always_eager = false [security] cors_origins = {{ include "fides.corsOrigins" . | trim }} diff --git a/fides/templates/fides/worker-config.yaml b/fides/templates/fides/worker-config.yaml index 84e9a3c..57525e3 100644 --- a/fides/templates/fides/worker-config.yaml +++ b/fides/templates/fides/worker-config.yaml @@ -15,7 +15,7 @@ data: [celery] event_queue_prefix = "fides_worker" task_default_queue = "fides" - task_always_eager = true + task_always_eager = false redis_socket_keepalive = true [security] diff --git a/fides/templates/fides/worker-deployment.yaml b/fides/templates/fides/worker-deployment.yaml index 262fd34..cbeb1c4 100644 --- a/fides/templates/fides/worker-deployment.yaml +++ b/fides/templates/fides/worker-deployment.yaml @@ -1,53 +1,109 @@ -{{- $_ := set $ "worker" ( ge (.Values.fides.workers.count | int) 1) }} +{{- include "fides.worker.validateUniqueNames" . }} +{{- $userWorkers := .Values.fides.workerConfiguration.workers | default list }} +{{- $isFidesplus := include "fides.isFidesplus" . }} + +{{/* Build default workers list */}} +{{- $defaultWorkers := list }} +{{/* Both Fides and Fidesplus get DSR and other workers */}} +{{- $defaultWorkers = list + (dict "name" "dsr" "count" 1 "queues" (list "fides.dsr") "resources" (dict "limits" (dict "memory" "512Mi") "requests" (dict "cpu" "100m" "memory" "256Mi"))) + (dict "name" "other" "count" 1 "excludeQueues" (list "fides.dsr" "fides.privacy_preferences") "resources" (dict "limits" (dict "memory" "512Mi") "requests" (dict "cpu" "100m" "memory" "256Mi"))) +}} +{{/* Fidesplus additionally gets classification, helios, and consent workers */}} +{{- if eq $isFidesplus "true" }} +{{- $defaultWorkers = concat $defaultWorkers (list + (dict "name" "classification" "count" 1 "queues" (list "fidesplus.discovery_monitors_classification") "resources" (dict "limits" (dict "memory" "512Mi") "requests" (dict "cpu" "100m" "memory" "256Mi"))) + (dict "name" "helios" "count" 1 "queues" (list "fidesplus.discovery_monitors_promotion" "fidesplus.discovery_monitors_detection") "resources" (dict "limits" (dict "memory" "512Mi") "requests" (dict "cpu" "100m" "memory" "256Mi"))) + (dict "name" "consent" "count" 1 "queues" (list "fides.privacy_preferences" "fides.consent_webhooks") "resources" (dict "limits" (dict "memory" "512Mi") "requests" (dict "cpu" "100m" "memory" "256Mi"))) +) }} +{{- end }} + +{{/* Merge user workers with defaults */}} +{{- $mergedWorkers := list }} +{{- $userWorkerNames := dict }} +{{- range $userWorkers }} +{{- $_ := set $userWorkerNames .name true }} +{{- $mergedWorkers = append $mergedWorkers . }} +{{- end }} +{{- range $defaultWorkers }} +{{- if not (hasKey $userWorkerNames .name) }} +{{- $mergedWorkers = append $mergedWorkers . }} +{{- end }} +{{- end }} + +{{/* Check if we have any active workers */}} +{{- $hasActiveWorkers := false }} +{{- range $mergedWorkers }} +{{- if gt (.count | int) 0 }} +{{- $hasActiveWorkers = true }} +{{- end }} +{{- end }} +{{- $_ := set $ "worker" $hasActiveWorkers }} + {{- if $.worker }} +{{- range $mergedWorkers }} +{{- $workerCount := .count | int }} +{{- if gt $workerCount 0 }} apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "fides.worker.fullname" . }} + name: {{ printf "%s-%s" (include "fides.worker.fullname" $) .name}} labels: - {{- include "fides.labels" . | nindent 4 }} + {{- include "fides.labels" $ | nindent 4 }} + fid.es/worker: {{ .name }} spec: - replicas: {{ .Values.fides.workers.count | int }} + replicas: {{ $workerCount | int }} selector: matchLabels: - {{- include "fides.worker.selectorLabels" . | nindent 6 }} + {{- include "fides.worker.selectorLabels" $ | nindent 6 }} + fid.es/worker: {{ .name }} strategy: - {{- include "fides.deploymentStrategy" . | nindent 4 }} + {{- include "fides.deploymentStrategy" $ | nindent 4 }} template: metadata: - {{- with .Values.podAnnotations }} + {{- with $.Values.podAnnotations }} annotations: {{- toYaml . | nindent 8 }} {{- end }} labels: - {{- include "fides.worker.selectorLabels" . | nindent 8 }} + {{- include "fides.worker.selectorLabels" $ | nindent 8 }} + fid.es/worker: {{ .name }} spec: - {{- with .Values.imagePullSecrets }} + {{- with $.Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "fides.serviceAccountName" . }} + serviceAccountName: {{ include "fides.serviceAccountName" $ }} securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml $.Values.podSecurityContext | nindent 8 }} containers: - name: fides securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: {{ printf "%s:%s" .Values.fides.image.repository ( include "fides.dockerTag" .)}} - imagePullPolicy: {{ .Values.fides.image.pullPolicy }} + {{- toYaml $.Values.securityContext | nindent 12 }} + image: {{ printf "%s:%s" $.Values.fides.image.repository (default (include "fides.dockerTag" $) .imageTagOverride) }} + imagePullPolicy: {{ $.Values.fides.image.pullPolicy }} command: ["fides"] - args: ["worker"] + {{- if and (hasKey . "queues") (hasKey . "excludeQueues") }} + {{- fail (printf "Worker '%s' cannot have both --queues and --exclude-queues passed" .name) }} + {{- end }} + args: + - worker + {{- if hasKey . "queues" }} + - {{ printf "--queues=%s" (join "," .queues) }} + {{- else if hasKey . "excludeQueues" }} + - {{ printf "--exclude-queues=%s" (join "," .excludeQueues) }} + {{- end }} env: - name: FIDES__CONFIG_PATH - value: {{ printf "%s/fides.toml" (include "fides.configPath" .) }} - {{- include "fides.env" . | nindent 12 }} - {{- include "custom_fides_secrets" . | indent 10 }} + value: {{ printf "%s/fides.toml" (include "fides.configPath" $) }} + {{- include "fides.env" $ | nindent 12 }} + {{- include "custom_fides_secrets" $ | indent 10 }} envFrom: - secretRef: - name: {{ include "fides.fidesSecuritySecretName" . }} - {{- if .Values.fides.configuration.additionalEnvVarsSecret }} + name: {{ include "fides.fidesSecuritySecretName" $ }} + {{- if $.Values.fides.configuration.additionalEnvVarsSecret }} - secretRef: - name: {{ .Values.fides.configuration.additionalEnvVarsSecret }} + name: {{ $.Values.fides.configuration.additionalEnvVarsSecret }} {{- end }} livenessProbe: exec: @@ -56,38 +112,41 @@ spec: "-c", "celery --quiet --no-color --app fides.api.tasks inspect ping --destination celery@$HOSTNAME --json" ] - initialDelaySeconds: {{ .Values.fides.startupTimeSeconds | default 30 }} + initialDelaySeconds: {{ $.Values.fides.startupTimeSeconds | default 30 }} periodSeconds: 60 - timeoutSeconds: {{ .Values.fides.healthCheckTimeoutSeconds | default 5 }} + timeoutSeconds: {{ $.Values.fides.healthCheckTimeoutSeconds | default 5 }} volumeMounts: - - name: {{ include "fides.configVolume" . }} - mountPath: {{ include "fides.configPath" . }} - {{- if .Values.fides.configuration.redisCaSecretName }} - - name: {{ include "fides.redisCaVolume" . }} - mountPath: {{ include "fides.redisCaPath" . }} + - name: {{ include "fides.configVolume" $ }} + mountPath: {{ include "fides.configPath" $ }} + {{- if $.Values.fides.configuration.redisCaSecretName }} + - name: {{ include "fides.redisCaVolume" $ }} + mountPath: {{ include "fides.redisCaPath" $ }} readOnly: true {{- end }} resources: - {{- toYaml .Values.fides.workers.resources | nindent 12 }} + {{- toYaml .resources | nindent 12 }} volumes: - - name: {{ include "fides.configVolume" . }} + - name: {{ include "fides.configVolume" $ }} configMap: - name: {{ include "fides.worker.tomlConfigMapName" . }} - {{- if .Values.fides.configuration.redisCaSecretName }} - - name: {{ include "fides.redisCaVolume" . }} + name: {{ include "fides.worker.tomlConfigMapName" $ }} + {{- if $.Values.fides.configuration.redisCaSecretName }} + - name: {{ include "fides.redisCaVolume" $ }} secret: - secretName: {{ .Values.fides.configuration.redisCaSecretName }} + secretName: {{ $.Values.fides.configuration.redisCaSecretName }} {{- end }} - {{- with .Values.nodeSelector }} + {{- with $.Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} + {{- with $.Values.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with $.Values.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} +--- +{{- end }} +{{- end }} {{- end }} diff --git a/fides/values.yaml b/fides/values.yaml index 03db7c8..2c8901d 100644 --- a/fides/values.yaml +++ b/fides/values.yaml @@ -36,6 +36,8 @@ fides: value: "false" - name: FIDES__REDIS__SSL_CERT_REQS # Accepted values include: none, optional and require. value: "none" + - name: FIDES__EXECUTION__USE_DSR_3_0 + value: "true" # Additional environment variables may be declared here. # fides.configuration.additionalEnvVarsSecret is an optional parameter representing the name of an existing secret containing environment variables to pass into the Fides containers. additionalEnvVarsSecret: "" @@ -43,8 +45,11 @@ fides: # FIDES__SECURITY__APP_ENCRYPTION_KEY, FIDES__SECURITY__OAUTH_ROOT_CLIENT_ID, FIDES__SECURITY__OAUTH_ROOT_CLIENT_SECRET, FIDES__SECURITY__DRP_JWT_SECRET fidesSecuritySecretName: "" # fides.configuration.additionalCORSOrigins is an optional parameter to configure allowed CORS origins in addition to the Fides and Privacy Center URLs. + # Note: When using LoadBalancer service type, the chart automatically includes both HTTP and HTTPS variants of your hostnames additionalCORSOrigins: [] # fides.publicHostname is used to set the allowed CORS origins for Fides, e.g. fides.example.com + # For LoadBalancer services: You can leave this empty initially and set it later once you know the LoadBalancer endpoint, + # or specify a CNAME/DNS name that points to your LoadBalancer publicHostname: "" fullnameOverride: "" count: 1 @@ -58,19 +63,35 @@ fides: startupTimeSeconds: 30 # fides.healthCheckTimeoutSeconds configures the timeoutSeconds of the liveness and readiness probes. healthCheckTimeoutSeconds: 5 - workers: - # fides.workers.count determines how many workers the deployment will use to process DSRs. - # To disable workers, set count to 0. This should be set to at least 1 in production environments. - count: 0 - resources: {} - # If you do want to specify resources for the worker, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 2048Mi - # requests: - # cpu: 100m - # memory: 2048Mi + # fides.workerConfiguration configures the Celery workers that process background tasks. + # + # DEFAULT WORKER CONFIGURATION: + # Both Fides and Fidesplus deployments automatically get: + # - 1 DSR worker + # - 1 other worker + # + # Fidesplus deployments additionally get: + # - 1 classification worker + # - 1 helios worker + # - 1 consent worker + # + # To override defaults, explicitly define workers below. To disable a worker, set count: 0. + # For more information, see: https://www.ethyca.com/docs/dev-docs/get-started/advanced#running-workers + workerConfiguration: + workers: [] + # Example worker override: + # - name: other + # count: 1 + # excludeQueues: + # - fides.dsr + # - fides.privacy_preferences + # resources: + # limits: + # memory: 512Mi + # requests: + # cpu: 100m + # memory: 256Mi + resources: {} # If you do want to specify resources for Fides, uncomment the following @@ -106,6 +127,8 @@ privacyCenter: additionalEnvVars: [] nameOverride: "" # privacyCenter.publicHostname is used to set the allowed CORS origins for Fides, e.g. privacy.example.com + # For LoadBalancer services: You can leave this empty initially and set it later once you know the LoadBalancer endpoint, + # or specify a CNAME/DNS name that points to your LoadBalancer publicHostname: "" fullnameOverride: "" count: 1