Added comprehensive Kerberos authentication support for Spark 3.1+

api/v1beta2/sparkapplication_types.go

@@ -142,6 +142,9 @@ type SparkApplicationSpec struct {
 	// scheduler backend since Spark 3.0.
 	// +optional
 	DynamicAllocation *DynamicAllocation `json:"dynamicAllocation,omitempty"`
+	// Kerberos configures Kerberos authentication for Hadoop access.
+	// +optional
+	Kerberos *KerberosSpec `json:"kerberos,omitempty"`
 }
 
 // SparkApplicationStatus defines the observed state of SparkApplication
@@ -604,6 +607,9 @@ const (
 	// SecretTypeHadoopDelegationToken is for secrets from an Hadoop delegation token that needs the
 	// environment variable HADOOP_TOKEN_FILE_LOCATION.
 	SecretTypeHadoopDelegationToken SecretType = "HadoopDelegationToken"
+	// SecretTypeKerberosKeytab is for secrets from a Kerberos keytab file that needs the
+	// environment variable KRB5_KEYTAB_FILE.
+	SecretTypeKerberosKeytab SecretType = "KerberosKeytab"
 	// SecretTypeGeneric is for secrets that needs no special handling.
 	SecretTypeGeneric SecretType = "Generic"
 )
@@ -717,3 +723,41 @@ type DynamicAllocation struct {
 	// +optional
 	ShuffleTrackingTimeout *int64 `json:"shuffleTrackingTimeout,omitempty"`
 }
+
+// KerberosSpec defines the Kerberos authentication configuration for Hadoop access.
+type KerberosSpec struct {
+	// Principal is the Kerberos principal name for authentication.
+	// +optional
+	Principal *string `json:"principal,omitempty"`
+	// KeytabSecret is the name of the secret containing the Kerberos keytab file.
+	// +optional
+	KeytabSecret *string `json:"keytabSecret,omitempty"`
+	// KeytabFile is the path to the keytab file within the keytab secret.
+	// Defaults to "krb5.keytab" if not specified.
+	// +optional
+	KeytabFile *string `json:"keytabFile,omitempty"`
+	// ConfigSecret is the name of the secret containing the Kerberos configuration file (krb5.conf).
+	// +optional
+	ConfigSecret *string `json:"configSecret,omitempty"`
+	// ConfigFile is the path to the krb5.conf file within the config secret.
+	// Defaults to "krb5.conf" if not specified.
+	// +optional
+	ConfigFile *string `json:"configFile,omitempty"`
+	// Realm is the Kerberos realm. This is optional and can be inferred from the principal.
+	// +optional
+	Realm *string `json:"realm,omitempty"`
+	// KDC is the Key Distribution Center address.
+	// +optional
+	KDC *string `json:"kdc,omitempty"`
+	// RenewalCredentials specifies the credential renewal strategy.
+	// Valid values are "keytab" (default) and "ccache".
+	// "keytab" enables automatic renewal using the provided keytab.
+	// "ccache" uses existing ticket cache (requires manual ticket management).
+	// +optional
+	// +kubebuilder:validation:Enum={keytab,ccache}
+	RenewalCredentials *string `json:"renewalCredentials,omitempty"`
+	// EnabledServices specifies which Hadoop services should have Kerberos credentials enabled.
+	// Defaults to ["hadoopfs", "hbase", "hive"] if not specified.
+	// +optional
+	EnabledServices []string `json:"enabledServices,omitempty"`
+}

charts/spark-operator-chart/README.md

@@ -173,6 +173,12 @@ See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall) for command docum
 | spark.serviceAccount.automountServiceAccountToken | bool | `true` | Auto-mount service account token to the spark applications pods. |
 | spark.rbac.create | bool | `true` | Specifies whether to create RBAC resources for spark applications. |
 | spark.rbac.annotations | object | `{}` | Optional annotations for the spark application RBAC resources. |
+| spark.kerberos.enable | bool | `false` | Enable Kerberos authentication support for Spark applications. |
+| spark.kerberos.defaultPrincipal | string | `""` | Default Kerberos principal for authentication (can be overridden per application). Example: [email protected] |
+| spark.kerberos.defaultRealm | string | `""` | Default Kerberos realm (can be overridden per application). Example: EXAMPLE.COM |
+| spark.kerberos.defaultKDC | string | `""` | Default Kerberos KDC address (can be overridden per application). Example: kdc.example.com:88 |
+| spark.kerberos.defaultKeytabSecret | string | `""` | Name of the secret containing the default Kerberos keytab file. This secret should contain a file named 'krb5.keytab' |
+| spark.kerberos.defaultConfigSecret | string | `""` | Name of the secret containing the default Kerberos configuration (krb5.conf). This secret should contain a file named 'krb5.conf' |
 | prometheus.metrics.enable | bool | `true` | Specifies whether to enable prometheus metrics scraping. |
 | prometheus.metrics.port | int | `8080` | Metrics port. |
 | prometheus.metrics.portName | string | `"metrics"` | Metrics port name. |

charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml

@@ -10342,6 +10342,57 @@ spec:
                     items:
                       type: string
                     type: array
+                  kerberos:
+                    description: Kerberos configures Kerberos authentication for Hadoop
+                      access.
+                    properties:
+                      configFile:
+                        description: |-
+                          ConfigFile is the path to the krb5.conf file within the config secret.
+                          Defaults to "krb5.conf" if not specified.
+                        type: string
+                      configSecret:
+                        description: ConfigSecret is the name of the secret containing
+                          the Kerberos configuration file (krb5.conf).
+                        type: string
+                      enabledServices:
+                        description: |-
+                          EnabledServices specifies which Hadoop services should have Kerberos credentials enabled.
+                          Defaults to ["hadoopfs", "hbase", "hive"] if not specified.
+                        items:
+                          type: string
+                        type: array
+                      kdc:
+                        description: KDC is the Key Distribution Center address.
+                        type: string
+                      keytabFile:
+                        description: |-
+                          KeytabFile is the path to the keytab file within the keytab secret.
+                          Defaults to "krb5.keytab" if not specified.
+                        type: string
+                      keytabSecret:
+                        description: KeytabSecret is the name of the secret containing
+                          the Kerberos keytab file.
+                        type: string
+                      principal:
+                        description: Principal is the Kerberos principal name for
+                          authentication.
+                        type: string
+                      realm:
+                        description: Realm is the Kerberos realm. This is optional
+                          and can be inferred from the principal.
+                        type: string
+                      renewalCredentials:
+                        description: |-
+                          RenewalCredentials specifies the credential renewal strategy.
+                          Valid values are "keytab" (default) and "ccache".
+                          "keytab" enables automatic renewal using the provided keytab.
+                          "ccache" uses existing ticket cache (requires manual ticket management).
+                        enum:
+                        - keytab
+                        - ccache
+                        type: string
+                    type: object
                   mainApplicationFile:
                     description: MainFile is the path to a bundled JAR, Python, or
                       R file of the application.

charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml

@@ -10260,6 +10260,56 @@ spec:
                 items:
                   type: string
                 type: array
+              kerberos:
+                description: Kerberos configures Kerberos authentication for Hadoop
+                  access.
+                properties:
+                  configFile:
+                    description: |-
+                      ConfigFile is the path to the krb5.conf file within the config secret.
+                      Defaults to "krb5.conf" if not specified.
+                    type: string
+                  configSecret:
+                    description: ConfigSecret is the name of the secret containing
+                      the Kerberos configuration file (krb5.conf).
+                    type: string
+                  enabledServices:
+                    description: |-
+                      EnabledServices specifies which Hadoop services should have Kerberos credentials enabled.
+                      Defaults to ["hadoopfs", "hbase", "hive"] if not specified.
+                    items:
+                      type: string
+                    type: array
+                  kdc:
+                    description: KDC is the Key Distribution Center address.
+                    type: string
+                  keytabFile:
+                    description: |-
+                      KeytabFile is the path to the keytab file within the keytab secret.
+                      Defaults to "krb5.keytab" if not specified.
+                    type: string
+                  keytabSecret:
+                    description: KeytabSecret is the name of the secret containing
+                      the Kerberos keytab file.
+                    type: string
+                  principal:
+                    description: Principal is the Kerberos principal name for authentication.
+                    type: string
+                  realm:
+                    description: Realm is the Kerberos realm. This is optional and
+                      can be inferred from the principal.
+                    type: string
+                  renewalCredentials:
+                    description: |-
+                      RenewalCredentials specifies the credential renewal strategy.
+                      Valid values are "keytab" (default) and "ccache".
+                      "keytab" enables automatic renewal using the provided keytab.
+                      "ccache" uses existing ticket cache (requires manual ticket management).
+                    enum:
+                    - keytab
+                    - ccache
+                    type: string
+                type: object
               mainApplicationFile:
                 description: MainFile is the path to a bundled JAR, Python, or R file
                   of the application.

charts/spark-operator-chart/values.yaml

@@ -409,6 +409,31 @@ spark:
     # -- Optional annotations for the spark application RBAC resources.
     annotations: {}
 
+  # Kerberos configuration for Spark applications
+  kerberos:
+    # -- Enable Kerberos authentication support for Spark applications.
+    enable: false
+
+    # -- Default Kerberos principal for authentication (can be overridden per application).
+    # Example: [email protected]
+    defaultPrincipal: ""
+
+    # -- Default Kerberos realm (can be overridden per application).
+    # Example: EXAMPLE.COM
+    defaultRealm: ""
+
+    # -- Default Kerberos KDC address (can be overridden per application).
+    # Example: kdc.example.com:88
+    defaultKDC: ""
+
+    # -- Name of the secret containing the default Kerberos keytab file.
+    # This secret should contain a file named 'krb5.keytab'
+    defaultKeytabSecret: ""
+
+    # -- Name of the secret containing the default Kerberos configuration (krb5.conf).
+    # This secret should contain a file named 'krb5.conf'
+    defaultConfigSecret: ""
+
 prometheus:
   metrics:
     # -- Specifies whether to enable prometheus metrics scraping.

config/crd/bases/sparkoperator.k8s.io_scheduledsparkapplications.yaml

@@ -10342,6 +10342,57 @@ spec:
                     items:
                       type: string
                     type: array
+                  kerberos:
+                    description: Kerberos configures Kerberos authentication for Hadoop
+                      access.
+                    properties:
+                      configFile:
+                        description: |-
+                          ConfigFile is the path to the krb5.conf file within the config secret.
+                          Defaults to "krb5.conf" if not specified.
+                        type: string
+                      configSecret:
+                        description: ConfigSecret is the name of the secret containing
+                          the Kerberos configuration file (krb5.conf).
+                        type: string
+                      enabledServices:
+                        description: |-
+                          EnabledServices specifies which Hadoop services should have Kerberos credentials enabled.
+                          Defaults to ["hadoopfs", "hbase", "hive"] if not specified.
+                        items:
+                          type: string
+                        type: array
+                      kdc:
+                        description: KDC is the Key Distribution Center address.
+                        type: string
+                      keytabFile:
+                        description: |-
+                          KeytabFile is the path to the keytab file within the keytab secret.
+                          Defaults to "krb5.keytab" if not specified.
+                        type: string
+                      keytabSecret:
+                        description: KeytabSecret is the name of the secret containing
+                          the Kerberos keytab file.
+                        type: string
+                      principal:
+                        description: Principal is the Kerberos principal name for
+                          authentication.
+                        type: string
+                      realm:
+                        description: Realm is the Kerberos realm. This is optional
+                          and can be inferred from the principal.
+                        type: string
+                      renewalCredentials:
+                        description: |-
+                          RenewalCredentials specifies the credential renewal strategy.
+                          Valid values are "keytab" (default) and "ccache".
+                          "keytab" enables automatic renewal using the provided keytab.
+                          "ccache" uses existing ticket cache (requires manual ticket management).
+                        enum:
+                        - keytab
+                        - ccache
+                        type: string
+                    type: object
                   mainApplicationFile:
                     description: MainFile is the path to a bundled JAR, Python, or
                       R file of the application.