Merge pull request #64 from Yelp/gcoll_COREML-2506_spark_run_k8s_master

Update spark_config to handle k8s container required volumes for paasta spark-run

Author: gcollura

service_configuration_lib/spark_config.py

@@ -62,7 +62,13 @@
     'spark.kubernetes.executor.label.paasta.yelp.com/instance',
     'spark.kubernetes.executor.label.paasta.yelp.com/cluster',
 }
-K8S_AUTH_FOLDER = '/etc/spark_k8s_secrets'
+K8S_AUTH_FOLDER = '/etc/pki/spark'
+DEFAULT_SPARK_K8S_SECRET_VOLUME = {
+    'hostPath': K8S_AUTH_FOLDER,
+    'containerPath': K8S_AUTH_FOLDER,
+    'mode': 'RO',
+}
+
 log = logging.Logger(__name__)
 
 
@@ -163,6 +169,23 @@ def _get_mesos_docker_volumes_conf(
     return {'spark.mesos.executor.docker.volumes': volume_str}
 
 
+def _get_k8s_docker_volumes_conf(
+    volumes: Optional[List[Mapping[str, str]]] = None,
+):
+    env = {}
+    k8s_volumes = volumes or []
+    k8s_volumes.append(DEFAULT_SPARK_K8S_SECRET_VOLUME)
+    k8s_volumes.append({'containerPath': '/etc/passwd', 'hostPath': '/etc/passwd', 'mode': 'RO'})
+    k8s_volumes.append({'containerPath': '/etc/group', 'hostPath': '/etc/group', 'mode': 'RO'})
+    for volume_name, volume in enumerate(k8s_volumes):
+        env[f'spark.kubernetes.executor.volumes.hostPath.{volume_name}.mount.path'] = volume['containerPath']
+        env[f'spark.kubernetes.executor.volumes.hostPath.{volume_name}.mount.readOnly'] = (
+            'true' if volume['mode'].lower() == 'ro' else 'false'
+        )
+        env[f'spark.kubernetes.executor.volumes.hostPath.{volume_name}.options.path'] = volume['hostPath']
+    return env
+
+
 def _append_sql_shuffle_partitions_conf(spark_opts: Dict[str, str]) -> Dict[str, str]:
     if 'spark.sql.shuffle.partitions' in spark_opts:
         return spark_opts
@@ -409,7 +432,7 @@ def _get_k8s_spark_env(
     paasta_pool: str,
 ) -> Dict[str, str]:
     spark_env = {
-        'spark.master': f'k8s://https://k8s.paasta-{paasta_cluster}.yelp:16443',
+        'spark.master': f'k8s://https://k8s.{paasta_cluster}.paasta:6443',
         'spark.executorEnv.PAASTA_SERVICE': paasta_service,
         'spark.executorEnv.PAASTA_INSTANCE': paasta_instance,
         'spark.executorEnv.PAASTA_CLUSTER': paasta_cluster,
@@ -430,14 +453,8 @@ def _get_k8s_spark_env(
         'spark.kubernetes.executor.label.paasta.yelp.com/cluster': paasta_cluster,
         'spark.kubernetes.node.selector.yelp.com/pool': paasta_pool,
         'spark.kubernetes.executor.label.yelp.com/pool': paasta_pool,
+        **_get_k8s_docker_volumes_conf(volumes),
     }
-    for i, volume in enumerate(volumes or []):
-        volume_name = i
-        spark_env[f'spark.kubernetes.executor.volumes.hostPath.{volume_name}.mount.path'] = volume['containerPath']
-        spark_env[f'spark.kubernetes.executor.volumes.hostPath.{volume_name}.mount.readOnly'] = (
-            'true' if volume['mode'].lower() == 'ro' else 'false'
-        )
-        spark_env[f'spark.kubernetes.executor.volumes.hostPath.{volume_name}.options.path'] = volume['hostPath']
     return spark_env
 
 

setup.py

@@ -17,7 +17,7 @@
 
 setup(
     name='service-configuration-lib',
-    version='2.5.5',
+    version='2.5.6',
     provides=['service_configuration_lib'],
     description='Start, stop, and inspect Yelp SOA services',
     url='https://github.com/Yelp/service_configuration_lib',