Merge pull request #74 from Yelp/gcoll_COREML-2697_infer_mesos_options_into_k8s

[COREML-2697] handle missing kubernetes config values for mass migration

Author: Giulio Collura

service_configuration_lib/spark_config.py

@@ -285,6 +285,17 @@ def _adjust_spark_requested_resources(
         max_cores = int(user_spark_opts.setdefault('spark.cores.max', str(DEFAULT_MAX_CORES)))
         executor_instances = max_cores / executor_cores
     elif cluster_manager == 'kubernetes':
+        # TODO(gcoll|COREML-2697): Consider cleaning this part of the code up
+        # once mesos is not longer around at Yelp.
+        if 'spark.executor.instances' not in user_spark_opts:
+            executor_instances = int(user_spark_opts.get('spark.cores.max', str(DEFAULT_MAX_CORES))) // executor_cores
+            user_spark_opts['spark.executor.instances'] = str(executor_instances)
+        if (
+            'spark.mesos.executor.memoryOverhead' in user_spark_opts and
+            'spark.executor.memoryOverhead' not in user_spark_opts
+        ):
+            user_spark_opts['spark.executor.memoryOverhead'] = user_spark_opts['spark.mesos.executor.memoryOverhead']
+
         executor_instances = int(
             user_spark_opts.setdefault('spark.executor.instances', str(DEFAULT_EXECUTOR_INSTANCES)),
         )

setup.py

@@ -17,7 +17,7 @@
 
 setup(
     name='service-configuration-lib',
-    version='2.9.0',
+    version='2.10.0',
     provides=['service_configuration_lib'],
     description='Start, stop, and inspect Yelp SOA services',
     url='https://github.com/Yelp/service_configuration_lib',

tests/spark_config_test.py

@@ -325,6 +325,27 @@ def gpu_pool(self, tmpdir, monkeypatch):
                     'spark.scheduler.maxRegisteredResourcesWaitingTime': '35min',
                 },
             ),
+            # kubernetes migration
+            (
+                'kubernetes',
+                {
+                    'spark.executor.memory': '2g',
+                    'spark.executor.cores': '4',
+                    'spark.cores.max': '12',
+                    'spark.mesos.executor.memoryOverhead': '4096',
+                },
+                {
+                    'spark.executor.memory': '2g',
+                    'spark.executor.cores': '4',
+                    'spark.executor.instances': '3',
+                    'spark.cores.max': '12',
+                    'spark.kubernetes.executor.limit.cores': '4',
+                    'spark.kubernetes.allocation.batch.size': '3',
+                    'spark.scheduler.maxRegisteredResourcesWaitingTime': '15min',
+                    'spark.executor.memoryOverhead': '4096',
+                    'spark.mesos.executor.memoryOverhead': '4096',
+                },
+            ),
             # use default mesos settings
             (
                 'mesos',