Coreml 3238 | set k8s allocation batch size accurately (#79)

* COREML-3238 | set spark.kubernetes.allocation.batch.size accurately

* COREML-3238 | add spark config tests

Co-authored-by: Sameer Sharma <[email protected]>
Co-authored-by: Sameer Sharma <[email protected]>

Author: 3 people

service_configuration_lib/spark_config.py

@@ -368,7 +368,7 @@ def _adjust_spark_requested_resources(
         max_cores = executor_instances * executor_cores
         user_spark_opts.setdefault(
             'spark.kubernetes.allocation.batch.size',
-            str(min(executor_instances, DEFAULT_K8S_BATCH_SIZE)),
+            str(DEFAULT_K8S_BATCH_SIZE),
         )
         user_spark_opts.setdefault('spark.kubernetes.executor.limit.cores', str(executor_cores))
         waiting_time = (

setup.py

@@ -17,7 +17,7 @@
 
 setup(
     name='service-configuration-lib',
-    version='2.10.4',
+    version='2.10.5',
     provides=['service_configuration_lib'],
     description='Start, stop, and inspect Yelp SOA services',
     url='https://github.com/Yelp/service_configuration_lib',

tests/spark_config_test.py

@@ -309,7 +309,7 @@ def gpu_pool(self, tmpdir, monkeypatch):
                     'spark.executor.cores': '4',
                     'spark.executor.instances': '2',
                     'spark.kubernetes.executor.limit.cores': '4',
-                    'spark.kubernetes.allocation.batch.size': '2',
+                    'spark.kubernetes.allocation.batch.size': '512',
                     'spark.scheduler.maxRegisteredResourcesWaitingTime': '15min',
                 },
             ),
@@ -327,7 +327,7 @@ def gpu_pool(self, tmpdir, monkeypatch):
                     'spark.executor.cores': '4',
                     'spark.executor.instances': '2',
                     'spark.kubernetes.executor.limit.cores': '4',
-                    'spark.kubernetes.allocation.batch.size': '2',
+                    'spark.kubernetes.allocation.batch.size': '512',
                     'spark.scheduler.maxRegisteredResourcesWaitingTime': '15min',
                 },
             ),
@@ -359,10 +359,33 @@ def gpu_pool(self, tmpdir, monkeypatch):
                     'spark.executor.cores': '4',
                     'spark.executor.instances': '32',
                     'spark.kubernetes.executor.limit.cores': '4',
-                    'spark.kubernetes.allocation.batch.size': '32',
+                    'spark.kubernetes.allocation.batch.size': '512',
                     'spark.scheduler.maxRegisteredResourcesWaitingTime': '16min',
                 },
             ),
+            # k8s allocation batch size not specified
+            (
+                'kubernetes',
+                {
+                    'spark.executor.cores': '4',
+                    'spark.cores.max': '128',
+                },
+                {
+                    'spark.kubernetes.allocation.batch.size': '512',
+                },
+            ),
+            # k8s allocation batch size specified
+            (
+                'kubernetes',
+                {
+                    'spark.executor.cores': '4',
+                    'spark.cores.max': '128',
+                    'spark.kubernetes.allocation.batch.size': '151',
+                },
+                {
+                    'spark.kubernetes.allocation.batch.size': '151',
+                },
+            ),
             # use default k8s settings
             (
                 'kubernetes',
@@ -372,7 +395,7 @@ def gpu_pool(self, tmpdir, monkeypatch):
                     'spark.executor.cores': '2',
                     'spark.executor.instances': '2',
                     'spark.kubernetes.executor.limit.cores': '2',
-                    'spark.kubernetes.allocation.batch.size': '2',
+                    'spark.kubernetes.allocation.batch.size': '512',
                     'spark.scheduler.maxRegisteredResourcesWaitingTime': '15min',
                 },
             ),
@@ -407,7 +430,7 @@ def gpu_pool(self, tmpdir, monkeypatch):
                     'spark.executor.instances': '3',
                     'spark.cores.max': '12',
                     'spark.kubernetes.executor.limit.cores': '4',
-                    'spark.kubernetes.allocation.batch.size': '3',
+                    'spark.kubernetes.allocation.batch.size': '512',
                     'spark.scheduler.maxRegisteredResourcesWaitingTime': '15min',
                     'spark.executor.memoryOverhead': '4096',
                     'spark.mesos.executor.memoryOverhead': '4096',