make Spark on Mesos use old spark shuffle fetch protocol (#85)

* make Spark on Mesos use old spark shuffle fetch protocol

* bump requirements to appease github CI; bump package patch version

Author: nanflasted

requirements.txt

@@ -1,3 +1,3 @@
-importlib-resources==1.5.0
+importlib-resources==5.4.0
 pyinotify==0.9.6
 pyyaml >= 3.0

service_configuration_lib/spark_config.py

@@ -616,6 +616,7 @@ def _get_mesos_spark_env(
         'spark.mesos.constraints': f'pool:{paasta_pool}',
         'spark.mesos.executor.docker.forcePullImage': 'true',
         'spark.mesos.principal': 'spark',
+        'spark.shuffle.useOldFetchProtocol': 'true',
         **auth_configs,
         **_get_mesos_docker_volumes_conf(
             user_spark_opts, extra_volumes,

setup.py

@@ -17,7 +17,7 @@
 
 setup(
     name='service-configuration-lib',
-    version='2.11.0',
+    version='2.11.1',
     provides=['service_configuration_lib'],
     description='Start, stop, and inspect Yelp SOA services',
     url='https://github.com/Yelp/service_configuration_lib',

tests/spark_config_test.py

@@ -929,6 +929,7 @@ def verify(output):
                 'spark.mesos.executor.docker.forcePullImage': 'true',
                 'spark.mesos.constraints': f'pool:{self.pool}',
                 'spark.mesos.principal': 'spark',
+                'spark.shuffle.useOldFetchProtocol': 'true',
             }
             for key, value in expected_output.items():
                 assert output[key] == value