Support Spark 2.4.7 & 3.0.1 (#667)

Author: imback82

azure-pipelines.yml

@@ -284,6 +284,15 @@ stages:
       env:
         SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.6-bin-hadoop2.7
 
+    - task: DotNetCoreCLI@2
+      displayName: 'E2E tests for Spark 2.4.7'
+      inputs:
+        command: test
+        projects: '**/Microsoft.Spark*.E2ETest/*.csproj'
+        arguments: '--configuration $(buildConfiguration)'
+      env:
+        SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.7-bin-hadoop2.7
+
     - task: DotNetCoreCLI@2
       displayName: 'E2E tests for Spark 3.0.0'
       inputs:
@@ -293,6 +302,16 @@ stages:
       env:
         SPARK_HOME: $(Build.BinariesDirectory)\spark-3.0.0-bin-hadoop2.7
 
+
+    - task: DotNetCoreCLI@2
+      displayName: 'E2E tests for Spark 3.0.1'
+      inputs:
+        command: test
+        projects: '**/Microsoft.Spark*.E2ETest/*.csproj'
+        arguments: '--configuration $(buildConfiguration)'
+      env:
+        SPARK_HOME: $(Build.BinariesDirectory)\spark-3.0.1-bin-hadoop2.7
+
 - stage: ForwardCompatibility
   displayName: E2E Forward Compatibility Tests
   dependsOn: Build
@@ -558,7 +577,16 @@ stages:
       env:
         SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.6-bin-hadoop2.7
 
-    # Spark 3.0.0 uses Arrow 0.15.1, which contains a new Arrow spec. This breaks backward
+    - task: DotNetCoreCLI@2
+      displayName: 'E2E tests for Spark 2.4.7'
+      inputs:
+        command: test
+        projects: '**/Microsoft.Spark*.E2ETest/*.csproj'
+        arguments: '--configuration $(buildConfiguration) --filter $(TestsToFilterOut)'
+      env:
+        SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.7-bin-hadoop2.7
+
+    # Spark 3.0.* uses Arrow 0.15.1, which contains a new Arrow spec. This breaks backward
     # compatibility when using Microsoft.Spark.Worker with incompatible versions of Arrow.
     # Skip Arrow tests until the backward compatibility Worker version is updated.
     - task: DotNetCoreCLI@2
@@ -571,3 +599,14 @@ stages:
         (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestVectorUdf)"
       env:
         SPARK_HOME: $(Build.BinariesDirectory)\spark-3.0.0-bin-hadoop2.7
+
+    - task: DotNetCoreCLI@2
+      displayName: 'E2E tests for Spark 3.0.1'
+      inputs:
+        command: test
+        projects: '**/Microsoft.Spark*.E2ETest/*.csproj'
+        arguments: "--configuration $(buildConfiguration) --filter $(TestsToFilterOut)&\
+        (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestGroupedMapUdf)&\
+        (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestVectorUdf)"
+      env:
+        SPARK_HOME: $(Build.BinariesDirectory)\spark-3.0.1-bin-hadoop2.7

script/download-spark-distros.cmd

@@ -24,6 +24,8 @@ curl -k -L -o spark-2.4.3.tgz https://archive.apache.org/dist/spark/spark-2.4.3/
 curl -k -L -o spark-2.4.4.tgz https://archive.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz && tar xzvf spark-2.4.4.tgz
 curl -k -L -o spark-2.4.5.tgz https://archive.apache.org/dist/spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz && tar xzvf spark-2.4.5.tgz
 curl -k -L -o spark-2.4.6.tgz https://archive.apache.org/dist/spark/spark-2.4.6/spark-2.4.6-bin-hadoop2.7.tgz && tar xzvf spark-2.4.6.tgz
+curl -k -L -o spark-2.4.7.tgz https://archive.apache.org/dist/spark/spark-2.4.7/spark-2.4.7-bin-hadoop2.7.tgz && tar xzvf spark-2.4.7.tgz
 curl -k -L -o spark-3.0.0.tgz https://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop2.7.tgz && tar xzvf spark-3.0.0.tgz
+curl -k -L -o spark-3.0.1.tgz https://archive.apache.org/dist/spark/spark-3.0.1/spark-3.0.1-bin-hadoop2.7.tgz && tar xzvf spark-3.0.1.tgz
 
 endlocal

src/scala/microsoft-spark-2.4.x/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala

@@ -35,7 +35,7 @@ import scala.util.Try
 object DotnetRunner extends Logging {
   private val DEBUG_PORT = 5567
   private val supportedSparkVersions =
-      Set[String]("2.4.0", "2.4.1", "2.4.3", "2.4.4", "2.4.5", "2.4.6")
+      Set[String]("2.4.0", "2.4.1", "2.4.3", "2.4.4", "2.4.5", "2.4.6", "2.4.7")
 
   val SPARK_VERSION = DotnetUtils.normalizeSparkVersion(spark.SPARK_VERSION)
 

src/scala/microsoft-spark-3.0.x/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala

@@ -34,7 +34,7 @@ import scala.util.Try
  */
 object DotnetRunner extends Logging {
   private val DEBUG_PORT = 5567
-  private val supportedSparkVersions = Set[String]("3.0.0")
+  private val supportedSparkVersions = Set[String]("3.0.0", "3.0.1")
 
   val SPARK_VERSION = DotnetUtils.normalizeSparkVersion(spark.SPARK_VERSION)