Spark 3.0.2 example changes (#9)

* test

* changing the spark version to 3.0.2

* removing test file

Co-authored-by: Mohan Kumar <[email protected]>

Author: mohanln-cloud

java/csv_to_parquet/pom.xml

@@ -10,14 +10,20 @@
 
 	<properties>
 		<oci-java-sdk-version>1.15.4</oci-java-sdk-version>
+		<oci-java-sdk-version>1.25.2</oci-java-sdk-version>
+		<spark.version>3.0.2</spark.version>
+		<oci-hdfs-version>3.2.1.3</oci-hdfs-version>
+		<ojdbc.version>18.3</ojdbc.version>
+		<maven.compiler.source>1.8</maven.compiler.source>
+		<maven.compiler.target>1.8</maven.compiler.target>
 	</properties>
 
 	<dependencies>
 		<!-- For interfacing with OCI -->
 		<dependency>
 			<groupId>com.oracle.oci.sdk</groupId>
 			<artifactId>oci-hdfs-connector</artifactId>
-			<version>2.9.2.6</version>
+			<version>${oci-hdfs-version}</version>
 			<scope>provided</scope>
 		</dependency>
 		<dependency>
@@ -30,13 +36,13 @@
 		<dependency>
 			<groupId>org.apache.spark</groupId>
 			<artifactId>spark-core_2.12</artifactId>
-			<version>2.4.4</version>
+			<version>${spark.version}</version>
 			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.spark</groupId>
 			<artifactId>spark-sql_2.12</artifactId>
-			<version>2.4.4</version>
+			<version>${spark.version}</version>
 			<scope>provided</scope>
 		</dependency>
 	</dependencies>
@@ -48,8 +54,8 @@
 				<artifactId>maven-compiler-plugin</artifactId>
 				<version>3.8.0</version>
 				<configuration>
-					<source>1.8</source>
-					<target>1.8</target>
+					<source>${maven.compiler.source}</source>
+					<target>${maven.compiler.target}</target>
 				</configuration>
 			</plugin>
 		</plugins>

scala/csv_to_parquet/csv_to_parquet.sbt

@@ -1,10 +1,10 @@
 // Copyright (c) 2016, 2020 Oracle and/or its affiliates.
 name := "csv_to_parquet"
 version := "1.0"
-scalaVersion := "2.11.0"
+scalaVersion := "2.12.0"
 
 libraryDependencies ++= Seq(
-  "org.apache.spark" %% "spark-core" % "2.4.4" % "provided",
-  "org.apache.spark" %% "spark-sql" % "2.4.4" % "provided",
-  "com.oracle.oci.sdk" % "oci-java-sdk-common" % "1.22.1",
+  "org.apache.spark" %% "spark-core" % "3.0.2" % "provided",
+  "org.apache.spark" %% "spark-sql" % "3.0.2" % "provided",
+  "com.oracle.oci.sdk" % "oci-java-sdk-common" % "1.25.2",
 )

scala/xml_to_parquet/build.sbt

@@ -1,3 +1,5 @@
+import sun.security.tools.PathList
+
 lazy val root = (project in file(".")).
   settings(
     inThisBuild(List(
@@ -6,11 +8,8 @@ lazy val root = (project in file(".")).
       version      := "0.6.0-SNAPSHOT"
     )),
     name := "IRS990",
-    libraryDependencies += "org.apache.spark" %% "spark-core" % "2.4.4",
-    libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.4.4",
+    libraryDependencies += "org.apache.spark" %% "spark-core" % "3.0.2",
+    libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.0.2",
     libraryDependencies += "org.scala-lang.modules" %% "scala-xml" % "1.1.0"
   )
-assemblyMergeStrategy in assembly := {
- case PathList("META-INF", xs @ _*) => MergeStrategy.discard
- case x => MergeStrategy.first
-}
+