diff --git a/core/pom.xml b/core/pom.xml index a5f2e756da0..93159995bac 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -112,7 +112,7 @@ io.netty netty-all - 4.1.17.Final + 4.1.74.Final org.lz4 diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml index 360fcff72b6..904b84d9f15 100644 --- a/examples/flink/pom.xml +++ b/examples/flink/pom.xml @@ -54,7 +54,7 @@ io.netty netty-all - 4.1.17.Final + 4.1.74.Final org.apache.carbondata diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml index 64f0d9f20c4..fc5ce57ce67 100644 --- a/examples/spark/pom.xml +++ b/examples/spark/pom.xml @@ -63,7 +63,7 @@ io.netty netty-all - 4.1.17.Final + 4.1.74.Final org.alluxio @@ -200,9 +200,6 @@ spark-2.3 - - true - 2.3 @@ -214,10 +211,13 @@ - spark-3.1 + spark-3.3 + + true + - 3.1 - 2.10.0 + 3.3 + 2.13.3 diff --git a/index/examples/pom.xml b/index/examples/pom.xml index bccbae1bc39..b84aa468044 100644 --- a/index/examples/pom.xml +++ b/index/examples/pom.xml @@ -92,9 +92,9 @@ - spark-3.1 + spark-3.3 - 3.1 + 3.3 diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml index b1a2414ffc3..b404ef36aa1 100644 --- a/index/secondary-index/pom.xml +++ b/index/secondary-index/pom.xml @@ -158,9 +158,6 @@ spark-2.3 - - true - 2.3 @@ -172,9 +169,12 @@ - spark-3.1 + spark-3.3 + + true + - 3.1 + 3.3 diff --git a/integration/flink/pom.xml b/integration/flink/pom.xml index 5eca6fbbdc0..a420c2e4dfd 100644 --- a/integration/flink/pom.xml +++ b/integration/flink/pom.xml @@ -212,7 +212,7 @@ io.netty netty-all - 4.1.17.Final + 4.1.74.Final test @@ -220,9 +220,6 @@ spark-2.3 - - true - 2.3 @@ -262,9 +259,12 @@ - spark-3.1 + spark-3.3 + + true + - 3.1 + 3.3 diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml index b317060f5f5..693ba8f4711 100644 --- a/integration/presto/pom.xml +++ b/integration/presto/pom.xml @@ -34,7 +34,7 @@ 4.4.9 ${basedir}/../../dev true - 2.10.0 + 2.13.3 0.193 diff --git a/integration/spark-common-cluster-test/pom.xml b/integration/spark-common-cluster-test/pom.xml index c520ef73dde..9811518fd77 100644 --- a/integration/spark-common-cluster-test/pom.xml +++ b/integration/spark-common-cluster-test/pom.xml @@ -65,13 +65,13 @@ io.netty netty - 3.9.9.Final + 4.1.74.Final test io.netty netty-all - 4.1.17.Final + 4.1.74.Final test diff --git a/integration/spark/pom.xml b/integration/spark/pom.xml index 6d278fcec12..c9bdfea4dba 100644 --- a/integration/spark/pom.xml +++ b/integration/spark/pom.xml @@ -608,9 +608,6 @@ spark-2.3 - - true - 2.3 @@ -621,9 +618,9 @@ maven-compiler-plugin - src/main/spark3.1 + src/main/spark3.3 src/main/spark2.4 - src/main/common2.4and3.1 + src/main/common2.4and3.3 @@ -662,7 +659,7 @@ maven-compiler-plugin - src/main/spark3.1 + src/main/spark3.3 src/main/spark2.3 @@ -682,7 +679,7 @@ src/main/spark2.4 src/main/common2.3and2.4 - src/main/common2.4and3.1 + src/main/common2.4and3.3 @@ -692,9 +689,12 @@ - spark-3.1 + spark-3.3 + + true + - 3.1 + 3.3 @@ -722,8 +722,8 @@ - src/main/spark3.1 - src/main/common2.4and3.1 + src/main/spark3.3 + src/main/common2.4and3.3 diff --git a/integration/spark/src/main/common2.3and2.4/org/apache/spark/sql/SparkVersionAdapter.scala b/integration/spark/src/main/common2.3and2.4/org/apache/spark/sql/SparkVersionAdapter.scala index e5cb5e0b13b..61e9a556b53 100644 --- a/integration/spark/src/main/common2.3and2.4/org/apache/spark/sql/SparkVersionAdapter.scala +++ b/integration/spark/src/main/common2.3and2.4/org/apache/spark/sql/SparkVersionAdapter.scala @@ -18,35 +18,39 @@ package org.apache.spark.sql import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer import org.antlr.v4.runtime.tree.TerminalNode import org.apache.spark.{SparkContext, TaskContext} import org.apache.spark.rdd.RDD import org.apache.spark.serializer.Serializer +import org.apache.spark.sql.carbondata.execution.datasources.tasklisteners.CarbonLoadTaskCompletionListener import org.apache.spark.sql.catalyst.{CarbonParserUtil, InternalRow, TableIdentifier} import org.apache.spark.sql.catalyst.analysis.{Analyzer, UnresolvedRelation} +import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.encoders.RowEncoder -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSeq, Expression, InterpretedPredicate, NamedExpression, SortOrder} +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSeq, Expression, InterpretedPredicate, NamedExpression, SortOrder, UnaryExpression} import org.apache.spark.sql.catalyst.expressions.codegen.GeneratePredicate import org.apache.spark.sql.catalyst.parser.ParserUtils.operationNotAllowed import org.apache.spark.sql.catalyst.parser.SqlBaseParser.{BucketSpecContext, ColTypeListContext, CreateTableHeaderContext, LocationSpecContext, QueryContext, SkewSpecContext, TablePropertyListContext} import org.apache.spark.sql.catalyst.plans.{logical, JoinType, QueryPlan} -import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoTable, Join, LogicalPlan, OneRowRelation} +import org.apache.spark.sql.catalyst.plans.logical.{Command, InsertIntoTable, Join, LogicalPlan, OneRowRelation} import org.apache.spark.sql.catalyst.plans.physical.SinglePartition import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.execution.{QueryExecution, ShuffledRowRDD, SparkPlan, SQLExecution, UnaryExecNode} -import org.apache.spark.sql.execution.command.{ExplainCommand, Field, PartitionerField, TableModel, TableNewProcessor} +import org.apache.spark.sql.execution.command.{AtomicRunnableCommand, DataCommand, DataWritingCommand, ExplainCommand, Field, MetadataCommand, PartitionerField, RunnableCommand, ShowPartitionsCommand, TableModel, TableNewProcessor} import org.apache.spark.sql.execution.command.table.{CarbonCreateTableAsSelectCommand, CarbonCreateTableCommand} -import org.apache.spark.sql.execution.datasources.{CreateTable, DataSourceStrategy, RefreshTable} +import org.apache.spark.sql.execution.datasources.{CreateTable, DataSource, DataSourceStrategy, FilePartition, FileScanRDD, OutputWriter, PartitionedFile, RefreshTable} import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight, BuildSide} import org.apache.spark.sql.execution.strategy.CarbonDataSourceScan import org.apache.spark.sql.internal.{SessionState, SharedState} import org.apache.spark.sql.parser.CarbonSpark2SqlParser import org.apache.spark.sql.parser.CarbonSparkSqlParserUtil.{checkIfDuplicateColumnExists, convertDbNameToLowerCase, validateStreamingProperty} -import org.apache.spark.sql.sources.Filter -import org.apache.spark.sql.types.{DataType, StructField} +import org.apache.spark.sql.sources.{BaseRelation, Filter} +import org.apache.spark.sql.types.{DataType, StructField, StructType} import org.apache.spark.unsafe.types.UTF8String +import org.apache.spark.util.TaskCompletionListener import org.apache.carbondata.common.exceptions.DeprecatedFeatureException import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException @@ -57,6 +61,7 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes import org.apache.carbondata.core.metadata.schema.SchemaReader import org.apache.carbondata.core.util.CarbonProperties import org.apache.carbondata.core.util.path.CarbonTablePath +import org.apache.carbondata.mv.plans.modular.ModularPlan import org.apache.carbondata.spark.CarbonOption import org.apache.carbondata.spark.util.CarbonScalaUtil @@ -451,6 +456,56 @@ trait SparkVersionAdapter { case others => others } } + + def showPartitionsCommand(spec: Option[TablePartitionSpec], + showPartitionsCommand: ShowPartitionsCommand): ShowPartitionsCommand = { + ShowPartitionsCommand(showPartitionsCommand.tableName, spec) + } + + def invokeWriteAndReadMethod(dataSourceObj: DataSource, + dataFrame: DataFrame, + data: LogicalPlan, + session: SparkSession, + mode: SaveMode, + query: LogicalPlan, + physicalPlan: SparkPlan): BaseRelation = { + dataSourceObj.writeAndRead(mode, query, query.output.map(_.name), physicalPlan) + } + + /** + * Parse a key-value map from a [[TablePropertyListContext]], assuming all values are specified. + * + * @param ctx Instance of TablePropertyListContext defining parser rule for the table + * properties. + * @param props Map of table property list + * @return Map of transformed table property. + */ + def visitPropertyKeyValues(ctx: TablePropertyListContext, + props: Map[String, String]): Map[String, String] = { + val badKeys = props.filter { case (_, v) => v == null }.keys + if (badKeys.nonEmpty) { + operationNotAllowed( + s"Values must be specified for key(s): ${ badKeys.mkString("[", ",", "]") }", ctx) + } + props.map { case (key, value) => + (key.toLowerCase, value) + } + } + + def getFileScanRDD(spark: SparkSession, readFunction: PartitionedFile => Iterator[InternalRow], + partitions: ArrayBuffer[FilePartition]) : FileScanRDD = { + new FileScanRDD(spark, readFunction, partitions) + } + + def check(context: TaskContext): Boolean = { + val onCompleteCallbacksField = + context.getClass.getDeclaredField("onCompleteCallbacks") + onCompleteCallbacksField.setAccessible(true) + val listeners = onCompleteCallbacksField.get(context) + .asInstanceOf[ArrayBuffer[TaskCompletionListener]] + listeners.exists(p => p.isInstanceOf[CarbonLoadTaskCompletionListener]) + } + } case class CarbonBuildSide(buildSide: BuildSide) { @@ -468,3 +523,43 @@ abstract class CarbonTakeOrderedAndProjectExecHelper(sortOrder: Seq[SortOrder], s"skipMapOrder=$skipMapOrder, readFromHead=$readFromHead, output=$outputString)" } } + +trait CarbonMergeIntoSQLCommandCarbon extends AtomicRunnableCommand { +} + +trait MvPlanWrapperCarbon extends ModularPlan { +} + +trait CarbonProjectForUpdate extends LogicalPlan { +} + +trait CarbonUpdateTable extends LogicalPlan { +} + +trait CarbonDeleteRecords extends LogicalPlan { +} + +trait CarbonInsertIntoCarbonTable extends Command { +} + +trait CarbonCustomDeterministicExpression extends Expression { +} + +abstract class OutputWriterCarbon(paths: String) extends OutputWriter { +} + +trait CarbonCommands extends MetadataCommand{ +} + +trait CarbonAtomicRunnableCommands extends AtomicRunnableCommand{ +} + +trait CarbonDataCommands extends DataCommand { +} + +trait CarbonDataWritingCommand extends DataWritingCommand { +} + +trait CarbonUnaryExpression extends UnaryExpression { +} + diff --git a/integration/spark/src/main/common2.3and2.4/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark/src/main/common2.3and2.4/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala index 6fbd4b62d3d..d463c18d66c 100644 --- a/integration/spark/src/main/common2.3and2.4/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala +++ b/integration/spark/src/main/common2.3and2.4/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala @@ -86,7 +86,7 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf, */ override def visitPropertyKeyValues(ctx: TablePropertyListContext): Map[String, String] = { val props = visitTablePropertyList(ctx) - CarbonSparkSqlParserUtil.visitPropertyKeyValues(ctx, props) + CarbonToSparkAdapter.visitPropertyKeyValues(ctx, props) } def getPropertyKeyValues(ctx: TablePropertyListContext): Map[String, String] diff --git a/integration/spark/src/main/common2.4and3.1/org/apache/spark/sql/CarbonBoundReference.scala b/integration/spark/src/main/common2.4and3.3/org/apache/spark/sql/CarbonBoundReference.scala similarity index 100% rename from integration/spark/src/main/common2.4and3.1/org/apache/spark/sql/CarbonBoundReference.scala rename to integration/spark/src/main/common2.4and3.3/org/apache/spark/sql/CarbonBoundReference.scala diff --git a/integration/spark/src/main/common2.4and3.1/org/apache/spark/sql/avro/AvroFileFormatFactory.scala b/integration/spark/src/main/common2.4and3.3/org/apache/spark/sql/avro/AvroFileFormatFactory.scala similarity index 100% rename from integration/spark/src/main/common2.4and3.1/org/apache/spark/sql/avro/AvroFileFormatFactory.scala rename to integration/spark/src/main/common2.4and3.3/org/apache/spark/sql/avro/AvroFileFormatFactory.scala diff --git a/integration/spark/src/main/common2.4and3.1/org/apache/spark/sql/execution/CreateDataSourceTableCommand.scala b/integration/spark/src/main/common2.4and3.3/org/apache/spark/sql/execution/CreateDataSourceTableCommand.scala similarity index 93% rename from integration/spark/src/main/common2.4and3.1/org/apache/spark/sql/execution/CreateDataSourceTableCommand.scala rename to integration/spark/src/main/common2.4and3.3/org/apache/spark/sql/execution/CreateDataSourceTableCommand.scala index c1a7d7f2e64..f09c5b364be 100644 --- a/integration/spark/src/main/common2.4and3.1/org/apache/spark/sql/execution/CreateDataSourceTableCommand.scala +++ b/integration/spark/src/main/common2.4and3.3/org/apache/spark/sql/execution/CreateDataSourceTableCommand.scala @@ -18,15 +18,14 @@ package org.apache.spark.sql.execution import org.apache.log4j.Logger -import org.apache.spark.sql.{AnalysisException, Row, SparkSession} +import org.apache.spark.sql.{AnalysisException, CarbonRunnableCommand, Row, SparkSession} import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType, CatalogUtils} -import org.apache.spark.sql.execution.command.RunnableCommand import org.apache.spark.sql.util.CreateTableCommonUtil.getCatalogTable import org.apache.carbondata.common.logging.LogServiceFactory case class CreateDataSourceTableCommand(table: CatalogTable, ignoreIfExists: Boolean) - extends RunnableCommand { + extends CarbonRunnableCommand { val LOGGER: Logger = LogServiceFactory.getLogService(this.getClass.getName) diff --git a/integration/spark/src/main/java/org/apache/spark/sql/CarbonMergeIntoSQLCommand.scala b/integration/spark/src/main/java/org/apache/spark/sql/CarbonMergeIntoSQLCommand.scala index 67611fdf726..71d5dbd72a1 100644 --- a/integration/spark/src/main/java/org/apache/spark/sql/CarbonMergeIntoSQLCommand.scala +++ b/integration/spark/src/main/java/org/apache/spark/sql/CarbonMergeIntoSQLCommand.scala @@ -26,7 +26,7 @@ import org.apache.spark.util.SparkUtil._ import org.apache.spark.util.TableAPIUtil case class CarbonMergeIntoSQLCommand(mergeInto: CarbonMergeIntoModel) - extends AtomicRunnableCommand { + extends CarbonMergeIntoSQLCommandCarbon { override def processMetadata(sparkSession: SparkSession): Seq[Row] = { Seq.empty diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/load/CsvRDDHelper.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/load/CsvRDDHelper.scala index 4867e5f7ded..c9ed2bd31f9 100644 --- a/integration/spark/src/main/scala/org/apache/carbondata/spark/load/CsvRDDHelper.scala +++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/load/CsvRDDHelper.scala @@ -111,7 +111,7 @@ object CsvRDDHelper { // 2. read function val readFunction = getReadFunction(hadoopConf) - new FileScanRDD(spark, readFunction, partitions) + CarbonToSparkAdapter.getFileScanRDD(spark, readFunction, partitions) } /** @@ -146,7 +146,7 @@ object CsvRDDHelper { // 2. read function val readFunction = getReadFunction(hadoopConf) - new FileScanRDD(spark, readFunction, partitions) + CarbonToSparkAdapter.getFileScanRDD(spark, readFunction, partitions) } private def getReadFunction(configuration: Configuration): (PartitionedFile => diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala index 90f1f8b5d3a..3f814f1e361 100644 --- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala +++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala @@ -644,13 +644,7 @@ class CarbonScanRDD[T: ClassTag]( // On success, // TODO: no memory leak should be there, resources should be freed on // success completion. - val onCompleteCallbacksField = - context.getClass.getDeclaredField("onCompleteCallbacks") - onCompleteCallbacksField.setAccessible(true) - val listeners = onCompleteCallbacksField.get(context) - .asInstanceOf[ArrayBuffer[TaskCompletionListener]] - - val isAdded = listeners.exists(p => p.isInstanceOf[CarbonLoadTaskCompletionListener]) + val isAdded = CarbonToSparkAdapter.check(context) model.setFreeUnsafeMemory(!isAdded) // add task completion before calling initialize as initialize method will internally // call for usage of unsafe method for processing of one blocklet and if there is any diff --git a/integration/spark/src/main/scala/org/apache/carbondata/view/MVPlanWrapper.scala b/integration/spark/src/main/scala/org/apache/carbondata/view/MVPlanWrapper.scala index c4c17a5f521..0c3996a07ee 100644 --- a/integration/spark/src/main/scala/org/apache/carbondata/view/MVPlanWrapper.scala +++ b/integration/spark/src/main/scala/org/apache/carbondata/view/MVPlanWrapper.scala @@ -17,12 +17,14 @@ package org.apache.carbondata.view +import org.apache.spark.sql.MvPlanWrapperCarbon import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.carbondata.core.view.MVSchema import org.apache.carbondata.mv.plans.modular.ModularPlan -case class MVPlanWrapper(modularPlan: ModularPlan, viewSchema: MVSchema) extends ModularPlan { +case class MVPlanWrapper(modularPlan: ModularPlan, viewSchema: MVSchema) + extends MvPlanWrapperCarbon { override def output: Seq[Attribute] = modularPlan.output diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala index 4813aa561d9..7dc0c8fb2b7 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala @@ -36,7 +36,7 @@ case class ExcludeProfile(attributes: Seq[Attribute]) extends CarbonProfile(attr case class ProjectForUpdate( table: UnresolvedRelation, columns: List[String], - children: Seq[LogicalPlan]) extends LogicalPlan { + children: Seq[LogicalPlan]) extends CarbonProjectForUpdate { override def output: Seq[Attribute] = Seq.empty } @@ -45,7 +45,7 @@ case class UpdateTable( columns: List[String], selectStmt: String, alias: Option[String] = None, - filer: String) extends LogicalPlan { + filer: String) extends CarbonUpdateTable { override def children: Seq[LogicalPlan] = Seq.empty override def output: Seq[Attribute] = Seq.empty } @@ -53,7 +53,7 @@ case class UpdateTable( case class DeleteRecords( statement: String, alias: Option[String] = None, - table: UnresolvedRelation) extends LogicalPlan { + table: UnresolvedRelation) extends CarbonDeleteRecords { override def children: Seq[LogicalPlan] = Seq.empty override def output: Seq[AttributeReference] = Seq.empty } @@ -69,7 +69,7 @@ case class InsertIntoCarbonTable (table: CarbonDatasourceHadoopRelation, overwrite: Boolean, ifNotExists: Boolean, containsMultipleInserts: Boolean) - extends Command { + extends CarbonInsertIntoCarbonTable { override def output: Seq[Attribute] = { Seq(AttributeReference("Segment ID", StringType, nullable = false)()) @@ -78,6 +78,7 @@ case class InsertIntoCarbonTable (table: CarbonDatasourceHadoopRelation, // This is the expected schema of the table prepared to be inserted into // including dynamic partition columns. val tableOutput = table.carbonRelation.output + } /** diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala index b73772f0975..61c381d2041 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala @@ -140,7 +140,7 @@ class CarbonEnv { .addNonSerializableProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, "true") Profiler.initialize(sparkSession.sparkContext) CarbonToSparkAdapter.addSparkSessionListener(sparkSession) - if(sparkSession.sparkContext.version.startsWith("3.1")) { + if(sparkSession.sparkContext.version.startsWith("3.3")) { CarbonProperties.getInstance().addProperty(CarbonCommonConstants .CARBON_SPARK_VERSION_SPARK3, "true") } diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala index c88828be235..d039900d17c 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala @@ -25,7 +25,8 @@ import org.apache.spark.sql.types.DataType /** * Custom expression to override the deterministic property . */ -case class CustomDeterministicExpression(nonDt: Expression ) extends Expression with Serializable{ +case class CustomDeterministicExpression(nonDt: Expression ) extends + CarbonCustomDeterministicExpression with Serializable{ override def nullable: Boolean = nonDt.nullable override def eval(input: InternalRow): Any = nonDt.eval(input) @@ -39,4 +40,5 @@ case class CustomDeterministicExpression(nonDt: Expression ) extends Expression override def genCode(ctx: CodegenContext): ExprCode = nonDt.genCode(ctx) override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = ev.copy() + } diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/carbondata/execution/datasources/SparkCarbonFileFormat.scala b/integration/spark/src/main/scala/org/apache/spark/sql/carbondata/execution/datasources/SparkCarbonFileFormat.scala index 30646c32726..ea2e1638274 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/carbondata/execution/datasources/SparkCarbonFileFormat.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/carbondata/execution/datasources/SparkCarbonFileFormat.scala @@ -193,7 +193,8 @@ class SparkCarbonFileFormat extends FileFormat */ private class CarbonOutputWriter(path: String, context: TaskAttemptContext, - fieldTypes: Array[StructField]) extends OutputWriter with AbstractCarbonOutputWriter { + fieldTypes: Array[StructField]) extends OutputWriterCarbon(path) + with AbstractCarbonOutputWriter { private val writable = new ObjectArrayWritable @@ -311,6 +312,7 @@ class SparkCarbonFileFormat extends FileFormat override def close(): Unit = { recordWriter.close(context) } + } override def shortName(): String = "carbon" diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/cache/CarbonDropCacheCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/cache/CarbonDropCacheCommand.scala index 3f38fbdd825..68edcc839f5 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/cache/CarbonDropCacheCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/cache/CarbonDropCacheCommand.scala @@ -19,9 +19,8 @@ package org.apache.spark.sql.execution.command.cache import scala.collection.JavaConverters._ -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.execution.command.MetadataCommand import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.core.cache.CacheProvider @@ -32,7 +31,7 @@ import org.apache.carbondata.events.{DropTableCacheEvent, OperationContext, Oper import org.apache.carbondata.view.MVManagerInSpark case class CarbonDropCacheCommand(tableIdentifier: TableIdentifier, internalCall: Boolean = false) - extends MetadataCommand { + extends CarbonCommands { val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/cache/CarbonShowCacheCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/cache/CarbonShowCacheCommand.scala index 0fd07425486..011fe1fe2ab 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/cache/CarbonShowCacheCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/cache/CarbonShowCacheCommand.scala @@ -20,8 +20,7 @@ package org.apache.spark.sql.execution.command.cache import scala.collection.JavaConverters._ import scala.collection.mutable -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} -import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.{AnalysisException, CarbonCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.NoSuchTableException import org.apache.spark.sql.catalyst.expressions.AttributeReference @@ -44,7 +43,7 @@ import org.apache.carbondata.spark.util.CommonUtil.bytesToDisplaySize case class CarbonShowCacheCommand(showExecutorCache: Boolean, tableIdentifier: Option[TableIdentifier], internalCall: Boolean = false) - extends MetadataCommand { + extends CarbonCommands { private lazy val cacheResult: Seq[(String, Int, Long, String)] = { executeJobToGetCache(List(), showExecutorCache) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/CarbonCreateIndexCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/CarbonCreateIndexCommand.scala index 19eeb519c87..71db2a2a7ad 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/CarbonCreateIndexCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/CarbonCreateIndexCommand.scala @@ -52,7 +52,7 @@ case class CarbonCreateIndexCommand( properties: Map[String, String], ifNotExistsSet: Boolean = false, var deferredRebuild: Boolean = false) - extends AtomicRunnableCommand { + extends CarbonAtomicRunnableCommands { private val LOGGER = LogServiceFactory.getLogService(this.getClass.getName) private var provider: IndexProvider = _ diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/CarbonRefreshIndexCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/CarbonRefreshIndexCommand.scala index fba56958ea3..f7126eb6e65 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/CarbonRefreshIndexCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/CarbonRefreshIndexCommand.scala @@ -21,10 +21,9 @@ import java.util import scala.util.control.Breaks.{break, breakable} -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonDataCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.NoSuchTableException -import org.apache.spark.sql.execution.command.DataCommand import org.apache.spark.sql.index.CarbonIndexUtil import org.apache.spark.sql.secondaryindex.command.SIRebuildSegmentRunner @@ -45,7 +44,7 @@ import org.apache.carbondata.index.IndexProvider case class CarbonRefreshIndexCommand( indexName: String, parentTableIdent: TableIdentifier, - segments: Option[List[String]]) extends DataCommand { + segments: Option[List[String]]) extends CarbonDataCommands { private val LOGGER = LogServiceFactory.getLogService(this.getClass.getName) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/DropIndexCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/DropIndexCommand.scala index f73ce3780be..4e4e467e1a2 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/DropIndexCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/DropIndexCommand.scala @@ -19,10 +19,9 @@ package org.apache.spark.sql.execution.command.index import scala.collection.mutable.ArrayBuffer -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonEnv, CarbonRunnableCommand, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.NoSuchTableException -import org.apache.spark.sql.execution.command.RunnableCommand import org.apache.spark.sql.hive.{CarbonHiveIndexMetadataUtil, CarbonMetaStore, CarbonRelation} import org.apache.spark.sql.index.CarbonIndexUtil import org.apache.spark.sql.secondaryindex.hive.CarbonInternalMetastore @@ -44,7 +43,7 @@ private[sql] case class DropIndexCommand( parentTableName: String = null, indexName: String, needLock: Boolean = true) - extends RunnableCommand { + extends CarbonRunnableCommand { def run(sparkSession: SparkSession): Seq[Row] = { var isSecondaryIndex = false diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/IndexRepairCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/IndexRepairCommand.scala index 6bcea43b815..345f06f7ca4 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/IndexRepairCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/IndexRepairCommand.scala @@ -19,9 +19,8 @@ package org.apache.spark.sql.execution.command.index import scala.collection.JavaConverters._ -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonDataCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.execution.command.DataCommand import org.apache.spark.sql.hive.CarbonRelation import org.apache.spark.sql.index.CarbonIndexUtil @@ -36,7 +35,7 @@ case class IndexRepairCommand( indexnameOp: Option[String], tableIdentifier: TableIdentifier, dbName: String, segments: Option[List[String]]) -extends DataCommand { +extends CarbonDataCommands { private val LOGGER = LogServiceFactory.getLogService(this.getClass.getName) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/ShowIndexesCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/ShowIndexesCommand.scala index 19ac6b1f6b6..3b952b63050 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/ShowIndexesCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/index/ShowIndexesCommand.scala @@ -17,24 +17,17 @@ package org.apache.spark.sql.execution.command.index -import java.util - import scala.collection.JavaConverters._ import scala.collection.immutable.ListMap -import scala.collection.mutable -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} -import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.{CarbonDataCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} -import org.apache.spark.sql.execution.command.DataCommand -import org.apache.spark.sql.index.CarbonIndexUtil import org.apache.spark.sql.secondaryindex.hive.CarbonInternalMetastore import org.apache.spark.sql.types.StringType import org.apache.carbondata.common.Strings import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.core.constants.CarbonCommonConstants -import org.apache.carbondata.core.index.IndexStoreManager import org.apache.carbondata.core.metadata.index.IndexType import org.apache.carbondata.core.metadata.schema.index.IndexProperty import org.apache.carbondata.core.metadata.schema.table.CarbonTable @@ -45,7 +38,7 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable case class ShowIndexesCommand( dbNameOp: Option[String], tableName: String) - extends DataCommand { + extends CarbonDataCommands { private val LOGGER = LogServiceFactory.getLogService(this.getClass.getName) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAddLoadCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAddLoadCommand.scala index b2d9fe1969e..ac7aae8bb03 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAddLoadCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAddLoadCommand.scala @@ -24,13 +24,12 @@ import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer import org.apache.hadoop.fs.FileStatus -import org.apache.spark.sql.{AnalysisException, CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{AnalysisException, CarbonCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.carbondata.execution.datasources.CarbonSparkDataSourceUtil.convertSparkToCarbonDataType import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.command.{Checker, MetadataCommand} import org.apache.spark.sql.execution.strategy.MixedFormatHandler import org.apache.spark.sql.hive.{CarbonHiveIndexMetadataUtil, CarbonRelation} -import org.apache.spark.sql.types.StructType import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException import org.apache.carbondata.common.logging.LogServiceFactory @@ -48,9 +47,8 @@ import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, SegmentUpdateDetails import org.apache.carbondata.core.reader.CarbonDeleteDeltaFileReaderImpl import org.apache.carbondata.core.statusmanager.{FileFormat, LoadMetadataDetails, SegmentStatus, SegmentStatusManager} import org.apache.carbondata.core.util.path.CarbonTablePath -import org.apache.carbondata.core.view.{MVSchema, MVStatus} -import org.apache.carbondata.events.{BuildIndexPostExecutionEvent, BuildIndexPreExecutionEvent, OperationContext, OperationListenerBus} -import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadTablePostExecutionEvent, LoadTablePostStatusUpdateEvent, LoadTablePreExecutionEvent, LoadTablePreStatusUpdateEvent} +import org.apache.carbondata.events.{OperationContext, OperationListenerBus} +import org.apache.carbondata.processing.loading.events.LoadEvents.LoadTablePostStatusUpdateEvent import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel} import org.apache.carbondata.processing.util.CarbonLoaderUtil import org.apache.carbondata.sdk.file.Schema @@ -67,7 +65,7 @@ case class CarbonAddLoadCommand( databaseNameOp: Option[String], tableName: String, options: Map[String, String]) - extends MetadataCommand { + extends CarbonCommands { private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala index 8a97296b172..5719445951a 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala @@ -22,11 +22,10 @@ import java.util import scala.collection.JavaConverters._ -import org.apache.spark.sql.{AnalysisException, CarbonEnv, Row, SparkSession, SQLContext} +import org.apache.spark.sql.{AnalysisException, CarbonAtomicRunnableCommands, CarbonEnv, Row, SparkSession, SQLContext} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.NoSuchTableException -import org.apache.spark.sql.execution.command.{AlterTableModel, AtomicRunnableCommand, CompactionModel} -import org.apache.spark.sql.hive.{CarbonHiveIndexMetadataUtil, CarbonRelation} +import org.apache.spark.sql.execution.command.{AlterTableModel, CompactionModel} import org.apache.spark.sql.optimizer.CarbonFilters import org.apache.spark.sql.util.CarbonException import org.apache.spark.util.AlterTableUtil @@ -58,7 +57,8 @@ import org.apache.carbondata.view.MVManagerInSpark case class CarbonAlterTableCompactionCommand( alterTableModel: AlterTableModel, tableInfoOp: Option[TableInfo] = None, - val operationContext: OperationContext = new OperationContext ) extends AtomicRunnableCommand { + val operationContext: OperationContext = new OperationContext ) + extends CarbonAtomicRunnableCommands { var table: CarbonTable = _ diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableFinishStreaming.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableFinishStreaming.scala index c1e8029b588..4f9c90f42d3 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableFinishStreaming.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableFinishStreaming.scala @@ -17,8 +17,7 @@ package org.apache.spark.sql.execution.command.management -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} -import org.apache.spark.sql.execution.command.MetadataCommand +import org.apache.spark.sql.{CarbonCommands, CarbonEnv, Row, SparkSession} import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.core.locks.{CarbonLockFactory, LockUsage} @@ -30,7 +29,7 @@ import org.apache.carbondata.streaming.segment.StreamSegment case class CarbonAlterTableFinishStreaming( dbName: Option[String], tableName: String) - extends MetadataCommand { + extends CarbonCommands { override def processMetadata(sparkSession: SparkSession): Seq[Row] = { val carbonTable = CarbonEnv.getCarbonTable(dbName, tableName)(sparkSession) setAuditTable(carbonTable) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala index d88cf212410..70dea71c0b4 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.execution.command.management import org.apache.log4j.Logger -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonDataCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression} import org.apache.spark.sql.execution.command.{Checker, DataCommand} import org.apache.spark.sql.optimizer.CarbonFilters @@ -40,7 +40,7 @@ case class CarbonCleanFilesCommand( tableName: String, options: Map[String, String] = Map.empty, isInternalCleanCall: Boolean = false) - extends DataCommand { + extends CarbonDataCommands { val LOGGER: Logger = LogServiceFactory.getLogService(this.getClass.getCanonicalName) val isDryRun: Boolean = options.getOrElse("dryrun", "false").toBoolean diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCliCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCliCommand.scala index e4fb725a4e7..d1d79200439 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCliCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCliCommand.scala @@ -21,7 +21,7 @@ import java.util import scala.collection.JavaConverters._ -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonDataCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} import org.apache.spark.sql.execution.command.{Checker, DataCommand} import org.apache.spark.sql.types.StringType @@ -38,7 +38,7 @@ case class CarbonCliCommand( databaseNameOp: Option[String], tableName: String, commandOptions: String) - extends DataCommand { + extends CarbonDataCommands { override def output: Seq[Attribute] = { Seq(AttributeReference("CarbonCli", StringType, nullable = false)()) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByIdCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByIdCommand.scala index 37117506086..ec87118473c 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByIdCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByIdCommand.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.execution.command.management -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonDataCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.execution.command.{Checker, DataCommand} import org.apache.carbondata.api.CarbonStore @@ -30,7 +30,7 @@ case class CarbonDeleteLoadByIdCommand( loadIds: Seq[String], databaseNameOp: Option[String], tableName: String) - extends DataCommand { + extends CarbonDataCommands { override def processData(sparkSession: SparkSession): Seq[Row] = { Checker.validateTableExists(databaseNameOp, tableName, sparkSession) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByLoadDateCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByLoadDateCommand.scala index e8db1bc2e4f..be01a79a4a9 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByLoadDateCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByLoadDateCommand.scala @@ -17,8 +17,8 @@ package org.apache.spark.sql.execution.command.management -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} -import org.apache.spark.sql.execution.command.{Checker, DataCommand} +import org.apache.spark.sql.{CarbonDataCommands, CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.execution.command.Checker import org.apache.carbondata.api.CarbonStore import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException @@ -31,7 +31,7 @@ case class CarbonDeleteLoadByLoadDateCommand( tableName: String, dateField: String, loadDate: String) - extends DataCommand { + extends CarbonDataCommands { override def processData(sparkSession: SparkSession): Seq[Row] = { Checker.validateTableExists(databaseNameOp, tableName, sparkSession) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteStageFilesCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteStageFilesCommand.scala index 0fc098ce1af..05b4720144c 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteStageFilesCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteStageFilesCommand.scala @@ -26,7 +26,7 @@ import scala.collection.JavaConverters._ import com.google.gson.Gson import org.apache.hadoop.conf.Configuration import org.apache.log4j.Logger -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonDataCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.execution.command.{Checker, DataCommand} import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException @@ -47,7 +47,7 @@ case class CarbonDeleteStageFilesCommand( databaseNameOp: Option[String], tableName: String, options: Map[String, String] -) extends DataCommand { +) extends CarbonDataCommands { @transient val LOGGER: Logger = LogServiceFactory.getLogService(this.getClass.getCanonicalName) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertFromStageCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertFromStageCommand.scala index 7b7bf3c51b1..c988ad7d0b5 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertFromStageCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertFromStageCommand.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.command.management import java.io.{DataInputStream, File, InputStreamReader, IOException} import java.util import java.util.Collections -import java.util.concurrent.{Callable, Executors, ExecutorService, TimeUnit} +import java.util.concurrent.{Callable, Executors, ExecutorService} import scala.collection.JavaConverters._ import scala.util.control.Breaks.{break, breakable} @@ -28,7 +28,7 @@ import scala.util.control.Breaks.{break, breakable} import com.google.gson.Gson import org.apache.hadoop.conf.Configuration import org.apache.hadoop.mapreduce.InputSplit -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonDataCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.execution.command.{Checker, DataCommand} import org.apache.spark.sql.hive.CarbonHiveIndexMetadataUtil import org.apache.spark.sql.util.SparkSQLUtil @@ -64,7 +64,7 @@ case class CarbonInsertFromStageCommand( databaseNameOp: Option[String], tableName: String, options: Map[String, String] -) extends DataCommand { +) extends CarbonDataCommands { private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala index 5a24aaaa0a5..39901f72e24 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala @@ -24,7 +24,7 @@ import scala.collection.JavaConverters._ import scala.collection.mutable import org.apache.spark.rdd.RDD -import org.apache.spark.sql.{AnalysisException, CarbonEnv, CarbonToSparkAdapter, DataFrame, Dataset, Row, SparkSession} +import org.apache.spark.sql.{AnalysisException, CarbonAtomicRunnableCommands, CarbonEnv, CarbonToSparkAdapter, DataFrame, Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Literal, NamedExpression} import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project} @@ -46,7 +46,6 @@ import org.apache.carbondata.core.util.{CarbonProperties, DataTypeUtil, ThreadLo import org.apache.carbondata.core.util.path.CarbonTablePath import org.apache.carbondata.events.OperationContext import org.apache.carbondata.events.exception.PreEventException -import org.apache.carbondata.processing.loading.TableProcessingOperations import org.apache.carbondata.processing.loading.exception.NoRetryException import org.apache.carbondata.processing.loading.model.CarbonLoadModel import org.apache.carbondata.processing.util.CarbonLoaderUtil @@ -67,7 +66,7 @@ case class CarbonInsertIntoCommand(databaseNameOp: Option[String], var partition: Map[String, Option[String]] = Map.empty, var operationContext: OperationContext = new OperationContext, var updateModel: Option[UpdateTableModel] = None) - extends AtomicRunnableCommand { + extends CarbonAtomicRunnableCommands { var table: CarbonTable = _ diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoHadoopFsRelationCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoHadoopFsRelationCommand.scala index cfa19dbd99e..085b16c9551 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoHadoopFsRelationCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoHadoopFsRelationCommand.scala @@ -62,7 +62,7 @@ case class CarbonInsertIntoHadoopFsRelationCommand( catalogTable: Option[CatalogTable], fileIndex: Option[FileIndex], outputColumnNames: Seq[String]) - extends DataWritingCommand { + extends CarbonDataWritingCommand { import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.escapePathName override def run(sparkSession: SparkSession, child: SparkPlan): Seq[Row] = { diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala index 1497740be1d..f314b6b0254 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala @@ -55,7 +55,7 @@ case class CarbonLoadDataCommand(databaseNameOp: Option[String], isOverwriteTable: Boolean, partition: Map[String, Option[String]] = Map.empty, var operationContext: OperationContext = new OperationContext) - extends AtomicRunnableCommand { + extends CarbonAtomicRunnableCommands { var table: CarbonTable = _ diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowSegmentsAsSelectCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowSegmentsAsSelectCommand.scala index 6bda29b3f7f..f69934a210e 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowSegmentsAsSelectCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowSegmentsAsSelectCommand.scala @@ -17,12 +17,12 @@ package org.apache.spark.sql.execution.command.management -import org.apache.spark.sql.{AnalysisException, CarbonEnv, DataFrame, Dataset, Row, SparkSession} +import org.apache.spark.sql.{AnalysisException, CarbonDataCommands, CarbonEnv, DataFrame, Dataset, Row, SparkSession} import org.apache.spark.sql.catalyst.analysis.UnresolvedStar import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, NamedExpression} import org.apache.spark.sql.catalyst.plans.logical.Project import org.apache.spark.sql.execution.command.{Checker, DataCommand} -import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, LongType, StringType} +import org.apache.spark.sql.types.DataTypes import org.apache.carbondata.api.CarbonStore import org.apache.carbondata.api.CarbonStore.readSegments @@ -42,7 +42,7 @@ case class CarbonShowSegmentsAsSelectCommand( limit: Option[Int], showHistory: Boolean = false, withStage: Boolean = false) - extends DataCommand { + extends CarbonDataCommands { private lazy val sparkSession = SparkSession.getActiveSession.get private lazy val carbonTable = { diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowSegmentsCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowSegmentsCommand.scala index be4d74e81f6..7783fe6c6f0 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowSegmentsCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowSegmentsCommand.scala @@ -19,8 +19,7 @@ package org.apache.spark.sql.execution.command.management import scala.collection.JavaConverters._ -import org.apache.hadoop.conf.Configuration -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonDataCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} import org.apache.spark.sql.execution.command.{Checker, DataCommand} import org.apache.spark.sql.types.StringType @@ -38,7 +37,7 @@ case class CarbonShowSegmentsCommand( limit: Option[Int], showHistory: Boolean = false, withStage: Boolean = false) - extends DataCommand { + extends CarbonDataCommands { // add new columns of show segments at last override def output: Seq[Attribute] = { diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala index 57c871332ff..6ef283bdfb4 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala @@ -50,7 +50,7 @@ import org.apache.carbondata.spark.util.CarbonScalaUtil case class RefreshCarbonTableCommand( databaseNameOp: Option[String], tableName: String) - extends MetadataCommand { + extends CarbonCommands { val LOGGER = LogServiceFactory.getLogService(this.getClass.getName) override def processMetadata(sparkSession: SparkSession): Seq[Row] = { diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala index 5435ed374da..cdfd1202f40 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala @@ -46,7 +46,7 @@ private[sql] case class CarbonProjectForDeleteCommand( databaseNameOp: Option[String], tableName: String, timestamp: String) - extends DataCommand { + extends CarbonDataCommands { override val output: Seq[Attribute] = { Seq(AttributeReference("Deleted Row Count", LongType, nullable = false)()) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala index 773c4a10691..dfdb74be375 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala @@ -50,7 +50,7 @@ private[sql] case class CarbonProjectForUpdateCommand( databaseNameOp: Option[String], tableName: String, columns: List[String]) - extends DataCommand { + extends CarbonDataCommands { override val output: Seq[Attribute] = { Seq(AttributeReference("Updated Row Count", LongType, nullable = false)()) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonTruncateCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonTruncateCommand.scala index 0bd36e7964d..7556420819a 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonTruncateCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonTruncateCommand.scala @@ -17,18 +17,14 @@ package org.apache.spark.sql.execution.command.mutation -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} -import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.execution.command.{Checker, DataCommand, TruncateTableCommand} -import org.apache.spark.sql.optimizer.CarbonFilters +import org.apache.spark.sql.{CarbonDataCommands, CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.execution.command.{Checker, TruncateTableCommand} import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.core.statusmanager.SegmentStatusManager -import org.apache.carbondata.core.util.CarbonProperties -import org.apache.carbondata.trash.DataTrashManager -case class CarbonTruncateCommand(child: TruncateTableCommand) extends DataCommand { +case class CarbonTruncateCommand(child: TruncateTableCommand) extends CarbonDataCommands { override def processData(sparkSession: SparkSession): Seq[Row] = { val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) Checker.validateTableExists(child.tableName.database, child.tableName.table, sparkSession) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/merge/CarbonMergeDataSetCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/merge/CarbonMergeDataSetCommand.scala index 53f5c26612f..246fc46e1ac 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/merge/CarbonMergeDataSetCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/merge/CarbonMergeDataSetCommand.scala @@ -29,7 +29,7 @@ import org.apache.hadoop.mapreduce.{JobID, TaskAttemptID, TaskID, TaskType} import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl import org.apache.spark.rdd.RDD -import org.apache.spark.sql.{AnalysisException, CarbonEnv, CarbonToSparkAdapter, Column, DataFrame, Dataset, Row, SparkSession} +import org.apache.spark.sql.{AnalysisException, CarbonDataCommands, CarbonEnv, CarbonToSparkAdapter, Column, DataFrame, Dataset, Row, SparkSession} import org.apache.spark.sql.avro.AvroFileFormatFactory import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute @@ -69,7 +69,7 @@ case class CarbonMergeDataSetCommand( var mergeMatches: MergeDataSetMatches = null, keyColumn: String = null, operationType: String = null) - extends DataCommand { + extends CarbonDataCommands { val LOGGER = LogServiceFactory.getLogService(this.getClass.getName) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableAddHivePartitionCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableAddHivePartitionCommand.scala index e77c6a9e26d..28021c6d5b8 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableAddHivePartitionCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableAddHivePartitionCommand.scala @@ -22,7 +22,7 @@ import java.util import scala.collection.JavaConverters._ import scala.util.control.Breaks.{break, breakable} -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonAtomicRunnableCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.execution.command.{AlterTableAddPartitionCommand, AlterTableDropPartitionCommand, AlterTableModel, AtomicRunnableCommand} @@ -53,7 +53,7 @@ case class CarbonAlterTableAddHivePartitionCommand( tableName: TableIdentifier, partitionSpecsAndLocs: Seq[(TablePartitionSpec, Option[String])], ifNotExists: Boolean) - extends AtomicRunnableCommand { + extends CarbonAtomicRunnableCommands { var partitionSpecsAndLocsTobeAdded : util.List[PartitionSpec] = _ var table: CarbonTable = _ diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala index d7d317f4f31..feddcfd4983 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala @@ -22,7 +22,7 @@ import java.util import scala.collection.JavaConverters._ import org.apache.hadoop.fs.Path -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonAtomicRunnableCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.execution.command.{AlterTableAddPartitionCommand, AlterTableDropPartitionCommand, AtomicRunnableCommand} @@ -33,7 +33,6 @@ import org.apache.spark.util.AlterTableUtil import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.core.datastore.impl.FileFactory import org.apache.carbondata.core.index.IndexStoreManager -import org.apache.carbondata.core.indexstore.PartitionSpec import org.apache.carbondata.core.locks.{ICarbonLock, LockUsage} import org.apache.carbondata.core.metadata.SegmentFileStore import org.apache.carbondata.core.metadata.schema.table.CarbonTable @@ -62,7 +61,7 @@ case class CarbonAlterTableDropHivePartitionCommand( purge: Boolean, retainData: Boolean, operationContext: OperationContext = new OperationContext) - extends AtomicRunnableCommand { + extends CarbonAtomicRunnableCommands { var carbonPartitionsTobeDropped : util.List[String] = _ var table: CarbonTable = _ diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala index 6aea625871f..b8c2f6b5c33 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala @@ -19,8 +19,8 @@ package org.apache.spark.sql.execution.command.schema import scala.collection.JavaConverters._ -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} -import org.apache.spark.sql.execution.command.{AlterTableAddColumnsModel, AlterTableColumnSchemaGenerator, MetadataCommand} +import org.apache.spark.sql.{CarbonCommands, CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.execution.command.{AlterTableAddColumnsModel, AlterTableColumnSchemaGenerator} import org.apache.spark.sql.hive.CarbonSessionCatalogUtil import org.apache.spark.util.{AlterTableUtil, SparkUtil} @@ -35,7 +35,7 @@ import org.apache.carbondata.format.TableInfo private[sql] case class CarbonAlterTableAddColumnCommand( alterTableAddColumnsModel: AlterTableAddColumnsModel) - extends MetadataCommand { + extends CarbonCommands { override def processMetadata(sparkSession: SparkSession): Seq[Row] = { val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableColRenameDataTypeChangeCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableColRenameDataTypeChangeCommand.scala index 01491fa79e4..c529c67a1bf 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableColRenameDataTypeChangeCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableColRenameDataTypeChangeCommand.scala @@ -23,7 +23,7 @@ import scala.collection.JavaConverters._ import scala.collection.mutable import org.apache.hadoop.hive.metastore.api.InvalidOperationException -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.execution.command.{AlterTableDataTypeChangeModel, DataTypeInfo, MetadataCommand} import org.apache.spark.sql.hive.CarbonSessionCatalogUtil import org.apache.spark.util.AlterTableUtil @@ -42,7 +42,7 @@ import org.apache.carbondata.format.{ColumnSchema, DataType, SchemaEvolutionEntr import org.apache.carbondata.spark.util.{CommonUtil, DataTypeConverterUtil} abstract class CarbonAlterTableColumnRenameCommand(oldColumnName: String, newColumnName: String) - extends MetadataCommand { + extends CarbonCommands { import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema protected def validColumnsForRenaming(columnSchemaList: mutable.Buffer[ColumnSchema], diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala index c4630917d49..0ff3853d1eb 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.command.schema import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.execution.command.{AlterTableDropColumnModel, MetadataCommand} import org.apache.spark.sql.hive.CarbonSessionCatalogUtil import org.apache.spark.util.{AlterTableUtil, SparkUtil} @@ -39,7 +39,7 @@ import org.apache.carbondata.format.SchemaEvolutionEntry private[sql] case class CarbonAlterTableDropColumnCommand( alterTableDropColumnModel: AlterTableDropColumnModel) - extends MetadataCommand { + extends CarbonCommands { override def processMetadata(sparkSession: SparkSession): Seq[Row] = { val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala index a8995725ae2..50440eccb76 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala @@ -17,9 +17,8 @@ package org.apache.spark.sql.execution.command.schema -import org.apache.spark.sql.{CarbonEnv, SparkSession} +import org.apache.spark.sql.{CarbonCommands, CarbonEnv, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.catalog.CatalogTablePartition import org.apache.spark.sql.execution.command.{AlterTableRenameModel, MetadataCommand} import org.apache.spark.sql.hive.{CarbonRelation, CarbonSessionCatalogUtil, MockClassForAlterRevertTests} import org.apache.spark.sql.index.CarbonIndexUtil @@ -41,7 +40,7 @@ import org.apache.carbondata.view.MVManagerInSpark private[sql] case class CarbonAlterTableRenameCommand( alterTableRenameModel: AlterTableRenameModel) - extends MetadataCommand { + extends CarbonCommands { override def processMetadata(sparkSession: SparkSession): Seq[Nothing] = { val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableSetCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableSetCommand.scala index 60751a2d510..dd76c729f6e 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableSetCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableSetCommand.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.execution.command.schema -import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.{CarbonCommands, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.command._ import org.apache.spark.util.AlterTableUtil @@ -26,7 +26,7 @@ private[sql] case class CarbonAlterTableSetCommand( tableIdentifier: TableIdentifier, properties: Map[String, String], isView: Boolean) - extends MetadataCommand { + extends CarbonCommands { override def processMetadata(sparkSession: SparkSession): Seq[Row] = { setAuditTable(tableIdentifier.database.getOrElse(sparkSession.catalog.currentDatabase), diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableUnsetCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableUnsetCommand.scala index 49f46791231..3c8bb627e34 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableUnsetCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableUnsetCommand.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.execution.command.schema -import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.{CarbonCommands, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.command._ import org.apache.spark.util.AlterTableUtil @@ -28,7 +28,7 @@ private[sql] case class CarbonAlterTableUnsetCommand( propKeys: Seq[String], ifExists: Boolean, isView: Boolean) - extends MetadataCommand { + extends CarbonCommands { override def processMetadata(sparkSession: SparkSession): Seq[Row] = { setAuditTable(tableIdentifier.database.getOrElse(sparkSession.catalog.currentDatabase), diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonCreateStreamCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonCreateStreamCommand.scala index cf91ba7889a..6e10a992127 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonCreateStreamCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonCreateStreamCommand.scala @@ -47,7 +47,7 @@ case class CarbonCreateStreamCommand( ifNotExists: Boolean, optionMap: Map[String, String], query: String -) extends DataCommand { +) extends CarbonDataCommands { override def output: Seq[Attribute] = Seq(AttributeReference("Stream Name", StringType, nullable = false)(), diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonDropStreamCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonDropStreamCommand.scala index 49f1d1128f1..17bdc21a280 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonDropStreamCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonDropStreamCommand.scala @@ -17,8 +17,7 @@ package org.apache.spark.sql.execution.command.stream -import org.apache.spark.sql.{Row, SparkSession} -import org.apache.spark.sql.execution.command.MetadataCommand +import org.apache.spark.sql.{CarbonCommands, Row, SparkSession} import org.apache.carbondata.stream.StreamJobManager @@ -28,7 +27,7 @@ import org.apache.carbondata.stream.StreamJobManager case class CarbonDropStreamCommand( streamName: String, ifExists: Boolean -) extends MetadataCommand { +) extends CarbonCommands { override def processMetadata(sparkSession: SparkSession): Seq[Row] = { setAuditInfo(Map("streamName" -> streamName)) StreamJobManager.stopStream(streamName, ifExists) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonShowStreamsCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonShowStreamsCommand.scala index ee749b3a56b..877dd118797 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonShowStreamsCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/stream/CarbonShowStreamsCommand.scala @@ -20,10 +20,9 @@ package org.apache.spark.sql.execution.command.stream import java.util.Date import java.util.concurrent.TimeUnit -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} -import org.apache.spark.sql.execution.command.MetadataCommand import org.apache.spark.sql.types.StringType import org.apache.carbondata.stream.StreamJobManager @@ -33,7 +32,7 @@ import org.apache.carbondata.stream.StreamJobManager */ case class CarbonShowStreamsCommand( tableOp: Option[TableIdentifier] -) extends MetadataCommand { +) extends CarbonCommands { override def output: Seq[Attribute] = { Seq(AttributeReference("Stream Name", StringType, nullable = false)(), AttributeReference("JobId", StringType, nullable = false)(), diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateDataSourceTableCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateDataSourceTableCommand.scala index 934812b4435..7af1262ccee 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateDataSourceTableCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateDataSourceTableCommand.scala @@ -18,10 +18,10 @@ package org.apache.spark.sql.execution.command.table import org.apache.hadoop.hive.metastore.api.AlreadyExistsException -import org.apache.spark.sql.{AnalysisException, CarbonEnv, CarbonSource, Row, SparkSession} +import org.apache.spark.sql.{AnalysisException, CarbonCommands, CarbonEnv, CarbonSource, Row, SparkSession} import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType} -import org.apache.spark.sql.execution.command.{CreateDataSourceTableCommand, DropTableCommand, MetadataCommand} +import org.apache.spark.sql.execution.command.DropTableCommand import org.apache.spark.sql.execution.datasources.PartitioningUtils import org.apache.carbondata.common.logging.LogServiceFactory @@ -34,7 +34,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory case class CarbonCreateDataSourceTableCommand( table: CatalogTable, ignoreIfExists: Boolean -) extends MetadataCommand { +) extends CarbonCommands { override protected def opName: String = "CREATE TABLE" override def processMetadata(sparkSession: SparkSession): Seq[Row] = { diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala index 2c6bbfb5921..8b3fdbeb21e 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala @@ -17,12 +17,11 @@ package org.apache.spark.sql.execution.command.table -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonAtomicRunnableCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.execution.command.AtomicRunnableCommand -import org.apache.spark.sql.execution.command.management.{CarbonInsertIntoCommand} +import org.apache.spark.sql.execution.command.management.CarbonInsertIntoCommand import org.apache.carbondata.core.metadata.schema.table.TableInfo @@ -39,7 +38,7 @@ case class CarbonCreateTableAsSelectCommand( tableInfo: TableInfo, query: LogicalPlan, ifNotExistsSet: Boolean = false, - tableLocation: Option[String] = None) extends AtomicRunnableCommand { + tableLocation: Option[String] = None) extends CarbonAtomicRunnableCommands { var insertIntoCommand: CarbonInsertIntoCommand = _ diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala index f14c3718d56..668e76a5ee7 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala @@ -42,7 +42,7 @@ case class CarbonCreateTableCommand( isExternal : Boolean = false, createDSTable: Boolean = true, isVisible: Boolean = true) - extends MetadataCommand { + extends CarbonCommands { override def processMetadata(sparkSession: SparkSession): Seq[Row] = { val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableLikeCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableLikeCommand.scala index 7be1215d974..28147a3638a 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableLikeCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableLikeCommand.scala @@ -20,9 +20,8 @@ package org.apache.spark.sql.execution.command.table import java.util import java.util.UUID -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.execution.command.MetadataCommand import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException import org.apache.carbondata.common.logging.LogServiceFactory @@ -40,7 +39,7 @@ import org.apache.carbondata.core.metadata.schema.table.{TableInfo, TableSchema} case class CarbonCreateTableLikeCommand( sourceTable: TableIdentifier, targetTable: TableIdentifier, - ifNotExists: Boolean = false) extends MetadataCommand { + ifNotExists: Boolean = false) extends CarbonCommands { private val LOGGER = LogServiceFactory.getLogService(this.getClass.getName) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala index 505c4810dac..7c889fc06cc 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala @@ -22,13 +22,12 @@ import java.util.Date import scala.collection.JavaConverters._ import scala.util.control.Breaks.{break, breakable} -import org.apache.spark.sql.{CarbonEnv, EnvHelper, Row, SparkSession} +import org.apache.spark.sql.{CarbonCommands, CarbonEnv, EnvHelper, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.command.Checker -import org.apache.spark.sql.execution.command.MetadataCommand import org.apache.spark.sql.hive.CarbonRelation import org.apache.spark.sql.types.{ArrayType, MapType, MetadataBuilder, StringType, StructField, StructType} @@ -47,7 +46,7 @@ private[sql] case class CarbonDescribeFormattedCommand( override val output: Seq[Attribute], partitionSpec: TablePartitionSpec, tblIdentifier: TableIdentifier) - extends MetadataCommand { + extends CarbonCommands { override def processMetadata(sparkSession: SparkSession): Seq[Row] = { val relation = CarbonEnv.getInstance(sparkSession).carbonMetaStore @@ -391,7 +390,7 @@ case class CarbonDescribeColumnCommand( databaseNameOp: Option[String], tableName: String, inputFieldNames: java.util.List[String]) - extends MetadataCommand { + extends CarbonCommands { override val output: Seq[Attribute] = Seq( // Column names are based on Hive. @@ -506,7 +505,7 @@ case class CarbonDescribeColumnCommand( case class CarbonDescribeShortCommand( databaseNameOp: Option[String], tableName: String) - extends MetadataCommand { + extends CarbonCommands { override val output: Seq[Attribute] = Seq( // Column names are based on Hive. diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala index 5217709d2f8..a33b35fb833 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala @@ -20,9 +20,8 @@ package org.apache.spark.sql.execution.command.table import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer -import org.apache.spark.sql.{CarbonEnv, EnvHelper, Row, SparkSession} +import org.apache.spark.sql.{CarbonAtomicRunnableCommands, CarbonEnv, EnvHelper, Row, SparkSession} import org.apache.spark.sql.catalyst.analysis.NoSuchTableException -import org.apache.spark.sql.execution.command.AtomicRunnableCommand import org.apache.spark.sql.execution.command.index.DropIndexCommand import org.apache.spark.sql.execution.command.view.CarbonDropMVCommand import org.apache.spark.sql.hive.CarbonFileMetastore @@ -45,7 +44,7 @@ case class CarbonDropTableCommand( tableName: String, dropChildTable: Boolean = false, isInternalCall: Boolean = false) - extends AtomicRunnableCommand { + extends CarbonAtomicRunnableCommands { var carbonTable: CarbonTable = _ var viewDropCommands : Seq[CarbonDropMVCommand] = Seq.empty diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala index 14fed7a46da..9e083602cf3 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.execution.command.table -import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.{CarbonCommands, Row, SparkSession} import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} import org.apache.spark.sql.catalyst.plans.logical.{Command, LogicalPlan, Union} import org.apache.spark.sql.execution.command.{ExplainCommand, MetadataCommand} @@ -30,7 +30,7 @@ case class CarbonExplainCommand( child: LogicalPlan, override val output: Seq[Attribute] = Seq(AttributeReference("plan", StringType, nullable = true)())) - extends MetadataCommand { + extends CarbonCommands { override def processMetadata(sparkSession: SparkSession): Seq[Row] = { val explainCommand = child.asInstanceOf[ExplainCommand] @@ -51,7 +51,7 @@ case class CarbonInternalExplainCommand( explainCommand: ExplainCommand, override val output: Seq[Attribute] = Seq(AttributeReference("plan", StringType, nullable = true)())) - extends MetadataCommand { + extends CarbonCommands { override def processMetadata(sparkSession: SparkSession): Seq[Row] = { CarbonExplainCommand diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonShowCreateTableCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonShowCreateTableCommand.scala index 8ac67d415ed..f2ecd21433a 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonShowCreateTableCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonShowCreateTableCommand.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.execution.command.table -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.analysis.NoSuchTableException import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} import org.apache.spark.sql.execution.command.{MetadataCommand, ShowCreateTableCommand} @@ -28,7 +28,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory case class CarbonShowCreateTableCommand( child: ShowCreateTableCommand -) extends MetadataCommand { +) extends CarbonCommands { override val output: Seq[Attribute] = Seq( AttributeReference("createtab_stmt", StringType, nullable = false)() ) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonShowTablesCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonShowTablesCommand.scala index 496a66d4bf1..f44992b9f53 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonShowTablesCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonShowTablesCommand.scala @@ -17,13 +17,13 @@ package org.apache.spark.sql.execution.command.table -import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.{CarbonCommands, Row, SparkSession} import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.execution.command.{MetadataCommand, ShowTablesCommand} private[sql] case class CarbonShowTablesCommand(showTablesCommand: ShowTablesCommand) - extends MetadataCommand { + extends CarbonCommands { override val output: Seq[Attribute] = showTablesCommand.output diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonCreateMVCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonCreateMVCommand.scala index e2d367db5a8..4d190db0aaf 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonCreateMVCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonCreateMVCommand.scala @@ -24,13 +24,13 @@ import scala.collection.mutable import scala.util.control.Breaks.{break, breakable} import org.apache.log4j.Logger -import org.apache.spark.sql.{CarbonEnv, CarbonSource, Row, SparkSession} +import org.apache.spark.sql.{CarbonAtomicRunnableCommands, CarbonEnv, CarbonSource, Row, SparkSession} import org.apache.spark.sql.catalyst.{CarbonParserUtil, TableIdentifier} import org.apache.spark.sql.catalyst.catalog.{CatalogTable, HiveTableRelation} import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, Cast, Coalesce, Expression, Literal, ScalaUDF} import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Average} import org.apache.spark.sql.catalyst.plans.logical.{Join, Limit, LogicalPlan, Sort} -import org.apache.spark.sql.execution.command.{AtomicRunnableCommand, Field, PartitionerField, TableModel, TableNewProcessor} +import org.apache.spark.sql.execution.command.{ Field, PartitionerField, TableModel, TableNewProcessor} import org.apache.spark.sql.execution.command.table.{CarbonCreateTableCommand, CarbonDropTableCommand} import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.parser.MVQueryParser @@ -46,7 +46,7 @@ import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, RelationId import org.apache.carbondata.core.statusmanager.SegmentStatusManager import org.apache.carbondata.core.util.CarbonProperties import org.apache.carbondata.core.view._ -import org.apache.carbondata.events.{withEvents, OperationContext, OperationListenerBus} +import org.apache.carbondata.events.withEvents import org.apache.carbondata.mv.plans.modular.{GroupBy, ModularPlan, SimpleModularizer} import org.apache.carbondata.mv.plans.util.{BirdcageOptimizer, SQLBuilder} import org.apache.carbondata.spark.util.CommonUtil @@ -64,7 +64,7 @@ case class CarbonCreateMVCommand( queryString: String, ifNotExistsSet: Boolean = false, deferredRefresh: Boolean = false) - extends AtomicRunnableCommand { + extends CarbonAtomicRunnableCommands { private val logger = CarbonCreateMVCommand.LOGGER diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonDropMVCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonDropMVCommand.scala index d54d81b647f..76d70ddeb8d 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonDropMVCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonDropMVCommand.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.execution.command.view import org.apache.log4j.Logger -import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.{CarbonAtomicRunnableCommands, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.command.AtomicRunnableCommand import org.apache.spark.sql.execution.command.table.CarbonDropTableCommand @@ -40,7 +40,7 @@ case class CarbonDropMVCommand( ifExistsSet: Boolean, forceDrop: Boolean = false, isLockAcquiredOnFactTable: String = null) - extends AtomicRunnableCommand { + extends CarbonAtomicRunnableCommands { private val logger = CarbonDropMVCommand.LOGGER diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonRefreshMVCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonRefreshMVCommand.scala index 6a6e6d5b91e..5dce418fb91 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonRefreshMVCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonRefreshMVCommand.scala @@ -17,9 +17,8 @@ package org.apache.spark.sql.execution.command.view -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonDataCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.execution.command.DataCommand import org.apache.carbondata.common.exceptions.sql.MalformedMVCommandException import org.apache.carbondata.core.view.MVStatus @@ -33,7 +32,7 @@ import org.apache.carbondata.view.{MVHelper, MVManagerInSpark, MVRefresher, Refr */ case class CarbonRefreshMVCommand( databaseNameOption: Option[String], - mvName: String) extends DataCommand { + mvName: String) extends CarbonDataCommands { override def processData(session: SparkSession): Seq[Row] = { val databaseName = diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonShowMVCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonShowMVCommand.scala index ccd289767e3..e4efc29b960 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonShowMVCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/view/CarbonShowMVCommand.scala @@ -21,7 +21,7 @@ import java.util import scala.collection.JavaConverters._ -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonDataCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} import org.apache.spark.sql.execution.command.{Checker, DataCommand} @@ -35,7 +35,7 @@ import org.apache.carbondata.view.MVManagerInSpark */ case class CarbonShowMVCommand( databaseNameOption: Option[String], - relatedTableIdentifier: Option[TableIdentifier]) extends DataCommand { + relatedTableIdentifier: Option[TableIdentifier]) extends CarbonDataCommands { override def output: Seq[Attribute] = { Seq( diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala index b0d1809f3b9..edf2cdecd10 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala @@ -32,7 +32,7 @@ import org.apache.spark.TaskContext import org.apache.spark.internal.Logging import org.apache.spark.internal.io.FileCommitProtocol import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonEnv, OutputWriterCarbon, Row, SparkSession} import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils import org.apache.spark.sql.internal.SQLConf @@ -477,7 +477,7 @@ private class CarbonOutputWriter(path: String, nonPartitionFieldTypes: Seq[DataType], taskNo : String, model: CarbonLoadModel) - extends OutputWriter with AbstractCarbonOutputWriter { + extends OutputWriterCarbon(path) with AbstractCarbonOutputWriter { var actualPath = path var tmpPath = context.getConfiguration.get("carbon.newTaskTempFile.path", actualPath) val converter = new DataTypeConverterImpl diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/joins/BroadCastPolygonFilterPushJoin.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/joins/BroadCastPolygonFilterPushJoin.scala index 1c083436f25..e2086de58b0 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/joins/BroadCastPolygonFilterPushJoin.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/joins/BroadCastPolygonFilterPushJoin.scala @@ -21,7 +21,7 @@ import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import org.apache.spark.rdd.RDD -import org.apache.spark.sql.CarbonToSparkAdapter +import org.apache.spark.sql.{CarbonBinaryExecNode, CarbonToSparkAdapter} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Attribute, BindReferences, Expression, JoinedRow, Literal, ScalaUDF} import org.apache.spark.sql.catalyst.plans.JoinType @@ -48,7 +48,7 @@ case class BroadCastPolygonFilterPushJoin( condition: Option[Expression], left: SparkPlan, right: SparkPlan -) extends BinaryExecNode { +) extends CarbonBinaryExecNode { // BuildSide will be BuildRight protected lazy val (buildPlan, streamedPlan) = (right, left) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala index 9564f446e7d..d3a76a262ae 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala @@ -74,7 +74,8 @@ private[sql] object CarbonSourceStrategy extends SparkStrategy { projects foreach { case alias: Alias if alias.child.isInstanceOf[Expression] => alias.child match { - case Cast(s: ScalaUDF, _, _) => validateGeoUtilUDFs(s) + case c: Cast if c.child.isInstanceOf[ScalaUDF] => validateGeoUtilUDFs( + c.child.asInstanceOf[ScalaUDF]) case s: ScalaUDF => validateGeoUtilUDFs(s) case _ => } diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/DDLHelper.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/DDLHelper.scala index 1839820b846..7bcdc883cea 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/DDLHelper.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/DDLHelper.scala @@ -321,10 +321,10 @@ object DDLHelper { showPartitionsCommand } else { if (cols.isDefined) { - ShowPartitionsCommand(tableName, - Option(CarbonSparkSqlParserUtil.copyTablePartition(cols.get))) + CarbonToSparkAdapter.showPartitionsCommand(Option(CarbonSparkSqlParserUtil + .copyTablePartition(cols.get)), showPartitionsCommand) } else { - ShowPartitionsCommand(tableName, None) + CarbonToSparkAdapter.showPartitionsCommand(None, showPartitionsCommand) } } } diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/DMLStrategy.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/DMLStrategy.scala index 9236e1e1dcb..d567206c228 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/DMLStrategy.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/DMLStrategy.scala @@ -23,7 +23,7 @@ import scala.collection.mutable import org.apache.log4j.Logger import org.apache.spark.rdd.RDD -import org.apache.spark.sql.{CarbonCountStar, CarbonDatasourceHadoopRelation, CarbonToSparkAdapter, CountStarPlan, InsertIntoCarbonTable, SparkSession, SparkVersionAdapter} +import org.apache.spark.sql.{CarbonCountStar, CarbonDatasourceHadoopRelation, CarbonToSparkAdapter, CountStarPlan, InsertIntoCarbonTable, SparkSession, SQLContext} import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} import org.apache.spark.sql.catalyst.analysis.UnresolvedAlias import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, Attribute, AttributeReference, Cast, Descending, Expression, IntegerLiteral, Literal, NamedExpression, ScalaUDF, SortOrder, UnsafeProjection} @@ -94,8 +94,8 @@ object DMLStrategy extends SparkStrategy { val dataType = StringType var children: Seq[Expression] = mutable.Seq.empty val geoHashColumn = condition.get.children.head match { - case Cast(attr: AttributeReference, _, _) => - attr + case c: Cast if c.child.isInstanceOf[AttributeReference] => + c.child.asInstanceOf[AttributeReference] case attr: AttributeReference => attr } @@ -169,7 +169,7 @@ object DMLStrategy extends SparkStrategy { ) condition.map(FilterExec(_, pushedDownJoin)).getOrElse(pushedDownJoin) :: Nil } - case CarbonExtractEquiJoinKeys(Inner, leftKeys, rightKeys, condition, left, right) + case CarbonExtractEquiJoinKeys(Inner, leftKeys, rightKeys, condition, _, left, right) if isCarbonPlan(left) && CarbonIndexUtil.checkIsIndexTable(right) => LOGGER.info(s"pushing down for ExtractEquiJoinKeys:right") val carbon = CarbonSourceStrategy.apply(left).head @@ -205,7 +205,7 @@ object DMLStrategy extends SparkStrategy { planLater(right), condition) condition.map(FilterExec(_, pushedDownJoin)).getOrElse(pushedDownJoin) :: Nil - case CarbonExtractEquiJoinKeys(Inner, leftKeys, rightKeys, condition, left, + case CarbonExtractEquiJoinKeys(Inner, leftKeys, rightKeys, condition, conditionOnJoin, left, right) if isCarbonPlan(right) && CarbonIndexUtil.checkIsIndexTable(left) => LOGGER.info(s"pushing down for ExtractEquiJoinKeys:left") @@ -220,7 +220,7 @@ object DMLStrategy extends SparkStrategy { carbon, condition) condition.map(FilterExec(_, pushedDownJoin)).getOrElse(pushedDownJoin) :: Nil - case CarbonExtractEquiJoinKeys(LeftSemi, leftKeys, rightKeys, condition, + case CarbonExtractEquiJoinKeys(LeftSemi, leftKeys, rightKeys, condition, conditionOnJoin, left, right) if isLeftSemiExistPushDownEnabled && isAllCarbonPlan(left) && isAllCarbonPlan(right) => @@ -247,12 +247,12 @@ object DMLStrategy extends SparkStrategy { object CarbonExtractEquiJoinKeys { def unapply(plan: LogicalPlan): Option[(JoinType, Seq[Expression], Seq[Expression], - Option[Expression], LogicalPlan, LogicalPlan)] = { + Option[Expression], Option[Expression], LogicalPlan, LogicalPlan)] = { plan match { case join: Join => ExtractEquiJoinKeys.unapply(join) match { // TODO: Spark is using hints now, carbon also should use join hints - case Some(x) => Some(x._1, x._2, x._3, x._4, x._5, x._6) + case Some(x) => Option(x._1, x._2, x._3, x._4, x._5, x._6, x._7) case None => None } case _ => None @@ -395,7 +395,8 @@ case class UnionCommandExec(cmd: RunnableCommand) extends LeafExecNode { protected[sql] lazy val sideEffectResult: Seq[InternalRow] = { val converter = CatalystTypeConverters.createToCatalystConverter(schema) - val internalRow = cmd.run(sqlContext.sparkSession).map(converter(_).asInstanceOf[InternalRow]) + val internalRow = cmd.run(SQLContext.getOrCreate(sparkContext).sparkSession) + .map(converter(_).asInstanceOf[InternalRow]) val unsafeProjection = UnsafeProjection.create(output.map(_.dataType).toArray) // To make GenericInternalRow to UnsafeRow val row = unsafeProjection(internalRow.head) @@ -405,7 +406,7 @@ case class UnionCommandExec(cmd: RunnableCommand) extends LeafExecNode { override def output: Seq[Attribute] = cmd.output protected override def doExecute(): RDD[InternalRow] = { - sqlContext.sparkContext.parallelize(sideEffectResult, 1) + SQLContext.getOrCreate(sparkContext).sparkContext.parallelize(sideEffectResult, 1) } } diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CreateCarbonSourceTableAsSelectCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CreateCarbonSourceTableAsSelectCommand.scala index 675b1b84195..1a60e43520a 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CreateCarbonSourceTableAsSelectCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CreateCarbonSourceTableAsSelectCommand.scala @@ -19,13 +19,12 @@ package org.apache.spark.sql.hive import java.net.URI -import org.apache.spark.sql.{AnalysisException, Dataset, Row, SaveMode, SparkSession} +import org.apache.spark.sql.{AnalysisException, CarbonAtomicRunnableCommands, CarbonToSparkAdapter, Dataset, Row, SaveMode, SparkSession} import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType, CatalogUtils} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.execution.command.{AlterTableRecoverPartitionsCommand, AtomicRunnableCommand} +import org.apache.spark.sql.execution.command.{AtomicRunnableCommand, RepairTableCommand} import org.apache.spark.sql.execution.datasources.{DataSource, HadoopFsRelation} import org.apache.spark.sql.sources.BaseRelation -import org.apache.spark.util.CarbonReflectionUtils /** * Create table 'using carbondata' and insert the query result into it. @@ -40,7 +39,7 @@ case class CreateCarbonSourceTableAsSelectCommand( table: CatalogTable, mode: SaveMode, query: LogicalPlan) - extends AtomicRunnableCommand { + extends CarbonAtomicRunnableCommands { override def processMetadata(sparkSession: SparkSession): Seq[Row] = { Seq.empty @@ -85,7 +84,8 @@ case class CreateCarbonSourceTableAsSelectCommand( case fs: HadoopFsRelation if table.partitionColumnNames.nonEmpty && sparkSession.sqlContext.conf.manageFilesourcePartitions => // Need to recover partitions into the metastore so our saved data is visible. - sessionState.executePlan(AlterTableRecoverPartitionsCommand(table.identifier)).toRdd + sessionState.executePlan(RepairTableCommand(table.identifier, enableAddPartitions = true + , enableDropPartitions = false)).toRdd } } @@ -115,7 +115,7 @@ case class CreateCarbonSourceTableAsSelectCommand( try { val physicalPlan = session.sessionState.executePlan(data).executedPlan - CarbonReflectionUtils.invokeWriteAndReadMethod(dataSource, + CarbonToSparkAdapter.invokeWriteAndReadMethod(dataSource, Dataset.ofRows(session, query), data, session, diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala index a3e3b41c109..0d8335b51a8 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.hive.execution.command import org.apache.hadoop.hive.metastore.api.InvalidOperationException -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonCommands, CarbonEnv, CarbonRunnableCommand, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchTableException} import org.apache.spark.sql.catalyst.expressions.Attribute @@ -31,7 +31,7 @@ import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOp import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil, SessionParams} case class CarbonDropDatabaseCommand(command: DropDatabaseCommand) - extends RunnableCommand { + extends CarbonRunnableCommand { override val output: Seq[Attribute] = command.output @@ -89,7 +89,7 @@ case class CarbonDropDatabaseCommand(command: DropDatabaseCommand) } case class CarbonSetCommand(command: SetCommand) - extends MetadataCommand { + extends CarbonCommands { override val output: Seq[Attribute] = command.output diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala index 2dd8c234f05..5c58881c077 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala @@ -89,17 +89,23 @@ object CarbonFilters { translateEqualTo(a.name, v, columnTypes) case EqualTo(Literal(v, _), a: Attribute) => translateEqualTo(a.name, v, columnTypes) - case c@EqualTo(Cast(_: Attribute, _, _), _: Literal) => + case c: EqualTo if c.left.isInstanceOf[Cast] && c.left.asInstanceOf[Cast].child + .isInstanceOf[Attribute] && c.right.isInstanceOf[Literal] => CastExpressionOptimization.checkIfCastCanBeRemove(c) - case c@EqualTo(_: Literal, Cast(_: Attribute, _, _)) => + case c: EqualTo if c.right.isInstanceOf[Cast] && c.right.asInstanceOf[Cast].child + .isInstanceOf[Attribute] && c.left.isInstanceOf[Literal] => CastExpressionOptimization.checkIfCastCanBeRemove(c) case Not(EqualTo(a: Attribute, Literal(v, _))) => translateNotEqualTo(a.name, v, columnTypes) case Not(EqualTo(Literal(v, _), a: Attribute)) => translateNotEqualTo(a.name, v, columnTypes) - case c@Not(EqualTo(Cast(_: Attribute, _, _), _: Literal)) => + case c: Not if c.child.isInstanceOf[EqualTo] && c.child.asInstanceOf[EqualTo].left + .isInstanceOf[Cast] && c.child.asInstanceOf[EqualTo].left.asInstanceOf[Cast].child + .isInstanceOf[Attribute] && c.child.asInstanceOf[EqualTo].right.isInstanceOf[Literal] => CastExpressionOptimization.checkIfCastCanBeRemove(c) - case c@Not(EqualTo(_: Literal, Cast(_: Attribute, _, _))) => + case c: Not if c.child.isInstanceOf[EqualTo] && c.child.asInstanceOf[EqualTo].right + .isInstanceOf[Cast] && c.child.asInstanceOf[EqualTo].right.asInstanceOf[Cast].child + .isInstanceOf[Attribute] && c.child.asInstanceOf[EqualTo].left.isInstanceOf[Literal] => CastExpressionOptimization.checkIfCastCanBeRemove(c) case IsNotNull(a: Attribute) => translateNotEqualTo(a.name, null, columnTypes, true) @@ -109,9 +115,12 @@ object CarbonFilters { translateNotIn(a.name, list.map(e => e.eval(EmptyRow)), columnTypes) case In(a: Attribute, list) if list.forall(_.isInstanceOf[Literal]) => translateIn(a.name, list.map(e => e.eval(EmptyRow)), columnTypes) - case c@Not(In(Cast(_: Attribute, _, _), list)) if list.forall(_.isInstanceOf[Literal]) => + case c: Not if c.child.isInstanceOf[In] && c.child.asInstanceOf[In].value.isInstanceOf[Cast] + && c.child.asInstanceOf[In].value.asInstanceOf[Cast].child.isInstanceOf[Attribute] + && c.child.asInstanceOf[In].list.forall(_.isInstanceOf[Literal]) => Some(transformExpression(c)) - case c@In(Cast(_: Attribute, _, _), list) if list.forall(_.isInstanceOf[Literal]) => + case c: In if c.value.isInstanceOf[Cast] && c.value.asInstanceOf[Cast].child + .isInstanceOf[Attribute] && c.list.forall(_.isInstanceOf[Literal]) => Some(transformExpression(c)) case InSet(a: Attribute, set) => translateIn(a.name, set.toSeq, columnTypes) @@ -121,33 +130,41 @@ object CarbonFilters { translateGreaterThan(a.name, v, columnTypes) case GreaterThan(Literal(v, _), a: Attribute) => translateLessThan(a.name, v, columnTypes) - case c@GreaterThan(Cast(_: Attribute, _, _), _: Literal) => + case c: GreaterThan if c.left.isInstanceOf[Cast] && c.left.asInstanceOf[Cast].child + .isInstanceOf[Attribute] && c.right.isInstanceOf[Literal] => CastExpressionOptimization.checkIfCastCanBeRemove(c) - case c@GreaterThan(_: Literal, Cast(_: Attribute, _, _)) => + case c: GreaterThan if c.right.isInstanceOf[Cast] && c.right.asInstanceOf[Cast].child + .isInstanceOf[Attribute] && c.left.isInstanceOf[Literal] => CastExpressionOptimization.checkIfCastCanBeRemove(c) case LessThan(a: Attribute, Literal(v, _)) => translateLessThan(a.name, v, columnTypes) case LessThan(Literal(v, _), a: Attribute) => translateGreaterThan(a.name, v, columnTypes) - case c@LessThan(Cast(_: Attribute, _, _), _: Literal) => + case c: LessThan if c.left.isInstanceOf[Cast] && c.left.asInstanceOf[Cast].child + .isInstanceOf[Attribute] && c.right.isInstanceOf[Literal] => CastExpressionOptimization.checkIfCastCanBeRemove(c) - case c@LessThan(_: Literal, Cast(_: Attribute, _, _)) => + case c: LessThan if c.right.isInstanceOf[Cast] && c.right.asInstanceOf[Cast].child + .isInstanceOf[Attribute] && c.left.isInstanceOf[Literal] => CastExpressionOptimization.checkIfCastCanBeRemove(c) case GreaterThanOrEqual(a: Attribute, Literal(v, _)) => translateGreaterThanEqual(a.name, v, columnTypes) case GreaterThanOrEqual(Literal(v, _), a: Attribute) => translateLessThanEqual(a.name, v, columnTypes) - case c@GreaterThanOrEqual(Cast(_: Attribute, _, _), _: Literal) => + case c: GreaterThanOrEqual if c.left.isInstanceOf[Cast] && c.left.asInstanceOf[Cast].child + .isInstanceOf[Attribute] && c.right.isInstanceOf[Literal] => CastExpressionOptimization.checkIfCastCanBeRemove(c) - case c@GreaterThanOrEqual(_: Literal, Cast(_: Attribute, _, _)) => + case c: GreaterThanOrEqual if c.right.isInstanceOf[Cast] && c.right.asInstanceOf[Cast].child + .isInstanceOf[Attribute] && c.left.isInstanceOf[Literal] => CastExpressionOptimization.checkIfCastCanBeRemove(c) case LessThanOrEqual(a: Attribute, Literal(v, _)) => translateLessThanEqual(a.name, v, columnTypes) case LessThanOrEqual(Literal(v, _), a: Attribute) => translateGreaterThanEqual(a.name, v, columnTypes) - case c@LessThanOrEqual(Cast(_: Attribute, _, _), Literal(v, t)) => + case c: LessThanOrEqual if c.left.isInstanceOf[Cast] && c.left.asInstanceOf[Cast].child + .isInstanceOf[Attribute] && c.right.isInstanceOf[Literal] => CastExpressionOptimization.checkIfCastCanBeRemove(c) - case c@LessThanOrEqual(_: Literal, Cast(_: Attribute, _, _)) => + case c: LessThanOrEqual if c.right.isInstanceOf[Cast] && c.right.asInstanceOf[Cast].child + .isInstanceOf[Attribute] && c.left.isInstanceOf[Literal] => CastExpressionOptimization.checkIfCastCanBeRemove(c) case StartsWith(a: Attribute, Literal(v, _)) if v.toString.nonEmpty => translateStartsWith(a.name, v, columnTypes) @@ -159,8 +176,10 @@ object CarbonFilters { Some(new FalseExpression(null)) case ArrayContains(a: Attribute, Literal(v, _)) => translateArrayContains(a, v, columnTypes) - case ac@ArrayContains(Cast(_: Attribute, _, _), _: Literal) => - CastExpressionOptimization.checkIfCastCanBeRemove(EqualTo(ac.left, ac.right)) + case c: ArrayContains if c.left.isInstanceOf[Cast] && c.left.asInstanceOf[Cast].child + .isInstanceOf[Attribute] && c.right.isInstanceOf[Literal] => + CastExpressionOptimization.checkIfCastCanBeRemove(EqualTo(c.left.asInstanceOf[Cast] + , c.right.asInstanceOf[Literal])) case _ => None } } diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala index 4b64ae357b9..f692f7e7ce7 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.optimizer import org.apache.log4j.Logger import org.apache.spark.internal.Logging -import org.apache.spark.sql.CarbonToSparkAdapter +import org.apache.spark.sql.{CarbonToSparkAdapter, CarbonUnaryExpression} import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeMap, AttributeReference, AttributeSet, Expression, PredicateHelper, _} import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction, Average, Count, Max, Min, NoOp, Sum} import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} @@ -1562,7 +1562,7 @@ private object SelectSelectGroupbyChildDelta } - case class AliasWrapper(alias: Alias) extends UnaryExpression { + case class AliasWrapper(alias: Alias) extends CarbonUnaryExpression { override def child: Expression = null override protected def doGenCode(ctx: CodegenContext, diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVRewrite.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVRewrite.scala index 48fb93f45c7..278bd363c0e 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVRewrite.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVRewrite.scala @@ -39,8 +39,8 @@ import org.apache.spark.unsafe.types.UTF8String import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.preagg.TimeSeriesFunctionEnum -import org.apache.carbondata.mv.expressions.modular.{ModularSubquery, ScalarModularSubquery} -import org.apache.carbondata.mv.plans.modular.{ExpressionHelper, GroupBy, HarmonizedRelation, LeafNode, Matchable, ModularPlan, ModularRelation, Select, SimpleModularizer} +import org.apache.carbondata.mv.expressions.modular.{ModularSubquery} +import org.apache.carbondata.mv.plans.modular.{ExpressionHelper, GroupBy, HarmonizedRelation, LeafNode, Matchable, ModularPlan, ModularRelation, ScalarModularSubquery, Select, SimpleModularizer} import org.apache.carbondata.mv.plans.util.BirdcageOptimizer import org.apache.carbondata.view.{MVCatalogInSpark, MVHelper, MVPlanWrapper, MVSchemaWrapper, MVTimeGranularity, TimeSeriesFunction} diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala b/integration/spark/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala index d4b313c2dd8..b5a03fc586b 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala @@ -331,26 +331,6 @@ object CarbonSparkSqlParserUtil { } } - /** - * Parse a key-value map from a [[TablePropertyListContext]], assuming all values are specified. - * - * @param ctx Instance of TablePropertyListContext defining parser rule for the table - * properties. - * @param props Map of table property list - * @return Map of transformed table property. - */ - def visitPropertyKeyValues(ctx: TablePropertyListContext, - props: Map[String, String]): Map[String, String] = { - val badKeys = props.filter { case (_, v) => v == null }.keys - if (badKeys.nonEmpty) { - operationNotAllowed( - s"Values must be specified for key(s): ${ badKeys.mkString("[", ",", "]") }", ctx) - } - props.map { case (key, value) => - (key.toLowerCase, value) - } - } - /** * check's whether need to convert to lower case * diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/command/RegisterIndexTableCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/command/RegisterIndexTableCommand.scala index 903506ab2b8..2ff32f2f313 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/command/RegisterIndexTableCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/command/RegisterIndexTableCommand.scala @@ -22,9 +22,8 @@ import java.util import scala.collection.JavaConverters._ import org.apache.log4j.Logger -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonDataCommands, CarbonEnv, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.execution.command.DataCommand import org.apache.spark.sql.util.CarbonException import org.apache.carbondata.common.logging.LogServiceFactory @@ -42,7 +41,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema */ case class RegisterIndexTableCommand(dbName: Option[String], indexTableName: String, parentTable: String) - extends DataCommand { + extends CarbonDataCommands { val LOGGER: Logger = LogServiceFactory.getLogService(this.getClass.getName) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/command/SICreationCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/command/SICreationCommand.scala index 06e1b51c0be..f9493bf2c1c 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/command/SICreationCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/command/SICreationCommand.scala @@ -76,7 +76,7 @@ private[sql] case class CarbonCreateSecondaryIndexCommand( ifNotExists: Boolean, isDeferredRefresh: Boolean, var isCreateSIndex: Boolean = true) - extends DataCommand { + extends CarbonDataCommands { val LOGGER: Logger = LogServiceFactory.getLogService(this.getClass.getCanonicalName) diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/command/SILoadCommand.scala b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/command/SILoadCommand.scala index 9bef2ae22aa..1282ba23ea9 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/command/SILoadCommand.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/command/SILoadCommand.scala @@ -19,8 +19,7 @@ package org.apache.spark.sql.secondaryindex.command import scala.collection.JavaConverters._ -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession, SQLContext} -import org.apache.spark.sql.execution.command.RunnableCommand +import org.apache.spark.sql.{CarbonEnv, CarbonRunnableCommand, Row, SparkSession, SQLContext} import org.apache.spark.sql.hive.CarbonRelation import org.apache.spark.sql.index.CarbonIndexUtil import org.apache.spark.sql.secondaryindex.load.CarbonInternalLoaderUtil @@ -50,7 +49,7 @@ case class SecondaryIndexModel(sqlContext: SQLContext, * */ private[sql] case class LoadDataForSecondaryIndex(indexModel: IndexModel) extends - RunnableCommand { + CarbonRunnableCommand { def run(sparkSession: SparkSession): Seq[Row] = { val tableName = indexModel.tableName diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/joins/BroadCastSIFilterPushJoin.scala b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/joins/BroadCastSIFilterPushJoin.scala index 527720aecc3..e74410cda3a 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/joins/BroadCastSIFilterPushJoin.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/joins/BroadCastSIFilterPushJoin.scala @@ -28,13 +28,11 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.mapreduce.JobContext import org.apache.log4j.Logger import org.apache.spark.rdd.RDD -import org.apache.spark.sql.{CarbonBuildSide, CarbonToSparkAdapter, SparkSession} +import org.apache.spark.sql.{CarbonBinaryExecNode, CarbonBuildSide, CarbonToSparkAdapter, SparkSession} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Alias, And, Attribute, AttributeReference, BindReferences, Expression, In, Literal, NamedExpression} -import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext import org.apache.spark.sql.catalyst.plans.JoinType import org.apache.spark.sql.execution.{BinaryExecNode, CarbonCodegenSupport, ProjectExec, RowDataSourceScanExec, SparkPlan} -import org.apache.spark.sql.execution.joins.HashJoin import org.apache.spark.sql.execution.metric.SQLMetrics import org.apache.spark.sql.execution.strategy.CarbonDataSourceScan import org.apache.spark.sql.optimizer.CarbonFilters @@ -64,7 +62,7 @@ case class BroadCastSIFilterPushJoin( buildSide: CarbonToSparkAdapter.CarbonBuildSideType, left: SparkPlan, right: SparkPlan, - condition: Option[Expression]) extends BinaryExecNode with CarbonCodegenSupport { + condition: Option[Expression]) extends CarbonBinaryExecNode with CarbonCodegenSupport { override def output: Seq[Attribute] = carbonScan.output diff --git a/integration/spark/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala b/integration/spark/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala index 107a86a91fd..dee011b1e64 100644 --- a/integration/spark/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala +++ b/integration/spark/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala @@ -160,16 +160,6 @@ object CarbonReflectionUtils { isFormatted } - def invokeWriteAndReadMethod(dataSourceObj: DataSource, - dataFrame: DataFrame, - data: LogicalPlan, - session: SparkSession, - mode: SaveMode, - query: LogicalPlan, - physicalPlan: SparkPlan): BaseRelation = { - dataSourceObj.writeAndRead(mode, query, query.output.map(_.name), physicalPlan) - } - /** * method to invoke alter table add columns for hive table from carbon session * @param table diff --git a/integration/spark/src/main/spark3.1/org/apache/spark/sql/CarbonDataSourceScanHelper.scala b/integration/spark/src/main/spark3.3/org/apache/spark/sql/CarbonDataSourceScanHelper.scala similarity index 98% rename from integration/spark/src/main/spark3.1/org/apache/spark/sql/CarbonDataSourceScanHelper.scala rename to integration/spark/src/main/spark3.3/org/apache/spark/sql/CarbonDataSourceScanHelper.scala index 20a8f5bf30a..f5f17aff09f 100644 --- a/integration/spark/src/main/spark3.1/org/apache/spark/sql/CarbonDataSourceScanHelper.scala +++ b/integration/spark/src/main/spark3.3/org/apache/spark/sql/CarbonDataSourceScanHelper.scala @@ -50,7 +50,7 @@ abstract class CarbonDataSourceScanHelper(relation: CarbonDatasourceHadoopRelati extends DataSourceScanExec { override lazy val supportsColumnar: Boolean = CarbonPlanHelper - .supportBatchedDataSource(sqlContext, output, extraRDD) + .supportBatchedDataSource( SQLContext.getOrCreate(sparkContext), output, extraRDD) lazy val supportsBatchOrColumnar: Boolean = supportsColumnar diff --git a/integration/spark/src/main/spark3.1/org/apache/spark/sql/CarbonToSparkAdapter.scala b/integration/spark/src/main/spark3.3/org/apache/spark/sql/CarbonToSparkAdapter.scala similarity index 98% rename from integration/spark/src/main/spark3.1/org/apache/spark/sql/CarbonToSparkAdapter.scala rename to integration/spark/src/main/spark3.3/org/apache/spark/sql/CarbonToSparkAdapter.scala index 9ab78880e2b..b3deb48c2f5 100644 --- a/integration/spark/src/main/spark3.1/org/apache/spark/sql/CarbonToSparkAdapter.scala +++ b/integration/spark/src/main/spark3.3/org/apache/spark/sql/CarbonToSparkAdapter.scala @@ -24,9 +24,9 @@ import scala.collection.mutable.ArrayBuffer import org.apache.spark.SparkContext import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd} import org.apache.spark.sql.carbondata.execution.datasources.CarbonFileIndexReplaceRule -import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, ExternalCatalogWithListener} -import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, AttributeSet, DynamicPruningSubquery, Expression, ExprId, NamedExpression, Predicate, ScalaUDF, SubqueryExpression} +import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, AttributeSet, DynamicPruningSubquery, Expression, ExprId, NamedExpression, Predicate, ScalaUDF} import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReference import org.apache.spark.sql.catalyst.expressions.codegen._ import org.apache.spark.sql.catalyst.expressions.codegen.Block._ diff --git a/integration/spark/src/main/spark3.1/org/apache/spark/sql/SparkSqlAdapter.scala b/integration/spark/src/main/spark3.3/org/apache/spark/sql/SparkSqlAdapter.scala similarity index 100% rename from integration/spark/src/main/spark3.1/org/apache/spark/sql/SparkSqlAdapter.scala rename to integration/spark/src/main/spark3.3/org/apache/spark/sql/SparkSqlAdapter.scala diff --git a/integration/spark/src/main/spark3.1/org/apache/spark/sql/SparkVersionAdapter.scala b/integration/spark/src/main/spark3.3/org/apache/spark/sql/SparkVersionAdapter.scala similarity index 75% rename from integration/spark/src/main/spark3.1/org/apache/spark/sql/SparkVersionAdapter.scala rename to integration/spark/src/main/spark3.3/org/apache/spark/sql/SparkVersionAdapter.scala index 805af13a683..89cbfb3e6c4 100644 --- a/integration/spark/src/main/spark3.1/org/apache/spark/sql/SparkVersionAdapter.scala +++ b/integration/spark/src/main/spark3.3/org/apache/spark/sql/SparkVersionAdapter.scala @@ -22,37 +22,42 @@ import java.time.ZoneId import javax.xml.bind.DatatypeConverter import scala.annotation.tailrec +import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer import org.antlr.v4.runtime.tree.TerminalNode import org.apache.spark.{SparkContext, TaskContext} import org.apache.spark.rdd.RDD import org.apache.spark.serializer.Serializer +import org.apache.spark.sql.carbondata.execution.datasources.tasklisteners.CarbonLoadTaskCompletionListener import org.apache.spark.sql.catalyst.{CarbonParserUtil, InternalRow, QueryPlanningTracker, TableIdentifier} import org.apache.spark.sql.catalyst.analysis.{Analyzer, UnresolvedRelation} +import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.encoders.RowEncoder -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSeq, Expression, Predicate, SortOrder} +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSeq, Expression, Predicate, SortOrder, UnaryExpression} import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} import org.apache.spark.sql.catalyst.parser.ParserUtils.operationNotAllowed -import org.apache.spark.sql.catalyst.parser.SqlBaseParser.{BucketSpecContext, ColTypeListContext, CreateTableHeaderContext, LocationSpecContext, PartitionFieldListContext, QueryContext, SkewSpecContext, TablePropertyListContext} +import org.apache.spark.sql.catalyst.parser.SqlBaseParser.{BucketSpecContext, ColTypeListContext, CreateTableHeaderContext, LocationSpecContext, PartitionFieldListContext, PropertyListContext, QueryContext, SkewSpecContext} import org.apache.spark.sql.catalyst.plans.{JoinType, QueryPlan} -import org.apache.spark.sql.catalyst.plans.logical.{CreateTableStatement, InsertIntoStatement, Join, JoinHint, LogicalPlan, OneRowRelation, QualifiedColType} +import org.apache.spark.sql.catalyst.plans.logical.{Command, CreateTable, InsertIntoStatement, Join, JoinHint, LogicalPlan, OneRowRelation, QualifiedColType} import org.apache.spark.sql.catalyst.plans.physical.SinglePartition import org.apache.spark.sql.catalyst.util.{DateTimeUtils, RebaseDateTime, TimestampFormatter} -import org.apache.spark.sql.execution.{ExplainMode, QueryExecution, ShuffledRowRDD, SimpleMode, SparkPlan, SQLExecution, UnaryExecNode} -import org.apache.spark.sql.execution.command.{ExplainCommand, Field, PartitionerField, RefreshTableCommand, TableModel, TableNewProcessor} +import org.apache.spark.sql.execution.{BinaryExecNode, ExplainMode, QueryExecution, ShuffledRowRDD, SimpleMode, SparkPlan, SQLExecution, UnaryExecNode} +import org.apache.spark.sql.execution.command.{AtomicRunnableCommand, DataCommand, DataWritingCommand, ExplainCommand, Field, MetadataCommand, PartitionerField, RefreshTableCommand, RunnableCommand, ShowPartitionsCommand, TableModel, TableNewProcessor} import org.apache.spark.sql.execution.command.table.{CarbonCreateTableAsSelectCommand, CarbonCreateTableCommand} -import org.apache.spark.sql.execution.datasources.DataSourceStrategy +import org.apache.spark.sql.execution.datasources.{BasicWriteJobStatsTracker, DataSource, DataSourceStrategy, FilePartition, FileScanRDD, OutputWriter, PartitionedFile} import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec import org.apache.spark.sql.execution.metric.SQLShuffleWriteMetricsReporter import org.apache.spark.sql.execution.strategy.CarbonDataSourceScan import org.apache.spark.sql.internal.{SessionState, SharedState} import org.apache.spark.sql.parser.CarbonSpark2SqlParser import org.apache.spark.sql.parser.CarbonSparkSqlParserUtil.{checkIfDuplicateColumnExists, convertDbNameToLowerCase, validateStreamingProperty} -import org.apache.spark.sql.sources.Filter -import org.apache.spark.sql.types.{AbstractDataType, CharType, DataType, StructField, VarcharType} +import org.apache.spark.sql.sources.{BaseRelation, Filter} +import org.apache.spark.sql.types.{AbstractDataType, CharType, DataType, StructField, StructType, VarcharType} import org.apache.spark.sql.util.SparkSQLUtil import org.apache.spark.unsafe.types.UTF8String +import org.apache.spark.util.TaskCompletionListener import org.apache.carbondata.common.exceptions.DeprecatedFeatureException import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException @@ -63,6 +68,7 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes import org.apache.carbondata.core.metadata.schema.SchemaReader import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo} import org.apache.carbondata.core.util.path.CarbonTablePath +import org.apache.carbondata.mv.plans.modular.ModularPlan import org.apache.carbondata.spark.CarbonOption import org.apache.carbondata.spark.util.CarbonScalaUtil @@ -255,7 +261,7 @@ trait SparkVersionAdapter { * */ def createCarbonTable(createTableTuple: (CreateTableHeaderContext, SkewSpecContext, - BucketSpecContext, PartitionFieldListContext, ColTypeListContext, TablePropertyListContext, + BucketSpecContext, PartitionFieldListContext, ColTypeListContext, PropertyListContext, LocationSpecContext, Option[String], TerminalNode, QueryContext, String), extraTableTuple: (Seq[StructField], Boolean, TableIdentifier, Boolean, Seq[String], Option[String], mutable.Map[String, String], Map[String, String], Seq[StructField], @@ -474,27 +480,75 @@ trait SparkVersionAdapter { def getUpdatedPlan(plan: LogicalPlan, sqlText: String): LogicalPlan = { plan match { - case create@CreateTableStatement(_, _, _, _, properties, _, _, - location, _, _, _, _) => - if ( location.isDefined && + case create@CreateTable(_, _, _, _, _) => + if ( create.tableSpec.location.isDefined && !sqlText.toUpperCase.startsWith("CREATE EXTERNAL TABLE ")) { // add a property to differentiate if create table statement has external keyword or not - val newProperties = properties. +("hasexternalkeyword" -> "false") - CreateTableStatement(create.tableName, create.tableSchema, create.partitioning, - create.bucketSpec, newProperties, create.provider, create.options, - location, create.comment, create.serde, create.external, create.ifNotExists) - } else if (create.options.contains("latestversion")) { + create.tableSpec.properties. +("hasexternalkeyword" -> "false") + CreateTable(create.name, create.tableSchema, create.partitioning, create.tableSpec, + create.ignoreIfExists) + } else if (create.tableSpec.options.contains("latestversion")) { // remove latestversion property in options if present - val newOptions = create.options.filterNot(_._1.equalsIgnoreCase("latestversion")) - CreateTableStatement(create.tableName, create.tableSchema, create.partitioning, - create.bucketSpec, properties, create.provider, newOptions, - location, create.comment, create.serde, create.external, create.ifNotExists) + create.tableSpec.options.-("latestversion") + CreateTable(create.name, create.tableSchema, create.partitioning, create.tableSpec, + create.ignoreIfExists) } else { create } case others => others } } + + def showPartitionsCommand(spec: Option[TablePartitionSpec], + showPartitionsCommand: ShowPartitionsCommand): ShowPartitionsCommand = { + ShowPartitionsCommand(showPartitionsCommand.tableName, showPartitionsCommand.output, spec) + } + + def invokeWriteAndReadMethod(dataSourceObj: DataSource, + dataFrame: DataFrame, + data: LogicalPlan, + session: SparkSession, + mode: SaveMode, + query: LogicalPlan, + physicalPlan: SparkPlan): BaseRelation = { + dataSourceObj.writeAndRead(mode, query, query.output.map(_.name), physicalPlan, + BasicWriteJobStatsTracker.metrics) + } + + /** + * Parse a key-value map from a [[TablePropertyListContext]], assuming all values are specified. + * + * @param ctx Instance of TablePropertyListContext defining parser rule for the table + * properties. + * @param props Map of table property list + * @return Map of transformed table property. + */ + def visitPropertyKeyValues(ctx: PropertyListContext, + props: Map[String, String]): Map[String, String] = { + val badKeys = props.filter { case (_, v) => v == null }.keys + if (badKeys.nonEmpty) { + operationNotAllowed( + s"Values must be specified for key(s): ${ badKeys.mkString("[", ",", "]") }", ctx) + } + props.map { case (key, value) => + (key.toLowerCase, value) + } + } + + def getFileScanRDD(spark: SparkSession, readFunction: PartitionedFile => Iterator[InternalRow], + partitions: ArrayBuffer[FilePartition]) : FileScanRDD = { + new FileScanRDD(spark, readFunction, partitions, StructType(Seq.empty)) + } + + def check(context: TaskContext): Boolean = { + val onCompleteCallbacksField = + context.getClass.getDeclaredField("onCompleteCallbacks") + onCompleteCallbacksField.setAccessible(true) + val listeners = onCompleteCallbacksField.get(context) + .asInstanceOf[java.util.Stack[TaskCompletionListener]] + listeners.exists(p => p.isInstanceOf[CarbonLoadTaskCompletionListener]) + } + } case class CarbonBuildSide(buildSide: BuildSide) { @@ -511,4 +565,80 @@ abstract class CarbonTakeOrderedAndProjectExecHelper(sortOrder: Seq[SortOrder], s"CarbonTakeOrderedAndProjectExec(limit=$limit, orderBy=$orderByString, " + s"skipMapOrder=$skipMapOrder, readFromHead=$readFromHead, output=$outputString)" } + + override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = newChild +} + +trait CarbonMergeIntoSQLCommandCarbon extends AtomicRunnableCommand { + override protected def withNewChildrenInternal( + newChildren: IndexedSeq[LogicalPlan]): LogicalPlan = newChildren.head +} + +trait MvPlanWrapperCarbon extends ModularPlan { + override protected def withNewChildrenInternal( + newChildren: IndexedSeq[ModularPlan]): ModularPlan = newChildren.head +} + +trait CarbonProjectForUpdate extends LogicalPlan { + override protected def withNewChildrenInternal(newChildren: IndexedSeq[LogicalPlan]): + LogicalPlan = newChildren.head } + +trait CarbonUpdateTable extends LogicalPlan { + override protected def withNewChildrenInternal(newChildren: IndexedSeq[LogicalPlan]): + LogicalPlan = newChildren.head +} + +trait CarbonDeleteRecords extends LogicalPlan { + override protected def withNewChildrenInternal(newChildren: IndexedSeq[LogicalPlan]): + LogicalPlan = newChildren.head +} + +trait CarbonInsertIntoCarbonTable extends Command { + override def children: Seq[LogicalPlan] = Seq.empty + + override protected def withNewChildrenInternal(newChildren: IndexedSeq[LogicalPlan]): + LogicalPlan = newChildren.head +} + +trait CarbonCustomDeterministicExpression extends Expression { + override protected def withNewChildrenInternal(newChildren: + IndexedSeq[Expression]): Expression = newChildren.head +} + +abstract class OutputWriterCarbon(paths: String) extends OutputWriter { + override def path(): String = paths +} + +trait CarbonCommands extends MetadataCommand { + override protected def withNewChildrenInternal(newChildren: + IndexedSeq[LogicalPlan]): LogicalPlan = newChildren.head +} + +trait CarbonAtomicRunnableCommands extends AtomicRunnableCommand { + override protected def withNewChildrenInternal(newChildren: + IndexedSeq[LogicalPlan]): LogicalPlan = newChildren.head +} + +trait CarbonRunnableCommand extends RunnableCommand { + override protected def withNewChildrenInternal(newChildren: + IndexedSeq[LogicalPlan]): LogicalPlan = newChildren.head +} + +trait CarbonDataCommands extends DataCommand { + override protected def withNewChildrenInternal(newChildren: + IndexedSeq[LogicalPlan]): LogicalPlan = newChildren.head +} + +trait CarbonDataWritingCommand extends DataWritingCommand { + override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = newChild +} + +trait CarbonUnaryExpression extends UnaryExpression { + override protected def withNewChildInternal(newChild: Expression): Expression = newChild +} +trait CarbonBinaryExecNode extends BinaryExecNode { + override protected def withNewChildrenInternal(newLeft: SparkPlan, + newRight: SparkPlan): SparkPlan = newLeft +} + diff --git a/integration/spark/src/main/spark3.1/org/apache/spark/sql/execution/CarbonCodegenSupport.scala b/integration/spark/src/main/spark3.3/org/apache/spark/sql/execution/CarbonCodegenSupport.scala similarity index 100% rename from integration/spark/src/main/spark3.1/org/apache/spark/sql/execution/CarbonCodegenSupport.scala rename to integration/spark/src/main/spark3.3/org/apache/spark/sql/execution/CarbonCodegenSupport.scala diff --git a/integration/spark/src/main/spark3.1/org/apache/spark/sql/hive/CarbonAnalyzer.scala b/integration/spark/src/main/spark3.3/org/apache/spark/sql/hive/CarbonAnalyzer.scala similarity index 100% rename from integration/spark/src/main/spark3.1/org/apache/spark/sql/hive/CarbonAnalyzer.scala rename to integration/spark/src/main/spark3.3/org/apache/spark/sql/hive/CarbonAnalyzer.scala diff --git a/integration/spark/src/main/spark3.1/org/apache/spark/sql/hive/CarbonSessionStateBuilder.scala b/integration/spark/src/main/spark3.3/org/apache/spark/sql/hive/CarbonSessionStateBuilder.scala similarity index 91% rename from integration/spark/src/main/spark3.1/org/apache/spark/sql/hive/CarbonSessionStateBuilder.scala rename to integration/spark/src/main/spark3.3/org/apache/spark/sql/hive/CarbonSessionStateBuilder.scala index 5ecc38183c4..2901bc9f5f0 100644 --- a/integration/spark/src/main/spark3.1/org/apache/spark/sql/hive/CarbonSessionStateBuilder.scala +++ b/integration/spark/src/main/spark3.3/org/apache/spark/sql/hive/CarbonSessionStateBuilder.scala @@ -23,9 +23,9 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.spark.sql.{CarbonEnv, SparkSession} import org.apache.spark.sql.catalyst.{QualifiedTableName, TableIdentifier} -import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry} -import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTablePartition, ExternalCatalogWithListener, FunctionResourceLoader, GlobalTempViewManager} -import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry, TableFunctionRegistry} +import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTablePartition, ExternalCatalogWithListener, FunctionExpressionBuilder, FunctionResourceLoader, GlobalTempViewManager} +import org.apache.spark.sql.catalyst.expressions.{ApplyFunctionExpression, Expression} import org.apache.spark.sql.catalyst.parser.ParserInterface import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.connector.catalog.CatalogManager @@ -54,19 +54,22 @@ class CarbonHiveSessionCatalog( globalTempViewManager: GlobalTempViewManager, functionRegistry: FunctionRegistry, sparkSession: SparkSession, + tableFunctionRegistry: TableFunctionRegistry, conf: SQLConf, hadoopConf: Configuration, parser: ParserInterface, - functionResourceLoader: FunctionResourceLoader) + functionResourceLoader: FunctionResourceLoader, + functionExpressionBuilder: FunctionExpressionBuilder) extends HiveSessionCatalog ( () => externalCatalog, () => globalTempViewManager, new HiveMetastoreCatalog(sparkSession), functionRegistry, + tableFunctionRegistry, hadoopConf, parser, - functionResourceLoader - ) with CarbonSessionCatalog { + functionResourceLoader, + functionExpressionBuilder) with CarbonSessionCatalog { private lazy val carbonEnv = { val env = new CarbonEnv @@ -144,7 +147,7 @@ class CarbonHiveSessionCatalog( */ class CarbonSessionStateBuilder(sparkSession: SparkSession, parentState: Option[SessionState] = None) - extends HiveSessionStateBuilder(sparkSession, parentState, Map.empty) { + extends HiveSessionStateBuilder(sparkSession, parentState) { override lazy val sqlParser: ParserInterface = new CarbonSparkSqlParser(conf, sparkSession) @@ -165,10 +168,12 @@ class CarbonSessionStateBuilder(sparkSession: SparkSession, session.sharedState.globalTempViewManager, functionRegistry, sparkSession, + tableFunctionRegistry, conf, SessionState.newHadoopConf(session.sparkContext.hadoopConfiguration, conf), sqlParser, - resourceLoader) + resourceLoader, + HiveUDFExpressionBuilder) parentState.foreach(_.catalog.copyStateTo(catalog)) catalog } diff --git a/integration/spark/src/main/spark3.1/org/apache/spark/sql/hive/CarbonSqlAstBuilder.scala b/integration/spark/src/main/spark3.3/org/apache/spark/sql/hive/CarbonSqlAstBuilder.scala similarity index 97% rename from integration/spark/src/main/spark3.1/org/apache/spark/sql/hive/CarbonSqlAstBuilder.scala rename to integration/spark/src/main/spark3.3/org/apache/spark/sql/hive/CarbonSqlAstBuilder.scala index fcfa54817cc..c426aafbe83 100644 --- a/integration/spark/src/main/spark3.1/org/apache/spark/sql/hive/CarbonSqlAstBuilder.scala +++ b/integration/spark/src/main/spark3.3/org/apache/spark/sql/hive/CarbonSqlAstBuilder.scala @@ -48,7 +48,7 @@ class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSes } val createTableTuple = (ctx.createTableHeader, createTableClauses.skewSpec(0), createTableClauses.bucketSpec(0), createTableClauses.partitioning, ctx.colTypeList(), - createTableClauses.tablePropertyList(0), createTableClauses.locationSpec(0), + createTableClauses.propertyList(0), createTableClauses.locationSpec(0), Option(commentSpecContext), ctx.AS, ctx.query, fileStorage) helper.createCarbonTable(createTableTuple) } else { diff --git a/integration/spark/src/main/spark3.1/org/apache/spark/sql/hive/SqlAstBuilderHelper.scala b/integration/spark/src/main/spark3.3/org/apache/spark/sql/hive/SqlAstBuilderHelper.scala similarity index 100% rename from integration/spark/src/main/spark3.1/org/apache/spark/sql/hive/SqlAstBuilderHelper.scala rename to integration/spark/src/main/spark3.3/org/apache/spark/sql/hive/SqlAstBuilderHelper.scala diff --git a/integration/spark/src/main/spark3.1/org/apache/spark/sql/hive/execution/command/CarbonResetCommand.scala b/integration/spark/src/main/spark3.3/org/apache/spark/sql/hive/execution/command/CarbonResetCommand.scala similarity index 92% rename from integration/spark/src/main/spark3.1/org/apache/spark/sql/hive/execution/command/CarbonResetCommand.scala rename to integration/spark/src/main/spark3.3/org/apache/spark/sql/hive/execution/command/CarbonResetCommand.scala index 784e8c7227d..83fbb5fa442 100644 --- a/integration/spark/src/main/spark3.1/org/apache/spark/sql/hive/execution/command/CarbonResetCommand.scala +++ b/integration/spark/src/main/spark3.3/org/apache/spark/sql/hive/execution/command/CarbonResetCommand.scala @@ -17,12 +17,12 @@ package org.apache.spark.sql.hive.execution.command -import org.apache.spark.sql.{CarbonEnv, Row, SparkSession} +import org.apache.spark.sql.{CarbonEnv, CarbonRunnableCommand, Row, SparkSession} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.execution.command.{ResetCommand, RunnableCommand} case class CarbonResetCommand() - extends RunnableCommand { + extends CarbonRunnableCommand { override val output = Seq() override def run(sparkSession: SparkSession): Seq[Row] = { diff --git a/integration/spark/src/main/spark3.1/org/apache/spark/sql/parser/CarbonExtensionSqlParser.scala b/integration/spark/src/main/spark3.3/org/apache/spark/sql/parser/CarbonExtensionSqlParser.scala similarity index 100% rename from integration/spark/src/main/spark3.1/org/apache/spark/sql/parser/CarbonExtensionSqlParser.scala rename to integration/spark/src/main/spark3.3/org/apache/spark/sql/parser/CarbonExtensionSqlParser.scala diff --git a/integration/spark/src/main/spark3.1/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark/src/main/spark3.3/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala similarity index 94% rename from integration/spark/src/main/spark3.1/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala rename to integration/spark/src/main/spark3.3/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala index 49978d4e3c2..95a26944577 100644 --- a/integration/spark/src/main/spark3.1/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala +++ b/integration/spark/src/main/spark3.3/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala @@ -83,20 +83,20 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf, sparkSession: SparkSession) extends SparkSqlAstBuilderWrapper(conf) { /** - * Parse a key-value map from a [[TablePropertyListContext]], assuming all values are specified. + * Parse a key-value map from a [[PropertyListContext]], assuming all values are specified. */ - override def visitPropertyKeyValues(ctx: TablePropertyListContext): Map[String, String] = { - val props = visitTablePropertyList(ctx) - CarbonSparkSqlParserUtil.visitPropertyKeyValues(ctx, props) + override def visitPropertyKeyValues(ctx: PropertyListContext): Map[String, String] = { + val props = visitPropertyList(ctx) + CarbonToSparkAdapter.visitPropertyKeyValues(ctx, props) } - def getPropertyKeyValues(ctx: TablePropertyListContext): Map[String, String] = { + def getPropertyKeyValues(ctx: PropertyListContext): Map[String, String] = { Option(ctx).map(visitPropertyKeyValues) .getOrElse(Map.empty) } def createCarbonTable(createTableTuple: (CreateTableHeaderContext, SkewSpecContext, - BucketSpecContext, PartitionFieldListContext, ColTypeListContext, TablePropertyListContext, + BucketSpecContext, PartitionFieldListContext, ColTypeListContext, PropertyListContext, LocationSpecContext, Option[String], TerminalNode, QueryContext, String)): LogicalPlan = { val (tableHeader, skewSpecContext, diff --git a/integration/spark/src/main/spark3.1/org/apache/spark/sql/parser/SparkSqlAstBuilderWrapper.scala b/integration/spark/src/main/spark3.3/org/apache/spark/sql/parser/SparkSqlAstBuilderWrapper.scala similarity index 93% rename from integration/spark/src/main/spark3.1/org/apache/spark/sql/parser/SparkSqlAstBuilderWrapper.scala rename to integration/spark/src/main/spark3.3/org/apache/spark/sql/parser/SparkSqlAstBuilderWrapper.scala index 7a576b16d8a..1492e8aad5b 100644 --- a/integration/spark/src/main/spark3.1/org/apache/spark/sql/parser/SparkSqlAstBuilderWrapper.scala +++ b/integration/spark/src/main/spark3.3/org/apache/spark/sql/parser/SparkSqlAstBuilderWrapper.scala @@ -26,5 +26,5 @@ import org.apache.spark.sql.internal.SQLConf */ abstract class SparkSqlAstBuilderWrapper(conf: SQLConf) extends SparkSqlAstBuilder { - def visitPropertyKeyValues(ctx: TablePropertyListContext): Map[String, String] + def visitPropertyKeyValues(ctx: PropertyListContext): Map[String, String] } diff --git a/integration/spark/src/test/scala/org/apache/carbondata/index/lucene/LuceneFineGrainIndexSuite.scala b/integration/spark/src/test/scala/org/apache/carbondata/index/lucene/LuceneFineGrainIndexSuite.scala index 8200563f26a..ed34adc02aa 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/index/lucene/LuceneFineGrainIndexSuite.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/index/lucene/LuceneFineGrainIndexSuite.scala @@ -356,7 +356,7 @@ class LuceneFineGrainIndexSuite extends QueryTest with BeforeAndAfterAll { sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE index_test_table OPTIONS('header'='false')") sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE index_test_table OPTIONS('header'='false')") sql("alter table index_test_table compact 'major'") - if (!sqlContext.sparkContext.version.startsWith("3.1")) { + if (!sqlContext.sparkContext.version.startsWith("3.3")) { checkAnswer(sql("SELECT COUNT(*) FROM index_test_table WHERE TEXT_MATCH('name:n10')"), sql("select COUNT(*) from index_test_table where name='n10'")) } @@ -384,7 +384,7 @@ class LuceneFineGrainIndexSuite extends QueryTest with BeforeAndAfterAll { sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE index_test_table OPTIONS('header'='false')") sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE index_test_table OPTIONS('header'='false')") sql("alter table index_test_table compact 'minor'") - if (!sqlContext.sparkContext.version.startsWith("3.1")) { + if (!sqlContext.sparkContext.version.startsWith("3.3")) { checkAnswer(sql("SELECT COUNT(*) FROM index_test_table WHERE TEXT_MATCH('name:n10')"), sql("select count(*) from index_test_table where name='n10'")) } @@ -433,7 +433,7 @@ class LuceneFineGrainIndexSuite extends QueryTest with BeforeAndAfterAll { """.stripMargin) sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE index_test_table OPTIONS('header'='false')") sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE index_test_table OPTIONS('header'='false')") - if (!sqlContext.sparkContext.version.startsWith("3.1")) { + if (!sqlContext.sparkContext.version.startsWith("3.3")) { checkAnswer(sql("SELECT count(*) FROM index_test_table WHERE TEXT_MATCH('name:n99*')"), sql("select count(*) from index_test_table where name like 'n99%'")) sql("delete from table index_test_table where SEGMENT.ID in (0) ") @@ -496,7 +496,7 @@ class LuceneFineGrainIndexSuite extends QueryTest with BeforeAndAfterAll { | AS | Select * from source_table where TEXT_MATCH('name:n1*') """.stripMargin) - if (!sqlContext.sparkContext.version.startsWith("3.1")) { + if (!sqlContext.sparkContext.version.startsWith("3.3")) { checkAnswer(sql("SELECT count(*) FROM target_table"), sql("select count(*) from source_table where name like 'n1%'")) } @@ -520,7 +520,7 @@ class LuceneFineGrainIndexSuite extends QueryTest with BeforeAndAfterAll { """.stripMargin) sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE index_test_limit OPTIONS('header'='false')") - if (!sqlContext.sparkContext.version.startsWith("3.1")) { + if (!sqlContext.sparkContext.version.startsWith("3.3")) { checkAnswer(sql( "select count(*) from index_test_limit where TEXT_MATCH_WITH_LIMIT('name:n10*',10)"), Seq(Row(10))) diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionWithMeasureSortColumns.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionWithMeasureSortColumns.scala index 80952a14945..05f1de3814b 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionWithMeasureSortColumns.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionWithMeasureSortColumns.scala @@ -93,7 +93,7 @@ class MajorCompactionWithMeasureSortColumns extends QueryTest with BeforeAndAfte val answer = sql("select * from store ").orderBy("code1") assert(csvRows.count() == answer.distinct().count()) - if (!sqlContext.sparkContext.version.startsWith("3.1")) { + if (!sqlContext.sparkContext.version.startsWith("3.3")) { checkAnswer(answer.distinct(), Seq(Row("51job, Inc.", "21695-534", "FR", 610, 60, Date.valueOf("2017-11-27"), 4483, 0, 510), Row("Intercontinental Exchange Inc.", "22100-020", "TH", 87, 4, diff --git a/integration/spark/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala b/integration/spark/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala index 909fc36ee3f..980ef002497 100644 --- a/integration/spark/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala +++ b/integration/spark/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala @@ -1297,7 +1297,7 @@ class SparkCarbonDataSourceTest extends QueryTest with BeforeAndAfterAll { sql("create table par (c1 string, c2 double, n int) using parquet") sql("create table car (c1 string, c2 double, n int) using carbon") - if (!sqlContext.sparkContext.version.startsWith("3.1")) { + if (!sqlContext.sparkContext.version.startsWith("3.3")) { sql("insert into par select 'a', 1.7986931348623157E308, 215565665556") sql("insert into car select 'a', 1.7986931348623157E308, 215565665556") } else { diff --git a/mv/plan/pom.xml b/mv/plan/pom.xml index 9865ff781cd..07984b333e6 100644 --- a/mv/plan/pom.xml +++ b/mv/plan/pom.xml @@ -145,9 +145,6 @@ spark-2.3 - - true - 2.3 @@ -158,7 +155,7 @@ maven-compiler-plugin - src/main/spark3.1 + src/main/spark3.3 src/main/spark2.4 @@ -198,7 +195,7 @@ maven-compiler-plugin - src/main/spark3.1 + src/main/spark3.3 src/main/spark2.3 @@ -227,9 +224,12 @@ - spark-3.1 + spark-3.3 + + true + - 3.1 + 3.3 @@ -240,7 +240,7 @@ src/main/spark2.3 src/main/spark2.4 - src/main/common2.3and2.4 + src/main/common2.3and2.4 @@ -257,7 +257,7 @@ - src/main/spark3.1 + src/main/spark3.3 diff --git a/mv/plan/src/main/common2.3and2.4/org/apache/carbondata/mv/plans/modular/SparkVersionHelper.scala b/mv/plan/src/main/common2.3and2.4/org/apache/carbondata/mv/plans/modular/SparkVersionHelper.scala index 78002f1198a..3cd8974a505 100644 --- a/mv/plan/src/main/common2.3and2.4/org/apache/carbondata/mv/plans/modular/SparkVersionHelper.scala +++ b/mv/plan/src/main/common2.3and2.4/org/apache/carbondata/mv/plans/modular/SparkVersionHelper.scala @@ -17,7 +17,7 @@ package org.apache.carbondata.mv.plans.modular -import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeMap, AttributeReference, AttributeSeq, Expression, ExprId, NamedExpression, SubqueryExpression} +import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeMap, AttributeReference, AttributeSeq, Exists, Expression, ExprId, ListQuery, NamedExpression, Predicate, SubqueryExpression, Unevaluable} import org.apache.spark.sql.catalyst.expressions.aggregate._ import org.apache.spark.sql.catalyst.optimizer.{BooleanSimplification, CollapseProject, CollapseRepartition, CollapseWindow, ColumnPruning, CombineFilters, CombineLimits, CombineUnions, ConstantFolding, EliminateOuterJoin, EliminateSerialization, EliminateSorts, FoldablePropagation, NullPropagation, PushDownPredicate, PushPredicateThroughJoin, PushProjectionThroughUnion, RemoveDispensableExpressions, RemoveRedundantAliases, RemoveRedundantProject, ReorderAssociativeOperator, ReorderJoin, RewriteCorrelatedScalarSubquery, SimplifyBinaryComparison, SimplifyCaseConversionExpressions, SimplifyCasts, SimplifyConditionals} import org.apache.spark.sql.catalyst.plans.{logical, JoinType, QueryPlan} @@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, Join, LogicalPla import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.types.{DataType, Metadata} +import org.apache.carbondata.mv.expressions.modular.ModularSubquery import org.apache.carbondata.mv.plans.util.BirdcageOptimizer @@ -114,6 +115,15 @@ trait UnionModularPlan extends ModularPlan { trait OneRowTableLeafNode extends LeafNode { } +trait ModularRelationHarmonizedRelation extends GetVerboseString { +} + +trait MVModularRelation extends GetVerboseString { +} + +trait MVModularizeLater extends LeafNode { +} + object MatchJoin { def unapply(plan : LogicalPlan): Option[(LogicalPlan, LogicalPlan, JoinType, Option[Expression], Option[Any])] = { @@ -135,3 +145,101 @@ object MatchAggregateExpression { } } } + +/** + * The [[Exists]] expression checks if a row exists in a subquery given some correlated condition. + * + * For example (SQL): + * {{{ + * SELECT * + * FROM a + * WHERE EXISTS (SELECT * + * FROM b + * WHERE b.id = a.id) + * }}} + */ +case class ModularExists( + plan: ModularPlan, + children: Seq[Expression] = Seq.empty, + exprId: ExprId = NamedExpression.newExprId) + extends ModularSubquery(plan, children, exprId) with Predicate with Unevaluable { + override def nullable: Boolean = false + + override def withNewPlan(plan: ModularPlan): ModularExists = copy(plan = plan) + + override def toString: String = s"modular-exists#${exprId.id} $conditionString" + + override lazy val canonicalized: Expression = { + ModularExists( + plan.canonicalizedDef, + children.map(_.canonicalized), + ExprId(0)) + } +} + +/** + * A [[ListQuery]] expression defines the query which we want to search in an IN subquery + * expression. It should and can only be used in conjunction with an IN expression. + * + * For example (SQL): + * {{{ + * SELECT * + * FROM a + * WHERE a.id IN (SELECT id + * FROM b) + * }}} + */ +case class ModularListQuery( + plan: ModularPlan, + children: Seq[Expression] = Seq.empty, + exprId: ExprId = NamedExpression.newExprId) + extends ModularSubquery(plan, children, exprId) with Unevaluable { + override def dataType: DataType = plan.schema.fields.head.dataType + + override def nullable: Boolean = false + + override def withNewPlan(plan: ModularPlan): ModularListQuery = copy(plan = plan) + + override def toString: String = s"modular-list#${exprId.id} $conditionString" + + override lazy val canonicalized: Expression = { + ModularListQuery( + plan.canonicalizedDef, + children.map(_.canonicalized), + ExprId(0)) + } +} + +/** + * A subquery that will return only one row and one column. This will be converted into a physical + * scalar subquery during planning. + * + * Note: `exprId` is used to have a unique name in explain string output. + */ +case class ScalarModularSubquery( + plan: ModularPlan, + children: Seq[Expression] = Seq.empty, + exprId: ExprId = NamedExpression.newExprId) + extends ModularSubquery(plan, children, exprId) with Unevaluable { + override def dataType: DataType = plan.schema.fields.head.dataType + + override def nullable: Boolean = true + + override def withNewPlan(plan: ModularPlan): ScalarModularSubquery = copy(plan = plan) + + override def toString: String = s"scalar-modular-subquery#${ exprId.id } $conditionString" + + override lazy val canonicalized: Expression = { + ScalarModularSubquery( + plan.canonicalizedDef, + children.map(_.canonicalized), + ExprId(0)) + } + + def hasCorrelatedScalarSubquery(e: Expression): Boolean = { + e.find { + case s: ScalarModularSubquery => s.children.nonEmpty + case _ => false + }.isDefined + } +} diff --git a/mv/plan/src/main/scala/org/apache/carbondata/mv/expressions/modular/subquery.scala b/mv/plan/src/main/scala/org/apache/carbondata/mv/expressions/modular/subquery.scala index 43551bba3c2..a04308dd349 100644 --- a/mv/plan/src/main/scala/org/apache/carbondata/mv/expressions/modular/subquery.scala +++ b/mv/plan/src/main/scala/org/apache/carbondata/mv/expressions/modular/subquery.scala @@ -39,16 +39,6 @@ abstract class ModularSubquery( override def withNewPlan(plan: ModularPlan): ModularSubquery - override def semanticEquals(o: Expression): Boolean = { - o match { - case p: ModularSubquery => - this.getClass.getName.equals(p.getClass.getName) && plan.sameResult(p.plan) && - children.length == p.children.length && - children.zip(p.children).forall(p => p._1.semanticEquals(p._2)) - case _ => false - } - } - def canonicalize(attrs: AttributeSeq): ModularSubquery = { // Normalize the outer references in the subquery plan. val normalizedPlan = plan.transformAllExpressions { @@ -59,105 +49,6 @@ abstract class ModularSubquery( } } -/** - * A subquery that will return only one row and one column. This will be converted into a physical - * scalar subquery during planning. - * - * Note: `exprId` is used to have a unique name in explain string output. - */ -case class ScalarModularSubquery( - plan: ModularPlan, - children: Seq[Expression] = Seq.empty, - exprId: ExprId = NamedExpression.newExprId) - extends ModularSubquery(plan, children, exprId) with Unevaluable { - override def dataType: DataType = plan.schema.fields.head.dataType - - override def nullable: Boolean = true - - override def withNewPlan(plan: ModularPlan): ScalarModularSubquery = copy(plan = plan) - - override def toString: String = s"scalar-modular-subquery#${ exprId.id } $conditionString" - - override lazy val canonicalized: Expression = { - ScalarModularSubquery( - plan.canonicalizedDef, - children.map(_.canonicalized), - ExprId(0)) - } -} - -object ScalarModularSubquery { - def hasCorrelatedScalarSubquery(e: Expression): Boolean = { - e.find { - case s: ScalarModularSubquery => s.children.nonEmpty - case _ => false - }.isDefined - } -} - -/** - * A [[ListQuery]] expression defines the query which we want to search in an IN subquery - * expression. It should and can only be used in conjunction with an IN expression. - * - * For example (SQL): - * {{{ - * SELECT * - * FROM a - * WHERE a.id IN (SELECT id - * FROM b) - * }}} - */ -case class ModularListQuery( - plan: ModularPlan, - children: Seq[Expression] = Seq.empty, - exprId: ExprId = NamedExpression.newExprId) - extends ModularSubquery(plan, children, exprId) with Unevaluable { - override def dataType: DataType = plan.schema.fields.head.dataType - - override def nullable: Boolean = false - - override def withNewPlan(plan: ModularPlan): ModularListQuery = copy(plan = plan) - - override def toString: String = s"modular-list#${ exprId.id } $conditionString" - - override lazy val canonicalized: Expression = { - ModularListQuery( - plan.canonicalizedDef, - children.map(_.canonicalized), - ExprId(0)) - } -} - -/** - * The [[Exists]] expression checks if a row exists in a subquery given some correlated condition. - * - * For example (SQL): - * {{{ - * SELECT * - * FROM a - * WHERE EXISTS (SELECT * - * FROM b - * WHERE b.id = a.id) - * }}} - */ -case class ModularExists( - plan: ModularPlan, - children: Seq[Expression] = Seq.empty, - exprId: ExprId = NamedExpression.newExprId) - extends ModularSubquery(plan, children, exprId) with Predicate with Unevaluable { - override def nullable: Boolean = false - - override def withNewPlan(plan: ModularPlan): ModularExists = copy(plan = plan) - - override def toString: String = s"modular-exists#${ exprId.id } $conditionString" - - override lazy val canonicalized: Expression = { - ModularExists( - plan.canonicalizedDef, - children.map(_.canonicalized), - ExprId(0)) - } -} /** * A place holder for generated SQL for subquery expression. diff --git a/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/AggregatePushDown.scala b/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/AggregatePushDown.scala index cbcd4248466..1cd8da6c4d3 100644 --- a/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/AggregatePushDown.scala +++ b/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/AggregatePushDown.scala @@ -128,7 +128,9 @@ trait AggregatePushDown { // self: ModularPlan => } else { Map.empty[Int, (NamedExpression, Seq[NamedExpression])] } - case sum@MatchAggregateExpression(Sum(cast@MatchCast(expr, dataType)), _, false, _, _) => + + case sum@MatchAggregateExpression(Sum(cast@MatchCast(expr, dataType), _) + , _, false, _, _) => val tAttr = selAliasMap.get(expr.asInstanceOf[Attribute]).getOrElse(expr) .asInstanceOf[Attribute] if (fact.outputSet.contains(tAttr)) { @@ -190,7 +192,8 @@ trait AggregatePushDown { // self: ModularPlan => } else { Map.empty[Int, (NamedExpression, Seq[NamedExpression])] } - case avg@MatchAggregateExpression(Average(cast@MatchCast(expr, dataType)), _, false, _, _) => + case avg@MatchAggregateExpression( + Sum(cast@MatchCast(expr, dataType), _), _, false, _, _) => val tAttr = selAliasMap.get(expr.asInstanceOf[Attribute]).getOrElse(expr) .asInstanceOf[Attribute] if (fact.outputSet.contains(tAttr)) { diff --git a/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPatterns.scala b/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPatterns.scala index 5fbfb9b6c7d..72a138b2d0f 100644 --- a/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPatterns.scala +++ b/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPatterns.scala @@ -71,7 +71,7 @@ abstract class ModularPattern extends GenericPattern[ModularPlan] { override protected def modularizeLater(plan: LogicalPlan): ModularPlan = ModularizeLater(plan) } -case class ModularizeLater(plan: LogicalPlan) extends LeafNode { +case class ModularizeLater(plan: LogicalPlan) extends MVModularizeLater { override def output: Seq[Attribute] = plan.output } diff --git a/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularRelation.scala b/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularRelation.scala index ba9eb4ba3ae..6df266acd10 100644 --- a/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularRelation.scala +++ b/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularRelation.scala @@ -41,7 +41,7 @@ case class ModularRelation( tableName: String, outputList: Seq[NamedExpression], flags: FlagSet, - rest: Seq[Seq[Any]]) extends GetVerboseString { + rest: Seq[Seq[Any]]) extends MVModularRelation { protected override def computeStats(spark: SparkSession): Statistics = { val plan = spark.table(s"${ databaseName }.${ tableName }").queryExecution.optimizedPlan val stats = plan.stats @@ -125,7 +125,7 @@ object HarmonizedRelation { } // support harmonization for dimension table -case class HarmonizedRelation(source: ModularPlan) extends GetVerboseString { +case class HarmonizedRelation(source: ModularPlan) extends ModularRelationHarmonizedRelation { require(HarmonizedRelation.canHarmonize(source), "invalid plan for harmonized relation") lazy val tableName = source.asInstanceOf[GroupBy].child.children(0).asInstanceOf[ModularRelation] .tableName diff --git a/mv/plan/src/main/spark3.1/org/apache/carbondata/mv/plans/modular/ExpressionHelper.scala b/mv/plan/src/main/spark3.3/org/apache/carbondata/mv/plans/modular/ExpressionHelper.scala similarity index 100% rename from mv/plan/src/main/spark3.1/org/apache/carbondata/mv/plans/modular/ExpressionHelper.scala rename to mv/plan/src/main/spark3.3/org/apache/carbondata/mv/plans/modular/ExpressionHelper.scala diff --git a/mv/plan/src/main/spark3.1/org/apache/carbondata/mv/plans/modular/SparkVersionHelper.scala b/mv/plan/src/main/spark3.3/org/apache/carbondata/mv/plans/modular/SparkVersionHelper.scala similarity index 57% rename from mv/plan/src/main/spark3.1/org/apache/carbondata/mv/plans/modular/SparkVersionHelper.scala rename to mv/plan/src/main/spark3.3/org/apache/carbondata/mv/plans/modular/SparkVersionHelper.scala index e52efc9d688..8d0b9a44d8d 100644 --- a/mv/plan/src/main/spark3.1/org/apache/carbondata/mv/plans/modular/SparkVersionHelper.scala +++ b/mv/plan/src/main/spark3.3/org/apache/carbondata/mv/plans/modular/SparkVersionHelper.scala @@ -19,13 +19,15 @@ package org.apache.carbondata.mv.plans.modular import scala.reflect.ClassTag -import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeMap, AttributeSeq, Expression, ExprId, NamedExpression, SubqueryExpression} +import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeMap, AttributeSeq, Exists, Expression, ExprId, ListQuery, NamedExpression, Predicate, SubqueryExpression, Unevaluable} import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction, AggregateMode} import org.apache.spark.sql.catalyst.optimizer.{BooleanSimplification, CollapseProject, CollapseRepartition, CollapseWindow, ColumnPruning, CombineFilters, CombineUnions, ConstantFolding, EliminateLimits, EliminateOuterJoin, EliminateSerialization, EliminateSorts, FoldablePropagation, NullPropagation, PushDownPredicates, PushPredicateThroughJoin, PushProjectionThroughUnion, RemoveDispensableExpressions, RemoveRedundantAliases, ReorderAssociativeOperator, ReorderJoin, RewriteCorrelatedScalarSubquery, SimplifyBinaryComparison, SimplifyCaseConversionExpressions, SimplifyCasts, SimplifyConditionals} import org.apache.spark.sql.catalyst.plans.{logical, JoinType, QueryPlan} import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, Join, LogicalPlan, Statistics, Subquery} import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.types.DataType +import org.apache.carbondata.mv.expressions.modular.ModularSubquery import org.apache.carbondata.mv.plans.util.BirdcageOptimizer object SparkVersionHelper { @@ -116,6 +118,9 @@ trait GroupByUnaryNode extends UnaryNode { } groupBy } + + override protected def withNewChildrenInternal( + newChildren: IndexedSeq[ModularPlan]): ModularPlan = newChildren.head } trait SelectModularPlan extends ModularPlan { @@ -132,14 +137,38 @@ trait SelectModularPlan extends ModularPlan { } select } + + override protected def withNewChildrenInternal( + newChildren: IndexedSeq[ModularPlan]): ModularPlan = newChildren.head } trait UnionModularPlan extends ModularPlan { override def verboseString(maxFields: Int): String = super.verboseString(maxFields) + + override protected def withNewChildrenInternal( + newChildren: IndexedSeq[ModularPlan]): ModularPlan = newChildren.head } trait OneRowTableLeafNode extends LeafNode { override def verboseString(maxFields: Int): String = super.verboseString(maxFields) + + override protected def withNewChildrenInternal( + newChildren: IndexedSeq[ModularPlan]): ModularPlan = newChildren.head +} + +trait ModularRelationHarmonizedRelation extends GetVerboseString { + override protected def withNewChildrenInternal( + newChildren: IndexedSeq[ModularPlan]): ModularPlan = newChildren.head +} + +trait MVModularRelation extends GetVerboseString { + override protected def withNewChildrenInternal( + newChildren: IndexedSeq[ModularPlan]): ModularPlan = newChildren.head +} + +trait MVModularizeLater extends LeafNode { + override protected def withNewChildrenInternal( + newChildren: IndexedSeq[ModularPlan]): ModularPlan = newChildren.head } object MatchJoin { @@ -164,3 +193,110 @@ object MatchAggregateExpression { } } } + +/** + * The [[Exists]] expression checks if a row exists in a subquery given some correlated condition. + * + * For example (SQL): + * {{{ + * SELECT * + * FROM a + * WHERE EXISTS (SELECT * + * FROM b + * WHERE b.id = a.id) + * }}} + */ +case class ModularExists( + plan: ModularPlan, + children: Seq[Expression] = Seq.empty, + exprId: ExprId = NamedExpression.newExprId) + extends ModularSubquery(plan, children, exprId) with Predicate with Unevaluable { + override def nullable: Boolean = false + + override def withNewPlan(plan: ModularPlan): ModularExists = copy(plan = plan) + + override def toString: String = s"modular-exists#${ exprId.id } $conditionString" + + override lazy val canonicalized: Expression = { + ModularExists( + plan.canonicalizedDef, + children.map(_.canonicalized), + ExprId(0)) + } + + override protected def withNewChildrenInternal( + newChildren: IndexedSeq[Expression]): Expression = newChildren.head +} + +/** + * A [[ListQuery]] expression defines the query which we want to search in an IN subquery + * expression. It should and can only be used in conjunction with an IN expression. + * + * For example (SQL): + * {{{ + * SELECT * + * FROM a + * WHERE a.id IN (SELECT id + * FROM b) + * }}} + */ +case class ModularListQuery( + plan: ModularPlan, + children: Seq[Expression] = Seq.empty, + exprId: ExprId = NamedExpression.newExprId) + extends ModularSubquery(plan, children, exprId) with Unevaluable { + override def dataType: DataType = plan.schema.fields.head.dataType + + override def nullable: Boolean = false + + override def withNewPlan(plan: ModularPlan): ModularListQuery = copy(plan = plan) + + override def toString: String = s"modular-list#${ exprId.id } $conditionString" + + override lazy val canonicalized: Expression = { + ModularListQuery( + plan.canonicalizedDef, + children.map(_.canonicalized), + ExprId(0)) + } + + override protected def withNewChildrenInternal( + newChildren: IndexedSeq[Expression]): Expression = newChildren.head +} + +/** + * A subquery that will return only one row and one column. This will be converted into a physical + * scalar subquery during planning. + * + * Note: `exprId` is used to have a unique name in explain string output. + */ +case class ScalarModularSubquery( + plan: ModularPlan, + children: Seq[Expression] = Seq.empty, + exprId: ExprId = NamedExpression.newExprId) + extends ModularSubquery(plan, children, exprId) with Unevaluable { + override def dataType: DataType = plan.schema.fields.head.dataType + + override def nullable: Boolean = true + + override def withNewPlan(plan: ModularPlan): ScalarModularSubquery = copy(plan = plan) + + override def toString: String = s"scalar-modular-subquery#${ exprId.id } $conditionString" + + override lazy val canonicalized: Expression = { + ScalarModularSubquery( + plan.canonicalizedDef, + children.map(_.canonicalized), + ExprId(0)) + } + + def hasCorrelatedScalarSubquery(e: Expression): Boolean = { + e.find { + case s: ScalarModularSubquery => s.children.nonEmpty + case _ => false + }.isDefined + } + + override protected def withNewChildrenInternal( + newChildren: IndexedSeq[Expression]): Expression = newChildren.head +} diff --git a/pom.xml b/pom.xml index 69dca97237d..57fcbde6f30 100644 --- a/pom.xml +++ b/pom.xml @@ -126,11 +126,11 @@ 2.7.2 4.3.4 4.3-alpha1 - 2.11 - 2.11.8 + 2.12 + 2.12.8 compile - 2.3.4 - 2.3 + 3.3.0 + 3.3 4.8 compile compile @@ -454,7 +454,7 @@ ${basedir}/src/main/scala ${basedir}/src/main/spark${spark.binary.version} ${basedir}/src/main/common2.3and2.4 - ${basedir}/src/main/common2.4and3.1 + ${basedir}/src/main/common2.4and3.3 ${basedir}/src/test/scala scalastyle-config.xml @@ -574,9 +574,6 @@ spark-2.3 - - true - 2.3 2.3.4 @@ -705,7 +702,7 @@ ${basedir}/integration/spark/src/main/scala ${basedir}/integration/spark/src/main/spark2.4 ${basedir}/integration/spark/src/main/common2.3and2.4 - ${basedir}/integration/spark/src/main/common2.4and3.1 + ${basedir}/integration/spark/src/main/common2.4and3.3 ${basedir}/integration/spark/src/main/java ${basedir}/integration/hive/src/main/scala ${basedir}/integration/hive/src/main/java @@ -723,13 +720,16 @@ - spark-3.1 + spark-3.3 + + true + - 3.1 - 3.1.1 + 3.3 + 3.3.0 2.12 2.12.8 - 2.10.0 + 2.13.3 @@ -778,8 +778,8 @@ ${basedir}/processing/src/main/java ${basedir}/hadoop/src/main/java ${basedir}/integration/spark/src/main/scala - ${basedir}/integration/spark/src/main/spark3.1 - ${basedir}/integration/spark/src/main/common2.4and3.1 + ${basedir}/integration/spark/src/main/spark3.3 + ${basedir}/integration/spark/src/main/common2.4and3.3 ${basedir}/integration/spark/src/main/java ${basedir}/integration/hive/src/main/scala ${basedir}/integration/hive/src/main/java diff --git a/streaming/pom.xml b/streaming/pom.xml index 3889bf1b80e..a7885ca9c6f 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -134,9 +134,6 @@ spark-2.3 - - true - 2.3 @@ -147,7 +144,7 @@ maven-compiler-plugin - src/main/spark3.1 + src/main/spark3.3 @@ -185,7 +182,7 @@ maven-compiler-plugin - src/main/spark3.1 + src/main/spark3.3 @@ -212,10 +209,13 @@ - spark-3.1 + spark-3.3 + + true + - 3.1.1 - 3.1 + 3.3.0 + 3.3 @@ -241,7 +241,7 @@ - src/main/spark3.1 + src/main/spark3.3 diff --git a/streaming/src/main/spark3.1/org/apache/carbondata/util/SparkStreamingUtil.scala b/streaming/src/main/spark3.3/org/apache/carbondata/util/SparkStreamingUtil.scala similarity index 100% rename from streaming/src/main/spark3.1/org/apache/carbondata/util/SparkStreamingUtil.scala rename to streaming/src/main/spark3.3/org/apache/carbondata/util/SparkStreamingUtil.scala