Skip to content

Commit 122a544

Browse files
QiangCaiQiangCai
committed
fix sdk test case
1 parent 2f1bea0 commit 122a544

File tree

26 files changed

+192
-485
lines changed

26 files changed

+192
-485
lines changed

‎common/pom.xml

+2-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@
5151
</dependency>
5252
<dependency>
5353
<groupId>org.apache.hadoop</groupId>
54-
<artifactId>hadoop-common</artifactId>
54+
<artifactId>hadoop-client-api</artifactId>
55+
<version>${hadoop.version}</version>
5556
</dependency>
5657
</dependencies>
5758

‎core/pom.xml

+32-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,38 @@
5252
</dependency>
5353
<dependency>
5454
<groupId>org.apache.hadoop</groupId>
55-
<artifactId>hadoop-client</artifactId>
55+
<artifactId>hadoop-client-runtime</artifactId>
56+
<version>${hadoop.version}</version>
57+
</dependency>
58+
<dependency>
59+
<groupId>commons-codec</groupId>
60+
<artifactId>commons-codec</artifactId>
61+
<version>1.16.0</version>
62+
</dependency>
63+
<dependency>
64+
<groupId>commons-io</groupId>
65+
<artifactId>commons-io</artifactId>
66+
<version>2.13.0</version>
67+
</dependency>
68+
<dependency>
69+
<groupId>com.google.guava</groupId>
70+
<artifactId>guava</artifactId>
71+
<version>14.0.1</version>
72+
</dependency>
73+
<dependency>
74+
<groupId>commons-collections</groupId>
75+
<artifactId>commons-collections</artifactId>
76+
<version>3.2.2</version>
77+
</dependency>
78+
<dependency>
79+
<groupId>org.apache.commons</groupId>
80+
<artifactId>commons-compress</artifactId>
81+
<version>1.23.0</version>
82+
</dependency>
83+
<dependency>
84+
<groupId>com.google.code.findbugs</groupId>
85+
<artifactId>jsr305</artifactId>
86+
<version>3.0.0</version>
5687
</dependency>
5788
<dependency>
5889
<groupId>org.xerial.snappy</groupId>

‎core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java

-12
Original file line numberDiff line numberDiff line change
@@ -2659,18 +2659,6 @@ private CarbonCommonConstants() {
26592659

26602660
public static final String CARBON_SDK_EMPTY_METADATA_PATH = "emptyMetadataFolder";
26612661

2662-
/**
2663-
* Property to identify if the spark version is above 3.x version
2664-
*/
2665-
public static final String CARBON_SPARK_VERSION_SPARK3 = "carbon.spark.version.spark3";
2666-
2667-
public static final String CARBON_SPARK_VERSION_SPARK3_DEFAULT = "false";
2668-
2669-
/**
2670-
* Carbon Spark 3.x supported data file written version
2671-
*/
2672-
public static final String CARBON_SPARK3_VERSION = "2.2.0";
2673-
26742662
/**
26752663
* This property is to enable the min max pruning of target carbon table based on input/source
26762664
* data

‎core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java

+3-43
Original file line numberDiff line numberDiff line change
@@ -544,50 +544,10 @@ private static long createTimeInstant(String dimensionValue, String dateFormat)
544544
}
545545

546546
private static Object parseTimestamp(String dimensionValue, String dateFormat) {
547-
Date dateToStr;
548-
DateFormat dateFormatter = null;
549-
long timeValue;
550547
try {
551-
if (Boolean.parseBoolean(CarbonProperties.getInstance()
552-
.getProperty(CarbonCommonConstants.CARBON_SPARK_VERSION_SPARK3,
553-
CarbonCommonConstants.CARBON_SPARK_VERSION_SPARK3_DEFAULT))) {
554-
try {
555-
return createTimeInstant(dimensionValue, dateFormat.trim());
556-
} catch (DateTimeParseException e) {
557-
throw new NumberFormatException(e.getMessage());
558-
}
559-
}
560-
if (null != dateFormat && !dateFormat.trim().isEmpty()) {
561-
dateFormatter = new SimpleDateFormat(dateFormat);
562-
dateFormatter.setLenient(false);
563-
} else {
564-
dateFormatter = timestampFormatter.get();
565-
}
566-
dateToStr = dateFormatter.parse(dimensionValue);
567-
timeValue = dateToStr.getTime();
568-
validateTimeStampRange(timeValue);
569-
return timeValue;
570-
} catch (ParseException e) {
571-
// If the parsing fails, try to parse again with setLenient to true if the property is set
572-
// (example: 1941-03-15 00:00:00 is invalid data and will fail to parse in Asia/Shanghai zone
573-
// as DST is observed and clocks were turned forward 1 hour to 1941-03-15 01:00:00)
574-
if (CarbonProperties.getInstance().isSetLenientEnabled()) {
575-
try {
576-
dateFormatter.setLenient(true);
577-
dateToStr = dateFormatter.parse(dimensionValue);
578-
timeValue = dateToStr.getTime();
579-
validateTimeStampRange(timeValue);
580-
LOGGER.info("Parsed data with lenience as true, setting back to default mode");
581-
return timeValue;
582-
} catch (ParseException ex) {
583-
LOGGER.info("Failed to parse data with lenience as true, setting back to default mode");
584-
throw new NumberFormatException(ex.getMessage());
585-
} finally {
586-
dateFormatter.setLenient(false);
587-
}
588-
} else {
589-
throw new NumberFormatException(e.getMessage());
590-
}
548+
return createTimeInstant(dimensionValue, dateFormat.trim());
549+
} catch (DateTimeParseException e) {
550+
throw new NumberFormatException(e.getMessage());
591551
}
592552
}
593553

‎examples/spark/pom.xml

+1
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@
157157
<plugin>
158158
<groupId>com.ning.maven.plugins</groupId>
159159
<artifactId>maven-duplicate-finder-plugin</artifactId>
160+
<version>1.0.9</version>
160161
<configuration>
161162
<skip>true</skip>
162163
</configuration>

‎geo/pom.xml

+1
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@
126126
<plugin>
127127
<groupId>com.ning.maven.plugins</groupId>
128128
<artifactId>maven-duplicate-finder-plugin</artifactId>
129+
<version>1.0.9</version>
129130
<configuration>
130131
<skip>true</skip>
131132
</configuration>

‎integration/spark/pom.xml

+48-45
Original file line numberDiff line numberDiff line change
@@ -88,41 +88,11 @@
8888
<artifactId>antlr4-runtime</artifactId>
8989
<version>${antlr4.version}</version>
9090
</dependency>
91-
<dependency>
92-
<groupId>org.apache.carbondata</groupId>
93-
<artifactId>carbondata-hive</artifactId>
94-
<version>${project.version}</version>
95-
<exclusions>
96-
<exclusion>
97-
<groupId>org.apache.commons</groupId>
98-
<artifactId>*</artifactId>
99-
</exclusion>
100-
<exclusion>
101-
<groupId>org.apache.hive</groupId>
102-
<artifactId>*</artifactId>
103-
</exclusion>
104-
<exclusion>
105-
<groupId>org.spark-project.hive</groupId>
106-
<artifactId>*</artifactId>
107-
</exclusion>
108-
</exclusions>
109-
</dependency>
11091
<dependency>
11192
<groupId>com.beust</groupId>
11293
<artifactId>jcommander</artifactId>
11394
<version>1.72</version>
11495
</dependency>
115-
<dependency>
116-
<groupId>org.apache.carbondata</groupId>
117-
<artifactId>carbondata-cli</artifactId>
118-
<version>${project.version}</version>
119-
<exclusions>
120-
<exclusion>
121-
<groupId>org.apache.hive</groupId>
122-
<artifactId>hive-exec</artifactId>
123-
</exclusion>
124-
</exclusions>
125-
</dependency>
12696
<dependency>
12797
<groupId>org.apache.carbondata</groupId>
12898
<artifactId>carbondata-lucene</artifactId>
@@ -241,21 +211,10 @@
241211
</exclusion>
242212
</exclusions>
243213
</dependency>
244-
<dependency>
245-
<groupId>com.databricks</groupId>
246-
<artifactId>spark-avro_2.11</artifactId>
247-
<version>4.0.0</version>
248-
<exclusions>
249-
<exclusion>
250-
<groupId>org.apache.avro</groupId>
251-
<artifactId>avro</artifactId>
252-
</exclusion>
253-
</exclusions>
254-
</dependency>
255214
<dependency>
256215
<groupId>org.apache.spark</groupId>
257216
<artifactId>spark-avro_${scala.binary.version}</artifactId>
258-
<version>2.4.5</version>
217+
<version>${spark.version}</version>
259218
<exclusions>
260219
<exclusion>
261220
<groupId>org.apache.avro</groupId>
@@ -277,6 +236,17 @@
277236
</exclusion>
278237
</exclusions>
279238
</dependency>
239+
<dependency>
240+
<groupId>org.apache.commons</groupId>
241+
<artifactId>commons-configuration2</artifactId>
242+
<version>2.1.1</version>
243+
<exclusions>
244+
<exclusion>
245+
<groupId>org.apache.commons</groupId>
246+
<artifactId>commons-lang3</artifactId>
247+
</exclusion>
248+
</exclusions>
249+
</dependency>
280250
<dependency>
281251
<groupId>commons-httpclient</groupId>
282252
<artifactId>commons-httpclient</artifactId>
@@ -304,6 +274,40 @@
304274
</exclusion>
305275
</exclusions>
306276
</dependency>
277+
<dependency>
278+
<groupId>org.apache.carbondata</groupId>
279+
<artifactId>carbondata-cli</artifactId>
280+
<version>${project.version}</version>
281+
<exclusions>
282+
<exclusion>
283+
<groupId>org.apache.hive</groupId>
284+
<artifactId>hive-exec</artifactId>
285+
</exclusion>
286+
</exclusions>
287+
</dependency>
288+
<dependency>
289+
<groupId>org.apache.carbondata</groupId>
290+
<artifactId>carbondata-hive</artifactId>
291+
<version>${project.version}</version>
292+
<exclusions>
293+
<exclusion>
294+
<groupId>org.apache.commons</groupId>
295+
<artifactId>*</artifactId>
296+
</exclusion>
297+
<exclusion>
298+
<groupId>org.apache.hive</groupId>
299+
<artifactId>*</artifactId>
300+
</exclusion>
301+
<exclusion>
302+
<groupId>org.apache.hadoop</groupId>
303+
<artifactId>*</artifactId>
304+
</exclusion>
305+
<exclusion>
306+
<groupId>org.spark-project.hive</groupId>
307+
<artifactId>*</artifactId>
308+
</exclusion>
309+
</exclusions>
310+
</dependency>
307311
<!-- scala -->
308312
<dependency>
309313
<groupId>org.scala-lang</groupId>
@@ -460,13 +464,12 @@
460464
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
461465
<junitxml>.</junitxml>
462466
<filereports>CarbonTestSuite.txt</filereports>
463-
<argLine> ${argLine} -ea -Xmx3g -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m
464-
</argLine>
467+
<argLine>-ea -Xmx3g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=512m</argLine>
465468
<stderr />
466469
<environmentVariables>
467470
</environmentVariables>
468471
<systemProperties>
469-
<java.awt.headless>true</java.awt.headless>
472+
<java.awt.headless>false</java.awt.headless>
470473
<spark.carbon.hive.schema.store>${carbon.hive.based.metastore}</spark.carbon.hive.schema.store>
471474
</systemProperties>
472475
</configuration>

‎integration/spark/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import org.apache.carbondata.core.scan.result.vector.CarbonDictionary;
2525
import org.apache.carbondata.core.scan.scanner.LazyPageLoader;
2626

27-
import org.apache.spark.sql.CarbonToSparkAdapter;
2827
import org.apache.spark.sql.CarbonVectorProxy;
2928
import org.apache.spark.sql.carbondata.execution.datasources.CarbonSparkDataSourceUtil;
3029
import org.apache.spark.sql.types.Decimal;
@@ -122,8 +121,7 @@ public void putInts(int rowId, int count, int value) {
122121
@Override
123122
public void putLong(int rowId, long value) {
124123
if (!filteredRows[rowId]) {
125-
sparkColumnVectorProxy
126-
.putLong(counter++, CarbonToSparkAdapter.rebaseTime(value, carbonDataFileWrittenVersion));
124+
sparkColumnVectorProxy.putLong(counter++, value);
127125
}
128126
}
129127

‎integration/spark/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapperDirect.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import org.apache.carbondata.core.scan.result.vector.CarbonDictionary;
2525
import org.apache.carbondata.core.scan.scanner.LazyPageLoader;
2626

27-
import org.apache.spark.sql.CarbonToSparkAdapter;
2827
import org.apache.spark.sql.CarbonVectorProxy;
2928
import org.apache.spark.sql.carbondata.execution.datasources.CarbonSparkDataSourceUtil;
3029
import org.apache.spark.sql.types.Decimal;
@@ -96,8 +95,7 @@ public void putInts(int rowId, int count, int value) {
9695

9796
@Override
9897
public void putLong(int rowId, long value) {
99-
sparkColumnVectorProxy
100-
.putLong(rowId, CarbonToSparkAdapter.rebaseTime(value, carbonDataFileWrittenVersion));
98+
sparkColumnVectorProxy.putLong(rowId, value);
10199
}
102100

103101
@Override

‎integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala

+1-32
Original file line numberDiff line numberDiff line change
@@ -570,27 +570,7 @@ class CarbonScanRDD[T: ClassTag](
570570
throw new java.util.NoSuchElementException("End of stream")
571571
}
572572
havePair = false
573-
val value = reader.getCurrentValue
574-
if (CarbonProperties.getInstance()
575-
.getProperty(CarbonCommonConstants.CARBON_SPARK_VERSION_SPARK3,
576-
CarbonCommonConstants.CARBON_SPARK_VERSION_SPARK3_DEFAULT).toBoolean &&
577-
timeStampProjectionColumns.nonEmpty) {
578-
value match {
579-
case row: GenericInternalRow if needRebaseTimeValue(reader) =>
580-
// rebase timestamp data by converting julian to Gregorian time
581-
timeStampProjectionColumns.foreach {
582-
projectionColumnWithIndex =>
583-
val timeStampData = row.get(projectionColumnWithIndex._2,
584-
org.apache.spark.sql.types.DataTypes.TimestampType)
585-
if (null != timeStampData) {
586-
row.update(projectionColumnWithIndex._2,
587-
CarbonToSparkAdapter.rebaseTime(timeStampData.asInstanceOf[Long]))
588-
}
589-
}
590-
case _ =>
591-
}
592-
}
593-
value
573+
reader.getCurrentValue
594574
}
595575

596576
}
@@ -622,17 +602,6 @@ class CarbonScanRDD[T: ClassTag](
622602
}
623603
}
624604

625-
def needRebaseTimeValue(reader: RecordReader[Void, Object]): Boolean = {
626-
// carbonDataFileWrittenVersion will be in the format x.x.x-SNAPSHOT
627-
// (eg., 2.1.0-SNAPSHOT), get the version name and check if the data file is
628-
// written before 2.2.0 version, then rebase timestamp value
629-
reader.isInstanceOf[CarbonRecordReader[T]] &&
630-
null != reader.asInstanceOf[CarbonRecordReader[T]].getCarbonDataFileWrittenVersion &&
631-
reader.asInstanceOf[CarbonRecordReader[T]].getCarbonDataFileWrittenVersion
632-
.split(CarbonCommonConstants.HYPHEN).head
633-
.compareTo(CarbonCommonConstants.CARBON_SPARK3_VERSION) < 0
634-
}
635-
636605
private def addTaskCompletionListener(split: Partition,
637606
context: TaskContext,
638607
queryStartTime: Long,

‎integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala

-4
Original file line numberDiff line numberDiff line change
@@ -139,10 +139,6 @@ class CarbonEnv {
139139
.addNonSerializableProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, "true")
140140
Profiler.initialize(sparkSession.sparkContext)
141141
CarbonToSparkAdapter.addSparkSessionListener(sparkSession)
142-
if(sparkSession.sparkContext.version.startsWith("3.1")) {
143-
CarbonProperties.getInstance().addProperty(CarbonCommonConstants
144-
.CARBON_SPARK_VERSION_SPARK3, "true")
145-
}
146142
initialized = true
147143
LOGGER.info("Initialize CarbonEnv completed...")
148144
}

‎integration/spark/src/main/scala/org/apache/spark/sql/SparkVersionAdapter.scala

-19
Original file line numberDiff line numberDiff line change
@@ -122,25 +122,6 @@ trait SparkVersionAdapter {
122122
DateTimeUtils.daysToLocalDate(date).toString
123123
}
124124

125-
/**
126-
* Rebase the timestamp value from julian to gregorian time micros
127-
*/
128-
def rebaseTime(timestamp: Long): Long = {
129-
RebaseDateTime.rebaseJulianToGregorianMicros(timestamp)
130-
}
131-
132-
def rebaseTime(timestamp: Long, carbonDataFileWrittenVersion: String): Long = {
133-
// carbonDataFileWrittenVersion will be in the format x.x.x-SNAPSHOT(eg., 2.1.0-SNAPSHOT),
134-
// get the version name and check if the data file is written before 2.2.0 version
135-
if (null != carbonDataFileWrittenVersion &&
136-
carbonDataFileWrittenVersion.split(CarbonCommonConstants.HYPHEN).head
137-
.compareTo(CarbonCommonConstants.CARBON_SPARK3_VERSION) < 0) {
138-
RebaseDateTime.rebaseJulianToGregorianMicros(timestamp)
139-
} else {
140-
timestamp
141-
}
142-
}
143-
144125
// Note that due to this scala bug: https://github.com/scala/bug/issues/11016, we need to make
145126
// this function polymorphic for every scala version >= 2.12, otherwise an overloaded method
146127
// resolution error occurs at compile time.

0 commit comments

Comments
 (0)