Skip to content

Commit cb96c8b

Browse files
QiangCaiQiangCai
committed
[WIP][ISSUE-4346] Support Spark 3.5
1 parent 71abab0 commit cb96c8b

File tree

76 files changed

+220
-3463
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

76 files changed

+220
-3463
lines changed

‎core/src/main/java/org/apache/carbondata/core/reader/CarbonIndexFileReader.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public void openThriftReader(String filePath) throws IOException {
8787
*
8888
* @param fileData
8989
*/
90-
public void openThriftReader(byte[] fileData) {
90+
public void openThriftReader(byte[] fileData) throws IOException {
9191
thriftReader = new ThriftReader(fileData);
9292
}
9393

‎core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java

+11-3
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,13 @@ public ThriftReader(String fileName, Configuration configuration) {
8787
/**
8888
* Constructor.
8989
*/
90-
public ThriftReader(byte[] fileData) {
90+
public ThriftReader(byte[] fileData) throws IOException {
9191
dataInputStream = new DataInputStream(new ByteArrayInputStream(fileData));
92-
binaryIn = new TCompactProtocol(new TIOStreamTransport(dataInputStream));
92+
try {
93+
binaryIn = new TCompactProtocol(new TIOStreamTransport(dataInputStream));
94+
} catch (TException e) {
95+
throw new IOException(e);
96+
}
9397
}
9498

9599
/**
@@ -98,7 +102,11 @@ public ThriftReader(byte[] fileData) {
98102
public void open() throws IOException {
99103
Configuration conf = configuration != null ? configuration : FileFactory.getConfiguration();
100104
dataInputStream = FileFactory.getDataInputStream(fileName, conf);
101-
binaryIn = new TCompactProtocol(new TIOStreamTransport(dataInputStream));
105+
try {
106+
binaryIn = new TCompactProtocol(new TIOStreamTransport(dataInputStream));
107+
} catch (TException e) {
108+
throw new IOException(e);
109+
}
102110
}
103111

104112
/**

‎core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -1413,8 +1413,8 @@ public static String convertToString(List<Segment> values) {
14131413
public static byte[] getByteArray(TBase t) {
14141414
ByteArrayOutputStream stream = new ByteArrayOutputStream();
14151415
byte[] thriftByteArray = null;
1416-
TProtocol binaryOut = new TCompactProtocol(new TIOStreamTransport(stream));
14171416
try {
1417+
TProtocol binaryOut = new TCompactProtocol(new TIOStreamTransport(stream));
14181418
t.write(binaryOut);
14191419
stream.flush();
14201420
thriftByteArray = stream.toByteArray();
@@ -1439,9 +1439,9 @@ public static DataChunk3 readDataChunk3(ByteBuffer dataChunkBuffer, int offset,
14391439

14401440
public static DataChunk3 readDataChunk3(InputStream stream) throws IOException {
14411441
TBaseCreator creator = DataChunk3::new;
1442-
TProtocol binaryIn = new TCompactProtocol(new TIOStreamTransport(stream));
14431442
TBase t = creator.create();
14441443
try {
1444+
TProtocol binaryIn = new TCompactProtocol(new TIOStreamTransport(stream));
14451445
t.read(binaryIn);
14461446
} catch (TException e) {
14471447
throw new IOException(e);
@@ -1461,9 +1461,9 @@ public static DataChunk3 readDataChunk3(InputStream stream) throws IOException {
14611461
private static TBase read(byte[] data, TBaseCreator creator, int offset, int length)
14621462
throws IOException {
14631463
ByteArrayInputStream stream = new ByteArrayInputStream(data, offset, length);
1464-
TProtocol binaryIn = new TCompactProtocol(new TIOStreamTransport(stream));
14651464
TBase t = creator.create();
14661465
try {
1466+
TProtocol binaryIn = new TCompactProtocol(new TIOStreamTransport(stream));
14671467
t.read(binaryIn);
14681468
} catch (TException e) {
14691469
throw new IOException(e);

‎core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java

+10-2
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,11 @@ public ThriftWriter(String fileName, boolean append) {
8080
*/
8181
public void open() throws IOException {
8282
dataOutputStream = FileFactory.getDataOutputStream(fileName, bufferSize, append);
83-
binaryOut = new TCompactProtocol(new TIOStreamTransport(dataOutputStream));
83+
try {
84+
binaryOut = new TCompactProtocol(new TIOStreamTransport(dataOutputStream));
85+
} catch (TException e) {
86+
throw new IOException(e);
87+
}
8488
}
8589

8690
/**
@@ -92,7 +96,11 @@ public void open() throws IOException {
9296
public void open(FileWriteOperation fileWriteOperation) throws IOException {
9397
atomicFileOperationsWriter = AtomicFileOperationFactory.getAtomicFileOperations(fileName);
9498
dataOutputStream = atomicFileOperationsWriter.openForWrite(fileWriteOperation);
95-
binaryOut = new TCompactProtocol(new TIOStreamTransport(dataOutputStream));
99+
try {
100+
binaryOut = new TCompactProtocol(new TIOStreamTransport(dataOutputStream));
101+
} catch (TException e) {
102+
throw new IOException(e);
103+
}
96104
}
97105

98106
/**

‎examples/spark/pom.xml

-22
Original file line numberDiff line numberDiff line change
@@ -198,27 +198,5 @@
198198
<maven.test.skip>true</maven.test.skip>
199199
</properties>
200200
</profile>
201-
<profile>
202-
<id>spark-2.3</id>
203-
<properties>
204-
<spark.binary.version>2.3</spark.binary.version>
205-
</properties>
206-
</profile>
207-
<profile>
208-
<id>spark-2.4</id>
209-
<activation>
210-
<activeByDefault>true</activeByDefault>
211-
</activation>
212-
<properties>
213-
<spark.binary.version>2.4</spark.binary.version>
214-
</properties>
215-
</profile>
216-
<profile>
217-
<id>spark-3.1</id>
218-
<properties>
219-
<spark.binary.version>3.1</spark.binary.version>
220-
<dep.jackson.version>2.10.0</dep.jackson.version>
221-
</properties>
222-
</profile>
223201
</profiles>
224202
</project>

‎format/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
<dependency>
3838
<groupId>org.apache.thrift</groupId>
3939
<artifactId>libthrift</artifactId>
40-
<version>0.9.3</version>
40+
<version>0.19.0</version>
4141
</dependency>
4242
</dependencies>
4343

‎index/examples/pom.xml

-22
Original file line numberDiff line numberDiff line change
@@ -77,26 +77,4 @@
7777
</plugin>
7878
</plugins>
7979
</build>
80-
81-
<profiles>
82-
<profile>
83-
<id>spark-2.3</id>
84-
<properties>
85-
<spark.binary.version>2.3</spark.binary.version>
86-
</properties>
87-
</profile>
88-
<profile>
89-
<id>spark-2.4</id>
90-
<properties>
91-
<spark.binary.version>2.4</spark.binary.version>
92-
</properties>
93-
</profile>
94-
<profile>
95-
<id>spark-3.1</id>
96-
<properties>
97-
<spark.binary.version>3.1</spark.binary.version>
98-
</properties>
99-
</profile>
100-
</profiles>
101-
10280
</project>

‎index/secondary-index/pom.xml

-21
Original file line numberDiff line numberDiff line change
@@ -156,27 +156,6 @@
156156
<maven.test.skip>true</maven.test.skip>
157157
</properties>
158158
</profile>
159-
<profile>
160-
<id>spark-2.3</id>
161-
<properties>
162-
<spark.binary.version>2.3</spark.binary.version>
163-
</properties>
164-
</profile>
165-
<profile>
166-
<id>spark-2.4</id>
167-
<activation>
168-
<activeByDefault>true</activeByDefault>
169-
</activation>
170-
<properties>
171-
<spark.binary.version>2.4</spark.binary.version>
172-
</properties>
173-
</profile>
174-
<profile>
175-
<id>spark-3.1</id>
176-
<properties>
177-
<spark.binary.version>3.1</spark.binary.version>
178-
</properties>
179-
</profile>
180159
</profiles>
181160

182161
</project>

‎integration/flink/pom.xml

+13-64
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,12 @@
2222
<dependencies>
2323
<dependency>
2424
<groupId>org.apache.carbondata</groupId>
25-
<artifactId>carbondata-flink-proxy</artifactId>
25+
<artifactId>carbondata-format</artifactId>
2626
<version>${project.version}</version>
2727
</dependency>
2828
<dependency>
2929
<groupId>org.apache.carbondata</groupId>
30-
<artifactId>carbondata-format</artifactId>
30+
<artifactId>carbondata-flink-proxy</artifactId>
3131
<version>${project.version}</version>
3232
</dependency>
3333
<dependency>
@@ -215,72 +215,21 @@
215215
<version>4.1.17.Final</version>
216216
<scope>test</scope>
217217
</dependency>
218-
</dependencies>
219-
220-
<profiles>
221-
<profile>
222-
<id>spark-2.3</id>
223-
<properties>
224-
<spark.binary.version>2.3</spark.binary.version>
225-
</properties>
226-
<dependencies>
227-
<dependency>
228-
<groupId>org.apache.carbondata</groupId>
229-
<artifactId>carbondata-spark_${spark.binary.version}</artifactId>
230-
<version>${project.version}</version>
231-
<scope>test</scope>
232-
<exclusions>
233-
<exclusion>
234-
<groupId>org.apache.hive</groupId>
235-
<artifactId>hive-exec</artifactId>
236-
</exclusion>
237-
</exclusions>
238-
</dependency>
239-
</dependencies>
240-
</profile>
241-
<profile>
242-
<id>spark-2.4</id>
243-
<activation>
244-
<activeByDefault>true</activeByDefault>
245-
</activation>
246-
<properties>
247-
<spark.binary.version>2.4</spark.binary.version>
248-
</properties>
249-
<dependencies>
250-
<dependency>
251-
<groupId>org.apache.carbondata</groupId>
252-
<artifactId>carbondata-spark_${spark.binary.version}</artifactId>
253-
<version>${project.version}</version>
254-
<scope>test</scope>
255-
<exclusions>
256-
<exclusion>
257-
<groupId>org.apache.hive</groupId>
258-
<artifactId>hive-exec</artifactId>
259-
</exclusion>
260-
</exclusions>
261-
</dependency>
262-
</dependencies>
263-
</profile>
264-
<profile>
265-
<id>spark-3.1</id>
266-
<properties>
267-
<spark.binary.version>3.1</spark.binary.version>
268-
</properties>
269-
<dependencies>
270-
<dependency>
218+
<dependency>
271219
<groupId>org.apache.carbondata</groupId>
272220
<artifactId>carbondata-spark_${spark.binary.version}</artifactId>
273221
<version>${project.version}</version>
274222
<scope>test</scope>
275-
<exclusions>
276-
<exclusion>
277-
<groupId>com.thoughtworks.paranamer</groupId>
278-
<artifactId>paranamer</artifactId>
279-
</exclusion>
280-
</exclusions>
281-
</dependency>
282-
</dependencies>
283-
</profile>
223+
<exclusions>
224+
<exclusion>
225+
<groupId>com.thoughtworks.paranamer</groupId>
226+
<artifactId>paranamer</artifactId>
227+
</exclusion>
228+
</exclusions>
229+
</dependency>
230+
</dependencies>
231+
232+
<profiles>
284233
<profile>
285234
<id>sdvtest</id>
286235
<properties>

0 commit comments

Comments
 (0)