Skip to content

Commit b796621

Browse files
jack86596jack86596
committed
[CARBONDATA-4291] Carbon hive table supports float datatype
1 parent ce860d0 commit b796621

File tree

77 files changed

+1482
-474
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

77 files changed

+1482
-474
lines changed

‎core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,8 @@ public Object getData(int rowId) {
445445
return getInt(rowId);
446446
} else if (dataType == LONG) {
447447
return getLong(rowId);
448+
} else if (dataType == DataTypes.FLOAT) {
449+
return getFloat(rowId);
448450
} else if (dataType == DataTypes.DOUBLE) {
449451
return getDouble(rowId);
450452
} else if (DataTypes.isDecimal(dataType)) {
@@ -560,6 +562,8 @@ private Object getNull(int rowId) {
560562
result = getInt(rowId);
561563
} else if (dataType == LONG) {
562564
result = getLong(rowId);
565+
} else if (dataType == DataTypes.FLOAT) {
566+
return getFloat(rowId);
563567
} else if (dataType == DataTypes.DOUBLE) {
564568
result = getDouble(rowId);
565569
} else if (DataTypes.isDecimal(dataType)) {

‎core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPageValueConverter.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,12 @@ public interface ColumnPageValueConverter {
4343

4444
long decodeLong(int value);
4545

46+
float decodeFloat(byte value);
47+
48+
float decodeFloat(short value);
49+
50+
float decodeFloat(int value);
51+
4652
double decodeDouble(byte value);
4753

4854
double decodeDouble(short value);

‎core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,22 @@ public double getDouble(int rowId) {
9494

9595
@Override
9696
public float getFloat(int rowId) {
97-
return (float) getDouble(rowId);
97+
DataType dataType = columnPage.getDataType();
98+
if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
99+
return converter.decodeFloat(columnPage.getByte(rowId));
100+
} else if (dataType == DataTypes.SHORT) {
101+
return converter.decodeFloat(columnPage.getShort(rowId));
102+
} else if (dataType == DataTypes.SHORT_INT) {
103+
return converter.decodeFloat(columnPage.getShortInt(rowId));
104+
} else if (dataType == DataTypes.INT) {
105+
return converter.decodeFloat(columnPage.getInt(rowId));
106+
} else if (dataType == DataTypes.FLOAT) {
107+
return columnPage.getFloat(rowId);
108+
} else if (dataType == DataTypes.BINARY) {
109+
return converter.decodeFloat(columnPage.getByte(rowId));
110+
} else {
111+
throw new RuntimeException("internal error: " + this);
112+
}
98113
}
99114

100115
@Override

‎core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageEncoderMeta.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,11 @@ private byte[] getValueAsBytes(Object value, DataType dataType) {
247247
b.putLong((long) value);
248248
b.flip();
249249
return b.array();
250+
} else if (dataType == DataTypes.FLOAT) {
251+
b = ByteBuffer.allocate(8);
252+
b.putFloat((float) value);
253+
b.flip();
254+
return b.array();
250255
} else if (dataType == DataTypes.DOUBLE) {
251256
b = ByteBuffer.allocate(8);
252257
b.putDouble((double) value);

‎core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,8 @@ private static DataType fitMinMax(DataType dataType, Object max, Object min) {
166166
return fitLongMinMax((int) max, (int) min);
167167
} else if ((dataType == DataTypes.LONG) || (dataType == DataTypes.TIMESTAMP)) {
168168
return fitLongMinMax((long) max, (long) min);
169+
} else if (dataType == DataTypes.FLOAT) {
170+
return fitLongMinMax((long) (float) max, (long) (float) min);
169171
} else if (dataType == DataTypes.DOUBLE) {
170172
return fitLongMinMax((long) (double) max, (long) (double) min);
171173
} else {
@@ -212,6 +214,8 @@ private static DataType fitDelta(DataType dataType, Object max, Object min) {
212214
value = (long) (short) max - (long) (short) min;
213215
} else if (dataType == DataTypes.INT) {
214216
value = (long) (int) max - (long) (int) min;
217+
} else if (dataType == DataTypes.FLOAT) {
218+
value = (long) (float) max - (long) (float) min;
215219
} else if (dataType == DataTypes.LONG || dataType == DataTypes.TIMESTAMP) {
216220
value = (long) max - (long) min;
217221
// The subtraction overflowed iff the operands have opposing signs
@@ -329,6 +333,9 @@ private static ColumnPageCodec getColumnPageCodec(SimpleStatsResult stats,
329333
// If absMaxValue exceeds LONG.MAX_VALUE, then go for direct compression
330334
if ((Math.pow(10, decimalCount) * absMaxValue) > Long.MAX_VALUE) {
331335
return new DirectCompressCodec(DataTypes.DOUBLE);
336+
} else if (srcDataType == DataTypes.FLOAT &&
337+
(Math.pow(10, decimalCount) * absMaxValue) > Integer.MAX_VALUE) {
338+
return new DirectCompressCodec(DataTypes.FLOAT);
332339
} else {
333340
long max = (long) (Math.pow(10, decimalCount) * absMaxValue);
334341
DataType adaptiveDataType = fitLongMinMax(max, 0);

‎core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
public class AdaptiveDeltaFloatingCodec extends AdaptiveCodec {
5454

5555
private Double factor;
56+
private float floatFactor;
5657
private long max;
5758

5859
public static ColumnPageCodec newInstance(DataType srcDataType, DataType targetDataType,
@@ -65,6 +66,7 @@ public AdaptiveDeltaFloatingCodec(DataType srcDataType, DataType targetDataType,
6566
SimpleStatsResult stats, boolean isInvertedIndex) {
6667
super(srcDataType, targetDataType, stats, isInvertedIndex);
6768
this.factor = Math.pow(10, stats.getDecimalCount());
69+
floatFactor = factor.floatValue();
6870
if (srcDataType == DataTypes.FLOAT) {
6971
this.max =
7072
(long) ((long) Math.pow(10, stats.getDecimalCount()) * ((float) stats.getMax()));
@@ -185,13 +187,15 @@ public void encode(int rowId, long value) {
185187
@Override
186188
public void encode(int rowId, float value) {
187189
if (targetDataType.equals(DataTypes.BYTE)) {
188-
encodedPage.putByte(rowId, (byte) (max - (value * factor)));
190+
encodedPage.putByte(rowId, (byte) (max - (value * floatFactor)));
189191
} else if (targetDataType.equals(DataTypes.SHORT)) {
190-
encodedPage.putShort(rowId, (short) (max - (value * factor)));
192+
encodedPage.putShort(rowId, (short) (max - (value * floatFactor)));
191193
} else if (targetDataType.equals(DataTypes.SHORT_INT)) {
192-
encodedPage.putShortInt(rowId, (int) (max - (value * factor)));
194+
encodedPage.putShortInt(rowId, (int) (max - (value * floatFactor)));
193195
} else if (targetDataType.equals(DataTypes.INT)) {
194-
encodedPage.putInt(rowId, (int) (max - (value * factor)));
196+
encodedPage.putInt(rowId, (int) (max - (value * floatFactor)));
197+
} else if (targetDataType.equals(DataTypes.FLOAT)) {
198+
encodedPage.putFloat(rowId, value);
195199
} else {
196200
throw new RuntimeException("internal error: " + debugInfo());
197201
}
@@ -229,6 +233,21 @@ public long decodeLong(int value) {
229233
throw new RuntimeException("internal error: " + debugInfo());
230234
}
231235

236+
@Override
237+
public float decodeFloat(byte value) {
238+
return (max - value) / floatFactor;
239+
}
240+
241+
@Override
242+
public float decodeFloat(short value) {
243+
return (max - value) / floatFactor;
244+
}
245+
246+
@Override
247+
public float decodeFloat(int value) {
248+
return (max - value) / floatFactor;
249+
}
250+
232251
@Override
233252
public double decodeDouble(byte value) {
234253
return (max - value) / factor;
@@ -265,7 +284,6 @@ public void decodeAndFillVector(byte[] pageData, ColumnVectorInfo vectorInfo, Bi
265284
int intSizeInBytes = DataTypes.INT.getSizeInBytes();
266285
int longSizeInBytes = DataTypes.LONG.getSizeInBytes();
267286
if (vectorDataType == DataTypes.FLOAT) {
268-
float floatFactor = factor.floatValue();
269287
if (pageDataType == DataTypes.BOOLEAN || pageDataType == DataTypes.BYTE) {
270288
for (int i = 0; i < pageSize; i++) {
271289
vector.putFloat(i, (max - pageData[i]) / floatFactor);

‎core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ public AdaptiveDeltaIntegralCodec(DataType srcDataType, DataType targetDataType,
6969
this.max = (int) stats.getMax();
7070
} else if (srcDataType == DataTypes.LONG || srcDataType == DataTypes.TIMESTAMP) {
7171
this.max = (long) stats.getMax();
72+
} else if (srcDataType == DataTypes.FLOAT) {
73+
this.max = (int) (float) stats.getMax();
7274
} else if (srcDataType == DataTypes.DOUBLE) {
7375
this.max = (long) (double) stats.getMax();
7476
} else if (DataTypes.isDecimal(srcDataType)) {
@@ -236,8 +238,6 @@ public void encode(int rowId, float value) {
236238
encodedPage.putShortInt(rowId, (int) (max - value));
237239
} else if (targetDataType == DataTypes.INT) {
238240
encodedPage.putInt(rowId, (int) (max - value));
239-
} else if (targetDataType == DataTypes.LONG) {
240-
encodedPage.putLong(rowId, (long) (max - value));
241241
} else {
242242
throw new RuntimeException("internal error");
243243
}
@@ -275,6 +275,21 @@ public long decodeLong(int value) {
275275
return max - value;
276276
}
277277

278+
@Override
279+
public float decodeFloat(byte value) {
280+
return max - value;
281+
}
282+
283+
@Override
284+
public float decodeFloat(short value) {
285+
return max - value;
286+
}
287+
288+
@Override
289+
public float decodeFloat(int value) {
290+
return max - value;
291+
}
292+
278293
@Override
279294
public double decodeDouble(byte value) {
280295
return max - value;

‎core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,19 @@ public void encode(int rowId, long value) {
172172

173173
@Override
174174
public void encode(int rowId, float value) {
175-
encode(rowId, (double) value);
175+
if (targetDataType == DataTypes.BYTE) {
176+
encodedPage.putByte(rowId, (byte) Math.round(value * floatFactor));
177+
} else if (targetDataType == DataTypes.SHORT) {
178+
encodedPage.putShort(rowId, (short) Math.round(value * floatFactor));
179+
} else if (targetDataType == DataTypes.SHORT_INT) {
180+
encodedPage.putShortInt(rowId, Math.round(value * floatFactor));
181+
} else if (targetDataType == DataTypes.INT) {
182+
encodedPage.putInt(rowId, Math.round(value * floatFactor));
183+
} else if (targetDataType == DataTypes.FLOAT) {
184+
encodedPage.putFloat(rowId, value);
185+
} else {
186+
throw new RuntimeException("internal error: " + debugInfo());
187+
}
176188
}
177189

178190
@Override
@@ -209,6 +221,21 @@ public long decodeLong(int value) {
209221
throw new RuntimeException("internal error: " + debugInfo());
210222
}
211223

224+
@Override
225+
public float decodeFloat(byte value) {
226+
return value / floatFactor;
227+
}
228+
229+
@Override
230+
public float decodeFloat(short value) {
231+
return value / floatFactor;
232+
}
233+
234+
@Override
235+
public float decodeFloat(int value) {
236+
return value / floatFactor;
237+
}
238+
212239
@Override
213240
public double decodeDouble(byte value) {
214241
return value / factor;

‎core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,21 @@ public long decodeLong(int value) {
251251
return value;
252252
}
253253

254+
@Override
255+
public float decodeFloat(byte value) {
256+
return value;
257+
}
258+
259+
@Override
260+
public float decodeFloat(short value) {
261+
return value;
262+
}
263+
264+
@Override
265+
public float decodeFloat(int value) {
266+
return value;
267+
}
268+
254269
@Override
255270
public double decodeDouble(byte value) {
256271
return value;

‎core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,21 @@ public long decodeLong(int value) {
215215
return value;
216216
}
217217

218+
@Override
219+
public float decodeFloat(byte value) {
220+
return value;
221+
}
222+
223+
@Override
224+
public float decodeFloat(short value) {
225+
return value;
226+
}
227+
228+
@Override
229+
public float decodeFloat(int value) {
230+
return value;
231+
}
232+
218233
@Override
219234
public double decodeDouble(byte value) {
220235
return value;

0 commit comments

Comments
 (0)