Skip to content

Commit 56cdfa7

Browse files
authored
[Variant][Shredding] Support typed_access for timestamp_micro/timestamp_nano (#8401)
# Which issue does this PR close? - Closes #8331 . # Rationale for this change - Add typed_access for `Timestamp(Micro, _)` and `Timestamp(Nano, -)` # What changes are included in this PR? - Extract some data gen logic in tests to simplify the test logic (commit 93090d5), but it based on some old code(before #8392), rebase the master in the last commit - Add typed_access for `Timestamp(Micro, _)` and `Timestamp(Nano, _)` - Add test for typed_access for `Timestamp(Micro, _)` and `Timestamp(Nano, _)` # Are these changes tested? Covered by existing and added tests # Are there any user-facing changes? No
1 parent b444ea7 commit 56cdfa7

File tree

3 files changed

+284
-191
lines changed

3 files changed

+284
-191
lines changed

parquet-variant-compute/src/variant_array.rs

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,17 @@
1717

1818
//! [`VariantArray`] implementation
1919
20-
use crate::type_conversion::primitive_conversion_single_value;
20+
use crate::type_conversion::{generic_conversion_single_value, primitive_conversion_single_value};
2121
use arrow::array::{Array, ArrayRef, AsArray, BinaryViewArray, StructArray};
2222
use arrow::buffer::NullBuffer;
2323
use arrow::compute::cast;
2424
use arrow::datatypes::{
2525
Date32Type, Float16Type, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type,
26+
TimestampMicrosecondType, TimestampNanosecondType,
2627
};
2728
use arrow_schema::extension::ExtensionType;
2829
use arrow_schema::{ArrowError, DataType, Field, FieldRef, Fields, TimeUnit};
30+
use chrono::DateTime;
2931
use parquet_variant::Uuid;
3032
use parquet_variant::Variant;
3133

@@ -837,6 +839,42 @@ fn typed_value_to_variant<'a>(
837839
DataType::Float64 => {
838840
primitive_conversion_single_value!(Float64Type, typed_value, index)
839841
}
842+
DataType::Timestamp(TimeUnit::Microsecond, Some(_)) => {
843+
generic_conversion_single_value!(
844+
TimestampMicrosecondType,
845+
as_primitive,
846+
|v| DateTime::from_timestamp_micros(v).unwrap(),
847+
typed_value,
848+
index
849+
)
850+
}
851+
DataType::Timestamp(TimeUnit::Microsecond, None) => {
852+
generic_conversion_single_value!(
853+
TimestampMicrosecondType,
854+
as_primitive,
855+
|v| DateTime::from_timestamp_micros(v).unwrap().naive_utc(),
856+
typed_value,
857+
index
858+
)
859+
}
860+
DataType::Timestamp(TimeUnit::Nanosecond, Some(_)) => {
861+
generic_conversion_single_value!(
862+
TimestampNanosecondType,
863+
as_primitive,
864+
DateTime::from_timestamp_nanos,
865+
typed_value,
866+
index
867+
)
868+
}
869+
DataType::Timestamp(TimeUnit::Nanosecond, None) => {
870+
generic_conversion_single_value!(
871+
TimestampNanosecondType,
872+
as_primitive,
873+
|v| DateTime::from_timestamp_nanos(v).naive_utc(),
874+
typed_value,
875+
index
876+
)
877+
}
840878
// todo other types here (note this is very similar to cast_to_variant.rs)
841879
// so it would be great to figure out how to share this code
842880
_ => {

0 commit comments

Comments
 (0)