Merge pull request #246 from aldanor/feature/f16

Float16 support

Author: aldanor

.github/workflows/ci.yml

@@ -46,7 +46,7 @@ jobs:
       - name: Document workspace
         env:
           RUSTDOCFLAGS: "--cfg docsrs"
-        run: cargo doc --features static,zlib,blosc,lzf
+        run: cargo doc --features static,zlib,blosc,lzf,f16,complex
 
   brew:
     name: brew
@@ -147,8 +147,8 @@ jobs:
         with: {toolchain: '${{matrix.rust}}'}
       - name: Build and test all crates
         run: cargo test --workspace -v --features hdf5-sys/static,hdf5-sys/zlib --exclude hdf5-derive
-      - name: Build and test with filters
-        run: cargo test --workspace -v --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc --exclude hdf5-derive
+      - name: Build and test with filters and other features
+        run: cargo test --workspace -v --features hdf5-sys/static,hdf5-sys/zlib,lzf,blosc,f16,complex --exclude hdf5-derive
         if: matrix.rust != 'stable-gnu'
       - name: Run examples
         run: |
@@ -276,7 +276,7 @@ jobs:
         with: {toolchain: 1.64}
       - name: Build and test all crates
         run:
-          cargo test --workspace -vv --features=hdf5-sys/static --exclude=hdf5-derive
+          cargo test --workspace -vv --features=hdf5-sys/static,hdf5-sys/zlib --exclude=hdf5-derive
 
   wine:
     name: wine

CHANGELOG.md

@@ -9,7 +9,8 @@
 - Add a `ByteReader` which implements `std::io::{Read, Seek}` for 1D `u8`
   datasets. Usage via `Dataset::as_byte_reader()`.
 - Add `chunk_visit` to visit all chunks in a dataset.
-- Implement `H5Type` for `num_complex::Complex`.
+- Add support for float16 values (`half::f16`), enabled via "f16" feature.
+- Add support for complex numbers (`num_complex::Complex`), enabled via "complex".
 - Adding feature `static` for the `hdf5` crate which downloads and builds a bundled HDF5.
 
 ### Changed

Cargo.toml

@@ -15,9 +15,11 @@ edition = "2021"
 [workspace.dependencies]
 # external
 cfg-if = "1.0"
+half = { version = "2.2", default-features = false }
 libc = "0.2"
 libz-sys = { version = "1.1", default-features = false }
 mpi-sys = "0.2"
+num-complex = { version = "0.4", default-features = false }
 regex = "1.8"
 # internal
 hdf5-derive = { version = "0.8.1", path = "hdf5-derive" }  # !V

hdf5-sys/build.rs

@@ -175,8 +175,8 @@ pub struct Header {
     pub have_direct: bool,
     pub have_parallel: bool,
     pub have_threadsafe: bool,
-    pub have_zlib: bool,
     pub have_no_deprecated: bool,
+    pub have_filter_deflate: bool,
     pub version: Version,
 }
 
@@ -203,7 +203,7 @@ impl Header {
             } else if name == "H5_HAVE_THREADSAFE" {
                 hdr.have_threadsafe = value > 0;
             } else if name == "H5_HAVE_FILTER_DEFLATE" {
-                hdr.have_zlib = value > 0;
+                hdr.have_filter_deflate = value > 0;
             } else if name == "H5_NO_DEPRECATED_SYMBOLS" {
                 hdr.have_no_deprecated = value > 0;
             }
@@ -680,14 +680,18 @@ impl Config {
             println!("cargo:rustc-cfg=feature=\"have-threadsafe\"");
             println!("cargo:have_threadsafe=1");
         }
+        if self.header.have_filter_deflate {
+            println!("cargo:rustc-cfg=feature=\"have-filter-deflate\"");
+            println!("cargo:have_filter_deflate=1");
+        }
     }
 
     fn check_against_features_required(&self) {
         let h = &self.header;
         for (flag, feature, native) in [
             (!h.have_no_deprecated, "deprecated", "HDF5_ENABLE_DEPRECATED_SYMBOLS"),
             (h.have_threadsafe, "threadsafe", "HDF5_ENABLE_THREADSAFE"),
-            (h.have_zlib, "zlib", "HDF5_ENABLE_Z_LIB_SUPPORT"),
+            (h.have_filter_deflate, "zlib", "HDF5_ENABLE_Z_LIB_SUPPORT"),
         ] {
             if feature_enabled(&feature.to_ascii_uppercase()) {
                 assert!(

hdf5-types/Cargo.toml

@@ -15,15 +15,20 @@ edition.workspace = true
 
 [features]
 h5-alloc = []
-complex = ["num-complex"]
+complex = ["dep:num-complex"]
+f16 = ["dep:half"]
 
 [dependencies]
 ascii = "1.1"
 cfg-if = { workspace = true }
 hdf5-sys = { workspace = true }
 libc = { workspace = true }
-num-complex = { version = "0.4", optional = true, default-features = false }
+num-complex = { workspace = true, optional = true }
+half = { workspace = true, optional = true }
 
 [dev-dependencies]
 quickcheck = { version = "1.0", default-features = false }
 unindent = "0.2"
+
+[package.metadata.docs.rs]
+features = ["f16", "complex"]

hdf5-types/src/dyn_value.rs

@@ -115,19 +115,76 @@ impl From<DynInteger> for DynValue<'_> {
 }
 
 #[derive(Copy, Clone, PartialEq)]
-pub enum DynScalar {
-    Integer(DynInteger),
+pub enum DynFloat {
+    #[cfg(feature = "f16")]
+    Float16(::half::f16),
     Float32(f32),
     Float64(f64),
+}
+
+impl DynFloat {
+    pub(self) fn read(buf: &[u8], size: FloatSize) -> Self {
+        match size {
+            #[cfg(feature = "f16")]
+            FloatSize::U2 => Self::Float16(read_raw(buf)),
+            FloatSize::U4 => Self::Float32(read_raw(buf)),
+            FloatSize::U8 => Self::Float64(read_raw(buf)),
+        }
+    }
+}
+
+unsafe impl DynClone for DynFloat {
+    fn dyn_clone(&mut self, out: &mut [u8]) {
+        match self {
+            #[cfg(feature = "f16")]
+            Self::Float16(x) => write_raw(out, *x),
+            Self::Float32(x) => write_raw(out, *x),
+            Self::Float64(x) => write_raw(out, *x),
+        }
+    }
+}
+
+impl Debug for DynFloat {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            #[cfg(feature = "f16")]
+            Self::Float16(x) => Debug::fmt(&x, f),
+            Self::Float32(x) => Debug::fmt(&x, f),
+            Self::Float64(x) => Debug::fmt(&x, f),
+        }
+    }
+}
+
+impl Display for DynFloat {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        Debug::fmt(self, f)
+    }
+}
+
+impl From<DynFloat> for DynScalar {
+    fn from(value: DynFloat) -> Self {
+        Self::Float(value)
+    }
+}
+
+impl From<DynFloat> for DynValue<'_> {
+    fn from(value: DynFloat) -> Self {
+        DynScalar::Float(value).into()
+    }
+}
+
+#[derive(Copy, Clone, PartialEq)]
+pub enum DynScalar {
+    Integer(DynInteger),
+    Float(DynFloat),
     Boolean(bool),
 }
 
 unsafe impl DynClone for DynScalar {
     fn dyn_clone(&mut self, out: &mut [u8]) {
         match self {
             Self::Integer(x) => x.dyn_clone(out),
-            Self::Float32(x) => write_raw(out, *x),
-            Self::Float64(x) => write_raw(out, *x),
+            Self::Float(x) => x.dyn_clone(out),
             Self::Boolean(x) => write_raw(out, *x),
         }
     }
@@ -137,8 +194,7 @@ impl Debug for DynScalar {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             Self::Integer(x) => Debug::fmt(&x, f),
-            Self::Float32(x) => Debug::fmt(&x, f),
-            Self::Float64(x) => Debug::fmt(&x, f),
+            Self::Float(x) => Debug::fmt(&x, f),
             Self::Boolean(x) => Debug::fmt(&x, f),
         }
     }
@@ -637,8 +693,7 @@ impl<'a> DynValue<'a> {
 
         match tp {
             Integer(size) | Unsigned(size) => DynInteger::read(buf, true, *size).into(),
-            Float(FloatSize::U4) => DynScalar::Float32(read_raw(buf)).into(),
-            Float(FloatSize::U8) => DynScalar::Float64(read_raw(buf)).into(),
+            Float(size) => DynFloat::read(buf, *size).into(),
             Boolean => DynScalar::Boolean(read_raw(buf)).into(),
             Enum(ref tp) => DynEnum::new(tp, DynInteger::read(buf, tp.signed, tp.size)).into(),
             Compound(ref tp) => DynCompound::new(tp, buf).into(),

hdf5-types/src/h5type.rs

@@ -40,18 +40,24 @@ impl IntSize {
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub enum FloatSize {
+    #[cfg(feature = "f16")]
+    U2 = 2,
     U4 = 4,
     U8 = 8,
 }
 
 impl FloatSize {
     pub const fn from_int(size: usize) -> Option<Self> {
-        if size == 4 {
-            Some(Self::U4)
-        } else if size == 8 {
-            Some(Self::U8)
-        } else {
-            None
+        #[cfg(feature = "f16")]
+        {
+            if size == 2 {
+                return Some(Self::U2);
+            }
+        }
+        match size {
+            4 => Some(Self::U4),
+            8 => Some(Self::U8),
+            _ => None,
         }
     }
 }
@@ -167,6 +173,8 @@ impl Display for TypeDescriptor {
             TypeDescriptor::Unsigned(IntSize::U2) => write!(f, "uint16"),
             TypeDescriptor::Unsigned(IntSize::U4) => write!(f, "uint32"),
             TypeDescriptor::Unsigned(IntSize::U8) => write!(f, "uint64"),
+            #[cfg(feature = "f16")]
+            TypeDescriptor::Float(FloatSize::U2) => write!(f, "float16"),
             TypeDescriptor::Float(FloatSize::U4) => write!(f, "float32"),
             TypeDescriptor::Float(FloatSize::U8) => write!(f, "float64"),
             TypeDescriptor::Boolean => write!(f, "bool"),
@@ -251,6 +259,8 @@ impl_h5type!(u8, Unsigned, IntSize::U1);
 impl_h5type!(u16, Unsigned, IntSize::U2);
 impl_h5type!(u32, Unsigned, IntSize::U4);
 impl_h5type!(u64, Unsigned, IntSize::U8);
+#[cfg(feature = "f16")]
+impl_h5type!(::half::f16, Float, FloatSize::U2);
 impl_h5type!(f32, Float, FloatSize::U4);
 impl_h5type!(f64, Float, FloatSize::U8);
 

hdf5/Cargo.toml

@@ -15,11 +15,21 @@ edition.workspace = true
 
 [features]
 default = []
-mpio = ["dep:mpi-sys", "hdf5-sys/mpio"]
-lzf = ["dep:lzf-sys", "dep:errno"]
-blosc = ["dep:blosc-sys"]
+# Compile and statically link bundled HDF5 library.
 static = ["hdf5-sys/static"]
+# Enable zlib compression filter.
 zlib = ["hdf5-sys/zlib"]
+# Enable LZF compression filter.
+lzf = ["dep:lzf-sys", "dep:errno"]
+# Enable blosc compression filters.
+blosc = ["dep:blosc-sys"]
+# Enable MPI support.
+mpio = ["dep:mpi-sys", "hdf5-sys/mpio"]
+# Enable complex number type support.
+complex = ["hdf5-types/complex"]
+# Enable float16 type support.
+f16 = ["hdf5-types/f16"]
+
 # The features with version numbers such as 1.10.3, 1.12.0 are metafeatures
 # and is only available when the HDF5 library is at least this version.
 # Features have_direct and have_parallel are also metafeatures and dependent
@@ -44,6 +54,8 @@ hdf5-sys = { workspace = true }
 hdf5-types = { workspace = true }
 
 [dev-dependencies]
+half = { workspace = true }
+num-complex = { workspace = true }
 paste = "1.0"
 pretty_assertions = "1.3"
 rand = { version = "0.8", features = ["small_rng"] }
@@ -52,5 +64,5 @@ scopeguard = "1.1"
 tempfile = "3.6"
 
 [package.metadata.docs.rs]
-features = ["hdf5-sys/static", "hdf5-sys/zlib", "blosc", "lzf"]
+features = ["static", "zlib", "blosc", "lzf", "f16", "complex"]
 rustdoc-args = ["--cfg", "docsrs"]

hdf5/build.rs

@@ -9,6 +9,7 @@ fn main() {
             "DEP_HDF5_HAVE_DIRECT" => print_feature("have-direct"),
             "DEP_HDF5_HAVE_PARALLEL" => print_feature("have-parallel"),
             "DEP_HDF5_HAVE_THREADSAFE" => print_feature("have-threadsafe"),
+            "DEP_HDF5_HAVE_FILTER_DEFLATE" => print_feature("have-filter-deflate"),
             // internal config flags
             "DEP_HDF5_MSVC_DLL_INDIRECTION" => print_cfg("msvc_dll_indirection"),
             // public version features

hdf5/src/hl/chunks.rs

@@ -72,7 +72,7 @@ mod v1_14_0 {
     use super::*;
     use hdf5_sys::h5d::H5Dchunk_iter;
 
-    /// Borrowed version of [ChunkInfo](crate::dataset::ChunkInfo)
+    /// Borrowed version of [`ChunkInfo`](crate::dataset::ChunkInfo)
     #[derive(Clone, Debug, PartialEq, Eq)]
     pub struct ChunkInfoRef<'a> {
         pub offset: &'a [hsize_t],

hdf5/src/hl/datatype.rs

@@ -327,6 +327,16 @@ impl Datatype {
             Ok(string_id)
         }
 
+        #[cfg(feature = "f16")]
+        unsafe fn f16_type() -> Result<hid_t> {
+            use hdf5_sys::h5t::{H5Tset_ebias, H5Tset_fields};
+            let f16_id = be_le!(H5T_IEEE_F32BE, H5T_IEEE_F32LE);
+            h5try!(H5Tset_fields(f16_id, 15, 10, 5, 0, 10)); // cf. h5py/h5py#339
+            h5try!(H5Tset_size(f16_id, 2));
+            h5try!(H5Tset_ebias(f16_id, 15));
+            Ok(f16_id)
+        }
+
         let datatype_id: Result<_> = h5lock!({
             match *desc {
                 TD::Integer(size) => Ok(match size {
@@ -342,6 +352,8 @@ impl Datatype {
                     IntSize::U8 => be_le!(H5T_STD_U64BE, H5T_STD_U64LE),
                 }),
                 TD::Float(size) => Ok(match size {
+                    #[cfg(feature = "f16")]
+                    FloatSize::U2 => f16_type()?,
                     FloatSize::U4 => be_le!(H5T_IEEE_F32BE, H5T_IEEE_F32LE),
                     FloatSize::U8 => be_le!(H5T_IEEE_I16BE, H5T_IEEE_F64LE),
                 }),

tests/common/gen.rs renamed to hdf5/tests/common/gen.rs

@@ -1,12 +1,16 @@
 use std::convert::TryFrom;
-use std::fmt;
+use std::fmt::{self, Debug};
 use std::iter;
 
 use hdf5::types::{FixedAscii, FixedUnicode, VarLenArray, VarLenAscii, VarLenUnicode};
 use hdf5::H5Type;
 
+use half::f16;
 use ndarray::{ArrayD, SliceInfo, SliceInfoElem};
+use num_complex::Complex;
+use rand::distributions::Standard;
 use rand::distributions::{Alphanumeric, Uniform};
+use rand::prelude::Distribution;
 use rand::prelude::{Rng, SliceRandom};
 
 pub fn gen_shape<R: Rng + ?Sized>(rng: &mut R, ndim: usize) -> Vec<usize> {
@@ -93,6 +97,21 @@ macro_rules! impl_gen_tuple {
 
 impl_gen_tuple! { A, B, C, D, E, F, G, H, I, J, K, L }
 
+impl Gen for f16 {
+    fn gen<R: Rng + ?Sized>(rng: &mut R) -> Self {
+        Self::from_f32(rng.gen())
+    }
+}
+
+impl<T: Debug> Gen for Complex<T>
+where
+    Standard: Distribution<T>,
+{
+    fn gen<R: Rng + ?Sized>(rng: &mut R) -> Self {
+        Self::new(rng.gen(), rng.gen())
+    }
+}
+
 pub fn gen_vec<R: Rng + ?Sized, T: Gen>(rng: &mut R, size: usize) -> Vec<T> {
     iter::repeat(()).map(|_| T::gen(rng)).take(size).collect()
 }