Update ICU data to 78 (#7025)

#6787

Author: robertbastian

Cargo.lock

@@ -140,7 +140,7 @@ icu_pattern = { version = "0.4.0", path = "components/pattern", default-features
 icu = { version = "~2.0.0", path = "components/icu", default-features = false }
 icu_calendar = { version = "~2.0.0", path = "components/calendar", default-features = false }
 icu_casemap = { version = "~2.0.0", path = "components/casemap", default-features = false }
-icu_collator = { version = "~2.0.0", path = "components/collator", default-features = false }
+icu_collator = { version = "~2.1.0-dev", path = "components/collator", default-features = false }
 icu_collections = { version = "~2.0.0", path = "components/collections", default-features = false }
 icu_codepointtrie_builder = { version = "~0.5.0", path = "components/collections/codepointtrie_builder", default-features = false }
 icu_datetime = { version = "~2.0.0", path = "components/datetime", default-features = false }
@@ -172,7 +172,7 @@ icu_provider_registry = { version = "~2.0.0", path = "provider/registry", defaul
 # Baked data
 icu_calendar_data = { version = "~2.0.0", path = "provider/data/calendar", default-features = false }
 icu_casemap_data = { version = "~2.0.0", path = "provider/data/casemap", default-features = false }
-icu_collator_data = { version = "~2.0.1", path = "provider/data/collator", default-features = false }
+icu_collator_data = { version = "~2.1.0-dev", path = "provider/data/collator", default-features = false }
 icu_datetime_data = { version = "~2.0.0", path = "provider/data/datetime", default-features = false }
 icu_decimal_data = { version = "~2.0.0", path = "provider/data/decimal", default-features = false }
 icu_list_data = { version = "~2.0.0", path = "provider/data/list", default-features = false }

Cargo.toml

@@ -5,7 +5,7 @@
 [package]
 name = "icu_collator"
 description = "API for comparing strings according to language-dependent conventions"
-version = "2.0.1"
+version = "2.1.0-dev"
 
 authors.workspace = true
 categories.workspace = true

components/collator/Cargo.toml

@@ -42,7 +42,6 @@ use crate::provider::CollationRootV1;
 use crate::provider::CollationSpecialPrimariesV1;
 use crate::provider::CollationSpecialPrimariesValidated;
 use crate::provider::CollationTailoringV1;
-use core::array;
 use core::cmp::Ordering;
 use core::convert::{Infallible, TryFrom};
 use icu_normalizer::provider::DecompositionData;
@@ -663,32 +662,25 @@ impl Collator {
             return Err(DataError::custom("invalid").with_marker(CollationSpecialPrimariesV1::INFO));
         }
         let special_primaries = special_primaries.map_project(|csp, _| {
-            if csp.last_primaries.len()
-                == (MaxVariable::Currency as usize)
-                    + core::mem::size_of_val(
-                        &CollationSpecialPrimariesValidated::HARDCODED_FALLBACK.compressible_bytes,
-                    ) / core::mem::size_of::<u16>()
-            {
-                CollationSpecialPrimariesValidated {
-                    compressible_bytes: array::from_fn(|i| {
-                        #[expect(clippy::unwrap_used)] // protected by the if
-                        {
-                            csp.last_primaries
-                                .get((MaxVariable::Currency as usize) + i)
-                                .unwrap()
-                        }
-                    }),
-                    last_primaries: csp.last_primaries.truncated(MaxVariable::Currency as usize),
-                    numeric_primary: csp.numeric_primary,
-                }
-            } else {
-                // Data without compressible bytes, add hardcoded data
-                CollationSpecialPrimariesValidated {
-                    last_primaries: csp.last_primaries,
-                    compressible_bytes: CollationSpecialPrimariesValidated::HARDCODED_FALLBACK
-                        .compressible_bytes,
-                    numeric_primary: csp.numeric_primary,
-                }
+            let compressible_bytes = (csp.last_primaries.len()
+                == MaxVariable::Currency as usize + 16)
+                .then(|| {
+                    csp.last_primaries
+                        .as_maybe_borrowed()?
+                        .as_ule_slice()
+                        .get((MaxVariable::Currency as usize)..)?
+                        .try_into()
+                        .ok()
+                })
+                .flatten()
+                .unwrap_or(
+                    CollationSpecialPrimariesValidated::HARDCODED_COMPRESSIBLE_BYTES_FALLBACK,
+                );
+
+            CollationSpecialPrimariesValidated {
+                last_primaries: csp.last_primaries.truncated(MaxVariable::Currency as usize),
+                numeric_primary: csp.numeric_primary,
+                compressible_bytes,
             }
         });
 
@@ -769,30 +761,70 @@ impl CollatorBorrowed<'static> {
             LocaleSpecificDataHolder::try_new_unstable_internal(provider, prefs, options)?;
 
         // TODO: redesign Korean search collation handling
-        if jamo.ce32s.len() != JAMO_COUNT {
-            return Err(DataError::custom("invalid").with_marker(CollationJamoV1::INFO));
-        }
+        const _: () = assert!(
+            crate::provider::Baked::SINGLETON_COLLATION_JAMO_V1
+                .ce32s
+                .as_slice()
+                .len()
+                == JAMO_COUNT
+        );
 
-        let special_primaries = crate::provider::Baked::SINGLETON_COLLATION_SPECIAL_PRIMARIES_V1;
         // `variant_count` isn't stable yet:
         // https://github.com/rust-lang/rust/issues/73662
-        if special_primaries.last_primaries.len() <= (MaxVariable::Currency as usize) {
-            return Err(DataError::custom("invalid").with_marker(CollationSpecialPrimariesV1::INFO));
-        } else if CollationSpecialPrimariesValidated::HARDCODED_FALLBACK.numeric_primary
-            != special_primaries.numeric_primary
-            || CollationSpecialPrimariesValidated::HARDCODED_FALLBACK
+        const _: () = assert!(
+            crate::provider::Baked::SINGLETON_COLLATION_SPECIAL_PRIMARIES_V1
                 .last_primaries
-                .iter()
-                .zip(special_primaries.last_primaries.iter())
-                .any(|(a, b)| a != b)
-        {
-            // Baked data without compressible bits, but not matching hardcoded data
-            return Err(
-                DataError::custom("cannot fall back to hardcoded compressible data")
-                    .with_marker(CollationSpecialPrimariesV1::INFO),
-            );
-        }
-        let special_primaries = CollationSpecialPrimariesValidated::HARDCODED_FALLBACK;
+                .as_slice()
+                .len()
+                > (MaxVariable::Currency as usize)
+        );
+
+        let special_primaries = const {
+            &CollationSpecialPrimariesValidated {
+                last_primaries: zerovec::ZeroSlice::from_ule_slice(
+                    crate::provider::Baked::SINGLETON_COLLATION_SPECIAL_PRIMARIES_V1
+                        .last_primaries
+                        .as_slice()
+                        .as_ule_slice()
+                        .split_at(MaxVariable::Currency as usize)
+                        .0,
+                )
+                .as_zerovec(),
+                numeric_primary: crate::provider::Baked::SINGLETON_COLLATION_SPECIAL_PRIMARIES_V1
+                    .numeric_primary,
+                compressible_bytes: {
+                    const C: &[<u16 as AsULE>::ULE] =
+                        crate::provider::Baked::SINGLETON_COLLATION_SPECIAL_PRIMARIES_V1
+                            .last_primaries
+                            .as_slice()
+                            .as_ule_slice();
+                    if C.len() == MaxVariable::Currency as usize + 16 {
+                        let i = MaxVariable::Currency as usize;
+                        #[allow(clippy::indexing_slicing)] // protected, const
+                        &[
+                            C[i],
+                            C[i + 1],
+                            C[i + 2],
+                            C[i + 3],
+                            C[i + 4],
+                            C[i + 5],
+                            C[i + 6],
+                            C[i + 7],
+                            C[i + 8],
+                            C[i + 9],
+                            C[i + 10],
+                            C[i + 11],
+                            C[i + 12],
+                            C[i + 13],
+                            C[i + 14],
+                            C[i + 15],
+                        ]
+                    } else {
+                        CollationSpecialPrimariesValidated::HARDCODED_COMPRESSIBLE_BYTES_FALLBACK
+                    }
+                },
+            }
+        };
 
         // Attribute belongs closer to `unwrap`, but
         // https://github.com/rust-lang/rust/issues/15701

components/collator/src/comparison.rs

@@ -21,7 +21,7 @@
 use icu_collections::char16trie::Char16TrieIterator;
 use icu_collections::codepointtrie::CodePointTrie;
 use icu_provider::prelude::*;
-use zerovec::ule::AsULE;
+use zerovec::ule::{AsULE, RawBytesULE};
 use zerovec::ZeroVec;
 use zerovec::{zeroslice, ZeroSlice};
 
@@ -577,39 +577,28 @@ pub(crate) struct CollationSpecialPrimariesValidated<'data> {
     pub numeric_primary: u8,
     /// 256 bits (packed in 16 u16s) to classify every possible
     /// byte into compressible or non-compressible.
-    pub compressible_bytes: [u16; 16],
+    pub compressible_bytes: &'data [<u16 as AsULE>::ULE; 16],
 }
 
 impl CollationSpecialPrimariesValidated<'static> {
-    pub(crate) const HARDCODED_FALLBACK: &Self = &Self {
-        last_primaries: zerovec::zerovec!(u16; <u16 as AsULE>::ULE::from_aligned; [
-          // Last primaries
-          1286,
-          3072,
-          3488,
-          3840,
-        ]),
-        numeric_primary: 16u8,
-        compressible_bytes: [
-            // Compressible bytes
-            0b0000_0000_0000_0000,
-            0b0000_0000_0000_0000,
-            0b0000_0000_0000_0000,
-            0b0000_0000_0000_0000,
-            0b0000_0000_0000_0000,
-            0b0000_0000_0000_0000,
-            0b1111_1111_1111_1110,
-            0b1111_1111_1111_1111,
-            0b0000_0000_0000_0001,
-            0b0000_0000_0000_0000,
-            0b0000_0000_0000_0000,
-            0b0000_0000_0000_0000,
-            0b0000_0000_0000_0000,
-            0b0000_0000_0000_0000,
-            0b0000_0000_0000_0000,
-            0b0100_0000_0000_0000,
-        ],
-    };
+    pub(crate) const HARDCODED_COMPRESSIBLE_BYTES_FALLBACK: &'static [<u16 as AsULE>::ULE; 16] = &[
+        RawBytesULE(0b0000_0000_0000_0000u16.to_le_bytes()),
+        RawBytesULE(0b0000_0000_0000_0000u16.to_le_bytes()),
+        RawBytesULE(0b0000_0000_0000_0000u16.to_le_bytes()),
+        RawBytesULE(0b0000_0000_0000_0000u16.to_le_bytes()),
+        RawBytesULE(0b0000_0000_0000_0000u16.to_le_bytes()),
+        RawBytesULE(0b0000_0000_0000_0000u16.to_le_bytes()),
+        RawBytesULE(0b1111_1111_1111_1110u16.to_le_bytes()),
+        RawBytesULE(0b1111_1111_1111_1111u16.to_le_bytes()),
+        RawBytesULE(0b0000_0000_0000_0001u16.to_le_bytes()),
+        RawBytesULE(0b0000_0000_0000_0000u16.to_le_bytes()),
+        RawBytesULE(0b0000_0000_0000_0000u16.to_le_bytes()),
+        RawBytesULE(0b0000_0000_0000_0000u16.to_le_bytes()),
+        RawBytesULE(0b0000_0000_0000_0000u16.to_le_bytes()),
+        RawBytesULE(0b0000_0000_0000_0000u16.to_le_bytes()),
+        RawBytesULE(0b0000_0000_0000_0000u16.to_le_bytes()),
+        RawBytesULE(0b0100_0000_0000_0000u16.to_le_bytes()),
+    ];
 }
 
 icu_provider::data_struct!(
@@ -633,7 +622,7 @@ impl CollationSpecialPrimariesValidated<'_> {
         // into Compiler Explorer shows that the panic
         // is optimized away.
         #[expect(clippy::indexing_slicing)]
-        let field = self.compressible_bytes[usize::from(b >> 4)];
+        let field = u16::from_unaligned(self.compressible_bytes[usize::from(b >> 4)]);
         let mask = 1 << (b & 0b1111);
         (field & mask) != 0
     }