[lib] Impl BidiInfo.reordered_levels_per_char()

This allows easier testing of the logic, specially having specific cases
from the conformance test as unit test, as corner bugs unfold.

Author: behnam

src/implicit.rs

@@ -26,6 +26,9 @@ pub fn resolve_weak(sequence: &IsolatingRunSequence, processing_classes: &mut [B
     // by an "earlier" rule.  We should either split this into separate passes, or preserve
     // extra state so each rule can see the correct previous class.
 
+    // FIXME: Also, this could be the cause of increased failure for using longer-UTF-8 chars in
+    // conformance tests, like BidiTest:69635 (AL ET EN)
+
     let mut prev_class = sequence.sos;
     let mut last_strong_is_al = false;
     let mut et_run_indices = Vec::new(); // for W5

src/lib.rs

@@ -291,12 +291,25 @@ impl<'text> BidiInfo<'text> {
         }
     }
 
-    /// Re-order a line based on resolved levels and return only the embedding levels.
+    /// Re-order a line based on resolved levels and return only the embedding levels, one `Level`
+    /// per *byte*.
     pub fn reordered_levels(&self, para: &ParagraphInfo, line: Range<usize>) -> Vec<Level> {
         let (levels, _) = self.visual_runs(para, line.clone());
         levels
     }
 
+    /// Re-order a line based on resolved levels and return only the embedding levels, one `Level`
+    /// per *character*.
+    pub fn reordered_levels_per_char(
+        &self,
+        para: &ParagraphInfo,
+        line: Range<usize>,
+    ) -> Vec<Level> {
+        let levels = self.reordered_levels(para, line);
+        self.text.char_indices().map(|(i, _)| levels[i]).collect()
+    }
+
+
     /// Re-order a line based on resolved levels and return the line in display order.
     pub fn reorder_line(&self, para: &ParagraphInfo, line: Range<usize>) -> Cow<'text, str> {
         let (levels, runs) = self.visual_runs(para, line.clone());
@@ -637,6 +650,10 @@ mod tests {
                 ],
             }
         );
+
+        /// BidiTest:69635 (AL ET EN)
+        let bidi_info = BidiInfo::new("\u{060B}\u{20CF}\u{06F9}", None);
+        assert_eq!(bidi_info.original_classes, vec![AL, AL, ET, ET, ET, EN, EN]);
     }
 
     #[test]
@@ -754,6 +771,60 @@ mod tests {
             vec!["ef].)gh&[דג(בא"]
         );
     }
+
+    fn reordered_levels_for_paras(text: &str) -> Vec<Vec<Level>> {
+        let bidi_info = BidiInfo::new(text, None);
+        bidi_info
+            .paragraphs
+            .iter()
+            .map(|para| bidi_info.reordered_levels(para, para.range.clone()))
+            .collect()
+    }
+
+    fn reordered_levels_per_char_for_paras(text: &str) -> Vec<Vec<Level>> {
+        let bidi_info = BidiInfo::new(text, None);
+        bidi_info
+            .paragraphs
+            .iter()
+            .map(|para| bidi_info.reordered_levels_per_char(para, para.range.clone()))
+            .collect()
+    }
+
+    #[test]
+    fn test_reordered_levels() {
+
+        /// BidiTest:946 (LRI PDI)
+        let text = "\u{2067}\u{2069}";
+        assert_eq!(
+            reordered_levels_for_paras(text),
+            vec![Level::vec(&[0, 0, 0, 0, 0, 0])]
+        );
+        assert_eq!(
+            reordered_levels_per_char_for_paras(text),
+            vec![Level::vec(&[0, 0])]
+        );
+
+        /* TODO
+        /// BidiTest:69635 (AL ET EN)
+        let text = "\u{060B}\u{20CF}\u{06F9}";
+        assert_eq!(
+            reordered_levels_for_paras(text),
+            vec![Level::vec(&[1, 1, 1, 1, 1, 2, 2])]
+        );
+        assert_eq!(
+            reordered_levels_per_char_for_paras(text),
+            vec![Level::vec(&[1, 1, 2])]
+        );
+         */
+
+        /* TODO
+        // BidiTest:291284 (AN RLI PDF R)
+        assert_eq!(
+            reordered_levels_per_char_for_paras("\u{0605}\u{2067}\u{202C}\u{0590}"),
+            vec![&["2", "0", "x", "1"]]
+        );
+         */
+    }
 }
 
 

tests/conformance_tests.rs

@@ -84,7 +84,8 @@ fn test_basic_conformance() {
 
                 // Check levels
                 let exp_levels: Vec<String> = exp_levels.iter().map(|x| x.to_owned()).collect();
-                let levels = gen_levels_list_from_bidi_info(&input_string, &bidi_info);
+                let para = &bidi_info.paragraphs[0];
+                let levels = bidi_info.reordered_levels_per_char(para, para.range.clone());
                 if levels != exp_levels {
                     fails.push(
                         Fail {
@@ -181,7 +182,8 @@ fn test_character_conformance() {
             let bidi_info = BidiInfo::new(&input_string, input_base_level);
 
             // Check levels
-            let levels = gen_levels_list_from_bidi_info(&input_string, &bidi_info);
+            let para = &bidi_info.paragraphs[0];
+            let levels = bidi_info.reordered_levels_per_char(para, para.range.clone());
             if levels != exp_levels {
                 fails.push(
                     Fail {
@@ -236,22 +238,13 @@ fn gen_base_level_for_characters_tests(idx: usize) -> Option<Level> {
 }
 
 
-/// We need to collaps levels to one-per-character from one-per-byte format.
-fn gen_levels_list_from_bidi_info(input_str: &str, bidi_info: &BidiInfo) -> Vec<Level> {
-    let para = &bidi_info.paragraphs[0];
-    let levels = bidi_info.reordered_levels(para, para.range.clone());
-    // TODO: Move to impl BidiInfo as pub api
-    input_str.char_indices().map(|(i, _)| levels[i]).collect()
-}
-
 fn get_sample_string_from_bidi_classes(class_names: &[&str]) -> String {
     class_names
         .iter()
         .map(|class_name| gen_char_from_bidi_class(class_name))
         .collect()
 }
 
-/// TODO: Auto-gen in tables.rs ?
 fn gen_char_from_bidi_class(class_name: &str) -> char {
     match class_name {
         "AL" => '\u{060B}',