Merge pull request #95 from developer0hye/perf/optimize-xlsx-conversion

perf(xlsx): stream large sheet rendering

Author: developer0hye

Cargo.lock

@@ -31,9 +31,9 @@ async-gemini = ["async", "dep:reqwest"]
 wasm = ["dep:wasm-bindgen", "dep:js-sys", "dep:wasm-bindgen-futures"]
 
 [dependencies]
-zip = { version = "2", default-features = false, features = ["deflate"] }
-quick-xml = "0.37"
-calamine = { version = "0.26", features = ["dates"] }
+zip = { version = "8.2", default-features = false, features = ["deflate"] }
+quick-xml = "0.41"
+calamine = { version = "0.36", features = ["dates"] }
 chrono = { version = "0.4", default-features = false }
 csv = "1"
 serde_json = "1"

Cargo.toml

@@ -18,8 +18,8 @@ A pure Rust tool and library that converts various document formats into Markdow
 |--------|-----------|-------|
 | DOCX | `.docx` | Headings, tables (incl. merged/layout tables), lists, bold/italic, hyperlinks, images, text boxes |
 | PPTX | `.pptx` | Slides, tables (incl. merged cells), speaker notes, images, group shapes |
-| XLSX | `.xlsx` | Multi-sheet, date/time handling, images |
-| XLS | `.xls` | Legacy Excel (via calamine) |
+| XLSX | `.xlsx` | Multi-sheet, date/time handling, images, single-pass table rendering |
+| XLS | `.xls` | Legacy Excel (via calamine), single-pass table rendering |
 | HTML | `.html`, `.htm` | Full DOM: headings, tables (incl. `colspan`/`rowspan`), lists, links, blockquotes, code blocks |
 | CSV | `.csv` | Converted to Markdown tables |
 | Jupyter Notebook | `.ipynb` | Markdown cells preserved, code cells in fenced blocks with language detection |
@@ -33,6 +33,8 @@ A pure Rust tool and library that converts various document formats into Markdow
 
 Format is auto-detected from magic bytes and file extension. ZIP-based formats (DOCX/PPTX/XLSX) are distinguished by inspecting internal archive structure.
 
+Large XLSX/XLS sheets render Markdown and plain text together in one row pass. Workbook parsing still uses memory proportional to the worksheet, but conversion does not build a second full cell matrix before producing output.
+
 ## Conversion Examples
 
 ### CSV

README.md

@@ -15,8 +15,8 @@ use zip::ZipArchive;
 
 use crate::converter::comments::{self, Comment};
 use crate::converter::ooxml_utils::{
-    ImageInfo, PendingImageResolution, Relationship, parse_relationships,
-    resolve_image_placeholders, resolve_relative_to_file,
+    ImageInfo, PendingImageResolution, Relationship, general_ref_value, parse_relationships,
+    resolve_image_placeholders, resolve_relative_to_file, text_value_unescaped,
 };
 use crate::converter::{
     ConversionOptions, ConversionResult, ConversionWarning, Converter, WarningCode,
@@ -1150,7 +1150,7 @@ fn parse_document(
                     continue;
                 }
                 if in_text && in_run {
-                    let text = e.unescape().unwrap_or_default().to_string();
+                    let text = text_value_unescaped(e);
                     let seg = RunSegment {
                         text,
                         bold: current_run_bold,
@@ -1165,6 +1165,20 @@ fn parse_document(
                     }
                 }
             }
+            Ok(Event::GeneralRef(ref e)) if !in_mc_choice && in_text && in_run => {
+                let seg = RunSegment {
+                    text: general_ref_value(e),
+                    bold: current_run_bold,
+                    italic: current_run_italic,
+                };
+                if in_hyperlink {
+                    hyperlink_runs.push(seg.clone());
+                    hyperlink_runs_plain.push(seg);
+                } else {
+                    current_para_runs.push(seg.clone());
+                    current_para_runs_plain.push(seg);
+                }
+            }
             Ok(Event::End(ref e)) => {
                 let local = e.local_name();
                 let local_str = std::str::from_utf8(local.as_ref()).unwrap_or("");
@@ -1737,7 +1751,10 @@ fn parse_comments_xml(xml: &str) -> HashMap<String, RawComment> {
                 }
             }
             Ok(Event::Text(ref e)) if in_text => {
-                cur_body.push_str(&e.unescape().unwrap_or_default());
+                cur_body.push_str(&text_value_unescaped(e));
+            }
+            Ok(Event::GeneralRef(ref e)) if in_text => {
+                cur_body.push_str(&general_ref_value(e));
             }
             Ok(Event::End(ref e)) => {
                 let local = e.local_name();
@@ -1933,7 +1950,13 @@ fn collect_ranges_in_part(xml: &str) -> (Vec<String>, HashMap<String, String>) {
                 }
             }
             Ok(Event::Text(ref e)) if in_text && in_run && !open.is_empty() => {
-                let t = e.unescape().unwrap_or_default();
+                let t = text_value_unescaped(e);
+                for id in &open {
+                    push_capped(text.entry(id.clone()).or_default(), &t);
+                }
+            }
+            Ok(Event::GeneralRef(ref e)) if in_text && in_run && !open.is_empty() => {
+                let t = general_ref_value(e);
                 for id in &open {
                     push_capped(text.entry(id.clone()).or_default(), &t);
                 }
@@ -2435,6 +2458,18 @@ mod tests {
         assert_eq!(result.markdown.trim(), "Hello, world!");
     }
 
+    #[test]
+    fn test_docx_text_entity_references_preserved() {
+        let doc = wrap_body(&para("R&amp;D &lt;ready&gt; &#x1F680;"));
+        let data = build_test_docx(&doc, None, None);
+        let result = DocxConverter
+            .convert(&data, &ConversionOptions::default())
+            .unwrap();
+
+        assert_eq!(result.markdown.trim(), "R&D <ready> 🚀");
+        assert_eq!(result.plain_text.trim(), "R&D <ready> 🚀");
+    }
+
     #[test]
     fn test_docx_multiple_paragraphs() {
         let body = format!("{}{}", para("First paragraph."), para("Second paragraph."));