JS: Avoid unnecessary stringification

Instead, allow the user to pass in a writer and write the tokens
incrementally. Or they can still stringify as before if they want.

Also remove some unneeded `#[inline]`.

Author: GuillaumeGomez

src/js/token.rs

@@ -1006,8 +1006,17 @@ pub fn tokenize(source: &str) -> Tokens<'_> {
 #[derive(Debug, PartialEq)]
 pub struct Tokens<'a>(pub Vec<Token<'a>>);
 
+impl<'a> Tokens<'a> {
+    pub(super) fn write<W: std::io::Write>(self, mut w: W) -> std::io::Result<()> {
+        for token in self.0.iter() {
+            write!(w, "{}", token)?;
+        }
+        Ok(())
+    }
+}
+
 impl<'a> fmt::Display for Tokens<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let tokens = &self.0;
         for i in 0..tokens.len() {
             if i > 0
@@ -1096,7 +1105,7 @@ impl<'a> From<&[Token<'a>]> for Tokens<'a> {
 fn check_regex() {
     let source = r#"var x = /"\.x/g;"#;
     let expected_result = r#"var x=/"\.x/g"#;
-    assert_eq!(::js::minify(source), expected_result);
+    assert_eq!(::js::minify(source).to_string(), expected_result);
 
     let v = tokenize(source).apply(::js::clean_tokens);
     assert_eq!(
@@ -1110,7 +1119,7 @@ fn check_regex() {
 
     let source = r#"var x = /"\.x/gigigigig;var x = "hello";"#;
     let expected_result = r#"var x=/"\.x/gi;var x="hello""#;
-    assert_eq!(::js::minify(source), expected_result);
+    assert_eq!(::js::minify(source).to_string(), expected_result);
 
     let v = tokenize(source).apply(::js::clean_tokens);
     assert_eq!(
@@ -1127,7 +1136,7 @@ fn check_regex() {
 fn more_regex() {
     let source = r#"var x = /"\.x\/a/i;"#;
     let expected_result = r#"var x=/"\.x\/a/i"#;
-    assert_eq!(::js::minify(source), expected_result);
+    assert_eq!(::js::minify(source).to_string(), expected_result);
 
     let v = tokenize(source).apply(::js::clean_tokens);
     assert_eq!(
@@ -1141,7 +1150,7 @@ fn more_regex() {
 
     let source = r#"var x = /\\/i;"#;
     let expected_result = r#"var x=/\\/i"#;
-    assert_eq!(::js::minify(source), expected_result);
+    assert_eq!(::js::minify(source).to_string(), expected_result);
 
     let v = tokenize(source).apply(::js::clean_tokens);
     assert_eq!(

src/js/tools.rs

@@ -1,5 +1,7 @@
 // Take a look at the license at the top of the repository in the LICENSE file.
 
+use std::{fmt, io};
+
 use js::token::{self, Keyword, ReservedChar, Token, Tokens};
 use js::utils::{get_array, get_variable_name_and_value_positions, VariableNameGenerator};
 
@@ -178,14 +180,26 @@ fn build_ast<'a>(v: &[token::Token<'a>]) -> Result<Elem<'a>, String> {
 ///                func(data[i]);
 ///            }
 ///         }"#.into();
-///     let js_minified = minify(js);
+///     let js_minified = minify(js).to_string();
 /// }
 /// ```
 #[inline]
-pub fn minify(source: &str) -> String {
-    token::tokenize(source)
-        .apply(::js::clean_tokens)
-        .to_string()
+pub fn minify<'a>(source: &'a str) -> Minified<'a> {
+    Minified(token::tokenize(source).apply(::js::clean_tokens))
+}
+
+pub struct Minified<'a>(token::Tokens<'a>);
+
+impl<'a> Minified<'a> {
+    pub fn write<W: io::Write>(self, w: W) -> io::Result<()> {
+        self.0.write(w)
+    }
+}
+
+impl<'a> fmt::Display for Minified<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
 }
 
 // TODO: No scope handling or anything. Might be nice as a second step to add it...
@@ -219,7 +233,6 @@ fn get_variables_name<'a>(
     (ret, variables)
 }
 
-#[inline]
 fn aggregate_strings_inner<'a, 'b: 'a>(
     mut tokens: Tokens<'a>,
     separation_token: Option<Token<'b>>,
@@ -386,7 +399,6 @@ pub fn aggregate_strings_with_separation<'a, 'b: 'a>(
     aggregate_strings_inner(tokens, Some(separation_token))
 }
 
-#[inline]
 fn aggregate_strings_into_array_inner<'a, 'b: 'a, T: Fn(&Tokens<'a>, usize) -> bool>(
     mut tokens: Tokens<'a>,
     array_name: &str,
@@ -906,7 +918,7 @@ fn name_generator() {
 fn simple_quote() {
     let source = r#"var x = "\\";"#;
     let expected_result = r#"var x="\\""#;
-    assert_eq!(minify(source), expected_result);
+    assert_eq!(minify(source).to_string(), expected_result);
 }
 
 #[test]
@@ -942,7 +954,7 @@ far_away(another_var, 12);
     let expected_result = "var foo=\"something\";var another_var=2348323;function far_away(x,y){\
                            var x2=x+4;return x*x2+y}far_away(another_var,12);far_away(another_var,\
                            12)";
-    assert_eq!(minify(source), expected_result);
+    assert_eq!(minify(source).to_string(), expected_result);
 }
 
 #[test]
@@ -975,7 +987,7 @@ console.log('done!');
  *
  * right?
  */function forEach(data,func){for(var i=0;i<data.length;++i){func(data[i])}}forEach([0,1,2,3,4,5,6,7,8,9],function(x){console.log(x)});console.log('done!')"#;
-    assert_eq!(minify(source), expected_result);
+    assert_eq!(minify(source).to_string(), expected_result);
 }
 
 #[test]
@@ -992,7 +1004,7 @@ search_input.onchange = function(e) {
 "#;
     let expected_result = "search_input.onchange=function(e){clearTimeout(searchTimeout);\
                            setTimeout(search,0)}";
-    assert_eq!(minify(source), expected_result);
+    assert_eq!(minify(source).to_string(), expected_result);
 }
 
 #[test]
@@ -1005,7 +1017,7 @@ for (var entry in results) {
 }"#;
     let expected_result = "for(var entry in results){if(results.hasOwnProperty(entry)){\
                            ar.push(results[entry])}}";
-    assert_eq!(minify(source), expected_result);
+    assert_eq!(minify(source).to_string(), expected_result);
 }
 
 #[test]
@@ -1015,37 +1027,37 @@ val = val.replace(/\_/g, "");
 
 var valGenerics = extractGenerics(val);"#;
     let expected_result = "val=val.replace(/\\_/g,\"\");var valGenerics=extractGenerics(val)";
-    assert_eq!(minify(source), expected_result);
+    assert_eq!(minify(source).to_string(), expected_result);
 }
 
 #[test]
 fn keep_space() {
     let source = "return 12;return x;";
 
     let expected_result = "return 12;return x";
-    assert_eq!(minify(source), expected_result);
-
-    assert_eq!("t in e", minify("t in e"));
-    assert_eq!("t+1 in e", minify("t + 1 in e"));
-    assert_eq!("t-1 in e", minify("t - 1 in e"));
-    assert_eq!("'a'in e", minify("'a' in e"));
-    assert_eq!("/a/g in e", minify("/a/g in e"));
-    assert_eq!("/a/i in e", minify("/a/i in e"));
-
-    assert_eq!("t instanceof e", minify("t instanceof e"));
-    assert_eq!("t+1 instanceof e", minify("t + 1 instanceof e"));
-    assert_eq!("t-1 instanceof e", minify("t - 1 instanceof e"));
-    assert_eq!("'a'instanceof e", minify("'a' instanceof e"));
-    assert_eq!("/a/g instanceof e", minify("/a/g instanceof e"));
-    assert_eq!("/a/i instanceof e", minify("/a/i instanceof e"));
+    assert_eq!(minify(source).to_string(), expected_result);
+
+    assert_eq!("t in e", minify("t in e").to_string());
+    assert_eq!("t+1 in e", minify("t + 1 in e").to_string());
+    assert_eq!("t-1 in e", minify("t - 1 in e").to_string());
+    assert_eq!("'a'in e", minify("'a' in e").to_string());
+    assert_eq!("/a/g in e", minify("/a/g in e").to_string());
+    assert_eq!("/a/i in e", minify("/a/i in e").to_string());
+
+    assert_eq!("t instanceof e", minify("t instanceof e").to_string());
+    assert_eq!("t+1 instanceof e", minify("t + 1 instanceof e").to_string());
+    assert_eq!("t-1 instanceof e", minify("t - 1 instanceof e").to_string());
+    assert_eq!("'a'instanceof e", minify("'a' instanceof e").to_string());
+    assert_eq!("/a/g instanceof e", minify("/a/g instanceof e").to_string());
+    assert_eq!("/a/i instanceof e", minify("/a/i instanceof e").to_string());
 }
 
 #[test]
 fn test_remove_extra_whitespace_before_typeof() {
     let source = "var x = typeof 'foo';var y = typeof x;case typeof 'foo': 'bla'";
 
     let expected_result = "var x=typeof'foo';var y=typeof x;case typeof'foo':'bla'";
-    assert_eq!(minify(source), expected_result);
+    assert_eq!(minify(source).to_string(), expected_result);
 }
 
 #[test]
@@ -1055,29 +1067,29 @@ if (x in ev && typeof ev) { return true; }
 if (true in ev) { return true; }"#;
 
     let expected_result = r#"if("key"in ev&&typeof ev){return true}if(x in ev&&typeof ev){return true}if(true in ev){return true}"#;
-    assert_eq!(minify(source), expected_result);
+    assert_eq!(minify(source).to_string(), expected_result);
 }
 
 #[test]
 fn test_remove_extra_whitespace_before_operator() {
     let source = "( x ) / 2; x / y;x /= y";
 
     let expected_result = "(x)/2;x/y;x/=y";
-    assert_eq!(minify(source), expected_result);
+    assert_eq!(minify(source).to_string(), expected_result);
 }
 
 #[test]
 fn check_regex_syntax() {
     let source = "console.log(/MSIE|Trident|Edge/.test(window.navigator.userAgent));";
     let expected = "console.log(/MSIE|Trident|Edge/.test(window.navigator.userAgent))";
-    assert_eq!(minify(source), expected);
+    assert_eq!(minify(source).to_string(), expected);
 }
 
 #[test]
 fn minify_minified() {
     let source = "function (i, n, a) { i[n].type.replace(/ *;(.|\\s)*/,\"\")===t&&a.push(i[n].MathJax.elementJax);return a}";
     let expected = "function(i,n,a){i[n].type.replace(/ *;(.|\\s)*/,\"\")===t&&a.push(i[n].MathJax.elementJax);return a}";
-    assert_eq!(minify(source), expected);
+    assert_eq!(minify(source).to_string(), expected);
 }
 
 #[test]
@@ -1090,7 +1102,7 @@ fn check_string() {
     "###;
     let expected = "const a=123;const b=\"123\";const c=`the number is ${a}  <-- note the spaces \
     here`;const d=`      ${a}         ${b}      `";
-    assert_eq!(minify(source), expected);
+    assert_eq!(minify(source).to_string(), expected);
 }
 
 // TODO: requires AST to fix this issue!
@@ -1102,7 +1114,7 @@ console.log(2)
 var x = 12;
 "#;
     let expected_result = r#"console.log(1);console.log(2);var x=12;"#;
-    assert_eq!(minify(source), expected_result);
+    assert_eq!(minify(source).to_string(), expected_result);
 }*/
 
 // TODO: requires AST to fix this issue!
@@ -1115,5 +1127,5 @@ function foo() {
 }
 "#;
     let expected_result = r#"function foo(){return 12;}"#;
-    assert_eq!(minify(source), expected_result);
+    assert_eq!(minify(source).to_string(), expected_result);
 }*/

src/js/utils.rs

@@ -174,7 +174,6 @@ pub fn replace_token_with<'a, 'b: 'a, F: Fn(&Token<'a>) -> Option<Token<'b>>>(
 ///     assert_eq!(result, vec![(2, Some(6)), (10, None), (14, Some(22))]);
 /// }
 /// ```
-#[inline]
 pub fn get_variable_name_and_value_positions<'a>(
     tokens: &'a Tokens<'a>,
     pos: usize,
@@ -275,7 +274,6 @@ fn get_next<'a>(it: &mut IntoIter<Token<'a>>) -> Option<Token<'a>> {
 ///     println!("result: {:?}", s);
 /// }
 /// ```
-#[inline]
 pub fn clean_tokens(tokens: Tokens<'_>) -> Tokens<'_> {
     let mut v = Vec::with_capacity(tokens.len() / 3 * 2);
     let mut it = tokens.0.into_iter();
@@ -307,7 +305,6 @@ pub fn clean_tokens(tokens: Tokens<'_>) -> Tokens<'_> {
 
 /// Returns true if the token is a "useful" one (so not a comment or a "useless"
 /// character).
-#[inline]
 pub fn clean_token(token: &Token<'_>, next_token: &Option<&Token<'_>>) -> bool {
     !token.is_comment() && {
         if let Some(x) = token.get_char() {
@@ -358,7 +355,6 @@ fn get_next_except<'a, F: Fn(&Token<'a>) -> bool>(
 ///     println!("result: {:?}", s);
 /// }
 /// ```
-#[inline]
 pub fn clean_tokens_except<'a, F: Fn(&Token<'a>) -> bool>(tokens: Tokens<'a>, f: F) -> Tokens<'a> {
     let mut v = Vec::with_capacity(tokens.len() / 3 * 2);
     let mut it = tokens.0.into_iter();
@@ -410,7 +406,6 @@ pub fn clean_token_except<'a, F: Fn(&Token<'a>) -> bool>(
     }
 }
 
-#[inline]
 pub(crate) fn get_array<'a>(
     tokens: &'a Tokens<'a>,
     array_name: &str,

src/json/read/byte_to_char.rs

@@ -39,7 +39,6 @@ impl<R: Read + fmt::Debug> fmt::Debug for ByteToChar<R> {
 impl<R: Read> Iterator for ByteToChar<R> {
     type Item = Result<char, CharsError>;
 
-    #[inline]
     fn next(&mut self) -> Option<Result<char, CharsError>> {
         let first_byte = match self.get_next() {
             Err(err) => return Some(Err(err)),

src/main.rs

@@ -85,7 +85,7 @@ fn main() {
             "css" => call_minifier(arg, |s| {
                 css::minify(s).expect("css minification failed").to_string()
             }),
-            "js" => call_minifier(arg, js::minify),
+            "js" => call_minifier(arg, |s| js::minify(s).to_string()),
             "json" => call_minifier(arg, json::minify),
             // "html" | "htm" => call_minifier(arg, html::minify),
             x => println!("\"{}\": this format isn't supported", x),

tests/js_minify.rs

@@ -57,5 +57,5 @@ fn test_minification() {
 
     let source = include_str!("./files/main.js");
     let expected_result = include_str!("./files/minified_main.js");
-    compare_strs(&minify(source), expected_result);
+    compare_strs(&minify(source).to_string(), expected_result);
 }