feat: add |||- chomped text block syntax (#1175)

* feat: add |||- chomped text block syntax

Resolves #289

* revert MathJax change

The optable in the MathJax code is not related to Jsonnet's syntax, and
should not have `|||-` added.

---------

Co-authored-by: John Bartholomew <[email protected]>

Author: vergenzt

core/formatter.cpp

@@ -500,7 +500,11 @@ class Unparser {
                 o << encode_utf8(ast->value);
                 o << "'";
             } else if (ast->tokenKind == LiteralString::BLOCK) {
-                o << "|||\n";
+                o << "|||";
+                if (ast->value.back() != U'\n') {
+                    o << "-";
+                }
+                o << "\n";
                 if (ast->value.c_str()[0] != U'\n')
                     o << ast->blockIndent;
                 for (const char32_t *cp = ast->value.c_str(); *cp != U'\0'; ++cp) {
@@ -513,6 +517,9 @@ class Unparser {
                         o << ast->blockIndent;
                     }
                 }
+                if (ast->value.back() != U'\n') {
+                    o << "\n";
+                }
                 o << ast->blockTermIndent << "|||";
             } else if (ast->tokenKind == LiteralString::VERBATIM_DOUBLE) {
                 o << "@\"";

core/lexer.cpp

@@ -704,6 +704,13 @@ Tokens jsonnet_lex(const std::string &filename, const char *input)
                     // Text block
                     if (*c == '|' && *(c + 1) == '|' && *(c + 2) == '|') {
                         c += 3;  // Skip the "|||".
+
+                        bool chomp_trailing_nl = false;
+                        if (*c == '-') {
+                            chomp_trailing_nl = true;
+                            c++;
+                        }
+
                         while (is_horz_ws(*c)) ++c;  // Chomp whitespace at end of line.
                         if (*c != '\n') {
                             auto msg = "text block syntax requires new line after |||.";
@@ -762,6 +769,10 @@ Tokens jsonnet_lex(const std::string &filename, const char *input)
                                 c += 3;  // Leave after the last |
                                 data = block.str();
                                 kind = Token::STRING_BLOCK;
+                                if (chomp_trailing_nl) {
+                                    assert(data.back() == '\n');
+                                    data.pop_back();
+                                }
                                 break;  // Out of the while loop.
                             }
                         }

doc/js/codemirror-mode-jsonnet.js

@@ -176,7 +176,7 @@
         }
 
         // Enter text block.
-        if (stream.match(/\|\|\|/)) {
+        if (stream.match(/\|\|\|-?/)) {
           state.textBlock = true;
           state.textBlockIndent = null;
           return "string";

doc/ref/spec.html

@@ -168,15 +168,19 @@ <h2 id="lexing">Lexing</h2>
               subsequent non-quoted <code>'</code>
             </li>
             <li>
-              Text block, beginning with <code>|||</code>, followed by optional whitespace and a
-              new-line.  The next non-empty line must be prefixed with some non-zero length
-              whitespace <i>W</i>.  The block ends at the first subsequent line that is non-empty
-              and does not begin with <i>W</i>, and it is an error if this line does not contain
-              some optional whitespace followed by <code>|||</code>.  The content of the string is
-              the concatenation of all the lines between the two <code>|||</code>, which either
-              begin with <i>W</i> (in which case that prefix is stripped) or they are empty lines
-              (in which case they remain as empty lines).  The line ending style in the file is
-              preserved in the string.  This form cannot be used in <code>import</code> statements.
+              Text block, beginning with <code>|||</code> followed by an optional
+              <code>-</code>, then optional whitespace and a new-line.  The next non-empty
+              line must be prefixed with some non-zero length whitespace <i>W</i>.  The
+              block ends at the first subsequent line that is non-empty and does not begin
+              with <i>W</i>, and it is an error if this line does not contain some
+              optional whitespace followed by <code>|||</code>.  The content of the string
+              is the concatenation of all the lines between the two <code>|||</code>,
+              which either begin with <i>W</i> (in which case that prefix is stripped) or
+              they are empty lines (in which case they remain as empty lines).  The line
+              ending style in the file is preserved in the string.  If the beginning
+              <code>|||</code> was followed by <code>-</code> then the final new-line is
+              stripped from the resulting string.  This form cannot be used in
+              <code>import</code> statements.
             </li>
           </ul>
           <p>

test_suite/text_block.jsonnet

@@ -58,6 +58,33 @@ local bash_mixed = |||
 std.assertEqual(bash_golden, bash_mixed) &&
 
 
+// Chomp trailing newline
+local str1 = |||-
+  foo bar baz
+|||;
+
+std.assertEqual(str1, "foo bar baz") &&
+
+
+// Chomp just one trailing newline
+local str1 = |||-
+  foo bar baz
+
+|||;
+
+std.assertEqual(str1, "foo bar baz\n") &&
+
+
+// Concatenate chomped blocks
+local str1 = |||-
+    foo bar baz
+||| + " " + |||-
+    biz buzz
+|||;
+
+std.assertEqual(str1, "foo bar baz biz buzz") &&
+
+
 // More indent
 local str1 = |||
         text

test_suite/text_block.jsonnet.fmt.golden

@@ -58,6 +58,32 @@ local bash_mixed = |||
 std.assertEqual(bash_golden, bash_mixed) &&
 
 
+// Chomp trailing newline
+local str1 = |||-
+  foo bar baz
+|||;
+
+std.assertEqual(str1, 'foo bar baz') &&
+
+
+// Chomp just one trailing newline
+local str1 = |||
+  foo bar baz
+|||;
+
+std.assertEqual(str1, 'foo bar baz\n') &&
+
+
+// Concatenate chomped blocks
+local str1 = |||-
+  foo bar baz
+||| + ' ' + |||-
+  biz buzz
+|||;
+
+std.assertEqual(str1, 'foo bar baz biz buzz') &&
+
+
 // More indent
 local str1 = |||
   text