scanner: accept octal and multi-byte escapes in rune literals (fix #28880) (#28882)

* scanner: accept octal and multi-byte escapes in rune literals (fix #28880)

`validate_char_literal` (added in 3b7b5ee) splits a rune literal into
characters to check it holds only one, but it gave every escape other
than `\x`, `\u` and `\U` a single character after the backslash, and
counted every byte escape as a character of its own. So `\141` was
three characters and `\xe2\x98\x85` / `\342\230\205` were three and
nine, although each is one character and strings accept all of them.
The rune example in doc/docs.md, which documents both forms, stopped
compiling, which is one of the two `check-markdown` failures on master.

An octal escape is now three octal digits, as the parser's string
decoder reads it, and a byte escape holding a UTF-8 lead byte takes the
continuation byte escapes that complete its character. Incomplete,
invalid and overlong sequences are still rejected.

Code generation needed no change: with the validator fixed, the whole
docs example runs and passes under tcc and gcc.

Co-Authored-By: WOZCODE <contact@withwoz.com>

* scanner: accept only well-formed UTF-8 byte escapes as one rune

The previous commit merged a UTF-8 lead byte escape with the escapes
after it whenever each continuation byte was in 0x80..0xbf. That is not
enough: the byte after E0, ED, F0 and F4 has a narrower range, so these
were all accepted as one character (found by JalonSolov on #28882):

    `\xe0\x80\x80`       overlong
    `\xf0\x80\x80\x80`   overlong
    `\xed\xa0\x80`       surrogate
    `\xf4\x90\x80\x80`   above U+10FFFF

The commit message of that commit and the PR description said overlong
and invalid sequences stayed rejected; that only held for the C0/C1
leads.

Decode the code point and apply the range `check_string_escape` already
enforces for `\u`/`\U` (at most U+10FFFF, no surrogates), plus the
smallest code point each length may encode, which rules out overlong
forms. With leads limited to C2..F4 and continuation bytes to 80..BF,
that is exactly the well-formed set of Unicode Table 3-7.

The new test checks the rule rather than examples: every lead byte
C0..F7, every boundary of the second-byte ranges, and a valid and an
invalid final byte, in hex and octal spelling, against
`encoding.utf8.validate_str`. It fails on the previous commit, and
removing any one of the four checks (shortest form, surrogate, maximum,
continuation byte) makes it fail on that class.

Also restore the `\u0061` case in the accept-list test: it had been
written into the file as a plain `a`, so `\u` was never tested.

Co-Authored-By: WOZCODE <contact@withwoz.com>

---------

Co-authored-by: Richard Wheeler <18647491+PythonWillRule@users.noreply.github.com>
Co-authored-by: WOZCODE <contact@withwoz.com>

Author: 3 people

vlib/v/scanner/scanner.v

@@ -217,13 +217,12 @@ fn (mut s Scanner) validate_char_literal(start int, content_start int, content_e
 			if s.diagnostics.len > before_errors {
 				return
 			}
-			extra := match escape {
-				`x` { 2 }
-				`u` { 4 }
-				`U` { 8 }
-				else { 0 }
+			offset = s.char_literal_escape_end(offset, content_end)
+			// A character can be spelled as its UTF-8 bytes (`\xe2\x98\x85`, `\342\230\205`),
+			// the same as in a string, so those escapes together are still one character.
+			if lead := s.char_literal_escape_byte(part_start, offset) {
+				offset = s.char_literal_utf8_escapes_end(lead, offset, content_end)
 			}
-			offset = int_min(content_end, offset + 2 + extra)
 		} else {
 			offset = int_min(content_end, offset + utf8_char_len(s.src[offset]))
 		}
@@ -235,6 +234,113 @@ fn (mut s Scanner) validate_char_literal(start int, content_start int, content_e
 	}
 }
 
+// char_literal_escape_end returns where the escape that starts at the backslash at
+// `offset` ends. An octal escape is exactly three octal digits, which is how string
+// literals decode it; without three, the backslash escapes only the next character.
+fn (s &Scanner) char_literal_escape_end(offset int, content_end int) int {
+	extra := match s.src[offset + 1] {
+		`x` {
+			2
+		}
+		`u` {
+			4
+		}
+		`U` {
+			8
+		}
+		else {
+			if s.char_literal_octal_escape_at(offset, content_end) { 2 } else { 0 }
+		}
+	}
+	return int_min(content_end, offset + 2 + extra)
+}
+
+fn (s &Scanner) char_literal_octal_escape_at(offset int, content_end int) bool {
+	if offset + 3 >= content_end {
+		return false
+	}
+	for i in 1 .. 4 {
+		c := s.src[offset + i]
+		if c < `0` || c > `7` {
+			return false
+		}
+	}
+	return true
+}
+
+// char_literal_escape_byte returns the byte that a `\xHH` or three-digit octal escape
+// spanning `start` to `end` stands for.
+fn (s &Scanner) char_literal_escape_byte(start int, end int) ?u8 {
+	if end - start != 4 {
+		return none
+	}
+	escape := s.src[start + 1]
+	mut value := 0
+	if escape == `x` {
+		for i in start + 2 .. end {
+			if !s.src[i].is_hex_digit() {
+				return none
+			}
+			value = value * 16 + int(string_escape_hex_value(s.src[i]))
+		}
+	} else if escape >= `0` && escape <= `7` {
+		for i in start + 1 .. end {
+			if s.src[i] < `0` || s.src[i] > `7` {
+				return none
+			}
+			value = value * 8 + int(s.src[i] - `0`)
+		}
+	} else {
+		return none
+	}
+	if value > 0xff {
+		return none
+	}
+	return u8(value)
+}
+
+// char_literal_utf8_escapes_end extends a byte escape holding the UTF-8 lead byte `lead`
+// over the continuation byte escapes that complete its character, and returns where
+// they end. The bytes have to be one well-formed UTF-8 sequence: if they are missing, are
+// not continuation bytes, or spell an overlong form, a surrogate or a code point above
+// U+10FFFF, the escape stays on its own.
+fn (s &Scanner) char_literal_utf8_escapes_end(lead u8, offset int, content_end int) int {
+	continuation_bytes := if lead >= 0xc2 && lead <= 0xdf {
+		1
+	} else if lead >= 0xe0 && lead <= 0xef {
+		2
+	} else if lead >= 0xf0 && lead <= 0xf4 {
+		3
+	} else {
+		0
+	}
+	if continuation_bytes == 0 {
+		return offset
+	}
+	mut code_point := u32(lead) & (u32(0x7f) >> (continuation_bytes + 1))
+	mut end := offset
+	for _ in 0 .. continuation_bytes {
+		if end + 1 >= content_end || s.src[end] != `\\` {
+			return offset
+		}
+		next_end := s.char_literal_escape_end(end, content_end)
+		b := s.char_literal_escape_byte(end, next_end) or { return offset }
+		if b < 0x80 || b > 0xbf {
+			return offset
+		}
+		code_point = (code_point << 6) | u32(b & 0x3f)
+		end = next_end
+	}
+	// The same range `check_string_escape` enforces for `\u`/`\U`, plus the smallest code
+	// point each length may encode, which rules out overlong forms.
+	shortest_form_minimum := [u32(0x80), 0x800, 0x10000][continuation_bytes - 1]
+	if code_point < shortest_form_minimum || code_point > 0x10ffff
+		|| (code_point >= 0xd800 && code_point <= 0xdfff) {
+		return offset
+	}
+	return end
+}
+
 // current_file returns current file data for Scanner.
 pub fn (s &Scanner) current_file() &token.File {
 	return unsafe { s.file }

vlib/v/scanner/scanner_test.v

@@ -1,5 +1,6 @@
 module scanner
 
+import encoding.utf8
 import v.pref
 import v.token
 
@@ -170,3 +171,105 @@ fn test_keyword_enum_selector_inserts_semicolon() {
 		assert scanner.scan() == expected
 	}
 }
+
+fn char_literal_diagnostics(source string) []string {
+	mut files := token.FileSet.new()
+	mut file := files.add_file('char_literal.v', source.len)
+	file.index_lines(source)
+	preferences := &pref.Preferences{}
+	mut scanner := new_scanner(preferences, .normal)
+	scanner.init(file, source)
+	for scanner.scan() != .eof {
+	}
+	return scanner.diagnostics.map(it.message)
+}
+
+// A character literal holds one character however it is spelled: a three-digit octal
+// escape is one byte, and the byte escapes of one UTF-8 sequence are one character, the
+// same as in a string.
+fn test_char_literal_escapes_that_spell_one_character_are_accepted() {
+	for source in [
+		r'`\141`',
+		r'`\x61`',
+		r'`\u0061`',
+		r'`\U0001F680`',
+		r'`\0`',
+		r'`\xc3\xa9`',
+		r'`\xe2\x98\x85`',
+		r'`\342\230\205`',
+		r'`\342\x98\205`',
+		r'`\xf0\x9f\x9a\x80`',
+	] {
+		assert char_literal_diagnostics(source) == [], source
+	}
+}
+
+fn test_char_literal_with_more_than_one_character_is_still_rejected() {
+	for source in [
+		r'`\141b`',
+		r'`\x61\x62`',
+		// A lead byte whose continuation bytes are missing, invalid, or overlong is not
+		// one character.
+		r'`\xe2\x98`',
+		r'`\xe2\x98\x41`',
+		r'`\xc0\x80`',
+		r'`\xc3\xa9\xa9`',
+	] {
+		diagnostics := char_literal_diagnostics(source)
+		assert diagnostics.len == 1, source
+		assert diagnostics[0].ends_with('(more than one character)'), '${source}: ${diagnostics[0]}'
+	}
+}
+
+fn utf8_sequence_len(lead u8) int {
+	return if lead >= 0xf0 {
+		4
+	} else if lead >= 0xe0 {
+		3
+	} else {
+		2
+	}
+}
+
+// Byte escapes are one character exactly when they spell one well-formed UTF-8 sequence,
+// which is decided here by `encoding.utf8`, independently of the scanner. The first
+// continuation byte is where the lead-specific limits live (overlong forms after `E0` and
+// `F0`, surrogates after `ED`, code points above U+10FFFF after `F4`), so it walks every
+// boundary of those ranges for every lead byte, together with a valid and an invalid final
+// continuation byte, in both hex and octal spelling.
+fn test_char_literal_byte_escapes_are_one_character_only_when_well_formed_utf8() {
+	second_bytes := [u8(0x7f), 0x80, 0x8f, 0x90, 0x9f, 0xa0, 0xbf, 0xc0]
+	last_bytes := [u8(0x7f), 0x80, 0xbf, 0xc0]
+	mut sequences := [][]u8{}
+	for lead in u8(0xc0) .. u8(0xf8) {
+		for second in second_bytes {
+			if utf8_sequence_len(lead) == 2 {
+				sequences << [lead, second]
+				continue
+			}
+			for last in last_bytes {
+				mut bytes := [lead, second]
+				for bytes.len < utf8_sequence_len(lead) - 1 {
+					bytes << u8(0x80)
+				}
+				bytes << last
+				sequences << bytes
+			}
+		}
+	}
+	for bytes in sequences {
+		well_formed := utf8.validate_str(bytes.bytestr())
+		hex := bytes.map('\\x${it.hex()}').join('')
+		octal := bytes.map('\\${it:o}').join('')
+		for spelling in [hex, octal] {
+			source := '`${spelling}`'
+			diagnostics := char_literal_diagnostics(source)
+			if well_formed {
+				assert diagnostics == [], source
+			} else {
+				assert diagnostics.len == 1, source
+				assert diagnostics[0].ends_with('(more than one character)'), '${source}: ${diagnostics[0]}'
+			}
+		}
+	}
+}