scanner: accept octal and multi-byte escapes in rune literals (fix #28880)

vlib/v/scanner/scanner.v

@@ -217,13 +217,12 @@ fn (mut s Scanner) validate_char_literal(start int, content_start int, content_e
 			if s.diagnostics.len > before_errors {
 				return
 			}
-			extra := match escape {
-				`x` { 2 }
-				`u` { 4 }
-				`U` { 8 }
-				else { 0 }
+			offset = s.char_literal_escape_end(offset, content_end)
+			// A character can be spelled as its UTF-8 bytes (`\xe2\x98\x85`, `\342\230\205`),
+			// the same as in a string, so those escapes together are still one character.
+			if lead := s.char_literal_escape_byte(part_start, offset) {
+				offset = s.char_literal_utf8_escapes_end(lead, offset, content_end)
 			}
-			offset = int_min(content_end, offset + 2 + extra)
 		} else {
 			offset = int_min(content_end, offset + utf8_char_len(s.src[offset]))
 		}
@@ -235,6 +234,113 @@ fn (mut s Scanner) validate_char_literal(start int, content_start int, content_e
 	}
 }
 
+// char_literal_escape_end returns where the escape that starts at the backslash at
+// `offset` ends. An octal escape is exactly three octal digits, which is how string
+// literals decode it; without three, the backslash escapes only the next character.
+fn (s &Scanner) char_literal_escape_end(offset int, content_end int) int {
+	extra := match s.src[offset + 1] {
+		`x` {
+			2
+		}
+		`u` {
+			4
+		}
+		`U` {
+			8
+		}
+		else {
+			if s.char_literal_octal_escape_at(offset, content_end) { 2 } else { 0 }
+		}
+	}
+	return int_min(content_end, offset + 2 + extra)
+}
+
+fn (s &Scanner) char_literal_octal_escape_at(offset int, content_end int) bool {
+	if offset + 3 >= content_end {
+		return false
+	}
+	for i in 1 .. 4 {
+		c := s.src[offset + i]
+		if c < `0` || c > `7` {
+			return false
+		}
+	}
+	return true
+}
+
+// char_literal_escape_byte returns the byte that a `\xHH` or three-digit octal escape
+// spanning `start` to `end` stands for.
+fn (s &Scanner) char_literal_escape_byte(start int, end int) ?u8 {
+	if end - start != 4 {
+		return none
+	}
+	escape := s.src[start + 1]
+	mut value := 0
+	if escape == `x` {
+		for i in start + 2 .. end {
+			if !s.src[i].is_hex_digit() {
+				return none
+			}
+			value = value * 16 + int(string_escape_hex_value(s.src[i]))
+		}
+	} else if escape >= `0` && escape <= `7` {
+		for i in start + 1 .. end {
+			if s.src[i] < `0` || s.src[i] > `7` {
+				return none
+			}
+			value = value * 8 + int(s.src[i] - `0`)
+		}
+	} else {
+		return none
+	}
+	if value > 0xff {
+		return none
+	}
+	return u8(value)
+}
+
+// char_literal_utf8_escapes_end extends a byte escape holding the UTF-8 lead byte `lead`
+// over the continuation byte escapes that complete its character, and returns where
+// they end. The bytes have to be one well-formed UTF-8 sequence: if they are missing, are
+// not continuation bytes, or spell an overlong form, a surrogate or a code point above
+// U+10FFFF, the escape stays on its own.
+fn (s &Scanner) char_literal_utf8_escapes_end(lead u8, offset int, content_end int) int {
+	continuation_bytes := if lead >= 0xc2 && lead <= 0xdf {
+		1
+	} else if lead >= 0xe0 && lead <= 0xef {
+		2
+	} else if lead >= 0xf0 && lead <= 0xf4 {
+		3
+	} else {
+		0
+	}
+	if continuation_bytes == 0 {
+		return offset
+	}
+	mut code_point := u32(lead) & (u32(0x7f) >> (continuation_bytes + 1))
+	mut end := offset
+	for _ in 0 .. continuation_bytes {
+		if end + 1 >= content_end || s.src[end] != `\\` {
+			return offset
+		}
+		next_end := s.char_literal_escape_end(end, content_end)
+		b := s.char_literal_escape_byte(end, next_end) or { return offset }
+		if b < 0x80 || b > 0xbf {
+			return offset
+		}
+		code_point = (code_point << 6) | u32(b & 0x3f)
+		end = next_end
+	}
+	// The same range `check_string_escape` enforces for `\u`/`\U`, plus the smallest code
+	// point each length may encode, which rules out overlong forms.
+	shortest_form_minimum := [u32(0x80), 0x800, 0x10000][continuation_bytes - 1]
+	if code_point < shortest_form_minimum || code_point > 0x10ffff
+		|| (code_point >= 0xd800 && code_point <= 0xdfff) {
+		return offset
+	}
+	return end
+}
+
 // current_file returns current file data for Scanner.
 pub fn (s &Scanner) current_file() &token.File {
 	return unsafe { s.file }

vlib/v/scanner/scanner_test.v

@@ -1,5 +1,6 @@
 module scanner
 
+import encoding.utf8
 import v.pref
 import v.token
 
@@ -170,3 +171,105 @@ fn test_keyword_enum_selector_inserts_semicolon() {
 		assert scanner.scan() == expected
 	}
 }
+
+fn char_literal_diagnostics(source string) []string {
+	mut files := token.FileSet.new()
+	mut file := files.add_file('char_literal.v', source.len)
+	file.index_lines(source)
+	preferences := &pref.Preferences{}
+	mut scanner := new_scanner(preferences, .normal)
+	scanner.init(file, source)
+	for scanner.scan() != .eof {
+	}
+	return scanner.diagnostics.map(it.message)
+}
+
+// A character literal holds one character however it is spelled: a three-digit octal
+// escape is one byte, and the byte escapes of one UTF-8 sequence are one character, the
+// same as in a string.
+fn test_char_literal_escapes_that_spell_one_character_are_accepted() {
+	for source in [
+		r'`\141`',
+		r'`\x61`',
+		r'`\u0061`',
+		r'`\U0001F680`',
+		r'`\0`',
+		r'`\xc3\xa9`',
+		r'`\xe2\x98\x85`',
+		r'`\342\230\205`',
+		r'`\342\x98\205`',
+		r'`\xf0\x9f\x9a\x80`',
+	] {
+		assert char_literal_diagnostics(source) == [], source
+	}
+}
+
+fn test_char_literal_with_more_than_one_character_is_still_rejected() {
+	for source in [
+		r'`\141b`',
+		r'`\x61\x62`',
+		// A lead byte whose continuation bytes are missing, invalid, or overlong is not
+		// one character.
+		r'`\xe2\x98`',
+		r'`\xe2\x98\x41`',
+		r'`\xc0\x80`',
+		r'`\xc3\xa9\xa9`',
+	] {
+		diagnostics := char_literal_diagnostics(source)
+		assert diagnostics.len == 1, source
+		assert diagnostics[0].ends_with('(more than one character)'), '${source}: ${diagnostics[0]}'
+	}
+}
+
+fn utf8_sequence_len(lead u8) int {
+	return if lead >= 0xf0 {
+		4
+	} else if lead >= 0xe0 {
+		3
+	} else {
+		2
+	}
+}
+
+// Byte escapes are one character exactly when they spell one well-formed UTF-8 sequence,
+// which is decided here by `encoding.utf8`, independently of the scanner. The first
+// continuation byte is where the lead-specific limits live (overlong forms after `E0` and
+// `F0`, surrogates after `ED`, code points above U+10FFFF after `F4`), so it walks every
+// boundary of those ranges for every lead byte, together with a valid and an invalid final
+// continuation byte, in both hex and octal spelling.
+fn test_char_literal_byte_escapes_are_one_character_only_when_well_formed_utf8() {
+	second_bytes := [u8(0x7f), 0x80, 0x8f, 0x90, 0x9f, 0xa0, 0xbf, 0xc0]
+	last_bytes := [u8(0x7f), 0x80, 0xbf, 0xc0]
+	mut sequences := [][]u8{}
+	for lead in u8(0xc0) .. u8(0xf8) {
+		for second in second_bytes {
+			if utf8_sequence_len(lead) == 2 {
+				sequences << [lead, second]
+				continue
+			}
+			for last in last_bytes {
+				mut bytes := [lead, second]
+				for bytes.len < utf8_sequence_len(lead) - 1 {
+					bytes << u8(0x80)
+				}
+				bytes << last
+				sequences << bytes
+			}
+		}
+	}
+	for bytes in sequences {
+		well_formed := utf8.validate_str(bytes.bytestr())
+		hex := bytes.map('\\x${it.hex()}').join('')
+		octal := bytes.map('\\${it:o}').join('')
+		for spelling in [hex, octal] {
+			source := '`${spelling}`'
+			diagnostics := char_literal_diagnostics(source)
+			if well_formed {
+				assert diagnostics == [], source
+			} else {
+				assert diagnostics.len == 1, source
+				assert diagnostics[0].ends_with('(more than one character)'), '${source}: ${diagnostics[0]}'
+			}
+		}
+	}
+}