Skip to content

Commit e4977e7

Browse files
borsbors
committed
Auto merge of #14836 - Veykril:rustc-lexer, r=Veykril
internal: Bump rustc_lexer
2 parents 1c0235e + 099b5b3 commit e4977e7

26 files changed

+134
-73
lines changed

‎Cargo.lock

Lines changed: 53 additions & 11 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,3 +89,5 @@ text-size = "1.1.0"
8989
serde = { version = "=1.0.156", features = ["derive"] }
9090
serde_json = "1.0.94"
9191
triomphe = { version = "0.1.8", default-features = false, features = ["std"] }
92+
93+
rustc_lexer = { version = "0.1.0", package = "ra-ap-rustc_lexer" }

‎crates/parser/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ doctest = false
1313

1414
[dependencies]
1515
drop_bomb = "0.1.5"
16-
rustc_lexer = { version = "727.0.0", package = "rustc-ap-rustc_lexer" }
16+
rustc_lexer.workspace = true
1717

1818
limit.workspace = true
1919

‎crates/parser/src/lexed_str.rs

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ impl<'a> LexedStr<'a> {
3636
};
3737

3838
for token in rustc_lexer::tokenize(&text[conv.offset..]) {
39-
let token_text = &text[conv.offset..][..token.len];
39+
let token_text = &text[conv.offset..][..token.len as usize];
4040

4141
conv.extend_token(&token.kind, token_text);
4242
}
@@ -49,8 +49,8 @@ impl<'a> LexedStr<'a> {
4949
return None;
5050
}
5151

52-
let token = rustc_lexer::first_token(text);
53-
if token.len != text.len() {
52+
let token = rustc_lexer::tokenize(text).next()?;
53+
if token.len as usize != text.len() {
5454
return None;
5555
}
5656

@@ -175,6 +175,10 @@ impl<'a> Converter<'a> {
175175
rustc_lexer::TokenKind::Ident => {
176176
SyntaxKind::from_keyword(token_text).unwrap_or(IDENT)
177177
}
178+
rustc_lexer::TokenKind::InvalidIdent => {
179+
err = "Ident contains invalid characters";
180+
IDENT
181+
}
178182

179183
rustc_lexer::TokenKind::RawIdent => IDENT,
180184
rustc_lexer::TokenKind::Literal { kind, .. } => {
@@ -221,6 +225,7 @@ impl<'a> Converter<'a> {
221225
err = "unknown literal prefix";
222226
IDENT
223227
}
228+
rustc_lexer::TokenKind::Eof => EOF,
224229
}
225230
};
226231

@@ -268,35 +273,30 @@ impl<'a> Converter<'a> {
268273
}
269274
BYTE_STRING
270275
}
271-
rustc_lexer::LiteralKind::RawStr { err: raw_str_err, .. } => {
272-
if let Some(raw_str_err) = raw_str_err {
273-
err = match raw_str_err {
274-
rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw string literal",
275-
rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found {
276-
"Missing trailing `\"` to terminate the raw string literal"
277-
} else {
278-
"Missing trailing `\"` with `#` symbols to terminate the raw string literal"
279-
},
280-
rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols",
281-
};
282-
};
276+
rustc_lexer::LiteralKind::CStr { terminated } => {
277+
if !terminated {
278+
err = "Missing trailing `\"` symbol to terminate the string literal";
279+
}
283280
STRING
284281
}
285-
rustc_lexer::LiteralKind::RawByteStr { err: raw_str_err, .. } => {
286-
if let Some(raw_str_err) = raw_str_err {
287-
err = match raw_str_err {
288-
rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw byte string literal",
289-
rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found {
290-
"Missing trailing `\"` to terminate the raw byte string literal"
291-
} else {
292-
"Missing trailing `\"` with `#` symbols to terminate the raw byte string literal"
293-
},
294-
rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw byte strings may be delimited by up to 65535 `#` symbols",
295-
};
296-
};
297-
282+
rustc_lexer::LiteralKind::RawStr { n_hashes } => {
283+
if n_hashes.is_none() {
284+
err = "Invalid raw string literal";
285+
}
286+
STRING
287+
}
288+
rustc_lexer::LiteralKind::RawByteStr { n_hashes } => {
289+
if n_hashes.is_none() {
290+
err = "Invalid raw string literal";
291+
}
298292
BYTE_STRING
299293
}
294+
rustc_lexer::LiteralKind::RawCStr { n_hashes } => {
295+
if n_hashes.is_none() {
296+
err = "Invalid raw string literal";
297+
}
298+
STRING
299+
}
300300
};
301301

302302
let err = if err.is_empty() { None } else { Some(err) };

‎crates/parser/src/syntax_kind/generated.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ pub enum SyntaxKind {
117117
BYTE,
118118
STRING,
119119
BYTE_STRING,
120+
C_STRING,
120121
ERROR,
121122
IDENT,
122123
WHITESPACE,
@@ -379,7 +380,7 @@ impl SyntaxKind {
379380
)
380381
}
381382
pub fn is_literal(self) -> bool {
382-
matches!(self, INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | BYTE_STRING)
383+
matches!(self, INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | BYTE_STRING | C_STRING)
383384
}
384385
pub fn from_keyword(ident: &str) -> Option<SyntaxKind> {
385386
let kw = match ident {

‎crates/parser/test_data/lexer/err/unclosed_raw_byte_string_at_eof.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
BYTE_STRING "br##\"" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
1+
BYTE_STRING "br##\"" error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ascii_escape.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
BYTE_STRING "br##\"\\x7f" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
1+
BYTE_STRING "br##\"\\x7f" error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ferris.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
BYTE_STRING "br##\"🦀" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
1+
BYTE_STRING "br##\"🦀" error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
BYTE_STRING "br##\"\\" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
1+
BYTE_STRING "br##\"\\" error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash_n.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
BYTE_STRING "br##\"\\n" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
1+
BYTE_STRING "br##\"\\n" error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_space.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
BYTE_STRING "br##\" " error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
1+
BYTE_STRING "br##\" " error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_unicode_escape.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
BYTE_STRING "br##\"\\u{20AA}" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
1+
BYTE_STRING "br##\"\\u{20AA}" error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unclosed_raw_string_at_eof.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
STRING "r##\"" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
1+
STRING "r##\"" error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unclosed_raw_string_with_ascii_escape.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
STRING "r##\"\\x7f" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
1+
STRING "r##\"\\x7f" error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unclosed_raw_string_with_ferris.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
STRING "r##\"🦀" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
1+
STRING "r##\"🦀" error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
STRING "r##\"\\" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
1+
STRING "r##\"\\" error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash_n.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
STRING "r##\"\\n" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
1+
STRING "r##\"\\n" error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unclosed_raw_string_with_space.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
STRING "r##\" " error: Missing trailing `"` with `#` symbols to terminate the raw string literal
1+
STRING "r##\" " error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unclosed_raw_string_with_unicode_escape.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
STRING "r##\"\\u{20AA}" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
1+
STRING "r##\"\\u{20AA}" error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unstarted_raw_byte_string_at_eof.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
BYTE_STRING "br##" error: Missing `"` symbol after `#` symbols to begin the raw byte string literal
1+
BYTE_STRING "br##" error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unstarted_raw_byte_string_with_ascii.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
BYTE_STRING "br## " error: Missing `"` symbol after `#` symbols to begin the raw byte string literal
1+
BYTE_STRING "br## " error: Invalid raw string literal
22
IDENT "I"
33
WHITESPACE " "
44
IDENT "lack"

‎crates/parser/test_data/lexer/err/unstarted_raw_string_at_eof.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
STRING "r##" error: Missing `"` symbol after `#` symbols to begin the raw string literal
1+
STRING "r##" error: Invalid raw string literal

‎crates/parser/test_data/lexer/err/unstarted_raw_string_with_ascii.rast

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
STRING "r## " error: Missing `"` symbol after `#` symbols to begin the raw string literal
1+
STRING "r## " error: Invalid raw string literal
22
IDENT "I"
33
WHITESPACE " "
44
IDENT "lack"

‎crates/syntax/Cargo.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,14 @@ cov-mark = "2.0.0-pre.1"
1717
either = "1.7.0"
1818
itertools = "0.10.5"
1919
rowan = "0.15.11"
20-
rustc_lexer = { version = "727.0.0", package = "rustc-ap-rustc_lexer" }
2120
rustc-hash = "1.1.0"
2221
once_cell = "1.17.0"
2322
indexmap = "1.9.1"
2423
smol_str.workspace = true
2524
triomphe.workspace = true
2625

26+
rustc_lexer.workspace = true
27+
2728
parser.workspace = true
2829
profile.workspace = true
2930
stdx.workspace = true

‎crates/syntax/src/tests/ast_src.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ pub(crate) const KINDS_SRC: KindsSrc<'_> = KindsSrc {
7171
"super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield",
7272
],
7373
contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules", "yeet"],
74-
literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"],
74+
literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING", "C_STRING"],
7575
tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"],
7676
nodes: &[
7777
"SOURCE_FILE",

0 commit comments

Comments
 (0)