Skip to content

Commit c8b7188

Browse files
bvanjoibvanjoi
committed
refactor(es/lexer): token eof
1 parent 2865141 commit c8b7188

File tree

34 files changed

+1020
-1142
lines changed

34 files changed

+1020
-1142
lines changed

‎crates/swc_common/src/syntax_pos.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,7 @@ impl Span {
407407
}
408408

409409
#[inline]
410+
#[track_caller]
410411
pub fn new_with_checked(lo: BytePos, hi: BytePos) -> Self {
411412
debug_assert!(lo <= hi, "lo: {lo:#?}, hi: {hi:#?}");
412413
Span { lo, hi }

‎crates/swc_ecma_lexer/src/common/lexer/mod.rs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1730,7 +1730,7 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
17301730
}
17311731

17321732
/// `#`
1733-
fn read_token_number_sign(&mut self) -> LexResult<Option<Self::Token>> {
1733+
fn read_token_number_sign(&mut self) -> LexResult<Self::Token> {
17341734
debug_assert!(self.cur().is_some_and(|c| c == '#'));
17351735

17361736
self.bump(); // '#'
@@ -1741,7 +1741,7 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
17411741
!self.input().is_at_start() || self.cur() != Some('!'),
17421742
"#! should have already been handled by read_shebang()"
17431743
);
1744-
Ok(Some(Self::Token::HASH))
1744+
Ok(Self::Token::HASH)
17451745
}
17461746

17471747
/// Read a token given `.`.
@@ -1936,14 +1936,14 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
19361936
}
19371937

19381938
#[inline(never)]
1939-
fn read_slash(&mut self) -> LexResult<Option<Self::Token>> {
1939+
fn read_slash(&mut self) -> LexResult<Self::Token> {
19401940
debug_assert_eq!(self.cur(), Some('/'));
19411941
self.bump(); // '/'
1942-
Ok(Some(if self.eat(b'=') {
1942+
Ok(if self.eat(b'=') {
19431943
Self::Token::DIV_EQ
19441944
} else {
19451945
Self::Token::DIV
1946-
}))
1946+
})
19471947
}
19481948

19491949
/// This can be used if there's no keyword starting with the first
@@ -2084,7 +2084,7 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
20842084
fn read_keyword_with(
20852085
&mut self,
20862086
convert: &dyn Fn(&str) -> Option<Self::Token>,
2087-
) -> LexResult<Option<Self::Token>> {
2087+
) -> LexResult<Self::Token> {
20882088
debug_assert!(self.cur().is_some());
20892089

20902090
let start = self.cur_pos();
@@ -2100,11 +2100,11 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
21002100
SyntaxError::EscapeInReservedWord { word: Atom::new(s) },
21012101
)
21022102
} else {
2103-
Ok(Some(word))
2103+
Ok(word)
21042104
}
21052105
} else {
21062106
let atom = self.atom(s);
2107-
Ok(Some(Self::Token::unknown_ident(atom, self)))
2107+
Ok(Self::Token::unknown_ident(atom, self))
21082108
}
21092109
}
21102110

‎crates/swc_ecma_lexer/src/common/lexer/token.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ pub trait TokenFactory<'a, TokenAndSpan, I: Tokens<TokenAndSpan>>: Sized + Parti
149149
const CASE: Self;
150150
const DEFAULT: Self;
151151
const DEBUGGER: Self;
152+
const EOF: Self;
152153

153154
fn jsx_name(name: &'a str, lexer: &mut Self::Lexer) -> Self;
154155
fn is_jsx_name(&self) -> bool;
@@ -586,6 +587,10 @@ pub trait TokenFactory<'a, TokenAndSpan, I: Tokens<TokenAndSpan>>: Sized + Parti
586587
fn is_exp(&self) -> bool {
587588
Self::EXP.eq(self)
588589
}
590+
#[inline(always)]
591+
fn is_eof(&self) -> bool {
592+
Self::EOF.eq(self)
593+
}
589594
fn is_no_substitution_template_literal(&self) -> bool;
590595
fn is_template_head(&self) -> bool;
591596
}

‎crates/swc_ecma_lexer/src/common/parser/buffer.rs

Lines changed: 34 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
1-
use debug_unreachable::debug_unreachable;
1+
use swc_atoms::Atom;
22
use swc_common::{BytePos, Span};
33
use swc_ecma_ast::EsVersion;
44

55
use super::token_and_span::TokenAndSpan as TokenAndSpanTrait;
66
use crate::common::{
7-
context::Context, input::Tokens, lexer::token::TokenFactory, syntax::SyntaxFlags,
7+
context::Context,
8+
input::Tokens,
9+
lexer::{token::TokenFactory, LexResult},
10+
syntax::SyntaxFlags,
811
};
912

1013
pub trait NextTokenAndSpan {
@@ -30,9 +33,9 @@ pub trait Buffer<'a> {
3033
fn set_next(&mut self, token: Option<Self::Next>);
3134
fn next_mut(&mut self) -> &mut Option<Self::Next>;
3235

33-
fn cur(&mut self) -> Option<&Self::Token>;
34-
fn get_cur(&self) -> Option<&Self::TokenAndSpan>;
35-
fn get_cur_mut(&mut self) -> &mut Option<Self::TokenAndSpan>;
36+
fn cur(&self) -> &Self::Token;
37+
fn get_cur(&self) -> &Self::TokenAndSpan;
38+
fn get_cur_mut(&mut self) -> &mut Self::TokenAndSpan;
3639

3740
fn prev_span(&self) -> Span;
3841
fn set_prev_span(&mut self, span: Span);
@@ -43,47 +46,38 @@ pub trait Buffer<'a> {
4346

4447
fn store(&mut self, token: Self::Token) {
4548
debug_assert!(self.next().is_none());
46-
debug_assert!(self.get_cur().is_none());
49+
debug_assert!(!self.cur().is_eof());
4750
let span = self.prev_span();
4851
let token = Self::TokenAndSpan::new(token, span, false);
4952
self.set_cur(token);
5053
}
5154

52-
#[allow(dead_code)]
53-
fn cur_debug<'b>(&'b self) -> Option<&'b Self::Token>
54-
where
55-
Self::TokenAndSpan: 'b,
56-
{
57-
self.get_cur().map(|it| it.token())
55+
#[cold]
56+
#[inline(never)]
57+
fn dump_cur(&self) -> String {
58+
format!("{:?}", self.cur())
5859
}
5960

60-
fn dump_cur(&mut self) -> String;
61-
62-
/// Returns current token.
63-
fn bump(&mut self) -> Self::Token {
64-
let prev = match self.get_cur_mut().take() {
65-
Some(t) => t,
66-
None => unsafe {
67-
debug_unreachable!(
68-
"Current token is `None`. Parser should not call bump() without knowing \
69-
current token"
70-
)
71-
},
72-
};
73-
self.set_prev_span(prev.span());
74-
prev.take_token()
75-
}
61+
/// find next token.
62+
fn bump(&mut self);
63+
fn expect_word_token_and_bump(&mut self) -> Atom;
64+
fn expect_number_token_and_bump(&mut self) -> (f64, Atom);
65+
fn expect_string_token_and_bump(&mut self) -> (Atom, Atom);
66+
fn expect_bigint_token_and_bump(&mut self) -> (Box<num_bigint::BigInt>, Atom);
67+
fn expect_regex_token_and_bump(&mut self) -> (Atom, Atom);
68+
fn expect_template_token_and_bump(&mut self) -> (LexResult<Atom>, Atom);
69+
fn expect_error_token_and_bump(&mut self) -> crate::error::Error;
70+
fn expect_jsx_name_token_and_bump(&mut self) -> Atom;
71+
fn expect_jsx_text_token_and_bump(&mut self) -> (Atom, Atom);
72+
fn expect_shebang_token_and_bump(&mut self) -> Atom;
7673

7774
#[inline]
7875
fn knows_cur(&self) -> bool {
79-
self.get_cur().is_some()
76+
!self.cur().is_eof()
8077
}
8178

82-
fn had_line_break_before_cur(&mut self) -> bool {
83-
self.cur();
84-
self.get_cur()
85-
.map(|it| it.had_line_break())
86-
.unwrap_or_else(|| true)
79+
fn had_line_break_before_cur(&self) -> bool {
80+
self.get_cur().had_line_break()
8781
}
8882

8983
/// This returns true on eof.
@@ -118,8 +112,8 @@ pub trait Buffer<'a> {
118112
if span.hi != next.span().lo {
119113
return;
120114
}
121-
let cur = self.get_cur_mut().take().unwrap();
122115
let next = self.next_mut().take().unwrap();
116+
let cur = self.get_cur();
123117
let cur_token = cur.token();
124118
let token = if cur_token.is_greater() {
125119
let next_token = next.token();
@@ -139,7 +133,6 @@ pub trait Buffer<'a> {
139133
// >>>=
140134
Self::Token::ZERO_FILL_RSHIFT_EQ
141135
} else {
142-
self.set_cur(cur);
143136
self.set_next(Some(next));
144137
return;
145138
}
@@ -155,12 +148,10 @@ pub trait Buffer<'a> {
155148
// <<=
156149
Self::Token::LSHIFT_EQ
157150
} else {
158-
self.set_cur(cur);
159151
self.set_next(Some(next));
160152
return;
161153
}
162154
} else {
163-
self.set_cur(cur);
164155
self.set_next(Some(next));
165156
return;
166157
};
@@ -170,8 +161,8 @@ pub trait Buffer<'a> {
170161
}
171162

172163
#[inline(always)]
173-
fn is(&mut self, expected: &Self::Token) -> bool {
174-
self.cur().is_some_and(|cur| cur == expected)
164+
fn is(&self, expected: &Self::Token) -> bool {
165+
self.cur() == expected
175166
}
176167

177168
#[inline(always)]
@@ -185,23 +176,13 @@ pub trait Buffer<'a> {
185176

186177
/// Returns start of current token.
187178
#[inline]
188-
fn cur_pos(&mut self) -> BytePos {
189-
let _ = self.cur();
190-
self.get_cur()
191-
.map(|item| item.span().lo)
192-
.unwrap_or_else(|| {
193-
// eof
194-
self.last_pos()
195-
})
179+
fn cur_pos(&self) -> BytePos {
180+
self.get_cur().span().lo
196181
}
197182

198183
#[inline]
199184
fn cur_span(&self) -> Span {
200-
let data = self
201-
.get_cur()
202-
.map(|item| item.span())
203-
.unwrap_or(self.prev_span());
204-
Span::new_with_checked(data.lo, data.hi)
185+
self.get_cur().span()
205186
}
206187

207188
/// Returns last byte position of previous token.

‎crates/swc_ecma_lexer/src/common/parser/class_and_fn.rs

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -187,25 +187,19 @@ pub fn parse_super_class<'a, P: Parser<'a>>(
187187
}
188188

189189
pub fn is_class_method<'a, P: Parser<'a>>(p: &mut P) -> bool {
190-
p.input_mut().is(&P::Token::LPAREN)
191-
|| (p.input().syntax().typescript()
192-
&& p.input_mut()
193-
.cur()
194-
.is_some_and(|cur| cur.is_less() || cur.is_jsx_tag_start()))
190+
let cur = p.input().cur();
191+
cur == &P::Token::LPAREN
192+
|| (p.input().syntax().typescript() && (cur.is_less() || cur.is_jsx_tag_start()))
195193
}
196194

197195
pub fn is_class_property<'a, P: Parser<'a>>(p: &mut P, asi: bool) -> bool {
198-
(p.input().syntax().typescript()
199-
&& p.input_mut()
200-
.cur()
201-
.is_some_and(|cur| cur.is_bang() || cur.is_colon()))
202-
|| p.input_mut()
203-
.cur()
204-
.is_some_and(|cur| cur.is_equal() || cur.is_rbrace())
196+
let cur = p.input().cur();
197+
(p.input().syntax().typescript() && (cur.is_bang() || cur.is_colon()))
198+
|| (cur.is_equal() || cur.is_rbrace())
205199
|| if asi {
206200
p.is_general_semi()
207201
} else {
208-
p.input_mut().is(&P::Token::SEMI)
202+
p.input().is(&P::Token::SEMI)
209203
}
210204
}
211205

@@ -986,11 +980,7 @@ fn parse_class_member_with_is_static<'a, P: Parser<'a>>(
986980
}
987981

988982
trace_cur!(p, parse_class_member_with_is_static__normal_class_member);
989-
let key = if readonly.is_some()
990-
&& p.input_mut()
991-
.cur()
992-
.is_some_and(|cur| cur.is_bang() || cur.is_colon())
993-
{
983+
let key = if readonly.is_some() && (p.input().cur().is_bang() || p.input().cur().is_colon()) {
994984
Key::Public(PropName::Ident(IdentName::new(
995985
atom!("readonly"),
996986
readonly.unwrap(),
@@ -1657,7 +1647,7 @@ fn parse_class_inner<'a, P: Parser<'a>>(
16571647
p.do_outside_of_context(Context::HasSuperClass, parse_class_body)?
16581648
};
16591649

1660-
if p.input_mut().cur().is_none() {
1650+
if p.input_mut().cur().is_eof() {
16611651
let eof_text = p.input_mut().dump_cur();
16621652
p.emit_err(
16631653
p.input().cur_span(),

0 commit comments

Comments
 (0)