Skip to content

Commit 6081335

Browse files
bvanjoibvanjoi
committed
refactor(es/lexer): token eof
1 parent 2865141 commit 6081335

File tree

33 files changed

+901
-1112
lines changed

33 files changed

+901
-1112
lines changed

‎crates/swc_common/src/syntax_pos.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,7 @@ impl Span {
407407
}
408408

409409
#[inline]
410+
#[track_caller]
410411
pub fn new_with_checked(lo: BytePos, hi: BytePos) -> Self {
411412
debug_assert!(lo <= hi, "lo: {lo:#?}, hi: {hi:#?}");
412413
Span { lo, hi }

‎crates/swc_ecma_lexer/src/common/lexer/mod.rs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1730,7 +1730,7 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
17301730
}
17311731

17321732
/// `#`
1733-
fn read_token_number_sign(&mut self) -> LexResult<Option<Self::Token>> {
1733+
fn read_token_number_sign(&mut self) -> LexResult<Self::Token> {
17341734
debug_assert!(self.cur().is_some_and(|c| c == '#'));
17351735

17361736
self.bump(); // '#'
@@ -1741,7 +1741,7 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
17411741
!self.input().is_at_start() || self.cur() != Some('!'),
17421742
"#! should have already been handled by read_shebang()"
17431743
);
1744-
Ok(Some(Self::Token::HASH))
1744+
Ok(Self::Token::HASH)
17451745
}
17461746

17471747
/// Read a token given `.`.
@@ -1936,14 +1936,14 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
19361936
}
19371937

19381938
#[inline(never)]
1939-
fn read_slash(&mut self) -> LexResult<Option<Self::Token>> {
1939+
fn read_slash(&mut self) -> LexResult<Self::Token> {
19401940
debug_assert_eq!(self.cur(), Some('/'));
19411941
self.bump(); // '/'
1942-
Ok(Some(if self.eat(b'=') {
1942+
Ok(if self.eat(b'=') {
19431943
Self::Token::DIV_EQ
19441944
} else {
19451945
Self::Token::DIV
1946-
}))
1946+
})
19471947
}
19481948

19491949
/// This can be used if there's no keyword starting with the first
@@ -2084,7 +2084,7 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
20842084
fn read_keyword_with(
20852085
&mut self,
20862086
convert: &dyn Fn(&str) -> Option<Self::Token>,
2087-
) -> LexResult<Option<Self::Token>> {
2087+
) -> LexResult<Self::Token> {
20882088
debug_assert!(self.cur().is_some());
20892089

20902090
let start = self.cur_pos();
@@ -2100,11 +2100,11 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
21002100
SyntaxError::EscapeInReservedWord { word: Atom::new(s) },
21012101
)
21022102
} else {
2103-
Ok(Some(word))
2103+
Ok(word)
21042104
}
21052105
} else {
21062106
let atom = self.atom(s);
2107-
Ok(Some(Self::Token::unknown_ident(atom, self)))
2107+
Ok(Self::Token::unknown_ident(atom, self))
21082108
}
21092109
}
21102110

‎crates/swc_ecma_lexer/src/common/lexer/token.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ pub trait TokenFactory<'a, TokenAndSpan, I: Tokens<TokenAndSpan>>: Sized + Parti
149149
const CASE: Self;
150150
const DEFAULT: Self;
151151
const DEBUGGER: Self;
152+
const EOF: Self;
152153

153154
fn jsx_name(name: &'a str, lexer: &mut Self::Lexer) -> Self;
154155
fn is_jsx_name(&self) -> bool;
@@ -586,6 +587,10 @@ pub trait TokenFactory<'a, TokenAndSpan, I: Tokens<TokenAndSpan>>: Sized + Parti
586587
fn is_exp(&self) -> bool {
587588
Self::EXP.eq(self)
588589
}
590+
#[inline(always)]
591+
fn is_eof(&self) -> bool {
592+
Self::EOF.eq(self)
593+
}
589594
fn is_no_substitution_template_literal(&self) -> bool;
590595
fn is_template_head(&self) -> bool;
591596
}

‎crates/swc_ecma_lexer/src/common/parser/buffer.rs

Lines changed: 19 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
use debug_unreachable::debug_unreachable;
21
use swc_common::{BytePos, Span};
32
use swc_ecma_ast::EsVersion;
43

@@ -30,9 +29,9 @@ pub trait Buffer<'a> {
3029
fn set_next(&mut self, token: Option<Self::Next>);
3130
fn next_mut(&mut self) -> &mut Option<Self::Next>;
3231

33-
fn cur(&mut self) -> Option<&Self::Token>;
34-
fn get_cur(&self) -> Option<&Self::TokenAndSpan>;
35-
fn get_cur_mut(&mut self) -> &mut Option<Self::TokenAndSpan>;
32+
fn cur(&self) -> &Self::Token;
33+
fn get_cur(&self) -> &Self::TokenAndSpan;
34+
fn get_cur_mut(&mut self) -> &mut Self::TokenAndSpan;
3635

3736
fn prev_span(&self) -> Span;
3837
fn set_prev_span(&mut self, span: Span);
@@ -43,47 +42,28 @@ pub trait Buffer<'a> {
4342

4443
fn store(&mut self, token: Self::Token) {
4544
debug_assert!(self.next().is_none());
46-
debug_assert!(self.get_cur().is_none());
45+
debug_assert!(!self.cur().is_eof());
4746
let span = self.prev_span();
4847
let token = Self::TokenAndSpan::new(token, span, false);
4948
self.set_cur(token);
5049
}
5150

52-
#[allow(dead_code)]
53-
fn cur_debug<'b>(&'b self) -> Option<&'b Self::Token>
54-
where
55-
Self::TokenAndSpan: 'b,
56-
{
57-
self.get_cur().map(|it| it.token())
51+
#[cold]
52+
#[inline(never)]
53+
fn dump_cur(&self) -> String {
54+
format!("{:?}", self.cur())
5855
}
5956

60-
fn dump_cur(&mut self) -> String;
61-
62-
/// Returns current token.
63-
fn bump(&mut self) -> Self::Token {
64-
let prev = match self.get_cur_mut().take() {
65-
Some(t) => t,
66-
None => unsafe {
67-
debug_unreachable!(
68-
"Current token is `None`. Parser should not call bump() without knowing \
69-
current token"
70-
)
71-
},
72-
};
73-
self.set_prev_span(prev.span());
74-
prev.take_token()
75-
}
57+
/// find next token.
58+
fn bump(&mut self);
7659

7760
#[inline]
7861
fn knows_cur(&self) -> bool {
79-
self.get_cur().is_some()
62+
!self.cur().is_eof()
8063
}
8164

82-
fn had_line_break_before_cur(&mut self) -> bool {
83-
self.cur();
84-
self.get_cur()
85-
.map(|it| it.had_line_break())
86-
.unwrap_or_else(|| true)
65+
fn had_line_break_before_cur(&self) -> bool {
66+
self.get_cur().had_line_break()
8767
}
8868

8969
/// This returns true on eof.
@@ -118,8 +98,8 @@ pub trait Buffer<'a> {
11898
if span.hi != next.span().lo {
11999
return;
120100
}
121-
let cur = self.get_cur_mut().take().unwrap();
122101
let next = self.next_mut().take().unwrap();
102+
let cur = self.get_cur();
123103
let cur_token = cur.token();
124104
let token = if cur_token.is_greater() {
125105
let next_token = next.token();
@@ -139,7 +119,6 @@ pub trait Buffer<'a> {
139119
// >>>=
140120
Self::Token::ZERO_FILL_RSHIFT_EQ
141121
} else {
142-
self.set_cur(cur);
143122
self.set_next(Some(next));
144123
return;
145124
}
@@ -155,12 +134,10 @@ pub trait Buffer<'a> {
155134
// <<=
156135
Self::Token::LSHIFT_EQ
157136
} else {
158-
self.set_cur(cur);
159137
self.set_next(Some(next));
160138
return;
161139
}
162140
} else {
163-
self.set_cur(cur);
164141
self.set_next(Some(next));
165142
return;
166143
};
@@ -170,8 +147,8 @@ pub trait Buffer<'a> {
170147
}
171148

172149
#[inline(always)]
173-
fn is(&mut self, expected: &Self::Token) -> bool {
174-
self.cur().is_some_and(|cur| cur == expected)
150+
fn is(&self, expected: &Self::Token) -> bool {
151+
self.cur() == expected
175152
}
176153

177154
#[inline(always)]
@@ -185,23 +162,13 @@ pub trait Buffer<'a> {
185162

186163
/// Returns start of current token.
187164
#[inline]
188-
fn cur_pos(&mut self) -> BytePos {
189-
let _ = self.cur();
190-
self.get_cur()
191-
.map(|item| item.span().lo)
192-
.unwrap_or_else(|| {
193-
// eof
194-
self.last_pos()
195-
})
165+
fn cur_pos(&self) -> BytePos {
166+
self.get_cur().span().lo
196167
}
197168

198169
#[inline]
199170
fn cur_span(&self) -> Span {
200-
let data = self
201-
.get_cur()
202-
.map(|item| item.span())
203-
.unwrap_or(self.prev_span());
204-
Span::new_with_checked(data.lo, data.hi)
171+
self.get_cur().span()
205172
}
206173

207174
/// Returns last byte position of previous token.

‎crates/swc_ecma_lexer/src/common/parser/class_and_fn.rs

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -187,25 +187,19 @@ pub fn parse_super_class<'a, P: Parser<'a>>(
187187
}
188188

189189
pub fn is_class_method<'a, P: Parser<'a>>(p: &mut P) -> bool {
190-
p.input_mut().is(&P::Token::LPAREN)
191-
|| (p.input().syntax().typescript()
192-
&& p.input_mut()
193-
.cur()
194-
.is_some_and(|cur| cur.is_less() || cur.is_jsx_tag_start()))
190+
let cur = p.input().cur();
191+
cur == &P::Token::LPAREN
192+
|| (p.input().syntax().typescript() && (cur.is_less() || cur.is_jsx_tag_start()))
195193
}
196194

197195
pub fn is_class_property<'a, P: Parser<'a>>(p: &mut P, asi: bool) -> bool {
198-
(p.input().syntax().typescript()
199-
&& p.input_mut()
200-
.cur()
201-
.is_some_and(|cur| cur.is_bang() || cur.is_colon()))
202-
|| p.input_mut()
203-
.cur()
204-
.is_some_and(|cur| cur.is_equal() || cur.is_rbrace())
196+
let cur = p.input().cur();
197+
(p.input().syntax().typescript() && (cur.is_bang() || cur.is_colon()))
198+
|| (cur.is_equal() || cur.is_rbrace())
205199
|| if asi {
206200
p.is_general_semi()
207201
} else {
208-
p.input_mut().is(&P::Token::SEMI)
202+
p.input().is(&P::Token::SEMI)
209203
}
210204
}
211205

@@ -986,11 +980,7 @@ fn parse_class_member_with_is_static<'a, P: Parser<'a>>(
986980
}
987981

988982
trace_cur!(p, parse_class_member_with_is_static__normal_class_member);
989-
let key = if readonly.is_some()
990-
&& p.input_mut()
991-
.cur()
992-
.is_some_and(|cur| cur.is_bang() || cur.is_colon())
993-
{
983+
let key = if readonly.is_some() && (p.input().cur().is_bang() || p.input().cur().is_colon()) {
994984
Key::Public(PropName::Ident(IdentName::new(
995985
atom!("readonly"),
996986
readonly.unwrap(),
@@ -1657,7 +1647,7 @@ fn parse_class_inner<'a, P: Parser<'a>>(
16571647
p.do_outside_of_context(Context::HasSuperClass, parse_class_body)?
16581648
};
16591649

1660-
if p.input_mut().cur().is_none() {
1650+
if p.input_mut().cur().is_eof() {
16611651
let eof_text = p.input_mut().dump_cur();
16621652
p.emit_err(
16631653
p.input().cur_span(),

0 commit comments

Comments
 (0)