Skip to content

Commit

Permalink
chore: more tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@plusvic
plusvic committed Jul 5, 2024
1 parent 9fc14cc commit 5895564
Show file tree
Hide file tree
Showing 10 changed files with 132 additions and 94 deletions.
168 changes: 76 additions & 92 deletions parser-ng/src/parser/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,44 +322,7 @@ impl<'src> InternalParser<'src> {
let span = token.span();
let token_str = token.as_str();

let tokens = self.expected_tokens.entry(span.start()).or_default();
tokens.extend(expected_tokens.iter().map(|t| t.as_str()));

let (last, all_except_last) = tokens.split_last().unwrap();

let error_msg = if all_except_last.is_empty() {
format!("expecting {last}, found {}", token_str)
} else {
format!(
"expecting {} or {last}, found {}",
all_except_last.join(", "),
token_str,
)
};

self.pending_errors.push((error_msg, span));

if self.opt_depth == 0 {
// Find the pending error starting at the largest offset. If several
// errors start at the same offset, the last one is used (this is
// guaranteed by the `max_by_key` function). `self.pending_errors`
// is left empty.
if let Some((error, span)) = self
.pending_errors
.drain(0..)
.max_by_key(|(_, span)| span.start())
{
match self.ready_errors.entry(span) {
Entry::Occupied(_) => {
// already present, don't replace.
}
Entry::Vacant(v) => {
v.insert(error);
}
}
//self.output.push_error(error, span);
}
}
self.unexpected_token_error(token_str, span, expected_tokens);

self.failed = true;
self
Expand Down Expand Up @@ -464,45 +427,82 @@ impl<'src> InternalParser<'src> {
self
}*/

fn unexpected_token_error(
&mut self,
token_str: &str,
span: Span,
expected_tokens: &TokenSet,
) {
let tokens = self.expected_tokens.entry(span.start()).or_default();
tokens.extend(expected_tokens.iter().map(|t| t.as_str()));

let (last, all_except_last) = tokens.split_last().unwrap();

let error_msg = if all_except_last.is_empty() {
format!("expecting {last}, found {}", token_str)
} else {
format!(
"expecting {} or {last}, found {}",
all_except_last.join(", "),
token_str,
)
};

self.pending_errors.push((error_msg, span));

if self.opt_depth == 0 {
// Find the pending error starting at the largest offset. If several
// errors start at the same offset, the last one is used (this is
// guaranteed by the `max_by_key` function). `self.pending_errors`
// is left empty.
if let Some((error, span)) = self
.pending_errors
.drain(0..)
.max_by_key(|(_, span)| span.start())
{
match self.ready_errors.entry(span) {
Entry::Occupied(_) => {
// already present, don't replace.
}
Entry::Vacant(v) => {
v.insert(error);
}
}
}
}
}

fn recover(&mut self) -> &mut Self {
self.failed = false;
self
}

fn sync(&mut self, expected_tokens: &TokenSet) -> &mut Self {
let failed = self.failed;
self.check(expected_tokens);
if self.failed {
self.ws();
self.output.begin(SyntaxKind::ERROR);
while let Some(token) = self.peek() {
if expected_tokens.contains(token) {
break;
} else {
self.bump();
}
self.ws();
match self.peek() {
None => return self,
Some(token) if expected_tokens.contains(token) => return self,
Some(token) => {
let span = token.span();
let token_str = token.as_str();
self.unexpected_token_error(token_str, span, expected_tokens);
}
}
self.output.begin(SyntaxKind::ERROR);
while let Some(token) = self.peek() {
if expected_tokens.contains(token) {
break;
} else {
self.bump();
}
self.output.end();
}
self.failed = failed;
self.output.end();
self
}

fn sync_and_recover(&mut self, expected_tokens: &TokenSet) -> &mut Self {
self.check(expected_tokens);
if self.failed {
self.recover();
self.ws();
self.output.begin(SyntaxKind::ERROR);
while let Some(token) = self.peek() {
if expected_tokens.contains(token) {
break;
} else {
self.bump();
}
}
self.output.end();
}
fn recover_and_sync(&mut self, expected_tokens: &TokenSet) -> &mut Self {
self.recover();
self.sync(expected_tokens);
self
}

Expand Down Expand Up @@ -546,30 +546,6 @@ impl<'src> InternalParser<'src> {
self
}

/*
/// Similar to [`InternalParser::check_and_recover`], but also consumes the
/// expected token.
fn expect_and_recover(&mut self, expected_tokens: &TokenSet) -> &mut Self {
self.failed = false;
self.expect(expected_tokens);
if self.failed {
self.failed = false;
self.ws();
self.output.begin(SyntaxKind::ERROR);
while let Some(token) = self.peek() {
if expected_tokens.contains(token) {
break;
} else {
self.bump();
}
}
self.output.end();
self.bump();
}
self
}
*/

/// Begins an alternative.
///
/// # Example
Expand Down Expand Up @@ -860,13 +836,13 @@ impl<'src> InternalParser<'src> {
.expect(t!(IDENT))
.ws()
.if_found(t!(COLON), |p| p.rule_tags())
.sync_and_recover(t!(L_BRACE))
.recover_and_sync(t!(L_BRACE))
.expect(t!(L_BRACE))
.ws()
.if_found(t!(META_KW), |p| p.meta_blk())
.ws()
.if_found(t!(STRINGS_KW), |p| p.patterns_blk())
.sync_and_recover(t!(CONDITION_KW))
.recover_and_sync(t!(CONDITION_KW))
.ws() // todo: remove
.one(|p| p.condition_blk())
.expect(t!(R_BRACE))
Expand Down Expand Up @@ -909,6 +885,14 @@ impl<'src> InternalParser<'src> {
.expect(t!(META_KW))
.expect(t!(COLON))
.one_or_more(|p| p.ws().meta_def())
/*.one(|p| {
while !p.failed {
p.ws()
.meta_def()
.sync(t!(IDENT | STRINGS_KW | CONDITION_KW));
}
p.recover()
})*/
.end()
}

Expand Down
8 changes: 6 additions & 2 deletions parser-ng/src/parser/tests/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,13 @@ fn cst() {
fn test() {
let cst = CST::from(Parser::new(
r#"
rule test : = {
rule test {
strings:
$a = "foo"
$b = bar
$c = "baz"
condition:
true
true
}
"#
.as_bytes(),
Expand Down
0 ...rc/parser/tests/testdata/meta-errors-1.in → ...src/parser/tests/testdata/meta-error-1.in
View file
File renamed without changes.
0 ...c/parser/tests/testdata/meta-errors-1.out → ...rc/parser/tests/testdata/meta-error-1.out
View file
File renamed without changes.
0 ...rc/parser/tests/testdata/meta-errors-2.in → ...src/parser/tests/testdata/meta-error-2.in
View file
File renamed without changes.
0 ...c/parser/tests/testdata/meta-errors-2.out → ...rc/parser/tests/testdata/meta-error-2.out
View file
File renamed without changes.
0 ...rc/parser/tests/testdata/meta-errors-3.in → ...src/parser/tests/testdata/meta-error-3.in
View file
File renamed without changes.
0 ...c/parser/tests/testdata/meta-errors-3.out → ...rc/parser/tests/testdata/meta-error-3.out
View file
File renamed without changes.
7 changes: 7 additions & 0 deletions parser-ng/src/parser/tests/testdata/meta-error-4.in
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
rule test {
meta:
foo = meta
bar = "bar"
condition:
true
}
43 changes: 43 additions & 0 deletions parser-ng/src/parser/tests/testdata/meta-error-4.out
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
[email protected]
[email protected]
[email protected] "rule"
[email protected] " "
[email protected] "test"
[email protected] " "
[email protected] "{"
[email protected] "\n"
[email protected] " "
[email protected]
[email protected] "meta"
[email protected] ":"
[email protected] "\n"
[email protected] " "
[email protected]
[email protected] "foo"
[email protected] " "
[email protected] "="
[email protected] " "
[email protected]
[email protected] "meta"
[email protected] "\n"
[email protected] " "
[email protected] "bar"
[email protected] " "
[email protected] "="
[email protected] " "
[email protected] "\"bar\""
[email protected] "\n"
[email protected] " "
[email protected]
[email protected] "condition"
[email protected] ":"
[email protected] "\n"
[email protected] "\t "
[email protected]
[email protected]
[email protected] "true"
[email protected] "\n"
[email protected] "}"

ERRORS:
- [30..34]: expecting `true`, `false`, INTEGER, FLOAT or STRING, found `meta`

0 comments on commit 5895564

Please sign in to comment.