refactor: start using the new parser

cli/Cargo.toml

@@ -49,7 +49,6 @@ protobuf-json-mapping = { workspace = true }
 serde_json = { workspace = true, features = ["preserve_order"] }
 yansi = { workspace = true }
 yara-x = { workspace = true }
-yara-x-parser = { workspace = true, features = ["ascii-tree"] }
 yara-x-parser-ng = { workspace = true }
 yara-x-proto-yaml = { workspace = true }
 yara-x-fmt = { workspace = true }

cli/src/commands/check.rs

@@ -8,7 +8,7 @@ use crossterm::tty::IsTty;
 use superconsole::{Component, Line, Lines, Span};
 use yansi::Color::{Green, Red, Yellow};
 use yansi::Paint;
-use yara_x_parser::SourceCode;
+use yara_x::SourceCode;
 
 use crate::walk::Message;
 use crate::{help, walk};

cli/src/commands/debug.rs

@@ -4,8 +4,8 @@ use std::path::PathBuf;
 use anyhow::Context;
 use clap::{arg, value_parser, ArgMatches, Command};
 
-use yara_x::Compiler;
-use yara_x_parser::{Parser, SourceCode};
+use yara_x::{Compiler, SourceCode};
+use yara_x_parser_ng::Parser;
 
 pub fn ast() -> Command {
     super::command("ast")
@@ -64,15 +64,10 @@ pub fn exec_ast(args: &ArgMatches) -> anyhow::Result<()> {
     let src = fs::read(rules_path)
         .with_context(|| format!("can not read `{}`", rules_path.display()))?;
 
-    let src = SourceCode::from(src.as_slice())
-        .with_origin(rules_path.as_os_str().to_str().unwrap());
-
-    let ast = Parser::new().colorize_errors(true).build_ast(src)?;
-
-    let mut output = String::new();
-    ascii_tree::write_tree(&mut output, &ast.ascii_tree())?;
+    let parser = Parser::new(src.as_slice());
+    let ast = parser.into_ast();
 
-    println!("{output}");
+    println!("{ast:?}");
     Ok(())
 }
 
@@ -82,7 +77,7 @@ pub fn exec_cst(args: &ArgMatches) -> anyhow::Result<()> {
     let src = fs::read(rules_path)
         .with_context(|| format!("can not read `{}`", rules_path.display()))?;
 
-    let parser = yara_x_parser_ng::Parser::new(src.as_slice());
+    let parser = Parser::new(src.as_slice());
     let cst = parser.into_cst();
 
     println!("{cst:?}");

cli/src/commands/mod.rs

@@ -29,8 +29,7 @@ use yansi::Color::Green;
 use yansi::Paint;
 
 use crate::{commands, APP_HELP_TEMPLATE};
-use yara_x::{Compiler, Rules};
-use yara_x_parser::SourceCode;
+use yara_x::{Compiler, Rules, SourceCode};
 
 use crate::walk::Walker;
 

cli/src/main.rs

@@ -78,8 +78,7 @@ fn main() -> anyhow::Result<()> {
         match err.downcast_ref::<yara_x::Error>() {
             // Errors produced by the compiler already have colors and start
             // with "error:", in such cases the error is printed as is.
-            Some(yara_x::Error::ParseError(_))
-            | Some(yara_x::Error::CompileError(_)) => {
+            Some(yara_x::Error::CompileError(_)) => {
                 eprintln!("{}", err);
             }
             // In all other cases imitate the style of compiler errors, so that

lib/src/compiler/errors.rs

@@ -4,6 +4,7 @@ use std::io;
 use thiserror::Error;
 
 use yara_x_macros::Error as DeriveError;
+use yara_x_parser_ng::ast;
 
 use crate::compiler::report::{Level, ReportBuilder, SourceRef};
 use crate::compiler::warnings::InvalidWarningCode;
@@ -303,6 +304,100 @@ pub enum CompileError {
         error_msg: String,
         error_span: SourceRef,
     },
+
+    #[error("E012", "invalid integer")]
+    #[label("{error_msg}", error_span)]
+    InvalidInteger {
+        detailed_report: String,
+        error_msg: String,
+        error_span: SourceRef,
+    },
+
+    #[error("E013", "invalid float")]
+    #[label("{error_msg}", error_span)]
+    InvalidFloat {
+        detailed_report: String,
+        error_msg: String,
+        error_span: SourceRef,
+    },
+
+    #[error("E014", "invalid escape sequence")]
+    #[label("{error_msg}", error_span)]
+    InvalidEscapeSequence {
+        detailed_report: String,
+        error_msg: String,
+        error_span: SourceRef,
+    },
+
+    #[error("E016", "invalid regexp modifier `{modifier}`")]
+    #[label("invalid modifier", error_span)]
+    InvalidRegexpModifier {
+        detailed_report: String,
+        modifier: String,
+        error_span: SourceRef,
+    },
+
+    #[error("E015", "unexpected escape sequence")]
+    #[label("escape sequences are not allowed in this string", error_span)]
+    UnexpectedEscapeSequence { detailed_report: String, error_span: SourceRef },
+
+    #[error("E017", "invalid UTF-8")]
+    #[label("invalid UTF-8 character", error_span)]
+    InvalidUTF8 { detailed_report: String, error_span: SourceRef },
+}
+
+impl CompileError {
+    pub(crate) fn from(
+        report_builder: &ReportBuilder,
+        err: ast::Error,
+    ) -> Self {
+        match err {
+            ast::Error::SyntaxError { message, span } => {
+                CompileError::syntax_error(
+                    report_builder,
+                    message,
+                    span.into(),
+                )
+            }
+            ast::Error::InvalidInteger { message, span } => {
+                CompileError::invalid_integer(
+                    report_builder,
+                    message,
+                    span.into(),
+                )
+            }
+            ast::Error::InvalidFloat { message, span } => {
+                CompileError::invalid_float(
+                    report_builder,
+                    message,
+                    span.into(),
+                )
+            }
+            ast::Error::InvalidRegexpModifier { message, span } => {
+                CompileError::invalid_regexp_modifier(
+                    report_builder,
+                    message,
+                    span.into(),
+                )
+            }
+            ast::Error::InvalidEscapeSequence { message, span } => {
+                CompileError::invalid_escape_sequence(
+                    report_builder,
+                    message,
+                    span.into(),
+                )
+            }
+            ast::Error::UnexpectedEscapeSequence(span) => {
+                CompileError::unexpected_escape_sequence(
+                    report_builder,
+                    span.into(),
+                )
+            }
+            ast::Error::InvalidUTF8(span) => {
+                CompileError::invalid_utf_8(report_builder, span.into())
+            }
+        }
+    }
 }
 
 impl CompileError {

lib/src/compiler/ir/ast2ir.rs

@@ -203,14 +203,14 @@ pub(in crate::compiler) fn text_pattern_from_ast<'src>(
         (1, None)
     };
 
-    let text: BString = pattern.text.as_ref().into();
+    let text: BString = pattern.text.value.as_ref().into();
 
     if text.len() < min_len {
         return Err(Box::new(CompileError::invalid_pattern(
             ctx.report_builder,
             pattern.identifier.name.to_string(),
             "this pattern is too short".to_string(),
-            pattern.span().into(),
+            pattern.text.span().into(),
             note,
         )));
     }
@@ -253,7 +253,7 @@ pub(in crate::compiler) fn hex_pattern_from_ast<'src>(
         in_use: false,
         pattern: Pattern::Regexp(RegexpPattern {
             flags: PatternFlagSet::from(PatternFlags::Ascii),
-            hir: re::hir::Hir::from(hex_pattern_hir_from_ast(ctx, pattern)),
+            hir: re::hir::Hir::from(hex_pattern_hir_from_ast(ctx, pattern)?),
             anchored_at: None,
         }),
     })
@@ -313,7 +313,7 @@ pub(in crate::compiler) fn regexp_pattern_from_ast<'src>(
             Warning::redundant_case_modifier(
                 ctx.report_builder,
                 pattern.modifiers.nocase().unwrap().span().into(),
-                pattern.span().subspan(i_pos, i_pos + 1).into(),
+                pattern.regexp.span().subspan(i_pos, i_pos + 1).into(),
             )
         });
     }
@@ -1117,7 +1117,7 @@ fn range_from_ast(
 
     // If both the lower and upper bounds are known at compile time, make sure
     // that lower_bound <= upper_bound. If they are not know (because they are
-    // variables, for example) we can't raise an error at compile time but it
+    // variables, for example) we can't raise an error at compile time, but it
     // will be handled at scan time.
     if let (
         TypeValue::Integer(Value::Const(lower_bound)),
@@ -1249,10 +1249,17 @@ fn pattern_set_from_ast(
                     return Err(Box::new(CompileError::empty_pattern_set(
                         ctx.report_builder,
                         item.span().into(),
-                        Some(format!(
-                            "`{}` doesn't match any pattern identifier",
-                            item.identifier,
-                        )),
+                        Some(if item.wildcard {
+                            format!(
+                                "`{}*` doesn't match any pattern identifier",
+                                item.identifier,
+                            )
+                        } else {
+                            format!(
+                                "`{}` doesn't match any pattern identifier",
+                                item.identifier,
+                            )
+                        }),
                     )));
                 }
             }
@@ -1449,12 +1456,19 @@ fn re_error_to_compile_error(
             CompileError::invalid_regexp(
                 report_builder,
                 msg,
-                // the error span is relative to the start of the regexp, not to
+                // The error span is relative to the start of the regexp, not to
                 // the start of the source file, here we make it relative to the
-                // source file.
+                // source file. Notice that the resulting span must be shifted one
+                // character to the left, because the error span doesn't include
+                // the opening slash (/) but the regexp span does.
+                //
+                // /someregexp/
+                //  ^ this is position 0 for error spans
+                // ^ this is where the regexp starts according to the regexp span
                 regexp
                     .span()
                     .subspan(span.start.offset, span.end.offset)
+                    .offset(1)
                     .into(),
                 note,
             )
@@ -1471,10 +1485,12 @@ fn re_error_to_compile_error(
             regexp
                 .span()
                 .subspan(span_1.start.offset, span_1.end.offset)
+                .offset(1)
                 .into(),
             regexp
                 .span()
                 .subspan(span_2.start.offset, span_2.end.offset)
+                .offset(1)
                 .into(),
         ),
     }
@@ -1506,7 +1522,7 @@ pub(in crate::compiler) fn warn_if_not_bool(
             Warning::non_boolean_as_boolean(
                 ctx.report_builder,
                 ty.to_string(),
-                span.clone().into(),
+                span.into(),
                 note,
             )
         });