Skip to content

internal: Add offset param to token descending API #15466

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 16, 2023
Merged

internal: Add offset param to token descending API #15466

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 3 additions & 11 deletions crates/hir-expand/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ use either::Either;
use syntax::{
algo::{self, skip_trivia_token},
ast::{self, AstNode, HasDocComments},
AstPtr, Direction, SyntaxNode, SyntaxNodePtr, SyntaxToken,
AstPtr, Direction, SyntaxNode, SyntaxNodePtr, SyntaxToken, TextSize,
};

use crate::{
Expand Down Expand Up @@ -642,6 +642,8 @@ impl ExpansionInfo {
db: &dyn db::ExpandDatabase,
item: Option<ast::Item>,
token: InFile<&SyntaxToken>,
// FIXME: use this for range mapping, so that we can resolve inline format args
_relative_token_offset: Option<TextSize>,
) -> Option<impl Iterator<Item = InFile<SyntaxToken>> + '_> {
assert_eq!(token.file_id, self.arg.file_id);
let token_id_in_attr_input = if let Some(item) = item {
Expand Down Expand Up @@ -1051,16 +1053,6 @@ impl InFile<SyntaxToken> {
}
}
}

pub fn ancestors_with_macros(
self,
db: &dyn db::ExpandDatabase,
) -> impl Iterator<Item = InFile<SyntaxNode>> + '_ {
self.value.parent().into_iter().flat_map({
let file_id = self.file_id;
move |parent| InFile::new(file_id, &parent).ancestors_with_macros(db)
})
}
}

#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
Expand Down
81 changes: 60 additions & 21 deletions crates/hir/src/semantics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,8 @@ impl<'db, DB: HirDatabase> Semantics<'db, DB> {
self.imp.is_derive_annotated(item)
}

/// Expand the macro call with a different token tree, mapping the `token_to_map` down into the
/// expansion. `token_to_map` should be a token from the `speculative args` node.
pub fn speculative_expand(
&self,
actual_macro_call: &ast::MacroCall,
Expand All @@ -179,6 +181,8 @@ impl<'db, DB: HirDatabase> Semantics<'db, DB> {
self.imp.speculative_expand(actual_macro_call, speculative_args, token_to_map)
}

/// Expand the macro call with a different item as the input, mapping the `token_to_map` down into the
/// expansion. `token_to_map` should be a token from the `speculative args` node.
pub fn speculative_expand_attr_macro(
&self,
actual_macro_call: &ast::Item,
Expand All @@ -201,14 +205,22 @@ impl<'db, DB: HirDatabase> Semantics<'db, DB> {
)
}

/// Descend the token into macrocalls to its first mapped counterpart.
pub fn descend_into_macros_single(&self, token: SyntaxToken) -> SyntaxToken {
self.imp.descend_into_macros_single(token)
/// Descend the token into its macro call if it is part of one, returning the token in the
/// expansion that it is associated with. If `offset` points into the token's range, it will
/// be considered for the mapping in case of inline format args.
pub fn descend_into_macros_single(&self, token: SyntaxToken, offset: TextSize) -> SyntaxToken {
self.imp.descend_into_macros_single(token, offset)
}

/// Descend the token into macrocalls to all its mapped counterparts.
pub fn descend_into_macros(&self, token: SyntaxToken) -> SmallVec<[SyntaxToken; 1]> {
self.imp.descend_into_macros(token)
/// Descend the token into its macro call if it is part of one, returning the tokens in the
/// expansion that it is associated with. If `offset` points into the token's range, it will
/// be considered for the mapping in case of inline format args.
pub fn descend_into_macros(
&self,
token: SyntaxToken,
offset: TextSize,
) -> SmallVec<[SyntaxToken; 1]> {
self.imp.descend_into_macros(token, offset)
}

/// Descend the token into macrocalls to all its mapped counterparts that have the same text as the input token.
Expand All @@ -217,12 +229,17 @@ impl<'db, DB: HirDatabase> Semantics<'db, DB> {
pub fn descend_into_macros_with_same_text(
&self,
token: SyntaxToken,
offset: TextSize,
) -> SmallVec<[SyntaxToken; 1]> {
self.imp.descend_into_macros_with_same_text(token)
self.imp.descend_into_macros_with_same_text(token, offset)
}

pub fn descend_into_macros_with_kind_preference(&self, token: SyntaxToken) -> SyntaxToken {
self.imp.descend_into_macros_with_kind_preference(token)
pub fn descend_into_macros_with_kind_preference(
&self,
token: SyntaxToken,
offset: TextSize,
) -> SyntaxToken {
self.imp.descend_into_macros_with_kind_preference(token, offset)
}

/// Maps a node down by mapping its first and last token down.
Expand Down Expand Up @@ -665,7 +682,7 @@ impl<'db> SemanticsImpl<'db> {
};

if first == last {
self.descend_into_macros_impl(first, &mut |InFile { value, .. }| {
self.descend_into_macros_impl(first, 0.into(), &mut |InFile { value, .. }| {
if let Some(node) = value.parent_ancestors().find_map(N::cast) {
res.push(node)
}
Expand All @@ -674,14 +691,15 @@ impl<'db> SemanticsImpl<'db> {
} else {
// Descend first and last token, then zip them to look for the node they belong to
let mut scratch: SmallVec<[_; 1]> = smallvec![];
self.descend_into_macros_impl(first, &mut |token| {
self.descend_into_macros_impl(first, 0.into(), &mut |token| {
scratch.push(token);
false
});

let mut scratch = scratch.into_iter();
self.descend_into_macros_impl(
last,
0.into(),
&mut |InFile { value: last, file_id: last_fid }| {
if let Some(InFile { value: first, file_id: first_fid }) = scratch.next() {
if first_fid == last_fid {
Expand All @@ -705,19 +723,27 @@ impl<'db> SemanticsImpl<'db> {
res
}

fn descend_into_macros(&self, token: SyntaxToken) -> SmallVec<[SyntaxToken; 1]> {
fn descend_into_macros(
&self,
token: SyntaxToken,
offset: TextSize,
) -> SmallVec<[SyntaxToken; 1]> {
let mut res = smallvec![];
self.descend_into_macros_impl(token, &mut |InFile { value, .. }| {
self.descend_into_macros_impl(token, offset, &mut |InFile { value, .. }| {
res.push(value);
false
});
res
}

fn descend_into_macros_with_same_text(&self, token: SyntaxToken) -> SmallVec<[SyntaxToken; 1]> {
fn descend_into_macros_with_same_text(
&self,
token: SyntaxToken,
offset: TextSize,
) -> SmallVec<[SyntaxToken; 1]> {
let text = token.text();
let mut res = smallvec![];
self.descend_into_macros_impl(token.clone(), &mut |InFile { value, .. }| {
self.descend_into_macros_impl(token.clone(), offset, &mut |InFile { value, .. }| {
if value.text() == text {
res.push(value);
}
Expand All @@ -729,7 +755,11 @@ impl<'db> SemanticsImpl<'db> {
res
}

fn descend_into_macros_with_kind_preference(&self, token: SyntaxToken) -> SyntaxToken {
fn descend_into_macros_with_kind_preference(
&self,
token: SyntaxToken,
offset: TextSize,
) -> SyntaxToken {
let fetch_kind = |token: &SyntaxToken| match token.parent() {
Some(node) => match node.kind() {
kind @ (SyntaxKind::NAME | SyntaxKind::NAME_REF) => {
Expand All @@ -741,7 +771,7 @@ impl<'db> SemanticsImpl<'db> {
};
let preferred_kind = fetch_kind(&token);
let mut res = None;
self.descend_into_macros_impl(token.clone(), &mut |InFile { value, .. }| {
self.descend_into_macros_impl(token.clone(), offset, &mut |InFile { value, .. }| {
if fetch_kind(&value) == preferred_kind {
res = Some(value);
true
Expand All @@ -755,9 +785,9 @@ impl<'db> SemanticsImpl<'db> {
res.unwrap_or(token)
}

fn descend_into_macros_single(&self, token: SyntaxToken) -> SyntaxToken {
fn descend_into_macros_single(&self, token: SyntaxToken, offset: TextSize) -> SyntaxToken {
let mut res = token.clone();
self.descend_into_macros_impl(token, &mut |InFile { value, .. }| {
self.descend_into_macros_impl(token, offset, &mut |InFile { value, .. }| {
res = value;
true
});
Expand All @@ -767,9 +797,13 @@ impl<'db> SemanticsImpl<'db> {
fn descend_into_macros_impl(
&self,
token: SyntaxToken,
// FIXME: We might want this to be Option<TextSize> to be able to opt out of subrange
// mapping, specifically for node downmapping
offset: TextSize,
f: &mut dyn FnMut(InFile<SyntaxToken>) -> bool,
) {
let _p = profile::span("descend_into_macros");
let relative_token_offset = token.text_range().start().checked_sub(offset);
let parent = match token.parent() {
Some(it) => it,
None => return,
Expand All @@ -796,7 +830,12 @@ impl<'db> SemanticsImpl<'db> {
self.cache(value, file_id);
}

let mapped_tokens = expansion_info.map_token_down(self.db.upcast(), item, token)?;
let mapped_tokens = expansion_info.map_token_down(
self.db.upcast(),
item,
token,
relative_token_offset,
)?;
let len = stack.len();

// requeue the tokens we got from mapping our current token down
Expand Down Expand Up @@ -943,7 +982,7 @@ impl<'db> SemanticsImpl<'db> {
offset: TextSize,
) -> impl Iterator<Item = impl Iterator<Item = SyntaxNode> + '_> + '_ {
node.token_at_offset(offset)
.map(move |token| self.descend_into_macros(token))
.map(move |token| self.descend_into_macros(token, offset))
.map(|descendants| {
descendants.into_iter().map(move |it| self.token_ancestors_with_macros(it))
})
Expand Down
2 changes: 1 addition & 1 deletion crates/ide-assists/src/handlers/extract_expressions_from_format_string.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ pub(crate) fn extract_expressions_from_format_string(
let tt = fmt_string.syntax().parent().and_then(ast::TokenTree::cast)?;

let expanded_t = ast::String::cast(
ctx.sema.descend_into_macros_with_kind_preference(fmt_string.syntax().clone()),
ctx.sema.descend_into_macros_with_kind_preference(fmt_string.syntax().clone(), 0.into()),
)?;
if !is_format_string(&expanded_t) {
return None;
Expand Down
2 changes: 1 addition & 1 deletion crates/ide-assists/src/handlers/extract_function.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -750,7 +750,7 @@ impl FunctionBody {
.descendants_with_tokens()
.filter_map(SyntaxElement::into_token)
.filter(|it| matches!(it.kind(), SyntaxKind::IDENT | T![self]))
.flat_map(|t| sema.descend_into_macros(t))
.flat_map(|t| sema.descend_into_macros(t, 0.into()))
.for_each(|t| add_name_if_local(t.parent().and_then(ast::NameRef::cast)));
}
}
Expand Down
2 changes: 1 addition & 1 deletion crates/ide-db/src/helpers.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ pub fn get_definition(
sema: &Semantics<'_, RootDatabase>,
token: SyntaxToken,
) -> Option<Definition> {
for token in sema.descend_into_macros(token) {
for token in sema.descend_into_macros(token, 0.into()) {
let def = IdentClass::classify_token(sema, &token).map(IdentClass::definitions_no_ops);
if let Some(&[x]) = def.as_deref() {
return Some(x);
Expand Down
4 changes: 2 additions & 2 deletions crates/ide-db/src/search.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -456,14 +456,14 @@ impl<'a> FindUsages<'a> {
it.text().trim_start_matches("r#") == name
})
.into_iter()
.flat_map(|token| {
.flat_map(move |token| {
// FIXME: There should be optimization potential here
// Currently we try to descend everything we find which
// means we call `Semantics::descend_into_macros` on
// every textual hit. That function is notoriously
// expensive even for things that do not get down mapped
// into macros.
sema.descend_into_macros(token).into_iter().filter_map(|it| it.parent())
sema.descend_into_macros(token, offset).into_iter().filter_map(|it| it.parent())
})
};

Expand Down
10 changes: 6 additions & 4 deletions crates/ide/src/call_hierarchy.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,18 +74,20 @@ pub(crate) fn incoming_calls(
Some(calls.into_items())
}

pub(crate) fn outgoing_calls(db: &RootDatabase, position: FilePosition) -> Option<Vec<CallItem>> {
pub(crate) fn outgoing_calls(
db: &RootDatabase,
FilePosition { file_id, offset }: FilePosition,
) -> Option<Vec<CallItem>> {
let sema = Semantics::new(db);
let file_id = position.file_id;
let file = sema.parse(file_id);
let file = file.syntax();
let token = pick_best_token(file.token_at_offset(position.offset), |kind| match kind {
let token = pick_best_token(file.token_at_offset(offset), |kind| match kind {
IDENT => 1,
_ => 0,
})?;
let mut calls = CallLocations::default();

sema.descend_into_macros(token)
sema.descend_into_macros(token, offset)
.into_iter()
.filter_map(|it| it.parent_ancestors().nth(1).and_then(ast::Item::cast))
.filter_map(|item| match item {
Expand Down
10 changes: 5 additions & 5 deletions crates/ide/src/doc_links.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,19 +131,19 @@ pub(crate) fn remove_links(markdown: &str) -> String {
// |===
pub(crate) fn external_docs(
db: &RootDatabase,
position: &FilePosition,
FilePosition { file_id, offset }: FilePosition,
target_dir: Option<&OsStr>,
sysroot: Option<&OsStr>,
) -> Option<DocumentationLinks> {
let sema = &Semantics::new(db);
let file = sema.parse(position.file_id).syntax().clone();
let token = pick_best_token(file.token_at_offset(position.offset), |kind| match kind {
let file = sema.parse(file_id).syntax().clone();
let token = pick_best_token(file.token_at_offset(offset), |kind| match kind {
IDENT | INT_NUMBER | T![self] => 3,
T!['('] | T![')'] => 2,
kind if kind.is_trivia() => 0,
_ => 1,
})?;
let token = sema.descend_into_macros_single(token);
let token = sema.descend_into_macros_single(token, offset);

let node = token.parent()?;
let definition = match_ast! {
Expand Down Expand Up @@ -285,7 +285,7 @@ impl DocCommentToken {
let original_start = doc_token.text_range().start();
let relative_comment_offset = offset - original_start - prefix_len;

sema.descend_into_macros(doc_token).into_iter().find_map(|t| {
sema.descend_into_macros(doc_token, offset).into_iter().find_map(|t| {
let (node, descended_prefix_len) = match_ast! {
match t {
ast::Comment(comment) => (t.parent()?, TextSize::try_from(comment.prefix().len()).ok()?),
Expand Down
47 changes: 26 additions & 21 deletions crates/ide/src/expand_macro.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,28 +40,33 @@ pub(crate) fn expand_macro(db: &RootDatabase, position: FilePosition) -> Option<
// struct Bar;
// ```

let derive = sema.descend_into_macros(tok.clone()).into_iter().find_map(|descended| {
let hir_file = sema.hir_file_for(&descended.parent()?);
if !hir_file.is_derive_attr_pseudo_expansion(db) {
return None;
}
let derive =
sema.descend_into_macros(tok.clone(), 0.into()).into_iter().find_map(|descended| {
let hir_file = sema.hir_file_for(&descended.parent()?);
if !hir_file.is_derive_attr_pseudo_expansion(db) {
return None;
}

let name = descended.parent_ancestors().filter_map(ast::Path::cast).last()?.to_string();
// up map out of the #[derive] expansion
let token = hir::InFile::new(hir_file, descended).upmap(db)?.value;
let attr = token.parent_ancestors().find_map(ast::Attr::cast)?;
let expansions = sema.expand_derive_macro(&attr)?;
let idx = attr
.token_tree()?
.token_trees_and_tokens()
.filter_map(NodeOrToken::into_token)
.take_while(|it| it != &token)
.filter(|it| it.kind() == T![,])
.count();
let expansion =
format(db, SyntaxKind::MACRO_ITEMS, position.file_id, expansions.get(idx).cloned()?);
Some(ExpandedMacro { name, expansion })
});
let name = descended.parent_ancestors().filter_map(ast::Path::cast).last()?.to_string();
// up map out of the #[derive] expansion
let token = hir::InFile::new(hir_file, descended).upmap(db)?.value;
let attr = token.parent_ancestors().find_map(ast::Attr::cast)?;
let expansions = sema.expand_derive_macro(&attr)?;
let idx = attr
.token_tree()?
.token_trees_and_tokens()
.filter_map(NodeOrToken::into_token)
.take_while(|it| it != &token)
.filter(|it| it.kind() == T![,])
.count();
let expansion = format(
db,
SyntaxKind::MACRO_ITEMS,
position.file_id,
expansions.get(idx).cloned()?,
);
Some(ExpandedMacro { name, expansion })
});

if derive.is_some() {
return derive;
Expand Down
Loading