diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 7b6847200b43..ade20191e25c 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -881,7 +881,12 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, if (ret) goto defer; - range_tree_set(&arena->rt, pgoff, page_cnt); + ret = range_tree_set(&arena->rt, pgoff, page_cnt); + if (ret) { + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + bpf_map_memcg_exit(old_memcg, new_memcg); + return; + } init_llist_head(&free_pages); cdata.arena = arena; @@ -1004,11 +1009,17 @@ static void arena_free_worker(struct work_struct *work) kaddr = arena_vm_start + s->uaddr; pgoff = compute_pgoff(arena, s->uaddr); + /* Mark the range as free before clearing PTEs so that + * if the range tree update fails we leave the PTEs + * intact, avoiding an unrecoverable state where pages + * are freed but the arena free tree does not track them. + */ + if (range_tree_set(&arena->rt, pgoff, page_cnt)) + continue; + /* clear ptes and collect pages in free_pages llist */ apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT, apply_range_clear_cb, &cdata); - - range_tree_set(&arena->rt, pgoff, page_cnt); } raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); diff --git a/kernel/bpf/range_tree.c b/kernel/bpf/range_tree.c index 2f28886f3ff7..45560e9f0d07 100644 --- a/kernel/bpf/range_tree.c +++ b/kernel/bpf/range_tree.c @@ -143,16 +143,21 @@ int range_tree_clear(struct range_tree *rt, u32 start, u32 len) if (rn->rn_start < start && rn->rn_last > last) { u32 old_last = rn->rn_last; + /* Pre-allocate the right-half node before modifying + * the tree. If allocation fails we return -ENOMEM + * without altering the range tree. + */ + new_rn = kmalloc_nolock(sizeof(struct range_node), + __GFP_ACCOUNT, NUMA_NO_NODE); + if (!new_rn) + return -ENOMEM; + /* Overlaps with the entire clearing range */ range_it_remove(rn, rt); rn->rn_last = start - 1; range_it_insert(rn, rt); - /* Add a range */ - new_rn = kmalloc_nolock(sizeof(struct range_node), __GFP_ACCOUNT, - NUMA_NO_NODE); - if (!new_rn) - return -ENOMEM; + /* Add right-half range */ new_rn->rn_start = last + 1; new_rn->rn_last = old_last; range_it_insert(new_rn, rt); @@ -193,6 +198,7 @@ int is_range_tree_set(struct range_tree *rt, u32 start, u32 len) int range_tree_set(struct range_tree *rt, u32 start, u32 len) { u32 last = start + len - 1; + struct range_node *new_rn = NULL; struct range_node *right; struct range_node *left; int err; @@ -202,20 +208,40 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len) if (left && left->rn_start <= start && left->rn_last >= last) return 0; + /* + * A new node is needed only when the range has no adjacent free + * range on either side. This is known before clearing: any range + * covering start - 1 or last + 1 survives the clear as an adjacent + * piece. Allocate only in that case, before modifying the tree, so + * a failure leaves the range tree unmodified + */ + left = range_it_iter_first(rt, start - 1, start - 1); + right = range_it_iter_first(rt, last + 1, last + 1); + if (!left && !right) { + new_rn = kmalloc_nolock(sizeof(struct range_node), + __GFP_ACCOUNT, NUMA_NO_NODE); + if (!new_rn) + return -ENOMEM; + } + /* Clear out everything in the range we want to set. */ err = range_tree_clear(rt, start, len); if (err) - return err; + goto out_free_new; /* Do we have a left-adjacent range ? */ left = range_it_iter_first(rt, start - 1, start - 1); - if (left && left->rn_last + 1 != start) - return -EFAULT; + if (left && left->rn_last + 1 != start) { + err = -EFAULT; + goto out_free_new; + } /* Do we have a right-adjacent range ? */ right = range_it_iter_first(rt, last + 1, last + 1); - if (right && right->rn_start != last + 1) - return -EFAULT; + if (right && right->rn_start != last + 1) { + err = -EFAULT; + goto out_free_new; + } if (left && right) { /* Combine left and right adjacent ranges */ @@ -235,14 +261,16 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len) right->rn_start = start; range_it_insert(right, rt); } else { - left = kmalloc_nolock(sizeof(struct range_node), __GFP_ACCOUNT, NUMA_NO_NODE); - if (!left) - return -ENOMEM; - left->rn_start = start; - left->rn_last = last; - range_it_insert(left, rt); + /* No adjacent ranges; use the pre-allocated node */ + new_rn->rn_start = start; + new_rn->rn_last = last; + range_it_insert(new_rn, rt); } return 0; + +out_free_new: + kfree_nolock(new_rn); + return err; } void range_tree_destroy(struct range_tree *rt)