diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 78068ae8f28a..6e879490c811 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -56,6 +56,8 @@ #include #include #include +#include +#include #include "workqueue_internal.h" @@ -8473,3 +8475,557 @@ static int __init workqueue_unbound_cpus_setup(char *str) return 1; } __setup("workqueue.unbound_cpus=", workqueue_unbound_cpus_setup); + +#ifdef CONFIG_BPF_SYSCALL + +/* + * BPF open-coded iterator over all workqueues. + * + * Walks the global @workqueues list, letting BPF programs read live workqueue + * state. + * + * The caller must hold bpf_rcu_read_lock() across new()->next()->destroy(). + */ + +/* Sentinel for "new() done, next() not yet called" vs "iteration ended". */ +#define BPF_WQ_ITER_START ((void *)1L) + +struct bpf_iter_workqueue { + __u64 __opaque[1]; +} __aligned(8); + +struct bpf_iter_workqueue_kern { + struct workqueue_struct *pos; +} __aligned(8); + +struct bpf_iter_worker_pool { + __u64 __opaque[1]; +} __aligned(8); + +struct bpf_iter_worker_pool_kern { + int id; /* next worker_pool_idr id to search from */ +} __aligned(8); + +__bpf_kfunc_start_defs(); + +/** + * bpf_iter_workqueue_new() - Initialize a workqueue iterator + * @it: The new bpf_iter_workqueue to initialize + * + * Iterates every workqueue on the global @workqueues list. Must be used inside + * a bpf_rcu_read_lock() region (KF_RCU_PROTECTED). + */ +__bpf_kfunc int bpf_iter_workqueue_new(struct bpf_iter_workqueue *it) +{ + struct bpf_iter_workqueue_kern *kit = (void *)it; + + BUILD_BUG_ON(sizeof(struct bpf_iter_workqueue_kern) > + sizeof(struct bpf_iter_workqueue)); + BUILD_BUG_ON(__alignof__(struct bpf_iter_workqueue_kern) != + __alignof__(struct bpf_iter_workqueue)); + + kit->pos = BPF_WQ_ITER_START; + return 0; +} + +/** + * bpf_iter_workqueue_next() - Get the next workqueue + * @it: The bpf_iter_workqueue + * + * Returns a pointer to the next struct workqueue_struct, or NULL when done. + */ +__bpf_kfunc struct workqueue_struct * +bpf_iter_workqueue_next(struct bpf_iter_workqueue *it) +{ + struct bpf_iter_workqueue_kern *kit = (void *)it; + + if (!kit->pos) /* already reached the end */ + return NULL; + + if (kit->pos == BPF_WQ_ITER_START) + kit->pos = list_first_or_null_rcu(&workqueues, + struct workqueue_struct, list); + else + kit->pos = list_next_or_null_rcu(&workqueues, &kit->pos->list, + struct workqueue_struct, list); + return kit->pos; +} + +/** + * bpf_iter_workqueue_destroy() - Destroy a workqueue iterator + * @it: The bpf_iter_workqueue to destroy + */ +__bpf_kfunc void bpf_iter_workqueue_destroy(struct bpf_iter_workqueue *it) +{ + /* No references taken; RCU is held by the caller. */ +} + +/** + * bpf_iter_worker_pool_new() - Initialize a worker_pool iterator + * @it: The new bpf_iter_worker_pool to initialize + * + * Iterates every worker_pool in the system (worker_pool_idr) in ascending + * pool-id order. + * The caller should hold RCU read lock. + */ +__bpf_kfunc int bpf_iter_worker_pool_new(struct bpf_iter_worker_pool *it) +{ + struct bpf_iter_worker_pool_kern *kit = (void *)it; + + BUILD_BUG_ON(sizeof(struct bpf_iter_worker_pool_kern) > + sizeof(struct bpf_iter_worker_pool)); + BUILD_BUG_ON(__alignof__(struct bpf_iter_worker_pool_kern) != + __alignof__(struct bpf_iter_worker_pool)); + + kit->id = 0; + return 0; +} + +/** + * bpf_iter_worker_pool_next() - Get the next worker_pool + * @it: The bpf_iter_worker_pool + * + * Returns a pointer to the next struct worker_pool, or NULL when done. + */ +__bpf_kfunc struct worker_pool * +bpf_iter_worker_pool_next(struct bpf_iter_worker_pool *it) +{ + struct bpf_iter_worker_pool_kern *kit = (void *)it; + struct worker_pool *pool; + + /* idr_get_next() is RCU-safe; see the for_each_pool() lock legend. */ + pool = idr_get_next(&worker_pool_idr, &kit->id); + if (pool) + kit->id++; /* advance past this pool for the next call */ + return pool; +} + +/** + * bpf_iter_worker_pool_destroy() - Destroy a worker_pool iterator + * @it: The bpf_iter_worker_pool to destroy + */ +__bpf_kfunc void bpf_iter_worker_pool_destroy(struct bpf_iter_worker_pool *it) +{ + /* Nothing to be done; RCU is held by the caller. */ +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(workqueue_iter_kfunc_ids) +BTF_ID_FLAGS(func, bpf_iter_workqueue_new, KF_ITER_NEW | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, bpf_iter_workqueue_next, KF_ITER_NEXT | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_iter_workqueue_destroy, KF_ITER_DESTROY) +BTF_ID_FLAGS(func, bpf_iter_worker_pool_new, KF_ITER_NEW | KF_RCU_PROTECTED) +BTF_ID_FLAGS(func, bpf_iter_worker_pool_next, KF_ITER_NEXT | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_iter_worker_pool_destroy, KF_ITER_DESTROY) +BTF_KFUNCS_END(workqueue_iter_kfunc_ids) + +static const struct btf_kfunc_id_set workqueue_iter_kfunc_set = { + .owner = THIS_MODULE, + .set = &workqueue_iter_kfunc_ids, +}; + +/* + * seq_file form of the workqueue iterator. It reuses the open-coded + * bpf_iter_workqueue_next(). + * + * Since the open-coded next() is KF_RCU_PROTECTED, the seq path supplies + * the RCU section itself. + * seq_start() takes rcu_read_lock() and holds it across the whole read chunk + * (start..stop). Position is re-derived from *pos on each start, so nothing + * is dereferenced across the rcu gap between chunks. The attached BPF program + * must be non-sleepable. + */ +union workqueue_iter_priv { + struct bpf_iter_workqueue it; + struct bpf_iter_workqueue_kern kit; +}; + +struct bpf_iter__workqueue { + __bpf_md_ptr(struct bpf_iter_meta *, meta); + __bpf_md_ptr(struct workqueue_struct *, wq); +}; + +static void *workqueue_iter_seq_start(struct seq_file *seq, loff_t *pos) +{ + union workqueue_iter_priv *p = seq->private; + struct workqueue_struct *wq; + loff_t cnt = 0; + + rcu_read_lock(); /* held until seq_stop() */ + list_for_each_entry_rcu(wq, &workqueues, list) { + if (cnt == *pos) { + p->kit.pos = wq; + return wq; + } + cnt++; + } + p->kit.pos = NULL; + return NULL; +} + +static void *workqueue_iter_seq_next(struct seq_file *seq, void *v, loff_t *pos) +{ + union workqueue_iter_priv *p = seq->private; + + ++*pos; + return bpf_iter_workqueue_next(&p->it); +} + +static int workqueue_iter_seq_show(struct seq_file *seq, void *v) +{ + struct bpf_iter__workqueue ctx; + struct bpf_iter_meta meta; + struct bpf_prog *prog; + + meta.seq = seq; + prog = bpf_iter_get_info(&meta, false); + if (!prog) + return 0; + ctx.meta = &meta; + ctx.wq = v; + return bpf_iter_run_prog(prog, &ctx); +} + +static void workqueue_iter_seq_stop(struct seq_file *seq, void *v) +{ + struct bpf_iter__workqueue ctx; + struct bpf_iter_meta meta; + struct bpf_prog *prog; + + if (!v) { + meta.seq = seq; + prog = bpf_iter_get_info(&meta, true); + if (prog) { + ctx.meta = &meta; + ctx.wq = NULL; + bpf_iter_run_prog(prog, &ctx); + } + } + rcu_read_unlock(); /* paired with seq_start() */ +} + +static const struct seq_operations workqueue_iter_seq_ops = { + .start = workqueue_iter_seq_start, + .next = workqueue_iter_seq_next, + .stop = workqueue_iter_seq_stop, + .show = workqueue_iter_seq_show, +}; + +DEFINE_BPF_ITER_FUNC(workqueue, struct bpf_iter_meta *meta, + struct workqueue_struct *wq) + +static const struct bpf_iter_seq_info workqueue_iter_seq_info = { + .seq_ops = &workqueue_iter_seq_ops, + .seq_priv_size = sizeof(union workqueue_iter_priv), +}; + +BTF_ID_LIST_SINGLE(workqueue_btf_id, struct, workqueue_struct) + +static struct bpf_iter_reg workqueue_iter_reg_info = { + .target = "workqueue", + .ctx_arg_info_size = 1, + .ctx_arg_info = { + { offsetof(struct bpf_iter__workqueue, wq), + PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED }, + }, + .seq_info = &workqueue_iter_seq_info, +}; + +/* + * seq_file form of the worker_pool iterator; same RCU-per-chunk scheme as the + * workqueue one, reusing the open-coded bpf_iter_worker_pool_next(). + */ +union worker_pool_iter_priv { + struct bpf_iter_worker_pool it; + struct bpf_iter_worker_pool_kern kit; +}; + +struct bpf_iter__worker_pool { + __bpf_md_ptr(struct bpf_iter_meta *, meta); + __bpf_md_ptr(struct worker_pool *, pool); +}; + +static void *worker_pool_iter_seq_start(struct seq_file *seq, loff_t *pos) +{ + union worker_pool_iter_priv *p = seq->private; + struct worker_pool *pool; + loff_t cnt = 0; + int id = 0; + + rcu_read_lock(); + while ((pool = idr_get_next(&worker_pool_idr, &id))) { + if (cnt == *pos) { + p->kit.id = id + 1; + return pool; + } + cnt++; + id++; + } + return NULL; +} + +static void *worker_pool_iter_seq_next(struct seq_file *seq, void *v, loff_t *pos) +{ + union worker_pool_iter_priv *p = seq->private; + + ++*pos; + return bpf_iter_worker_pool_next(&p->it); +} + +static int worker_pool_iter_seq_show(struct seq_file *seq, void *v) +{ + struct bpf_iter__worker_pool ctx; + struct bpf_iter_meta meta; + struct bpf_prog *prog; + + meta.seq = seq; + prog = bpf_iter_get_info(&meta, false); + if (!prog) + return 0; + ctx.meta = &meta; + ctx.pool = v; + return bpf_iter_run_prog(prog, &ctx); +} + +static void worker_pool_iter_seq_stop(struct seq_file *seq, void *v) +{ + struct bpf_iter__worker_pool ctx; + struct bpf_iter_meta meta; + struct bpf_prog *prog; + + if (!v) { + meta.seq = seq; + prog = bpf_iter_get_info(&meta, true); + if (prog) { + ctx.meta = &meta; + ctx.pool = NULL; + bpf_iter_run_prog(prog, &ctx); + } + } + rcu_read_unlock(); +} + +static const struct seq_operations worker_pool_iter_seq_ops = { + .start = worker_pool_iter_seq_start, + .next = worker_pool_iter_seq_next, + .stop = worker_pool_iter_seq_stop, + .show = worker_pool_iter_seq_show, +}; + +DEFINE_BPF_ITER_FUNC(worker_pool, struct bpf_iter_meta *meta, + struct worker_pool *pool) + +static const struct bpf_iter_seq_info worker_pool_iter_seq_info = { + .seq_ops = &worker_pool_iter_seq_ops, + .seq_priv_size = sizeof(union worker_pool_iter_priv), +}; + +BTF_ID_LIST_SINGLE(worker_pool_btf_id, struct, worker_pool) + +static struct bpf_iter_reg worker_pool_iter_reg_info = { + .target = "worker_pool", + .ctx_arg_info_size = 1, + .ctx_arg_info = { + { offsetof(struct bpf_iter__worker_pool, pool), + PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED }, + }, + .seq_info = &worker_pool_iter_seq_info, +}; + +/* + * seq-file BPF iterator over the pending work items of every worker_pool. + * + * pool->worklist is protected by pool->lock and work_struct has neither a + * refcount nor RCU-freeing, so a pending work cannot be handed to a suspended + * (open-coded) iterator safely. + * Instead, for each pool a bounded snapshot of its pending works is copied out + * while pool->lock is held, mirroring the printk_deferred section of + * show_one_worker_pool() and then the lock is dropped. + * The BPF program then runs over the stable snapshot with no lock held. + * + * Pools are visited in worker_pool_idr order (a stable resume key); each pool's + * snapshot lives in the persistent seq_private, so the walk resumes correctly + * across read() chunks. A pool with more than WQ_PENDING_SNAP_MAX pending works + * is truncated (best-effort, like the kernel's own worklist dump). + */ +#define WQ_PENDING_SNAP_MAX 128 + +/* + * One projected pending work item, as seen by the BPF program. Addresses only + * -- the live work_struct is not exposed (it may be freed after the lock). + */ +struct wq_pending_work_info { + __u64 pool_id; + __u64 work; + __u64 func; +}; + +struct wq_pending_iter_priv { + int next_pool; /* worker_pool_idr cursor for the next fill */ + unsigned int idx; /* position within snap[] */ + unsigned int count; /* valid entries in snap[] */ + struct wq_pending_work_info snap[WQ_PENDING_SNAP_MAX]; +}; + +struct bpf_iter__workqueue_pending_work { + __bpf_md_ptr(struct bpf_iter_meta *, meta); + __bpf_md_ptr(struct wq_pending_work_info *, info); +}; + +/* + * Snapshot the next non-empty pool's pending works into priv->snap[], advancing + * priv->next_pool past it. Returns true if a pool was captured, false at end. + * pool->lock is held only for the field copy; RCU keeps each pool alive. + */ +static bool wq_pending_fill(struct wq_pending_iter_priv *priv) +{ + struct worker_pool *pool; + struct work_struct *work; + int id; + + rcu_read_lock(); + for (id = priv->next_pool; (pool = idr_get_next(&worker_pool_idr, &id)); id++) { + unsigned int n = 0; + + raw_spin_lock_irq(&pool->lock); + list_for_each_entry(work, &pool->worklist, entry) { + if (n >= WQ_PENDING_SNAP_MAX) + break; + priv->snap[n].pool_id = pool->id; + priv->snap[n].work = (__u64)(unsigned long)work; + priv->snap[n].func = (__u64)(unsigned long)work->func; + n++; + } + raw_spin_unlock_irq(&pool->lock); + + if (n) { + priv->count = n; + priv->idx = 0; + priv->next_pool = id + 1; + rcu_read_unlock(); + return true; + } + } + rcu_read_unlock(); + return false; +} + +static void *wq_pending_seq_start(struct seq_file *seq, loff_t *pos) +{ + struct wq_pending_iter_priv *priv = seq->private; + + if (*pos == 0) { + priv->next_pool = 0; + priv->idx = 0; + priv->count = 0; + } + while (priv->idx >= priv->count) { + if (!wq_pending_fill(priv)) + return NULL; + } + return &priv->snap[priv->idx]; +} + +static void *wq_pending_seq_next(struct seq_file *seq, void *v, loff_t *pos) +{ + struct wq_pending_iter_priv *priv = seq->private; + + ++*pos; + priv->idx++; + while (priv->idx >= priv->count) { + if (!wq_pending_fill(priv)) + return NULL; + } + return &priv->snap[priv->idx]; +} + +static int wq_pending_seq_show(struct seq_file *seq, void *v) +{ + struct bpf_iter__workqueue_pending_work ctx; + struct bpf_iter_meta meta; + struct bpf_prog *prog; + + meta.seq = seq; + prog = bpf_iter_get_info(&meta, false); + if (!prog) + return 0; + ctx.meta = &meta; + ctx.info = v; + return bpf_iter_run_prog(prog, &ctx); +} + +static void wq_pending_seq_stop(struct seq_file *seq, void *v) +{ + struct bpf_iter__workqueue_pending_work ctx; + struct bpf_iter_meta meta; + struct bpf_prog *prog; + + if (v) + return; + meta.seq = seq; + prog = bpf_iter_get_info(&meta, true); + if (prog) { + ctx.meta = &meta; + ctx.info = NULL; + bpf_iter_run_prog(prog, &ctx); + } +} + +static const struct seq_operations wq_pending_seq_ops = { + .start = wq_pending_seq_start, + .next = wq_pending_seq_next, + .stop = wq_pending_seq_stop, + .show = wq_pending_seq_show, +}; + +DEFINE_BPF_ITER_FUNC(workqueue_pending_work, struct bpf_iter_meta *meta, + struct wq_pending_work_info *info) + +static const struct bpf_iter_seq_info wq_pending_seq_info = { + .seq_ops = &wq_pending_seq_ops, + .seq_priv_size = sizeof(struct wq_pending_iter_priv), +}; + +BTF_ID_LIST_SINGLE(wq_pending_work_info_btf_id, struct, wq_pending_work_info) + +static struct bpf_iter_reg wq_pending_reg_info = { + .target = "workqueue_pending_work", + .feature = BPF_ITER_RESCHED, + .ctx_arg_info_size = 1, + .ctx_arg_info = { + { offsetof(struct bpf_iter__workqueue_pending_work, info), + PTR_TO_BTF_ID_OR_NULL }, + }, + .seq_info = &wq_pending_seq_info, +}; + +static int __init bpf_workqueue_iter_init(void) +{ + int ret; + + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, + &workqueue_iter_kfunc_set); + if (ret) + return ret; + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, + &workqueue_iter_kfunc_set); + if (ret) + return ret; + + workqueue_iter_reg_info.ctx_arg_info[0].btf_id = workqueue_btf_id[0]; + ret = bpf_iter_reg_target(&workqueue_iter_reg_info); + if (ret) + return ret; + + worker_pool_iter_reg_info.ctx_arg_info[0].btf_id = worker_pool_btf_id[0]; + ret = bpf_iter_reg_target(&worker_pool_iter_reg_info); + if (ret) + return ret; + + wq_pending_reg_info.ctx_arg_info[0].btf_id = wq_pending_work_info_btf_id[0]; + return bpf_iter_reg_target(&wq_pending_reg_info); +} +late_initcall(bpf_workqueue_iter_init); + +#endif /* CONFIG_BPF_SYSCALL */ diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h index ff37ae5a113d..bd04d1c623fe 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -361,6 +361,16 @@ extern int bpf_iter_dmabuf_new(struct bpf_iter_dmabuf *it) __weak __ksym; extern struct dma_buf *bpf_iter_dmabuf_next(struct bpf_iter_dmabuf *it) __weak __ksym; extern void bpf_iter_dmabuf_destroy(struct bpf_iter_dmabuf *it) __weak __ksym; +struct bpf_iter_workqueue; +extern int bpf_iter_workqueue_new(struct bpf_iter_workqueue *it) __weak __ksym; +extern struct workqueue_struct *bpf_iter_workqueue_next(struct bpf_iter_workqueue *it) __weak __ksym; +extern void bpf_iter_workqueue_destroy(struct bpf_iter_workqueue *it) __weak __ksym; + +struct bpf_iter_worker_pool; +extern int bpf_iter_worker_pool_new(struct bpf_iter_worker_pool *it) __weak __ksym; +extern struct worker_pool *bpf_iter_worker_pool_next(struct bpf_iter_worker_pool *it) __weak __ksym; +extern void bpf_iter_worker_pool_destroy(struct bpf_iter_worker_pool *it) __weak __ksym; + extern int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str, struct bpf_dynptr *value_p) __weak __ksym; diff --git a/tools/testing/selftests/bpf/prog_tests/wq_iter.c b/tools/testing/selftests/bpf/prog_tests/wq_iter.c new file mode 100644 index 000000000000..73383c263bee --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/wq_iter.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "wq_iter.skel.h" +#include "wq_iter_fail.skel.h" + +static void subtest_open_coded(struct wq_iter *skel) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int err; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.count_workqueues), + &opts); + if (!ASSERT_OK(err, "run count_workqueues")) + return; + /* There are always several system workqueues (events, ...). */ + ASSERT_GT(skel->bss->nr_workqueues, 0, "nr_workqueues"); + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.count_worker_pools), + &opts); + if (!ASSERT_OK(err, "run count_worker_pools")) + return; + /* At least the per-CPU normal/highpri pools exist. */ + ASSERT_GT(skel->bss->nr_worker_pools, 0, "nr_worker_pools"); + ASSERT_GT(skel->bss->nr_percpu_pools, 0, "nr_percpu_pools"); +} + +static void drain_iter(struct bpf_program *prog, const char *name) +{ + struct bpf_link *link; + char buf[512]; + int iter_fd; + ssize_t n; + + link = bpf_program__attach_iter(prog, NULL); + if (!ASSERT_OK_PTR(link, name)) + return; + iter_fd = bpf_iter_create(bpf_link__fd(link)); + if (!ASSERT_GE(iter_fd, 0, "iter_create")) + goto out; + while ((n = read(iter_fd, buf, sizeof(buf))) > 0) + ; + ASSERT_GE(n, 0, "read iter"); + close(iter_fd); +out: + bpf_link__destroy(link); +} + +static void subtest_seq(struct wq_iter *skel) +{ + /* seq forms are backed by the same open-coded next(). */ + drain_iter(skel->progs.dump_workqueues, "attach workqueue iter"); + ASSERT_GT(skel->bss->nr_wq_seq, 0, "nr_wq_seq"); + + drain_iter(skel->progs.dump_worker_pools, "attach worker_pool iter"); + ASSERT_GT(skel->bss->nr_pool_seq, 0, "nr_pool_seq"); + + /* + * worklists are usually empty on an idle system; this drives the whole + * per-pool snapshot path and verifies it drains cleanly. + */ + drain_iter(skel->progs.dump_pending, "attach pending_work iter"); +} + +void test_wq_iter(void) +{ + struct wq_iter *skel; + + skel = wq_iter__open_and_load(); + if (!ASSERT_OK_PTR(skel, "wq_iter__open_and_load")) + return; + + if (test__start_subtest("open_coded")) + subtest_open_coded(skel); + if (test__start_subtest("seq")) + subtest_seq(skel); + + wq_iter__destroy(skel); + + RUN_TESTS(wq_iter_fail); +} diff --git a/tools/testing/selftests/bpf/progs/wq_iter.c b/tools/testing/selftests/bpf/progs/wq_iter.c new file mode 100644 index 000000000000..15f081bc634b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/wq_iter.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include +#include +#include "bpf_experimental.h" + +char _license[] SEC("license") = "GPL"; + +void bpf_rcu_read_lock(void) __ksym; +void bpf_rcu_read_unlock(void) __ksym; + +/* Results, checked by userspace. */ +int nr_workqueues; +int nr_worker_pools; +int nr_percpu_pools; +int nr_wq_seq; +int nr_pool_seq; +int nr_pending; + +/* --- open-coded iterators (KF_RCU_PROTECTED) --- */ + +SEC("syscall") +int count_workqueues(const void *ctx) +{ + struct workqueue_struct *wq; + int n = 0; + + bpf_rcu_read_lock(); + bpf_for_each(workqueue, wq) + n++; + bpf_rcu_read_unlock(); + + nr_workqueues = n; + return 0; +} + +SEC("syscall") +int count_worker_pools(const void *ctx) +{ + struct worker_pool *pool; + int n = 0, percpu = 0; + + bpf_rcu_read_lock(); + bpf_for_each(worker_pool, pool) { + n++; + if (pool->cpu >= 0) /* per-CPU pool */ + percpu++; + } + bpf_rcu_read_unlock(); + + nr_worker_pools = n; + nr_percpu_pools = percpu; + return 0; +} + +/* --- seq_file iterators --- */ + +SEC("iter/workqueue") +int dump_workqueues(struct bpf_iter__workqueue *ctx) +{ + struct seq_file *seq = ctx->meta->seq; + struct workqueue_struct *wq = ctx->wq; + + if (!wq) + return 0; + nr_wq_seq++; + BPF_SEQ_PRINTF(seq, "%s\n", wq->name); + return 0; +} + +SEC("iter/worker_pool") +int dump_worker_pools(struct bpf_iter__worker_pool *ctx) +{ + struct worker_pool *pool = ctx->pool; + + if (!pool) + return 0; + nr_pool_seq++; + return 0; +} + +SEC("iter/workqueue_pending_work") +int dump_pending(struct bpf_iter__workqueue_pending_work *ctx) +{ + struct seq_file *seq = ctx->meta->seq; + struct wq_pending_work_info *info = ctx->info; + + if (!info) /* final call */ + return 0; + + nr_pending++; + BPF_SEQ_PRINTF(seq, "pool %llu work 0x%llx func 0x%llx\n", + info->pool_id, info->work, info->func); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/wq_iter_fail.c b/tools/testing/selftests/bpf/progs/wq_iter_fail.c new file mode 100644 index 000000000000..fb2dcf9847e2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/wq_iter_fail.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include +#include +#include "bpf_misc.h" +#include "bpf_experimental.h" + +char _license[] SEC("license") = "GPL"; + +/* + * The workqueue and worker_pool open-coded iterators are KF_RCU_PROTECTED: + * using them outside a bpf_rcu_read_lock() region must be rejected. + */ + +SEC("?syscall") +__failure __msg("kernel func bpf_iter_workqueue_new requires RCU critical section protection") +int workqueue_iter_no_rcu(const void *ctx) +{ + struct workqueue_struct *wq; + int n = 0; + + bpf_for_each(workqueue, wq) + n++; + return n; +} + +SEC("?syscall") +__failure __msg("kernel func bpf_iter_worker_pool_new requires RCU critical section protection") +int worker_pool_iter_no_rcu(const void *ctx) +{ + struct worker_pool *pool; + int n = 0; + + bpf_for_each(worker_pool, pool) + n++; + return n; +}