1// SPDX-License-Identifier: GPL-2.0
  2
  3#include <linux/mutex.h>
  4#include <linux/netdevice.h>
  5#include <linux/xarray.h>
  6#include <net/busy_poll.h>
  7#include <net/net_debug.h>
  8#include <net/netdev_rx_queue.h>
  9#include <net/page_pool/helpers.h>
 10#include <net/page_pool/types.h>
 11#include <net/sock.h>
 12
 13#include "devmem.h"
 14#include "page_pool_priv.h"
 15#include "netdev-genl-gen.h"
 16
 17static DEFINE_XARRAY_FLAGS(page_pools, XA_FLAGS_ALLOC1);
 18/* Protects: page_pools, netdevice->page_pools, pool->p.napi, pool->slow.netdev,
 19 *	pool->user.
 20 * Ordering: inside rtnl_lock
 21 */
 22DEFINE_MUTEX(page_pools_lock);
 23
 24/* Page pools are only reachable from user space (via netlink) if they are
 25 * linked to a netdev at creation time. Following page pool "visibility"
 26 * states are possible:
 27 *  - normal
 28 *    - user.list: linked to real netdev, netdev: real netdev
 29 *  - orphaned - real netdev has disappeared
 30 *    - user.list: linked to lo, netdev: lo
 31 *  - invisible - either (a) created without netdev linking, (b) unlisted due
 32 *      to error, or (c) the entire namespace which owned this pool disappeared
 33 *    - user.list: unhashed, netdev: unknown
 34 */
 35
 36typedef int (*pp_nl_fill_cb)(struct sk_buff *rsp, const struct page_pool *pool,
 37			     const struct genl_info *info);
 38
 39static int
 40netdev_nl_page_pool_get_do(struct genl_info *info, u32 id, pp_nl_fill_cb fill)
 41{
 42	struct page_pool *pool;
 43	struct sk_buff *rsp;
 44	int err;
 45
 46	mutex_lock(&page_pools_lock);
 47	pool = xa_load(&page_pools, id);
 48	if (!pool || hlist_unhashed(&pool->user.list) ||
 49	    !net_eq(dev_net(pool->slow.netdev), genl_info_net(info))) {
 50		err = -ENOENT;
 51		goto err_unlock;
 52	}
 53
 54	rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
 55	if (!rsp) {
 56		err = -ENOMEM;
 57		goto err_unlock;
 58	}
 59
 60	err = fill(rsp, pool, info);
 61	if (err)
 62		goto err_free_msg;
 63
 64	mutex_unlock(&page_pools_lock);
 65
 66	return genlmsg_reply(rsp, info);
 67
 68err_free_msg:
 69	nlmsg_free(rsp);
 70err_unlock:
 71	mutex_unlock(&page_pools_lock);
 72	return err;
 73}
 74
 75struct page_pool_dump_cb {
 76	unsigned long ifindex;
 77	u32 pp_id;
 78};
 79
 80static int
 81netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb,
 82			     pp_nl_fill_cb fill)
 83{
 84	struct page_pool_dump_cb *state = (void *)cb->ctx;
 85	const struct genl_info *info = genl_info_dump(cb);
 86	struct net *net = sock_net(skb->sk);
 87	struct net_device *netdev;
 88	struct page_pool *pool;
 89	int err = 0;
 90
 91	rtnl_lock();
 92	mutex_lock(&page_pools_lock);
 93	for_each_netdev_dump(net, netdev, state->ifindex) {
 94		hlist_for_each_entry(pool, &netdev->page_pools, user.list) {
 95			if (state->pp_id && state->pp_id < pool->user.id)
 96				continue;
 97
 98			state->pp_id = pool->user.id;
 99			err = fill(skb, pool, info);
100			if (err)
101				goto out;
102		}
103
104		state->pp_id = 0;
105	}
106out:
107	mutex_unlock(&page_pools_lock);
108	rtnl_unlock();
109
110	return err;
111}
112
113static int
114page_pool_nl_stats_fill(struct sk_buff *rsp, const struct page_pool *pool,
115			const struct genl_info *info)
116{
117#ifdef CONFIG_PAGE_POOL_STATS
118	struct page_pool_stats stats = {};
119	struct nlattr *nest;
120	void *hdr;
121
122	if (!page_pool_get_stats(pool, &stats))
123		return 0;
124
125	hdr = genlmsg_iput(rsp, info);
126	if (!hdr)
127		return -EMSGSIZE;
128
129	nest = nla_nest_start(rsp, NETDEV_A_PAGE_POOL_STATS_INFO);
130
131	if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id) ||
132	    (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX &&
133	     nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX,
134			 pool->slow.netdev->ifindex)))
135		goto err_cancel_nest;
136
137	nla_nest_end(rsp, nest);
138
139	if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_FAST,
140			 stats.alloc_stats.fast) ||
141	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW,
142			 stats.alloc_stats.slow) ||
143	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW_HIGH_ORDER,
144			 stats.alloc_stats.slow_high_order) ||
145	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_EMPTY,
146			 stats.alloc_stats.empty) ||
147	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_REFILL,
148			 stats.alloc_stats.refill) ||
149	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_WAIVE,
150			 stats.alloc_stats.waive) ||
151	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHED,
152			 stats.recycle_stats.cached) ||
153	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHE_FULL,
154			 stats.recycle_stats.cache_full) ||
155	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING,
156			 stats.recycle_stats.ring) ||
157	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING_FULL,
158			 stats.recycle_stats.ring_full) ||
159	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RELEASED_REFCNT,
160			 stats.recycle_stats.released_refcnt))
161		goto err_cancel_msg;
162
163	genlmsg_end(rsp, hdr);
164
165	return 0;
166err_cancel_nest:
167	nla_nest_cancel(rsp, nest);
168err_cancel_msg:
169	genlmsg_cancel(rsp, hdr);
170	return -EMSGSIZE;
171#else
172	GENL_SET_ERR_MSG(info, "kernel built without CONFIG_PAGE_POOL_STATS");
173	return -EOPNOTSUPP;
174#endif
175}
176
177int netdev_nl_page_pool_stats_get_doit(struct sk_buff *skb,
178				       struct genl_info *info)
179{
180	struct nlattr *tb[ARRAY_SIZE(netdev_page_pool_info_nl_policy)];
181	struct nlattr *nest;
182	int err;
183	u32 id;
184
185	if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_STATS_INFO))
186		return -EINVAL;
187
188	nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO];
189	err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest,
190			       netdev_page_pool_info_nl_policy,
191			       info->extack);
192	if (err)
193		return err;
194
195	if (NL_REQ_ATTR_CHECK(info->extack, nest, tb, NETDEV_A_PAGE_POOL_ID))
196		return -EINVAL;
197	if (tb[NETDEV_A_PAGE_POOL_IFINDEX]) {
198		NL_SET_ERR_MSG_ATTR(info->extack,
199				    tb[NETDEV_A_PAGE_POOL_IFINDEX],
200				    "selecting by ifindex not supported");
201		return -EINVAL;
202	}
203
204	id = nla_get_uint(tb[NETDEV_A_PAGE_POOL_ID]);
205
206	return netdev_nl_page_pool_get_do(info, id, page_pool_nl_stats_fill);
207}
208
209int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff *skb,
210					 struct netlink_callback *cb)
211{
212	return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill);
213}
214
215static int
216page_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool,
217		  const struct genl_info *info)
218{
219	struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
220	size_t inflight, refsz;
221	unsigned int napi_id;
222	void *hdr;
223
224	hdr = genlmsg_iput(rsp, info);
225	if (!hdr)
226		return -EMSGSIZE;
227
228	if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id))
229		goto err_cancel;
230
231	if (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX &&
232	    nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX,
233			pool->slow.netdev->ifindex))
234		goto err_cancel;
235
236	napi_id = pool->p.napi ? READ_ONCE(pool->p.napi->napi_id) : 0;
237	if (napi_id >= MIN_NAPI_ID &&
238	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_NAPI_ID, napi_id))
239		goto err_cancel;
240
241	inflight = page_pool_inflight(pool, false);
242	refsz =	PAGE_SIZE << pool->p.order;
243	if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT, inflight) ||
244	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT_MEM,
245			 inflight * refsz))
246		goto err_cancel;
247	if (pool->user.detach_time &&
248	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_DETACH_TIME,
249			 pool->user.detach_time))
250		goto err_cancel;
251
252	if (binding && nla_put_u32(rsp, NETDEV_A_PAGE_POOL_DMABUF, binding->id))
253		goto err_cancel;
254
255	genlmsg_end(rsp, hdr);
256
257	return 0;
258err_cancel:
259	genlmsg_cancel(rsp, hdr);
260	return -EMSGSIZE;
261}
262
263static void netdev_nl_page_pool_event(const struct page_pool *pool, u32 cmd)
264{
265	struct genl_info info;
266	struct sk_buff *ntf;
267	struct net *net;
268
269	lockdep_assert_held(&page_pools_lock);
270
271	/* 'invisible' page pools don't matter */
272	if (hlist_unhashed(&pool->user.list))
273		return;
274	net = dev_net(pool->slow.netdev);
275
276	if (!genl_has_listeners(&netdev_nl_family, net, NETDEV_NLGRP_PAGE_POOL))
277		return;
278
279	genl_info_init_ntf(&info, &netdev_nl_family, cmd);
280
281	ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
282	if (!ntf)
283		return;
284
285	if (page_pool_nl_fill(ntf, pool, &info)) {
286		nlmsg_free(ntf);
287		return;
288	}
289
290	genlmsg_multicast_netns(&netdev_nl_family, net, ntf,
291				0, NETDEV_NLGRP_PAGE_POOL, GFP_KERNEL);
292}
293
294int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info)
295{
296	u32 id;
297
298	if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_ID))
299		return -EINVAL;
300
301	id = nla_get_uint(info->attrs[NETDEV_A_PAGE_POOL_ID]);
302
303	return netdev_nl_page_pool_get_do(info, id, page_pool_nl_fill);
304}
305
306int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb,
307				   struct netlink_callback *cb)
308{
309	return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill);
310}
311
312int page_pool_list(struct page_pool *pool)
313{
314	static u32 id_alloc_next;
315	int err;
316
317	mutex_lock(&page_pools_lock);
318	err = xa_alloc_cyclic(&page_pools, &pool->user.id, pool, xa_limit_32b,
319			      &id_alloc_next, GFP_KERNEL);
320	if (err < 0)
321		goto err_unlock;
322
323	INIT_HLIST_NODE(&pool->user.list);
324	if (pool->slow.netdev) {
325		hlist_add_head(&pool->user.list,
326			       &pool->slow.netdev->page_pools);
327		netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_ADD_NTF);
328	}
329
330	mutex_unlock(&page_pools_lock);
331	return 0;
332
333err_unlock:
334	mutex_unlock(&page_pools_lock);
335	return err;
336}
337
338void page_pool_detached(struct page_pool *pool)
339{
340	mutex_lock(&page_pools_lock);
341	pool->user.detach_time = ktime_get_boottime_seconds();
342	netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF);
343	mutex_unlock(&page_pools_lock);
344}
345
346void page_pool_unlist(struct page_pool *pool)
347{
348	mutex_lock(&page_pools_lock);
349	netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_DEL_NTF);
350	xa_erase(&page_pools, pool->user.id);
351	if (!hlist_unhashed(&pool->user.list))
352		hlist_del(&pool->user.list);
353	mutex_unlock(&page_pools_lock);
354}
355
356int page_pool_check_memory_provider(struct net_device *dev,
357				    struct netdev_rx_queue *rxq)
358{
359	struct net_devmem_dmabuf_binding *binding = rxq->mp_params.mp_priv;
360	struct page_pool *pool;
361	struct hlist_node *n;
362
363	if (!binding)
364		return 0;
365
366	mutex_lock(&page_pools_lock);
367	hlist_for_each_entry_safe(pool, n, &dev->page_pools, user.list) {
368		if (pool->mp_priv != binding)
369			continue;
370
371		if (pool->slow.queue_idx == get_netdev_rx_queue_index(rxq)) {
372			mutex_unlock(&page_pools_lock);
373			return 0;
374		}
375	}
376	mutex_unlock(&page_pools_lock);
377	return -ENODATA;
378}
379
380static void page_pool_unreg_netdev_wipe(struct net_device *netdev)
381{
382	struct page_pool *pool;
383	struct hlist_node *n;
384
385	mutex_lock(&page_pools_lock);
386	hlist_for_each_entry_safe(pool, n, &netdev->page_pools, user.list) {
387		hlist_del_init(&pool->user.list);
388		pool->slow.netdev = NET_PTR_POISON;
389	}
390	mutex_unlock(&page_pools_lock);
391}
392
393static void page_pool_unreg_netdev(struct net_device *netdev)
394{
395	struct page_pool *pool, *last;
396	struct net_device *lo;
397
398	lo = dev_net(netdev)->loopback_dev;
399
400	mutex_lock(&page_pools_lock);
401	last = NULL;
402	hlist_for_each_entry(pool, &netdev->page_pools, user.list) {
403		pool->slow.netdev = lo;
404		netdev_nl_page_pool_event(pool,
405					  NETDEV_CMD_PAGE_POOL_CHANGE_NTF);
406		last = pool;
407	}
408	if (last)
409		hlist_splice_init(&netdev->page_pools, &last->user.list,
410				  &lo->page_pools);
411	mutex_unlock(&page_pools_lock);
412}
413
414static int
415page_pool_netdevice_event(struct notifier_block *nb,
416			  unsigned long event, void *ptr)
417{
418	struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
419
420	if (event != NETDEV_UNREGISTER)
421		return NOTIFY_DONE;
422
423	if (hlist_empty(&netdev->page_pools))
424		return NOTIFY_OK;
425
426	if (netdev->ifindex != LOOPBACK_IFINDEX)
427		page_pool_unreg_netdev(netdev);
428	else
429		page_pool_unreg_netdev_wipe(netdev);
430	return NOTIFY_OK;
431}
432
433static struct notifier_block page_pool_netdevice_nb = {
434	.notifier_call = page_pool_netdevice_event,
435};
436
437static int __init page_pool_user_init(void)
438{
439	return register_netdevice_notifier(&page_pool_netdevice_nb);
440}
441
442subsys_initcall(page_pool_user_init);