1/* SPDX-License-Identifier: GPL-2.0
  2 *
  3 *	Network memory
  4 *
  5 *	Author:	Mina Almasry <almasrymina@google.com>
  6 */
  7
  8#ifndef _NET_NETMEM_H
  9#define _NET_NETMEM_H
 10
 11#include <linux/mm.h>
 12#include <net/net_debug.h>
 13
 14/* net_iov */
 15
 16DECLARE_STATIC_KEY_FALSE(page_pool_mem_providers);
 17
 18/*  We overload the LSB of the struct page pointer to indicate whether it's
 19 *  a page or net_iov.
 20 */
 21#define NET_IOV 0x01UL
 22
 23struct net_iov {
 24	unsigned long __unused_padding;
 25	unsigned long pp_magic;
 26	struct page_pool *pp;
 27	struct dmabuf_genpool_chunk_owner *owner;
 28	unsigned long dma_addr;
 29	atomic_long_t pp_ref_count;
 30};
 31
 32/* These fields in struct page are used by the page_pool and net stack:
 33 *
 34 *        struct {
 35 *                unsigned long pp_magic;
 36 *                struct page_pool *pp;
 37 *                unsigned long _pp_mapping_pad;
 38 *                unsigned long dma_addr;
 39 *                atomic_long_t pp_ref_count;
 40 *        };
 41 *
 42 * We mirror the page_pool fields here so the page_pool can access these fields
 43 * without worrying whether the underlying fields belong to a page or net_iov.
 44 *
 45 * The non-net stack fields of struct page are private to the mm stack and must
 46 * never be mirrored to net_iov.
 47 */
 48#define NET_IOV_ASSERT_OFFSET(pg, iov)             \
 49	static_assert(offsetof(struct page, pg) == \
 50		      offsetof(struct net_iov, iov))
 51NET_IOV_ASSERT_OFFSET(pp_magic, pp_magic);
 52NET_IOV_ASSERT_OFFSET(pp, pp);
 53NET_IOV_ASSERT_OFFSET(dma_addr, dma_addr);
 54NET_IOV_ASSERT_OFFSET(pp_ref_count, pp_ref_count);
 55#undef NET_IOV_ASSERT_OFFSET
 56
 57/* netmem */
 58
 59/**
 60 * typedef netmem_ref - a nonexistent type marking a reference to generic
 61 * network memory.
 62 *
 63 * A netmem_ref currently is always a reference to a struct page. This
 64 * abstraction is introduced so support for new memory types can be added.
 65 *
 66 * Use the supplied helpers to obtain the underlying memory pointer and fields.
 67 */
 68typedef unsigned long __bitwise netmem_ref;
 69
 70static inline bool netmem_is_net_iov(const netmem_ref netmem)
 71{
 72	return (__force unsigned long)netmem & NET_IOV;
 73}
 74
 75/* This conversion fails (returns NULL) if the netmem_ref is not struct page
 76 * backed.
 77 */
 78static inline struct page *netmem_to_page(netmem_ref netmem)
 79{
 80	if (WARN_ON_ONCE(netmem_is_net_iov(netmem)))
 81		return NULL;
 82
 83	return (__force struct page *)netmem;
 84}
 85
 86static inline struct net_iov *netmem_to_net_iov(netmem_ref netmem)
 87{
 88	if (netmem_is_net_iov(netmem))
 89		return (struct net_iov *)((__force unsigned long)netmem &
 90					  ~NET_IOV);
 91
 92	DEBUG_NET_WARN_ON_ONCE(true);
 93	return NULL;
 94}
 95
 96static inline netmem_ref net_iov_to_netmem(struct net_iov *niov)
 97{
 98	return (__force netmem_ref)((unsigned long)niov | NET_IOV);
 99}
100
101static inline netmem_ref page_to_netmem(struct page *page)
102{
103	return (__force netmem_ref)page;
104}
105
106static inline int netmem_ref_count(netmem_ref netmem)
107{
108	/* The non-pp refcount of net_iov is always 1. On net_iov, we only
109	 * support pp refcounting which uses the pp_ref_count field.
110	 */
111	if (netmem_is_net_iov(netmem))
112		return 1;
113
114	return page_ref_count(netmem_to_page(netmem));
115}
116
117static inline unsigned long netmem_pfn_trace(netmem_ref netmem)
118{
119	if (netmem_is_net_iov(netmem))
120		return 0;
121
122	return page_to_pfn(netmem_to_page(netmem));
123}
124
125static inline struct net_iov *__netmem_clear_lsb(netmem_ref netmem)
126{
127	return (struct net_iov *)((__force unsigned long)netmem & ~NET_IOV);
128}
129
130static inline struct page_pool *netmem_get_pp(netmem_ref netmem)
131{
132	return __netmem_clear_lsb(netmem)->pp;
133}
134
135static inline atomic_long_t *netmem_get_pp_ref_count_ref(netmem_ref netmem)
136{
137	return &__netmem_clear_lsb(netmem)->pp_ref_count;
138}
139
140static inline bool netmem_is_pref_nid(netmem_ref netmem, int pref_nid)
141{
142	/* NUMA node preference only makes sense if we're allocating
143	 * system memory. Memory providers (which give us net_iovs)
144	 * choose for us.
145	 */
146	if (netmem_is_net_iov(netmem))
147		return true;
148
149	return page_to_nid(netmem_to_page(netmem)) == pref_nid;
150}
151
152static inline netmem_ref netmem_compound_head(netmem_ref netmem)
153{
154	/* niov are never compounded */
155	if (netmem_is_net_iov(netmem))
156		return netmem;
157
158	return page_to_netmem(compound_head(netmem_to_page(netmem)));
159}
160
161static inline void *netmem_address(netmem_ref netmem)
162{
163	if (netmem_is_net_iov(netmem))
164		return NULL;
165
166	return page_address(netmem_to_page(netmem));
167}
168
169static inline unsigned long netmem_get_dma_addr(netmem_ref netmem)
170{
171	return __netmem_clear_lsb(netmem)->dma_addr;
172}
173
174#endif /* _NET_NETMEM_H */