Linux Audio

Check our new training course

Loading...
v6.8
  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (c) 2019 Facebook
  4 * Copyright 2020 Google LLC.
  5 */
  6
  7#include <linux/rculist.h>
  8#include <linux/list.h>
  9#include <linux/hash.h>
 10#include <linux/types.h>
 11#include <linux/spinlock.h>
 12#include <linux/bpf.h>
 13#include <linux/bpf_local_storage.h>
 14#include <net/sock.h>
 15#include <uapi/linux/sock_diag.h>
 16#include <uapi/linux/btf.h>
 17#include <linux/bpf_lsm.h>
 18#include <linux/btf_ids.h>
 19#include <linux/fdtable.h>
 20#include <linux/rcupdate_trace.h>
 21
 22DEFINE_BPF_STORAGE_CACHE(inode_cache);
 23
 24static struct bpf_local_storage __rcu **
 25inode_storage_ptr(void *owner)
 26{
 27	struct inode *inode = owner;
 28	struct bpf_storage_blob *bsb;
 29
 30	bsb = bpf_inode(inode);
 31	if (!bsb)
 32		return NULL;
 33	return &bsb->storage;
 34}
 35
 36static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
 37							   struct bpf_map *map,
 38							   bool cacheit_lockit)
 39{
 40	struct bpf_local_storage *inode_storage;
 41	struct bpf_local_storage_map *smap;
 42	struct bpf_storage_blob *bsb;
 43
 44	bsb = bpf_inode(inode);
 45	if (!bsb)
 46		return NULL;
 47
 48	inode_storage =
 49		rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
 50	if (!inode_storage)
 51		return NULL;
 52
 53	smap = (struct bpf_local_storage_map *)map;
 54	return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
 55}
 56
 57void bpf_inode_storage_free(struct inode *inode)
 58{
 59	struct bpf_local_storage *local_storage;
 
 60	struct bpf_storage_blob *bsb;
 61
 62	bsb = bpf_inode(inode);
 63	if (!bsb)
 64		return;
 65
 66	rcu_read_lock();
 67
 68	local_storage = rcu_dereference(bsb->storage);
 69	if (!local_storage) {
 70		rcu_read_unlock();
 71		return;
 72	}
 73
 74	bpf_local_storage_destroy(local_storage);
 
 
 75	rcu_read_unlock();
 
 
 
 76}
 77
 78static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
 79{
 80	struct bpf_local_storage_data *sdata;
 81	struct fd f = fdget_raw(*(int *)key);
 
 82
 83	if (!f.file)
 
 
 84		return ERR_PTR(-EBADF);
 85
 86	sdata = inode_storage_lookup(file_inode(f.file), map, true);
 87	fdput(f);
 88	return sdata ? sdata->data : NULL;
 89}
 90
 91static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
 92					     void *value, u64 map_flags)
 93{
 94	struct bpf_local_storage_data *sdata;
 95	struct fd f = fdget_raw(*(int *)key);
 
 96
 97	if (!f.file)
 
 
 98		return -EBADF;
 99	if (!inode_storage_ptr(file_inode(f.file))) {
100		fdput(f);
101		return -EBADF;
102	}
103
104	sdata = bpf_local_storage_update(file_inode(f.file),
105					 (struct bpf_local_storage_map *)map,
106					 value, map_flags, GFP_ATOMIC);
107	fdput(f);
108	return PTR_ERR_OR_ZERO(sdata);
109}
110
111static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
112{
113	struct bpf_local_storage_data *sdata;
114
115	sdata = inode_storage_lookup(inode, map, false);
116	if (!sdata)
117		return -ENOENT;
118
119	bpf_selem_unlink(SELEM(sdata), false);
120
121	return 0;
122}
123
124static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
125{
126	struct fd f = fdget_raw(*(int *)key);
127	int err;
128
129	if (!f.file)
 
 
130		return -EBADF;
131
132	err = inode_storage_delete(file_inode(f.file), map);
133	fdput(f);
134	return err;
135}
136
137/* *gfp_flags* is a hidden argument provided by the verifier */
138BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
139	   void *, value, u64, flags, gfp_t, gfp_flags)
140{
141	struct bpf_local_storage_data *sdata;
142
143	WARN_ON_ONCE(!bpf_rcu_lock_held());
144	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
145		return (unsigned long)NULL;
146
147	/* explicitly check that the inode_storage_ptr is not
148	 * NULL as inode_storage_lookup returns NULL in this case and
149	 * bpf_local_storage_update expects the owner to have a
150	 * valid storage pointer.
151	 */
152	if (!inode || !inode_storage_ptr(inode))
153		return (unsigned long)NULL;
154
155	sdata = inode_storage_lookup(inode, map, true);
156	if (sdata)
157		return (unsigned long)sdata->data;
158
159	/* This helper must only called from where the inode is guaranteed
160	 * to have a refcount and cannot be freed.
161	 */
162	if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
163		sdata = bpf_local_storage_update(
164			inode, (struct bpf_local_storage_map *)map, value,
165			BPF_NOEXIST, gfp_flags);
166		return IS_ERR(sdata) ? (unsigned long)NULL :
167					     (unsigned long)sdata->data;
168	}
169
170	return (unsigned long)NULL;
171}
172
173BPF_CALL_2(bpf_inode_storage_delete,
174	   struct bpf_map *, map, struct inode *, inode)
175{
176	WARN_ON_ONCE(!bpf_rcu_lock_held());
177	if (!inode)
178		return -EINVAL;
179
180	/* This helper must only called from where the inode is guaranteed
181	 * to have a refcount and cannot be freed.
182	 */
183	return inode_storage_delete(inode, map);
184}
185
186static int notsupp_get_next_key(struct bpf_map *map, void *key,
187				void *next_key)
188{
189	return -ENOTSUPP;
190}
191
192static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
193{
194	return bpf_local_storage_map_alloc(attr, &inode_cache, false);
195}
196
197static void inode_storage_map_free(struct bpf_map *map)
198{
199	bpf_local_storage_map_free(map, &inode_cache, NULL);
200}
201
202const struct bpf_map_ops inode_storage_map_ops = {
203	.map_meta_equal = bpf_map_meta_equal,
204	.map_alloc_check = bpf_local_storage_map_alloc_check,
205	.map_alloc = inode_storage_map_alloc,
206	.map_free = inode_storage_map_free,
207	.map_get_next_key = notsupp_get_next_key,
208	.map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
209	.map_update_elem = bpf_fd_inode_storage_update_elem,
210	.map_delete_elem = bpf_fd_inode_storage_delete_elem,
211	.map_check_btf = bpf_local_storage_map_check_btf,
212	.map_mem_usage = bpf_local_storage_map_mem_usage,
213	.map_btf_id = &bpf_local_storage_map_btf_id[0],
214	.map_owner_storage_ptr = inode_storage_ptr,
215};
216
217BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
218
219const struct bpf_func_proto bpf_inode_storage_get_proto = {
220	.func		= bpf_inode_storage_get,
221	.gpl_only	= false,
222	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,
223	.arg1_type	= ARG_CONST_MAP_PTR,
224	.arg2_type	= ARG_PTR_TO_BTF_ID_OR_NULL,
225	.arg2_btf_id	= &bpf_inode_storage_btf_ids[0],
226	.arg3_type	= ARG_PTR_TO_MAP_VALUE_OR_NULL,
227	.arg4_type	= ARG_ANYTHING,
228};
229
230const struct bpf_func_proto bpf_inode_storage_delete_proto = {
231	.func		= bpf_inode_storage_delete,
232	.gpl_only	= false,
233	.ret_type	= RET_INTEGER,
234	.arg1_type	= ARG_CONST_MAP_PTR,
235	.arg2_type	= ARG_PTR_TO_BTF_ID_OR_NULL,
236	.arg2_btf_id	= &bpf_inode_storage_btf_ids[0],
237};
v6.2
  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (c) 2019 Facebook
  4 * Copyright 2020 Google LLC.
  5 */
  6
  7#include <linux/rculist.h>
  8#include <linux/list.h>
  9#include <linux/hash.h>
 10#include <linux/types.h>
 11#include <linux/spinlock.h>
 12#include <linux/bpf.h>
 13#include <linux/bpf_local_storage.h>
 14#include <net/sock.h>
 15#include <uapi/linux/sock_diag.h>
 16#include <uapi/linux/btf.h>
 17#include <linux/bpf_lsm.h>
 18#include <linux/btf_ids.h>
 19#include <linux/fdtable.h>
 20#include <linux/rcupdate_trace.h>
 21
 22DEFINE_BPF_STORAGE_CACHE(inode_cache);
 23
 24static struct bpf_local_storage __rcu **
 25inode_storage_ptr(void *owner)
 26{
 27	struct inode *inode = owner;
 28	struct bpf_storage_blob *bsb;
 29
 30	bsb = bpf_inode(inode);
 31	if (!bsb)
 32		return NULL;
 33	return &bsb->storage;
 34}
 35
 36static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
 37							   struct bpf_map *map,
 38							   bool cacheit_lockit)
 39{
 40	struct bpf_local_storage *inode_storage;
 41	struct bpf_local_storage_map *smap;
 42	struct bpf_storage_blob *bsb;
 43
 44	bsb = bpf_inode(inode);
 45	if (!bsb)
 46		return NULL;
 47
 48	inode_storage =
 49		rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
 50	if (!inode_storage)
 51		return NULL;
 52
 53	smap = (struct bpf_local_storage_map *)map;
 54	return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
 55}
 56
 57void bpf_inode_storage_free(struct inode *inode)
 58{
 59	struct bpf_local_storage *local_storage;
 60	bool free_inode_storage = false;
 61	struct bpf_storage_blob *bsb;
 62
 63	bsb = bpf_inode(inode);
 64	if (!bsb)
 65		return;
 66
 67	rcu_read_lock();
 68
 69	local_storage = rcu_dereference(bsb->storage);
 70	if (!local_storage) {
 71		rcu_read_unlock();
 72		return;
 73	}
 74
 75	raw_spin_lock_bh(&local_storage->lock);
 76	free_inode_storage = bpf_local_storage_unlink_nolock(local_storage);
 77	raw_spin_unlock_bh(&local_storage->lock);
 78	rcu_read_unlock();
 79
 80	if (free_inode_storage)
 81		kfree_rcu(local_storage, rcu);
 82}
 83
 84static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
 85{
 86	struct bpf_local_storage_data *sdata;
 87	struct file *f;
 88	int fd;
 89
 90	fd = *(int *)key;
 91	f = fget_raw(fd);
 92	if (!f)
 93		return ERR_PTR(-EBADF);
 94
 95	sdata = inode_storage_lookup(f->f_inode, map, true);
 96	fput(f);
 97	return sdata ? sdata->data : NULL;
 98}
 99
100static int bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
101					 void *value, u64 map_flags)
102{
103	struct bpf_local_storage_data *sdata;
104	struct file *f;
105	int fd;
106
107	fd = *(int *)key;
108	f = fget_raw(fd);
109	if (!f)
110		return -EBADF;
111	if (!inode_storage_ptr(f->f_inode)) {
112		fput(f);
113		return -EBADF;
114	}
115
116	sdata = bpf_local_storage_update(f->f_inode,
117					 (struct bpf_local_storage_map *)map,
118					 value, map_flags, GFP_ATOMIC);
119	fput(f);
120	return PTR_ERR_OR_ZERO(sdata);
121}
122
123static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
124{
125	struct bpf_local_storage_data *sdata;
126
127	sdata = inode_storage_lookup(inode, map, false);
128	if (!sdata)
129		return -ENOENT;
130
131	bpf_selem_unlink(SELEM(sdata), true);
132
133	return 0;
134}
135
136static int bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
137{
138	struct file *f;
139	int fd, err;
140
141	fd = *(int *)key;
142	f = fget_raw(fd);
143	if (!f)
144		return -EBADF;
145
146	err = inode_storage_delete(f->f_inode, map);
147	fput(f);
148	return err;
149}
150
151/* *gfp_flags* is a hidden argument provided by the verifier */
152BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
153	   void *, value, u64, flags, gfp_t, gfp_flags)
154{
155	struct bpf_local_storage_data *sdata;
156
157	WARN_ON_ONCE(!bpf_rcu_lock_held());
158	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
159		return (unsigned long)NULL;
160
161	/* explicitly check that the inode_storage_ptr is not
162	 * NULL as inode_storage_lookup returns NULL in this case and
163	 * bpf_local_storage_update expects the owner to have a
164	 * valid storage pointer.
165	 */
166	if (!inode || !inode_storage_ptr(inode))
167		return (unsigned long)NULL;
168
169	sdata = inode_storage_lookup(inode, map, true);
170	if (sdata)
171		return (unsigned long)sdata->data;
172
173	/* This helper must only called from where the inode is guaranteed
174	 * to have a refcount and cannot be freed.
175	 */
176	if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
177		sdata = bpf_local_storage_update(
178			inode, (struct bpf_local_storage_map *)map, value,
179			BPF_NOEXIST, gfp_flags);
180		return IS_ERR(sdata) ? (unsigned long)NULL :
181					     (unsigned long)sdata->data;
182	}
183
184	return (unsigned long)NULL;
185}
186
187BPF_CALL_2(bpf_inode_storage_delete,
188	   struct bpf_map *, map, struct inode *, inode)
189{
190	WARN_ON_ONCE(!bpf_rcu_lock_held());
191	if (!inode)
192		return -EINVAL;
193
194	/* This helper must only called from where the inode is guaranteed
195	 * to have a refcount and cannot be freed.
196	 */
197	return inode_storage_delete(inode, map);
198}
199
200static int notsupp_get_next_key(struct bpf_map *map, void *key,
201				void *next_key)
202{
203	return -ENOTSUPP;
204}
205
206static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
207{
208	return bpf_local_storage_map_alloc(attr, &inode_cache);
209}
210
211static void inode_storage_map_free(struct bpf_map *map)
212{
213	bpf_local_storage_map_free(map, &inode_cache, NULL);
214}
215
216const struct bpf_map_ops inode_storage_map_ops = {
217	.map_meta_equal = bpf_map_meta_equal,
218	.map_alloc_check = bpf_local_storage_map_alloc_check,
219	.map_alloc = inode_storage_map_alloc,
220	.map_free = inode_storage_map_free,
221	.map_get_next_key = notsupp_get_next_key,
222	.map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
223	.map_update_elem = bpf_fd_inode_storage_update_elem,
224	.map_delete_elem = bpf_fd_inode_storage_delete_elem,
225	.map_check_btf = bpf_local_storage_map_check_btf,
 
226	.map_btf_id = &bpf_local_storage_map_btf_id[0],
227	.map_owner_storage_ptr = inode_storage_ptr,
228};
229
230BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
231
232const struct bpf_func_proto bpf_inode_storage_get_proto = {
233	.func		= bpf_inode_storage_get,
234	.gpl_only	= false,
235	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,
236	.arg1_type	= ARG_CONST_MAP_PTR,
237	.arg2_type	= ARG_PTR_TO_BTF_ID,
238	.arg2_btf_id	= &bpf_inode_storage_btf_ids[0],
239	.arg3_type	= ARG_PTR_TO_MAP_VALUE_OR_NULL,
240	.arg4_type	= ARG_ANYTHING,
241};
242
243const struct bpf_func_proto bpf_inode_storage_delete_proto = {
244	.func		= bpf_inode_storage_delete,
245	.gpl_only	= false,
246	.ret_type	= RET_INTEGER,
247	.arg1_type	= ARG_CONST_MAP_PTR,
248	.arg2_type	= ARG_PTR_TO_BTF_ID,
249	.arg2_btf_id	= &bpf_inode_storage_btf_ids[0],
250};