Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2019 Facebook
4 * Copyright 2020 Google LLC.
5 */
6
7#include <linux/rculist.h>
8#include <linux/list.h>
9#include <linux/hash.h>
10#include <linux/types.h>
11#include <linux/spinlock.h>
12#include <linux/bpf.h>
13#include <linux/bpf_local_storage.h>
14#include <net/sock.h>
15#include <uapi/linux/sock_diag.h>
16#include <uapi/linux/btf.h>
17#include <linux/bpf_lsm.h>
18#include <linux/btf_ids.h>
19#include <linux/fdtable.h>
20#include <linux/rcupdate_trace.h>
21
22DEFINE_BPF_STORAGE_CACHE(inode_cache);
23
24static struct bpf_local_storage __rcu **
25inode_storage_ptr(void *owner)
26{
27 struct inode *inode = owner;
28 struct bpf_storage_blob *bsb;
29
30 bsb = bpf_inode(inode);
31 if (!bsb)
32 return NULL;
33 return &bsb->storage;
34}
35
36static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
37 struct bpf_map *map,
38 bool cacheit_lockit)
39{
40 struct bpf_local_storage *inode_storage;
41 struct bpf_local_storage_map *smap;
42 struct bpf_storage_blob *bsb;
43
44 bsb = bpf_inode(inode);
45 if (!bsb)
46 return NULL;
47
48 inode_storage =
49 rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
50 if (!inode_storage)
51 return NULL;
52
53 smap = (struct bpf_local_storage_map *)map;
54 return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
55}
56
57void bpf_inode_storage_free(struct inode *inode)
58{
59 struct bpf_local_storage *local_storage;
60 bool free_inode_storage = false;
61 struct bpf_storage_blob *bsb;
62
63 bsb = bpf_inode(inode);
64 if (!bsb)
65 return;
66
67 rcu_read_lock();
68
69 local_storage = rcu_dereference(bsb->storage);
70 if (!local_storage) {
71 rcu_read_unlock();
72 return;
73 }
74
75 raw_spin_lock_bh(&local_storage->lock);
76 free_inode_storage = bpf_local_storage_unlink_nolock(local_storage);
77 raw_spin_unlock_bh(&local_storage->lock);
78 rcu_read_unlock();
79
80 if (free_inode_storage)
81 kfree_rcu(local_storage, rcu);
82}
83
84static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
85{
86 struct bpf_local_storage_data *sdata;
87 struct file *f;
88 int fd;
89
90 fd = *(int *)key;
91 f = fget_raw(fd);
92 if (!f)
93 return ERR_PTR(-EBADF);
94
95 sdata = inode_storage_lookup(f->f_inode, map, true);
96 fput(f);
97 return sdata ? sdata->data : NULL;
98}
99
100static int bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
101 void *value, u64 map_flags)
102{
103 struct bpf_local_storage_data *sdata;
104 struct file *f;
105 int fd;
106
107 fd = *(int *)key;
108 f = fget_raw(fd);
109 if (!f)
110 return -EBADF;
111 if (!inode_storage_ptr(f->f_inode)) {
112 fput(f);
113 return -EBADF;
114 }
115
116 sdata = bpf_local_storage_update(f->f_inode,
117 (struct bpf_local_storage_map *)map,
118 value, map_flags, GFP_ATOMIC);
119 fput(f);
120 return PTR_ERR_OR_ZERO(sdata);
121}
122
123static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
124{
125 struct bpf_local_storage_data *sdata;
126
127 sdata = inode_storage_lookup(inode, map, false);
128 if (!sdata)
129 return -ENOENT;
130
131 bpf_selem_unlink(SELEM(sdata), true);
132
133 return 0;
134}
135
136static int bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
137{
138 struct file *f;
139 int fd, err;
140
141 fd = *(int *)key;
142 f = fget_raw(fd);
143 if (!f)
144 return -EBADF;
145
146 err = inode_storage_delete(f->f_inode, map);
147 fput(f);
148 return err;
149}
150
151/* *gfp_flags* is a hidden argument provided by the verifier */
152BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
153 void *, value, u64, flags, gfp_t, gfp_flags)
154{
155 struct bpf_local_storage_data *sdata;
156
157 WARN_ON_ONCE(!bpf_rcu_lock_held());
158 if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
159 return (unsigned long)NULL;
160
161 /* explicitly check that the inode_storage_ptr is not
162 * NULL as inode_storage_lookup returns NULL in this case and
163 * bpf_local_storage_update expects the owner to have a
164 * valid storage pointer.
165 */
166 if (!inode || !inode_storage_ptr(inode))
167 return (unsigned long)NULL;
168
169 sdata = inode_storage_lookup(inode, map, true);
170 if (sdata)
171 return (unsigned long)sdata->data;
172
173 /* This helper must only called from where the inode is guaranteed
174 * to have a refcount and cannot be freed.
175 */
176 if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
177 sdata = bpf_local_storage_update(
178 inode, (struct bpf_local_storage_map *)map, value,
179 BPF_NOEXIST, gfp_flags);
180 return IS_ERR(sdata) ? (unsigned long)NULL :
181 (unsigned long)sdata->data;
182 }
183
184 return (unsigned long)NULL;
185}
186
187BPF_CALL_2(bpf_inode_storage_delete,
188 struct bpf_map *, map, struct inode *, inode)
189{
190 WARN_ON_ONCE(!bpf_rcu_lock_held());
191 if (!inode)
192 return -EINVAL;
193
194 /* This helper must only called from where the inode is guaranteed
195 * to have a refcount and cannot be freed.
196 */
197 return inode_storage_delete(inode, map);
198}
199
200static int notsupp_get_next_key(struct bpf_map *map, void *key,
201 void *next_key)
202{
203 return -ENOTSUPP;
204}
205
206static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
207{
208 return bpf_local_storage_map_alloc(attr, &inode_cache);
209}
210
211static void inode_storage_map_free(struct bpf_map *map)
212{
213 bpf_local_storage_map_free(map, &inode_cache, NULL);
214}
215
216const struct bpf_map_ops inode_storage_map_ops = {
217 .map_meta_equal = bpf_map_meta_equal,
218 .map_alloc_check = bpf_local_storage_map_alloc_check,
219 .map_alloc = inode_storage_map_alloc,
220 .map_free = inode_storage_map_free,
221 .map_get_next_key = notsupp_get_next_key,
222 .map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
223 .map_update_elem = bpf_fd_inode_storage_update_elem,
224 .map_delete_elem = bpf_fd_inode_storage_delete_elem,
225 .map_check_btf = bpf_local_storage_map_check_btf,
226 .map_btf_id = &bpf_local_storage_map_btf_id[0],
227 .map_owner_storage_ptr = inode_storage_ptr,
228};
229
230BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
231
232const struct bpf_func_proto bpf_inode_storage_get_proto = {
233 .func = bpf_inode_storage_get,
234 .gpl_only = false,
235 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
236 .arg1_type = ARG_CONST_MAP_PTR,
237 .arg2_type = ARG_PTR_TO_BTF_ID,
238 .arg2_btf_id = &bpf_inode_storage_btf_ids[0],
239 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
240 .arg4_type = ARG_ANYTHING,
241};
242
243const struct bpf_func_proto bpf_inode_storage_delete_proto = {
244 .func = bpf_inode_storage_delete,
245 .gpl_only = false,
246 .ret_type = RET_INTEGER,
247 .arg1_type = ARG_CONST_MAP_PTR,
248 .arg2_type = ARG_PTR_TO_BTF_ID,
249 .arg2_btf_id = &bpf_inode_storage_btf_ids[0],
250};
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2019 Facebook
4 * Copyright 2020 Google LLC.
5 */
6
7#include <linux/rculist.h>
8#include <linux/list.h>
9#include <linux/hash.h>
10#include <linux/types.h>
11#include <linux/spinlock.h>
12#include <linux/bpf.h>
13#include <linux/bpf_local_storage.h>
14#include <net/sock.h>
15#include <uapi/linux/sock_diag.h>
16#include <uapi/linux/btf.h>
17#include <linux/bpf_lsm.h>
18#include <linux/btf_ids.h>
19#include <linux/fdtable.h>
20
21DEFINE_BPF_STORAGE_CACHE(inode_cache);
22
23static struct bpf_local_storage __rcu **
24inode_storage_ptr(void *owner)
25{
26 struct inode *inode = owner;
27 struct bpf_storage_blob *bsb;
28
29 bsb = bpf_inode(inode);
30 if (!bsb)
31 return NULL;
32 return &bsb->storage;
33}
34
35static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
36 struct bpf_map *map,
37 bool cacheit_lockit)
38{
39 struct bpf_local_storage *inode_storage;
40 struct bpf_local_storage_map *smap;
41 struct bpf_storage_blob *bsb;
42
43 bsb = bpf_inode(inode);
44 if (!bsb)
45 return NULL;
46
47 inode_storage = rcu_dereference(bsb->storage);
48 if (!inode_storage)
49 return NULL;
50
51 smap = (struct bpf_local_storage_map *)map;
52 return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
53}
54
55void bpf_inode_storage_free(struct inode *inode)
56{
57 struct bpf_local_storage_elem *selem;
58 struct bpf_local_storage *local_storage;
59 bool free_inode_storage = false;
60 struct bpf_storage_blob *bsb;
61 struct hlist_node *n;
62
63 bsb = bpf_inode(inode);
64 if (!bsb)
65 return;
66
67 rcu_read_lock();
68
69 local_storage = rcu_dereference(bsb->storage);
70 if (!local_storage) {
71 rcu_read_unlock();
72 return;
73 }
74
75 /* Neither the bpf_prog nor the bpf-map's syscall
76 * could be modifying the local_storage->list now.
77 * Thus, no elem can be added-to or deleted-from the
78 * local_storage->list by the bpf_prog or by the bpf-map's syscall.
79 *
80 * It is racing with bpf_local_storage_map_free() alone
81 * when unlinking elem from the local_storage->list and
82 * the map's bucket->list.
83 */
84 raw_spin_lock_bh(&local_storage->lock);
85 hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {
86 /* Always unlink from map before unlinking from
87 * local_storage.
88 */
89 bpf_selem_unlink_map(selem);
90 free_inode_storage = bpf_selem_unlink_storage_nolock(
91 local_storage, selem, false);
92 }
93 raw_spin_unlock_bh(&local_storage->lock);
94 rcu_read_unlock();
95
96 /* free_inoode_storage should always be true as long as
97 * local_storage->list was non-empty.
98 */
99 if (free_inode_storage)
100 kfree_rcu(local_storage, rcu);
101}
102
103static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
104{
105 struct bpf_local_storage_data *sdata;
106 struct file *f;
107 int fd;
108
109 fd = *(int *)key;
110 f = fget_raw(fd);
111 if (!f)
112 return ERR_PTR(-EBADF);
113
114 sdata = inode_storage_lookup(f->f_inode, map, true);
115 fput(f);
116 return sdata ? sdata->data : NULL;
117}
118
119static int bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
120 void *value, u64 map_flags)
121{
122 struct bpf_local_storage_data *sdata;
123 struct file *f;
124 int fd;
125
126 fd = *(int *)key;
127 f = fget_raw(fd);
128 if (!f)
129 return -EBADF;
130 if (!inode_storage_ptr(f->f_inode)) {
131 fput(f);
132 return -EBADF;
133 }
134
135 sdata = bpf_local_storage_update(f->f_inode,
136 (struct bpf_local_storage_map *)map,
137 value, map_flags);
138 fput(f);
139 return PTR_ERR_OR_ZERO(sdata);
140}
141
142static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
143{
144 struct bpf_local_storage_data *sdata;
145
146 sdata = inode_storage_lookup(inode, map, false);
147 if (!sdata)
148 return -ENOENT;
149
150 bpf_selem_unlink(SELEM(sdata));
151
152 return 0;
153}
154
155static int bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
156{
157 struct file *f;
158 int fd, err;
159
160 fd = *(int *)key;
161 f = fget_raw(fd);
162 if (!f)
163 return -EBADF;
164
165 err = inode_storage_delete(f->f_inode, map);
166 fput(f);
167 return err;
168}
169
170BPF_CALL_4(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
171 void *, value, u64, flags)
172{
173 struct bpf_local_storage_data *sdata;
174
175 if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
176 return (unsigned long)NULL;
177
178 /* explicitly check that the inode_storage_ptr is not
179 * NULL as inode_storage_lookup returns NULL in this case and
180 * bpf_local_storage_update expects the owner to have a
181 * valid storage pointer.
182 */
183 if (!inode || !inode_storage_ptr(inode))
184 return (unsigned long)NULL;
185
186 sdata = inode_storage_lookup(inode, map, true);
187 if (sdata)
188 return (unsigned long)sdata->data;
189
190 /* This helper must only called from where the inode is guaranteed
191 * to have a refcount and cannot be freed.
192 */
193 if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
194 sdata = bpf_local_storage_update(
195 inode, (struct bpf_local_storage_map *)map, value,
196 BPF_NOEXIST);
197 return IS_ERR(sdata) ? (unsigned long)NULL :
198 (unsigned long)sdata->data;
199 }
200
201 return (unsigned long)NULL;
202}
203
204BPF_CALL_2(bpf_inode_storage_delete,
205 struct bpf_map *, map, struct inode *, inode)
206{
207 if (!inode)
208 return -EINVAL;
209
210 /* This helper must only called from where the inode is guaranteed
211 * to have a refcount and cannot be freed.
212 */
213 return inode_storage_delete(inode, map);
214}
215
216static int notsupp_get_next_key(struct bpf_map *map, void *key,
217 void *next_key)
218{
219 return -ENOTSUPP;
220}
221
222static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
223{
224 struct bpf_local_storage_map *smap;
225
226 smap = bpf_local_storage_map_alloc(attr);
227 if (IS_ERR(smap))
228 return ERR_CAST(smap);
229
230 smap->cache_idx = bpf_local_storage_cache_idx_get(&inode_cache);
231 return &smap->map;
232}
233
234static void inode_storage_map_free(struct bpf_map *map)
235{
236 struct bpf_local_storage_map *smap;
237
238 smap = (struct bpf_local_storage_map *)map;
239 bpf_local_storage_cache_idx_free(&inode_cache, smap->cache_idx);
240 bpf_local_storage_map_free(smap, NULL);
241}
242
243static int inode_storage_map_btf_id;
244const struct bpf_map_ops inode_storage_map_ops = {
245 .map_meta_equal = bpf_map_meta_equal,
246 .map_alloc_check = bpf_local_storage_map_alloc_check,
247 .map_alloc = inode_storage_map_alloc,
248 .map_free = inode_storage_map_free,
249 .map_get_next_key = notsupp_get_next_key,
250 .map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
251 .map_update_elem = bpf_fd_inode_storage_update_elem,
252 .map_delete_elem = bpf_fd_inode_storage_delete_elem,
253 .map_check_btf = bpf_local_storage_map_check_btf,
254 .map_btf_name = "bpf_local_storage_map",
255 .map_btf_id = &inode_storage_map_btf_id,
256 .map_owner_storage_ptr = inode_storage_ptr,
257};
258
259BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
260
261const struct bpf_func_proto bpf_inode_storage_get_proto = {
262 .func = bpf_inode_storage_get,
263 .gpl_only = false,
264 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
265 .arg1_type = ARG_CONST_MAP_PTR,
266 .arg2_type = ARG_PTR_TO_BTF_ID,
267 .arg2_btf_id = &bpf_inode_storage_btf_ids[0],
268 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
269 .arg4_type = ARG_ANYTHING,
270};
271
272const struct bpf_func_proto bpf_inode_storage_delete_proto = {
273 .func = bpf_inode_storage_delete,
274 .gpl_only = false,
275 .ret_type = RET_INTEGER,
276 .arg1_type = ARG_CONST_MAP_PTR,
277 .arg2_type = ARG_PTR_TO_BTF_ID,
278 .arg2_btf_id = &bpf_inode_storage_btf_ids[0],
279};