Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2019 Facebook
4 * Copyright 2020 Google LLC.
5 */
6
7#include <linux/rculist.h>
8#include <linux/list.h>
9#include <linux/hash.h>
10#include <linux/types.h>
11#include <linux/spinlock.h>
12#include <linux/bpf.h>
13#include <linux/bpf_local_storage.h>
14#include <net/sock.h>
15#include <uapi/linux/sock_diag.h>
16#include <uapi/linux/btf.h>
17#include <linux/bpf_lsm.h>
18#include <linux/btf_ids.h>
19#include <linux/fdtable.h>
20#include <linux/rcupdate_trace.h>
21
22DEFINE_BPF_STORAGE_CACHE(inode_cache);
23
24static struct bpf_local_storage __rcu **
25inode_storage_ptr(void *owner)
26{
27 struct inode *inode = owner;
28 struct bpf_storage_blob *bsb;
29
30 bsb = bpf_inode(inode);
31 if (!bsb)
32 return NULL;
33 return &bsb->storage;
34}
35
36static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
37 struct bpf_map *map,
38 bool cacheit_lockit)
39{
40 struct bpf_local_storage *inode_storage;
41 struct bpf_local_storage_map *smap;
42 struct bpf_storage_blob *bsb;
43
44 bsb = bpf_inode(inode);
45 if (!bsb)
46 return NULL;
47
48 inode_storage =
49 rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
50 if (!inode_storage)
51 return NULL;
52
53 smap = (struct bpf_local_storage_map *)map;
54 return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
55}
56
57void bpf_inode_storage_free(struct inode *inode)
58{
59 struct bpf_local_storage *local_storage;
60 struct bpf_storage_blob *bsb;
61
62 bsb = bpf_inode(inode);
63 if (!bsb)
64 return;
65
66 rcu_read_lock();
67
68 local_storage = rcu_dereference(bsb->storage);
69 if (!local_storage) {
70 rcu_read_unlock();
71 return;
72 }
73
74 bpf_local_storage_destroy(local_storage);
75 rcu_read_unlock();
76}
77
78static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
79{
80 struct bpf_local_storage_data *sdata;
81 struct fd f = fdget_raw(*(int *)key);
82
83 if (!f.file)
84 return ERR_PTR(-EBADF);
85
86 sdata = inode_storage_lookup(file_inode(f.file), map, true);
87 fdput(f);
88 return sdata ? sdata->data : NULL;
89}
90
91static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
92 void *value, u64 map_flags)
93{
94 struct bpf_local_storage_data *sdata;
95 struct fd f = fdget_raw(*(int *)key);
96
97 if (!f.file)
98 return -EBADF;
99 if (!inode_storage_ptr(file_inode(f.file))) {
100 fdput(f);
101 return -EBADF;
102 }
103
104 sdata = bpf_local_storage_update(file_inode(f.file),
105 (struct bpf_local_storage_map *)map,
106 value, map_flags, GFP_ATOMIC);
107 fdput(f);
108 return PTR_ERR_OR_ZERO(sdata);
109}
110
111static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
112{
113 struct bpf_local_storage_data *sdata;
114
115 sdata = inode_storage_lookup(inode, map, false);
116 if (!sdata)
117 return -ENOENT;
118
119 bpf_selem_unlink(SELEM(sdata), false);
120
121 return 0;
122}
123
124static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
125{
126 struct fd f = fdget_raw(*(int *)key);
127 int err;
128
129 if (!f.file)
130 return -EBADF;
131
132 err = inode_storage_delete(file_inode(f.file), map);
133 fdput(f);
134 return err;
135}
136
137/* *gfp_flags* is a hidden argument provided by the verifier */
138BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
139 void *, value, u64, flags, gfp_t, gfp_flags)
140{
141 struct bpf_local_storage_data *sdata;
142
143 WARN_ON_ONCE(!bpf_rcu_lock_held());
144 if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
145 return (unsigned long)NULL;
146
147 /* explicitly check that the inode_storage_ptr is not
148 * NULL as inode_storage_lookup returns NULL in this case and
149 * bpf_local_storage_update expects the owner to have a
150 * valid storage pointer.
151 */
152 if (!inode || !inode_storage_ptr(inode))
153 return (unsigned long)NULL;
154
155 sdata = inode_storage_lookup(inode, map, true);
156 if (sdata)
157 return (unsigned long)sdata->data;
158
159 /* This helper must only called from where the inode is guaranteed
160 * to have a refcount and cannot be freed.
161 */
162 if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
163 sdata = bpf_local_storage_update(
164 inode, (struct bpf_local_storage_map *)map, value,
165 BPF_NOEXIST, gfp_flags);
166 return IS_ERR(sdata) ? (unsigned long)NULL :
167 (unsigned long)sdata->data;
168 }
169
170 return (unsigned long)NULL;
171}
172
173BPF_CALL_2(bpf_inode_storage_delete,
174 struct bpf_map *, map, struct inode *, inode)
175{
176 WARN_ON_ONCE(!bpf_rcu_lock_held());
177 if (!inode)
178 return -EINVAL;
179
180 /* This helper must only called from where the inode is guaranteed
181 * to have a refcount and cannot be freed.
182 */
183 return inode_storage_delete(inode, map);
184}
185
186static int notsupp_get_next_key(struct bpf_map *map, void *key,
187 void *next_key)
188{
189 return -ENOTSUPP;
190}
191
192static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
193{
194 return bpf_local_storage_map_alloc(attr, &inode_cache, false);
195}
196
197static void inode_storage_map_free(struct bpf_map *map)
198{
199 bpf_local_storage_map_free(map, &inode_cache, NULL);
200}
201
202const struct bpf_map_ops inode_storage_map_ops = {
203 .map_meta_equal = bpf_map_meta_equal,
204 .map_alloc_check = bpf_local_storage_map_alloc_check,
205 .map_alloc = inode_storage_map_alloc,
206 .map_free = inode_storage_map_free,
207 .map_get_next_key = notsupp_get_next_key,
208 .map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
209 .map_update_elem = bpf_fd_inode_storage_update_elem,
210 .map_delete_elem = bpf_fd_inode_storage_delete_elem,
211 .map_check_btf = bpf_local_storage_map_check_btf,
212 .map_mem_usage = bpf_local_storage_map_mem_usage,
213 .map_btf_id = &bpf_local_storage_map_btf_id[0],
214 .map_owner_storage_ptr = inode_storage_ptr,
215};
216
217BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
218
219const struct bpf_func_proto bpf_inode_storage_get_proto = {
220 .func = bpf_inode_storage_get,
221 .gpl_only = false,
222 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
223 .arg1_type = ARG_CONST_MAP_PTR,
224 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
225 .arg2_btf_id = &bpf_inode_storage_btf_ids[0],
226 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
227 .arg4_type = ARG_ANYTHING,
228};
229
230const struct bpf_func_proto bpf_inode_storage_delete_proto = {
231 .func = bpf_inode_storage_delete,
232 .gpl_only = false,
233 .ret_type = RET_INTEGER,
234 .arg1_type = ARG_CONST_MAP_PTR,
235 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
236 .arg2_btf_id = &bpf_inode_storage_btf_ids[0],
237};
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2019 Facebook
4 * Copyright 2020 Google LLC.
5 */
6
7#include <linux/rculist.h>
8#include <linux/list.h>
9#include <linux/hash.h>
10#include <linux/types.h>
11#include <linux/spinlock.h>
12#include <linux/bpf.h>
13#include <linux/bpf_local_storage.h>
14#include <net/sock.h>
15#include <uapi/linux/sock_diag.h>
16#include <uapi/linux/btf.h>
17#include <linux/bpf_lsm.h>
18#include <linux/btf_ids.h>
19#include <linux/rcupdate_trace.h>
20
21DEFINE_BPF_STORAGE_CACHE(inode_cache);
22
23static struct bpf_local_storage __rcu **
24inode_storage_ptr(void *owner)
25{
26 struct inode *inode = owner;
27 struct bpf_storage_blob *bsb;
28
29 bsb = bpf_inode(inode);
30 if (!bsb)
31 return NULL;
32 return &bsb->storage;
33}
34
35static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
36 struct bpf_map *map,
37 bool cacheit_lockit)
38{
39 struct bpf_local_storage *inode_storage;
40 struct bpf_local_storage_map *smap;
41 struct bpf_storage_blob *bsb;
42
43 bsb = bpf_inode(inode);
44 if (!bsb)
45 return NULL;
46
47 inode_storage =
48 rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
49 if (!inode_storage)
50 return NULL;
51
52 smap = (struct bpf_local_storage_map *)map;
53 return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
54}
55
56void bpf_inode_storage_free(struct inode *inode)
57{
58 struct bpf_local_storage *local_storage;
59 struct bpf_storage_blob *bsb;
60
61 bsb = bpf_inode(inode);
62 if (!bsb)
63 return;
64
65 rcu_read_lock();
66
67 local_storage = rcu_dereference(bsb->storage);
68 if (!local_storage) {
69 rcu_read_unlock();
70 return;
71 }
72
73 bpf_local_storage_destroy(local_storage);
74 rcu_read_unlock();
75}
76
77static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
78{
79 struct bpf_local_storage_data *sdata;
80 CLASS(fd_raw, f)(*(int *)key);
81
82 if (fd_empty(f))
83 return ERR_PTR(-EBADF);
84
85 sdata = inode_storage_lookup(file_inode(fd_file(f)), map, true);
86 return sdata ? sdata->data : NULL;
87}
88
89static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
90 void *value, u64 map_flags)
91{
92 struct bpf_local_storage_data *sdata;
93 CLASS(fd_raw, f)(*(int *)key);
94
95 if (fd_empty(f))
96 return -EBADF;
97 if (!inode_storage_ptr(file_inode(fd_file(f))))
98 return -EBADF;
99
100 sdata = bpf_local_storage_update(file_inode(fd_file(f)),
101 (struct bpf_local_storage_map *)map,
102 value, map_flags, false, GFP_ATOMIC);
103 return PTR_ERR_OR_ZERO(sdata);
104}
105
106static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
107{
108 struct bpf_local_storage_data *sdata;
109
110 sdata = inode_storage_lookup(inode, map, false);
111 if (!sdata)
112 return -ENOENT;
113
114 bpf_selem_unlink(SELEM(sdata), false);
115
116 return 0;
117}
118
119static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
120{
121 CLASS(fd_raw, f)(*(int *)key);
122
123 if (fd_empty(f))
124 return -EBADF;
125 return inode_storage_delete(file_inode(fd_file(f)), map);
126}
127
128/* *gfp_flags* is a hidden argument provided by the verifier */
129BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
130 void *, value, u64, flags, gfp_t, gfp_flags)
131{
132 struct bpf_local_storage_data *sdata;
133
134 WARN_ON_ONCE(!bpf_rcu_lock_held());
135 if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
136 return (unsigned long)NULL;
137
138 /* explicitly check that the inode_storage_ptr is not
139 * NULL as inode_storage_lookup returns NULL in this case and
140 * bpf_local_storage_update expects the owner to have a
141 * valid storage pointer.
142 */
143 if (!inode || !inode_storage_ptr(inode))
144 return (unsigned long)NULL;
145
146 sdata = inode_storage_lookup(inode, map, true);
147 if (sdata)
148 return (unsigned long)sdata->data;
149
150 /* This helper must only called from where the inode is guaranteed
151 * to have a refcount and cannot be freed.
152 */
153 if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
154 sdata = bpf_local_storage_update(
155 inode, (struct bpf_local_storage_map *)map, value,
156 BPF_NOEXIST, false, gfp_flags);
157 return IS_ERR(sdata) ? (unsigned long)NULL :
158 (unsigned long)sdata->data;
159 }
160
161 return (unsigned long)NULL;
162}
163
164BPF_CALL_2(bpf_inode_storage_delete,
165 struct bpf_map *, map, struct inode *, inode)
166{
167 WARN_ON_ONCE(!bpf_rcu_lock_held());
168 if (!inode)
169 return -EINVAL;
170
171 /* This helper must only called from where the inode is guaranteed
172 * to have a refcount and cannot be freed.
173 */
174 return inode_storage_delete(inode, map);
175}
176
177static int notsupp_get_next_key(struct bpf_map *map, void *key,
178 void *next_key)
179{
180 return -ENOTSUPP;
181}
182
183static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
184{
185 return bpf_local_storage_map_alloc(attr, &inode_cache, false);
186}
187
188static void inode_storage_map_free(struct bpf_map *map)
189{
190 bpf_local_storage_map_free(map, &inode_cache, NULL);
191}
192
193const struct bpf_map_ops inode_storage_map_ops = {
194 .map_meta_equal = bpf_map_meta_equal,
195 .map_alloc_check = bpf_local_storage_map_alloc_check,
196 .map_alloc = inode_storage_map_alloc,
197 .map_free = inode_storage_map_free,
198 .map_get_next_key = notsupp_get_next_key,
199 .map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
200 .map_update_elem = bpf_fd_inode_storage_update_elem,
201 .map_delete_elem = bpf_fd_inode_storage_delete_elem,
202 .map_check_btf = bpf_local_storage_map_check_btf,
203 .map_mem_usage = bpf_local_storage_map_mem_usage,
204 .map_btf_id = &bpf_local_storage_map_btf_id[0],
205 .map_owner_storage_ptr = inode_storage_ptr,
206};
207
208BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
209
210const struct bpf_func_proto bpf_inode_storage_get_proto = {
211 .func = bpf_inode_storage_get,
212 .gpl_only = false,
213 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
214 .arg1_type = ARG_CONST_MAP_PTR,
215 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
216 .arg2_btf_id = &bpf_inode_storage_btf_ids[0],
217 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
218 .arg4_type = ARG_ANYTHING,
219};
220
221const struct bpf_func_proto bpf_inode_storage_delete_proto = {
222 .func = bpf_inode_storage_delete,
223 .gpl_only = false,
224 .ret_type = RET_INTEGER,
225 .arg1_type = ARG_CONST_MAP_PTR,
226 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
227 .arg2_btf_id = &bpf_inode_storage_btf_ids[0],
228};