Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2016 Tom Herbert <tom@herbertland.com> */
3
4#include <linux/skbuff.h>
5#include <linux/workqueue.h>
6#include <net/strparser.h>
7#include <net/tcp.h>
8#include <net/sock.h>
9#include <net/tls.h>
10
11#include "tls.h"
12
13static struct workqueue_struct *tls_strp_wq;
14
15static void tls_strp_abort_strp(struct tls_strparser *strp, int err)
16{
17 if (strp->stopped)
18 return;
19
20 strp->stopped = 1;
21
22 /* Report an error on the lower socket */
23 strp->sk->sk_err = -err;
24 sk_error_report(strp->sk);
25}
26
27static void tls_strp_anchor_free(struct tls_strparser *strp)
28{
29 struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
30
31 DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1);
32 shinfo->frag_list = NULL;
33 consume_skb(strp->anchor);
34 strp->anchor = NULL;
35}
36
37/* Create a new skb with the contents of input copied to its page frags */
38static struct sk_buff *tls_strp_msg_make_copy(struct tls_strparser *strp)
39{
40 struct strp_msg *rxm;
41 struct sk_buff *skb;
42 int i, err, offset;
43
44 skb = alloc_skb_with_frags(0, strp->stm.full_len, TLS_PAGE_ORDER,
45 &err, strp->sk->sk_allocation);
46 if (!skb)
47 return NULL;
48
49 offset = strp->stm.offset;
50 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
51 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
52
53 WARN_ON_ONCE(skb_copy_bits(strp->anchor, offset,
54 skb_frag_address(frag),
55 skb_frag_size(frag)));
56 offset += skb_frag_size(frag);
57 }
58
59 skb_copy_header(skb, strp->anchor);
60 rxm = strp_msg(skb);
61 rxm->offset = 0;
62 return skb;
63}
64
65/* Steal the input skb, input msg is invalid after calling this function */
66struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx)
67{
68 struct tls_strparser *strp = &ctx->strp;
69
70#ifdef CONFIG_TLS_DEVICE
71 DEBUG_NET_WARN_ON_ONCE(!strp->anchor->decrypted);
72#else
73 /* This function turns an input into an output,
74 * that can only happen if we have offload.
75 */
76 WARN_ON(1);
77#endif
78
79 if (strp->copy_mode) {
80 struct sk_buff *skb;
81
82 /* Replace anchor with an empty skb, this is a little
83 * dangerous but __tls_cur_msg() warns on empty skbs
84 * so hopefully we'll catch abuses.
85 */
86 skb = alloc_skb(0, strp->sk->sk_allocation);
87 if (!skb)
88 return NULL;
89
90 swap(strp->anchor, skb);
91 return skb;
92 }
93
94 return tls_strp_msg_make_copy(strp);
95}
96
97/* Force the input skb to be in copy mode. The data ownership remains
98 * with the input skb itself (meaning unpause will wipe it) but it can
99 * be modified.
100 */
101int tls_strp_msg_cow(struct tls_sw_context_rx *ctx)
102{
103 struct tls_strparser *strp = &ctx->strp;
104 struct sk_buff *skb;
105
106 if (strp->copy_mode)
107 return 0;
108
109 skb = tls_strp_msg_make_copy(strp);
110 if (!skb)
111 return -ENOMEM;
112
113 tls_strp_anchor_free(strp);
114 strp->anchor = skb;
115
116 tcp_read_done(strp->sk, strp->stm.full_len);
117 strp->copy_mode = 1;
118
119 return 0;
120}
121
122/* Make a clone (in the skb sense) of the input msg to keep a reference
123 * to the underlying data. The reference-holding skbs get placed on
124 * @dst.
125 */
126int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst)
127{
128 struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
129
130 if (strp->copy_mode) {
131 struct sk_buff *skb;
132
133 WARN_ON_ONCE(!shinfo->nr_frags);
134
135 /* We can't skb_clone() the anchor, it gets wiped by unpause */
136 skb = alloc_skb(0, strp->sk->sk_allocation);
137 if (!skb)
138 return -ENOMEM;
139
140 __skb_queue_tail(dst, strp->anchor);
141 strp->anchor = skb;
142 } else {
143 struct sk_buff *iter, *clone;
144 int chunk, len, offset;
145
146 offset = strp->stm.offset;
147 len = strp->stm.full_len;
148 iter = shinfo->frag_list;
149
150 while (len > 0) {
151 if (iter->len <= offset) {
152 offset -= iter->len;
153 goto next;
154 }
155
156 chunk = iter->len - offset;
157 offset = 0;
158
159 clone = skb_clone(iter, strp->sk->sk_allocation);
160 if (!clone)
161 return -ENOMEM;
162 __skb_queue_tail(dst, clone);
163
164 len -= chunk;
165next:
166 iter = iter->next;
167 }
168 }
169
170 return 0;
171}
172
173static void tls_strp_flush_anchor_copy(struct tls_strparser *strp)
174{
175 struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
176 int i;
177
178 DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1);
179
180 for (i = 0; i < shinfo->nr_frags; i++)
181 __skb_frag_unref(&shinfo->frags[i], false);
182 shinfo->nr_frags = 0;
183 strp->copy_mode = 0;
184}
185
186static int tls_strp_copyin(read_descriptor_t *desc, struct sk_buff *in_skb,
187 unsigned int offset, size_t in_len)
188{
189 struct tls_strparser *strp = (struct tls_strparser *)desc->arg.data;
190 struct sk_buff *skb;
191 skb_frag_t *frag;
192 size_t len, chunk;
193 int sz;
194
195 if (strp->msg_ready)
196 return 0;
197
198 skb = strp->anchor;
199 frag = &skb_shinfo(skb)->frags[skb->len / PAGE_SIZE];
200
201 len = in_len;
202 /* First make sure we got the header */
203 if (!strp->stm.full_len) {
204 /* Assume one page is more than enough for headers */
205 chunk = min_t(size_t, len, PAGE_SIZE - skb_frag_size(frag));
206 WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
207 skb_frag_address(frag) +
208 skb_frag_size(frag),
209 chunk));
210
211 sz = tls_rx_msg_size(strp, strp->anchor);
212 if (sz < 0) {
213 desc->error = sz;
214 return 0;
215 }
216
217 /* We may have over-read, sz == 0 is guaranteed under-read */
218 if (sz > 0)
219 chunk = min_t(size_t, chunk, sz - skb->len);
220
221 skb->len += chunk;
222 skb->data_len += chunk;
223 skb_frag_size_add(frag, chunk);
224 frag++;
225 len -= chunk;
226 offset += chunk;
227
228 strp->stm.full_len = sz;
229 if (!strp->stm.full_len)
230 goto read_done;
231 }
232
233 /* Load up more data */
234 while (len && strp->stm.full_len > skb->len) {
235 chunk = min_t(size_t, len, strp->stm.full_len - skb->len);
236 chunk = min_t(size_t, chunk, PAGE_SIZE - skb_frag_size(frag));
237 WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
238 skb_frag_address(frag) +
239 skb_frag_size(frag),
240 chunk));
241
242 skb->len += chunk;
243 skb->data_len += chunk;
244 skb_frag_size_add(frag, chunk);
245 frag++;
246 len -= chunk;
247 offset += chunk;
248 }
249
250 if (strp->stm.full_len == skb->len) {
251 desc->count = 0;
252
253 strp->msg_ready = 1;
254 tls_rx_msg_ready(strp);
255 }
256
257read_done:
258 return in_len - len;
259}
260
261static int tls_strp_read_copyin(struct tls_strparser *strp)
262{
263 struct socket *sock = strp->sk->sk_socket;
264 read_descriptor_t desc;
265
266 desc.arg.data = strp;
267 desc.error = 0;
268 desc.count = 1; /* give more than one skb per call */
269
270 /* sk should be locked here, so okay to do read_sock */
271 sock->ops->read_sock(strp->sk, &desc, tls_strp_copyin);
272
273 return desc.error;
274}
275
276static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort)
277{
278 struct skb_shared_info *shinfo;
279 struct page *page;
280 int need_spc, len;
281
282 /* If the rbuf is small or rcv window has collapsed to 0 we need
283 * to read the data out. Otherwise the connection will stall.
284 * Without pressure threshold of INT_MAX will never be ready.
285 */
286 if (likely(qshort && !tcp_epollin_ready(strp->sk, INT_MAX)))
287 return 0;
288
289 shinfo = skb_shinfo(strp->anchor);
290 shinfo->frag_list = NULL;
291
292 /* If we don't know the length go max plus page for cipher overhead */
293 need_spc = strp->stm.full_len ?: TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE;
294
295 for (len = need_spc; len > 0; len -= PAGE_SIZE) {
296 page = alloc_page(strp->sk->sk_allocation);
297 if (!page) {
298 tls_strp_flush_anchor_copy(strp);
299 return -ENOMEM;
300 }
301
302 skb_fill_page_desc(strp->anchor, shinfo->nr_frags++,
303 page, 0, 0);
304 }
305
306 strp->copy_mode = 1;
307 strp->stm.offset = 0;
308
309 strp->anchor->len = 0;
310 strp->anchor->data_len = 0;
311 strp->anchor->truesize = round_up(need_spc, PAGE_SIZE);
312
313 tls_strp_read_copyin(strp);
314
315 return 0;
316}
317
318static bool tls_strp_check_no_dup(struct tls_strparser *strp)
319{
320 unsigned int len = strp->stm.offset + strp->stm.full_len;
321 struct sk_buff *skb;
322 u32 seq;
323
324 skb = skb_shinfo(strp->anchor)->frag_list;
325 seq = TCP_SKB_CB(skb)->seq;
326
327 while (skb->len < len) {
328 seq += skb->len;
329 len -= skb->len;
330 skb = skb->next;
331
332 if (TCP_SKB_CB(skb)->seq != seq)
333 return false;
334 }
335
336 return true;
337}
338
339static void tls_strp_load_anchor_with_queue(struct tls_strparser *strp, int len)
340{
341 struct tcp_sock *tp = tcp_sk(strp->sk);
342 struct sk_buff *first;
343 u32 offset;
344
345 first = tcp_recv_skb(strp->sk, tp->copied_seq, &offset);
346 if (WARN_ON_ONCE(!first))
347 return;
348
349 /* Bestow the state onto the anchor */
350 strp->anchor->len = offset + len;
351 strp->anchor->data_len = offset + len;
352 strp->anchor->truesize = offset + len;
353
354 skb_shinfo(strp->anchor)->frag_list = first;
355
356 skb_copy_header(strp->anchor, first);
357 strp->anchor->destructor = NULL;
358
359 strp->stm.offset = offset;
360}
361
362void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh)
363{
364 struct strp_msg *rxm;
365 struct tls_msg *tlm;
366
367 DEBUG_NET_WARN_ON_ONCE(!strp->msg_ready);
368 DEBUG_NET_WARN_ON_ONCE(!strp->stm.full_len);
369
370 if (!strp->copy_mode && force_refresh) {
371 if (WARN_ON(tcp_inq(strp->sk) < strp->stm.full_len))
372 return;
373
374 tls_strp_load_anchor_with_queue(strp, strp->stm.full_len);
375 }
376
377 rxm = strp_msg(strp->anchor);
378 rxm->full_len = strp->stm.full_len;
379 rxm->offset = strp->stm.offset;
380 tlm = tls_msg(strp->anchor);
381 tlm->control = strp->mark;
382}
383
384/* Called with lock held on lower socket */
385static int tls_strp_read_sock(struct tls_strparser *strp)
386{
387 int sz, inq;
388
389 inq = tcp_inq(strp->sk);
390 if (inq < 1)
391 return 0;
392
393 if (unlikely(strp->copy_mode))
394 return tls_strp_read_copyin(strp);
395
396 if (inq < strp->stm.full_len)
397 return tls_strp_read_copy(strp, true);
398
399 if (!strp->stm.full_len) {
400 tls_strp_load_anchor_with_queue(strp, inq);
401
402 sz = tls_rx_msg_size(strp, strp->anchor);
403 if (sz < 0) {
404 tls_strp_abort_strp(strp, sz);
405 return sz;
406 }
407
408 strp->stm.full_len = sz;
409
410 if (!strp->stm.full_len || inq < strp->stm.full_len)
411 return tls_strp_read_copy(strp, true);
412 }
413
414 if (!tls_strp_check_no_dup(strp))
415 return tls_strp_read_copy(strp, false);
416
417 strp->msg_ready = 1;
418 tls_rx_msg_ready(strp);
419
420 return 0;
421}
422
423void tls_strp_check_rcv(struct tls_strparser *strp)
424{
425 if (unlikely(strp->stopped) || strp->msg_ready)
426 return;
427
428 if (tls_strp_read_sock(strp) == -ENOMEM)
429 queue_work(tls_strp_wq, &strp->work);
430}
431
432/* Lower sock lock held */
433void tls_strp_data_ready(struct tls_strparser *strp)
434{
435 /* This check is needed to synchronize with do_tls_strp_work.
436 * do_tls_strp_work acquires a process lock (lock_sock) whereas
437 * the lock held here is bh_lock_sock. The two locks can be
438 * held by different threads at the same time, but bh_lock_sock
439 * allows a thread in BH context to safely check if the process
440 * lock is held. In this case, if the lock is held, queue work.
441 */
442 if (sock_owned_by_user_nocheck(strp->sk)) {
443 queue_work(tls_strp_wq, &strp->work);
444 return;
445 }
446
447 tls_strp_check_rcv(strp);
448}
449
450static void tls_strp_work(struct work_struct *w)
451{
452 struct tls_strparser *strp =
453 container_of(w, struct tls_strparser, work);
454
455 lock_sock(strp->sk);
456 tls_strp_check_rcv(strp);
457 release_sock(strp->sk);
458}
459
460void tls_strp_msg_done(struct tls_strparser *strp)
461{
462 WARN_ON(!strp->stm.full_len);
463
464 if (likely(!strp->copy_mode))
465 tcp_read_done(strp->sk, strp->stm.full_len);
466 else
467 tls_strp_flush_anchor_copy(strp);
468
469 strp->msg_ready = 0;
470 memset(&strp->stm, 0, sizeof(strp->stm));
471
472 tls_strp_check_rcv(strp);
473}
474
475void tls_strp_stop(struct tls_strparser *strp)
476{
477 strp->stopped = 1;
478}
479
480int tls_strp_init(struct tls_strparser *strp, struct sock *sk)
481{
482 memset(strp, 0, sizeof(*strp));
483
484 strp->sk = sk;
485
486 strp->anchor = alloc_skb(0, GFP_KERNEL);
487 if (!strp->anchor)
488 return -ENOMEM;
489
490 INIT_WORK(&strp->work, tls_strp_work);
491
492 return 0;
493}
494
495/* strp must already be stopped so that tls_strp_recv will no longer be called.
496 * Note that tls_strp_done is not called with the lower socket held.
497 */
498void tls_strp_done(struct tls_strparser *strp)
499{
500 WARN_ON(!strp->stopped);
501
502 cancel_work_sync(&strp->work);
503 tls_strp_anchor_free(strp);
504}
505
506int __init tls_strp_dev_init(void)
507{
508 tls_strp_wq = create_workqueue("tls-strp");
509 if (unlikely(!tls_strp_wq))
510 return -ENOMEM;
511
512 return 0;
513}
514
515void tls_strp_dev_exit(void)
516{
517 destroy_workqueue(tls_strp_wq);
518}
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2016 Tom Herbert <tom@herbertland.com> */
3
4#include <linux/skbuff.h>
5#include <linux/skbuff_ref.h>
6#include <linux/workqueue.h>
7#include <net/strparser.h>
8#include <net/tcp.h>
9#include <net/sock.h>
10#include <net/tls.h>
11
12#include "tls.h"
13
14static struct workqueue_struct *tls_strp_wq;
15
16static void tls_strp_abort_strp(struct tls_strparser *strp, int err)
17{
18 if (strp->stopped)
19 return;
20
21 strp->stopped = 1;
22
23 /* Report an error on the lower socket */
24 WRITE_ONCE(strp->sk->sk_err, -err);
25 /* Paired with smp_rmb() in tcp_poll() */
26 smp_wmb();
27 sk_error_report(strp->sk);
28}
29
30static void tls_strp_anchor_free(struct tls_strparser *strp)
31{
32 struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
33
34 DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1);
35 if (!strp->copy_mode)
36 shinfo->frag_list = NULL;
37 consume_skb(strp->anchor);
38 strp->anchor = NULL;
39}
40
41static struct sk_buff *
42tls_strp_skb_copy(struct tls_strparser *strp, struct sk_buff *in_skb,
43 int offset, int len)
44{
45 struct sk_buff *skb;
46 int i, err;
47
48 skb = alloc_skb_with_frags(0, len, TLS_PAGE_ORDER,
49 &err, strp->sk->sk_allocation);
50 if (!skb)
51 return NULL;
52
53 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
54 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
55
56 WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
57 skb_frag_address(frag),
58 skb_frag_size(frag)));
59 offset += skb_frag_size(frag);
60 }
61
62 skb->len = len;
63 skb->data_len = len;
64 skb_copy_header(skb, in_skb);
65 return skb;
66}
67
68/* Create a new skb with the contents of input copied to its page frags */
69static struct sk_buff *tls_strp_msg_make_copy(struct tls_strparser *strp)
70{
71 struct strp_msg *rxm;
72 struct sk_buff *skb;
73
74 skb = tls_strp_skb_copy(strp, strp->anchor, strp->stm.offset,
75 strp->stm.full_len);
76 if (!skb)
77 return NULL;
78
79 rxm = strp_msg(skb);
80 rxm->offset = 0;
81 return skb;
82}
83
84/* Steal the input skb, input msg is invalid after calling this function */
85struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx)
86{
87 struct tls_strparser *strp = &ctx->strp;
88
89#ifdef CONFIG_TLS_DEVICE
90 DEBUG_NET_WARN_ON_ONCE(!strp->anchor->decrypted);
91#else
92 /* This function turns an input into an output,
93 * that can only happen if we have offload.
94 */
95 WARN_ON(1);
96#endif
97
98 if (strp->copy_mode) {
99 struct sk_buff *skb;
100
101 /* Replace anchor with an empty skb, this is a little
102 * dangerous but __tls_cur_msg() warns on empty skbs
103 * so hopefully we'll catch abuses.
104 */
105 skb = alloc_skb(0, strp->sk->sk_allocation);
106 if (!skb)
107 return NULL;
108
109 swap(strp->anchor, skb);
110 return skb;
111 }
112
113 return tls_strp_msg_make_copy(strp);
114}
115
116/* Force the input skb to be in copy mode. The data ownership remains
117 * with the input skb itself (meaning unpause will wipe it) but it can
118 * be modified.
119 */
120int tls_strp_msg_cow(struct tls_sw_context_rx *ctx)
121{
122 struct tls_strparser *strp = &ctx->strp;
123 struct sk_buff *skb;
124
125 if (strp->copy_mode)
126 return 0;
127
128 skb = tls_strp_msg_make_copy(strp);
129 if (!skb)
130 return -ENOMEM;
131
132 tls_strp_anchor_free(strp);
133 strp->anchor = skb;
134
135 tcp_read_done(strp->sk, strp->stm.full_len);
136 strp->copy_mode = 1;
137
138 return 0;
139}
140
141/* Make a clone (in the skb sense) of the input msg to keep a reference
142 * to the underlying data. The reference-holding skbs get placed on
143 * @dst.
144 */
145int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst)
146{
147 struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
148
149 if (strp->copy_mode) {
150 struct sk_buff *skb;
151
152 WARN_ON_ONCE(!shinfo->nr_frags);
153
154 /* We can't skb_clone() the anchor, it gets wiped by unpause */
155 skb = alloc_skb(0, strp->sk->sk_allocation);
156 if (!skb)
157 return -ENOMEM;
158
159 __skb_queue_tail(dst, strp->anchor);
160 strp->anchor = skb;
161 } else {
162 struct sk_buff *iter, *clone;
163 int chunk, len, offset;
164
165 offset = strp->stm.offset;
166 len = strp->stm.full_len;
167 iter = shinfo->frag_list;
168
169 while (len > 0) {
170 if (iter->len <= offset) {
171 offset -= iter->len;
172 goto next;
173 }
174
175 chunk = iter->len - offset;
176 offset = 0;
177
178 clone = skb_clone(iter, strp->sk->sk_allocation);
179 if (!clone)
180 return -ENOMEM;
181 __skb_queue_tail(dst, clone);
182
183 len -= chunk;
184next:
185 iter = iter->next;
186 }
187 }
188
189 return 0;
190}
191
192static void tls_strp_flush_anchor_copy(struct tls_strparser *strp)
193{
194 struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
195 int i;
196
197 DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1);
198
199 for (i = 0; i < shinfo->nr_frags; i++)
200 __skb_frag_unref(&shinfo->frags[i], false);
201 shinfo->nr_frags = 0;
202 if (strp->copy_mode) {
203 kfree_skb_list(shinfo->frag_list);
204 shinfo->frag_list = NULL;
205 }
206 strp->copy_mode = 0;
207 strp->mixed_decrypted = 0;
208}
209
210static int tls_strp_copyin_frag(struct tls_strparser *strp, struct sk_buff *skb,
211 struct sk_buff *in_skb, unsigned int offset,
212 size_t in_len)
213{
214 size_t len, chunk;
215 skb_frag_t *frag;
216 int sz;
217
218 frag = &skb_shinfo(skb)->frags[skb->len / PAGE_SIZE];
219
220 len = in_len;
221 /* First make sure we got the header */
222 if (!strp->stm.full_len) {
223 /* Assume one page is more than enough for headers */
224 chunk = min_t(size_t, len, PAGE_SIZE - skb_frag_size(frag));
225 WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
226 skb_frag_address(frag) +
227 skb_frag_size(frag),
228 chunk));
229
230 skb->len += chunk;
231 skb->data_len += chunk;
232 skb_frag_size_add(frag, chunk);
233
234 sz = tls_rx_msg_size(strp, skb);
235 if (sz < 0)
236 return sz;
237
238 /* We may have over-read, sz == 0 is guaranteed under-read */
239 if (unlikely(sz && sz < skb->len)) {
240 int over = skb->len - sz;
241
242 WARN_ON_ONCE(over > chunk);
243 skb->len -= over;
244 skb->data_len -= over;
245 skb_frag_size_add(frag, -over);
246
247 chunk -= over;
248 }
249
250 frag++;
251 len -= chunk;
252 offset += chunk;
253
254 strp->stm.full_len = sz;
255 if (!strp->stm.full_len)
256 goto read_done;
257 }
258
259 /* Load up more data */
260 while (len && strp->stm.full_len > skb->len) {
261 chunk = min_t(size_t, len, strp->stm.full_len - skb->len);
262 chunk = min_t(size_t, chunk, PAGE_SIZE - skb_frag_size(frag));
263 WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
264 skb_frag_address(frag) +
265 skb_frag_size(frag),
266 chunk));
267
268 skb->len += chunk;
269 skb->data_len += chunk;
270 skb_frag_size_add(frag, chunk);
271 frag++;
272 len -= chunk;
273 offset += chunk;
274 }
275
276read_done:
277 return in_len - len;
278}
279
280static int tls_strp_copyin_skb(struct tls_strparser *strp, struct sk_buff *skb,
281 struct sk_buff *in_skb, unsigned int offset,
282 size_t in_len)
283{
284 struct sk_buff *nskb, *first, *last;
285 struct skb_shared_info *shinfo;
286 size_t chunk;
287 int sz;
288
289 if (strp->stm.full_len)
290 chunk = strp->stm.full_len - skb->len;
291 else
292 chunk = TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE;
293 chunk = min(chunk, in_len);
294
295 nskb = tls_strp_skb_copy(strp, in_skb, offset, chunk);
296 if (!nskb)
297 return -ENOMEM;
298
299 shinfo = skb_shinfo(skb);
300 if (!shinfo->frag_list) {
301 shinfo->frag_list = nskb;
302 nskb->prev = nskb;
303 } else {
304 first = shinfo->frag_list;
305 last = first->prev;
306 last->next = nskb;
307 first->prev = nskb;
308 }
309
310 skb->len += chunk;
311 skb->data_len += chunk;
312
313 if (!strp->stm.full_len) {
314 sz = tls_rx_msg_size(strp, skb);
315 if (sz < 0)
316 return sz;
317
318 /* We may have over-read, sz == 0 is guaranteed under-read */
319 if (unlikely(sz && sz < skb->len)) {
320 int over = skb->len - sz;
321
322 WARN_ON_ONCE(over > chunk);
323 skb->len -= over;
324 skb->data_len -= over;
325 __pskb_trim(nskb, nskb->len - over);
326
327 chunk -= over;
328 }
329
330 strp->stm.full_len = sz;
331 }
332
333 return chunk;
334}
335
336static int tls_strp_copyin(read_descriptor_t *desc, struct sk_buff *in_skb,
337 unsigned int offset, size_t in_len)
338{
339 struct tls_strparser *strp = (struct tls_strparser *)desc->arg.data;
340 struct sk_buff *skb;
341 int ret;
342
343 if (strp->msg_ready)
344 return 0;
345
346 skb = strp->anchor;
347 if (!skb->len)
348 skb_copy_decrypted(skb, in_skb);
349 else
350 strp->mixed_decrypted |= !!skb_cmp_decrypted(skb, in_skb);
351
352 if (IS_ENABLED(CONFIG_TLS_DEVICE) && strp->mixed_decrypted)
353 ret = tls_strp_copyin_skb(strp, skb, in_skb, offset, in_len);
354 else
355 ret = tls_strp_copyin_frag(strp, skb, in_skb, offset, in_len);
356 if (ret < 0) {
357 desc->error = ret;
358 ret = 0;
359 }
360
361 if (strp->stm.full_len && strp->stm.full_len == skb->len) {
362 desc->count = 0;
363
364 WRITE_ONCE(strp->msg_ready, 1);
365 tls_rx_msg_ready(strp);
366 }
367
368 return ret;
369}
370
371static int tls_strp_read_copyin(struct tls_strparser *strp)
372{
373 read_descriptor_t desc;
374
375 desc.arg.data = strp;
376 desc.error = 0;
377 desc.count = 1; /* give more than one skb per call */
378
379 /* sk should be locked here, so okay to do read_sock */
380 tcp_read_sock(strp->sk, &desc, tls_strp_copyin);
381
382 return desc.error;
383}
384
385static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort)
386{
387 struct skb_shared_info *shinfo;
388 struct page *page;
389 int need_spc, len;
390
391 /* If the rbuf is small or rcv window has collapsed to 0 we need
392 * to read the data out. Otherwise the connection will stall.
393 * Without pressure threshold of INT_MAX will never be ready.
394 */
395 if (likely(qshort && !tcp_epollin_ready(strp->sk, INT_MAX)))
396 return 0;
397
398 shinfo = skb_shinfo(strp->anchor);
399 shinfo->frag_list = NULL;
400
401 /* If we don't know the length go max plus page for cipher overhead */
402 need_spc = strp->stm.full_len ?: TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE;
403
404 for (len = need_spc; len > 0; len -= PAGE_SIZE) {
405 page = alloc_page(strp->sk->sk_allocation);
406 if (!page) {
407 tls_strp_flush_anchor_copy(strp);
408 return -ENOMEM;
409 }
410
411 skb_fill_page_desc(strp->anchor, shinfo->nr_frags++,
412 page, 0, 0);
413 }
414
415 strp->copy_mode = 1;
416 strp->stm.offset = 0;
417
418 strp->anchor->len = 0;
419 strp->anchor->data_len = 0;
420 strp->anchor->truesize = round_up(need_spc, PAGE_SIZE);
421
422 tls_strp_read_copyin(strp);
423
424 return 0;
425}
426
427static bool tls_strp_check_queue_ok(struct tls_strparser *strp)
428{
429 unsigned int len = strp->stm.offset + strp->stm.full_len;
430 struct sk_buff *first, *skb;
431 u32 seq;
432
433 first = skb_shinfo(strp->anchor)->frag_list;
434 skb = first;
435 seq = TCP_SKB_CB(first)->seq;
436
437 /* Make sure there's no duplicate data in the queue,
438 * and the decrypted status matches.
439 */
440 while (skb->len < len) {
441 seq += skb->len;
442 len -= skb->len;
443 skb = skb->next;
444
445 if (TCP_SKB_CB(skb)->seq != seq)
446 return false;
447 if (skb_cmp_decrypted(first, skb))
448 return false;
449 }
450
451 return true;
452}
453
454static void tls_strp_load_anchor_with_queue(struct tls_strparser *strp, int len)
455{
456 struct tcp_sock *tp = tcp_sk(strp->sk);
457 struct sk_buff *first;
458 u32 offset;
459
460 first = tcp_recv_skb(strp->sk, tp->copied_seq, &offset);
461 if (WARN_ON_ONCE(!first))
462 return;
463
464 /* Bestow the state onto the anchor */
465 strp->anchor->len = offset + len;
466 strp->anchor->data_len = offset + len;
467 strp->anchor->truesize = offset + len;
468
469 skb_shinfo(strp->anchor)->frag_list = first;
470
471 skb_copy_header(strp->anchor, first);
472 strp->anchor->destructor = NULL;
473
474 strp->stm.offset = offset;
475}
476
477void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh)
478{
479 struct strp_msg *rxm;
480 struct tls_msg *tlm;
481
482 DEBUG_NET_WARN_ON_ONCE(!strp->msg_ready);
483 DEBUG_NET_WARN_ON_ONCE(!strp->stm.full_len);
484
485 if (!strp->copy_mode && force_refresh) {
486 if (WARN_ON(tcp_inq(strp->sk) < strp->stm.full_len))
487 return;
488
489 tls_strp_load_anchor_with_queue(strp, strp->stm.full_len);
490 }
491
492 rxm = strp_msg(strp->anchor);
493 rxm->full_len = strp->stm.full_len;
494 rxm->offset = strp->stm.offset;
495 tlm = tls_msg(strp->anchor);
496 tlm->control = strp->mark;
497}
498
499/* Called with lock held on lower socket */
500static int tls_strp_read_sock(struct tls_strparser *strp)
501{
502 int sz, inq;
503
504 inq = tcp_inq(strp->sk);
505 if (inq < 1)
506 return 0;
507
508 if (unlikely(strp->copy_mode))
509 return tls_strp_read_copyin(strp);
510
511 if (inq < strp->stm.full_len)
512 return tls_strp_read_copy(strp, true);
513
514 if (!strp->stm.full_len) {
515 tls_strp_load_anchor_with_queue(strp, inq);
516
517 sz = tls_rx_msg_size(strp, strp->anchor);
518 if (sz < 0) {
519 tls_strp_abort_strp(strp, sz);
520 return sz;
521 }
522
523 strp->stm.full_len = sz;
524
525 if (!strp->stm.full_len || inq < strp->stm.full_len)
526 return tls_strp_read_copy(strp, true);
527 }
528
529 if (!tls_strp_check_queue_ok(strp))
530 return tls_strp_read_copy(strp, false);
531
532 WRITE_ONCE(strp->msg_ready, 1);
533 tls_rx_msg_ready(strp);
534
535 return 0;
536}
537
538void tls_strp_check_rcv(struct tls_strparser *strp)
539{
540 if (unlikely(strp->stopped) || strp->msg_ready)
541 return;
542
543 if (tls_strp_read_sock(strp) == -ENOMEM)
544 queue_work(tls_strp_wq, &strp->work);
545}
546
547/* Lower sock lock held */
548void tls_strp_data_ready(struct tls_strparser *strp)
549{
550 /* This check is needed to synchronize with do_tls_strp_work.
551 * do_tls_strp_work acquires a process lock (lock_sock) whereas
552 * the lock held here is bh_lock_sock. The two locks can be
553 * held by different threads at the same time, but bh_lock_sock
554 * allows a thread in BH context to safely check if the process
555 * lock is held. In this case, if the lock is held, queue work.
556 */
557 if (sock_owned_by_user_nocheck(strp->sk)) {
558 queue_work(tls_strp_wq, &strp->work);
559 return;
560 }
561
562 tls_strp_check_rcv(strp);
563}
564
565static void tls_strp_work(struct work_struct *w)
566{
567 struct tls_strparser *strp =
568 container_of(w, struct tls_strparser, work);
569
570 lock_sock(strp->sk);
571 tls_strp_check_rcv(strp);
572 release_sock(strp->sk);
573}
574
575void tls_strp_msg_done(struct tls_strparser *strp)
576{
577 WARN_ON(!strp->stm.full_len);
578
579 if (likely(!strp->copy_mode))
580 tcp_read_done(strp->sk, strp->stm.full_len);
581 else
582 tls_strp_flush_anchor_copy(strp);
583
584 WRITE_ONCE(strp->msg_ready, 0);
585 memset(&strp->stm, 0, sizeof(strp->stm));
586
587 tls_strp_check_rcv(strp);
588}
589
590void tls_strp_stop(struct tls_strparser *strp)
591{
592 strp->stopped = 1;
593}
594
595int tls_strp_init(struct tls_strparser *strp, struct sock *sk)
596{
597 memset(strp, 0, sizeof(*strp));
598
599 strp->sk = sk;
600
601 strp->anchor = alloc_skb(0, GFP_KERNEL);
602 if (!strp->anchor)
603 return -ENOMEM;
604
605 INIT_WORK(&strp->work, tls_strp_work);
606
607 return 0;
608}
609
610/* strp must already be stopped so that tls_strp_recv will no longer be called.
611 * Note that tls_strp_done is not called with the lower socket held.
612 */
613void tls_strp_done(struct tls_strparser *strp)
614{
615 WARN_ON(!strp->stopped);
616
617 cancel_work_sync(&strp->work);
618 tls_strp_anchor_free(strp);
619}
620
621int __init tls_strp_dev_init(void)
622{
623 tls_strp_wq = create_workqueue("tls-strp");
624 if (unlikely(!tls_strp_wq))
625 return -ENOMEM;
626
627 return 0;
628}
629
630void tls_strp_dev_exit(void)
631{
632 destroy_workqueue(tls_strp_wq);
633}