Loading...
Note: File does not exist in v4.17.
1/****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2010 Solarflare Communications Inc.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation, incorporated herein by reference.
8 */
9
10#include <linux/in.h>
11#include <net/ip.h>
12#include "efx.h"
13#include "filter.h"
14#include "io.h"
15#include "nic.h"
16#include "regs.h"
17
18/* "Fudge factors" - difference between programmed value and actual depth.
19 * Due to pipelined implementation we need to program H/W with a value that
20 * is larger than the hop limit we want.
21 */
22#define FILTER_CTL_SRCH_FUDGE_WILD 3
23#define FILTER_CTL_SRCH_FUDGE_FULL 1
24
25/* Hard maximum hop limit. Hardware will time-out beyond 200-something.
26 * We also need to avoid infinite loops in efx_filter_search() when the
27 * table is full.
28 */
29#define FILTER_CTL_SRCH_MAX 200
30
31/* Don't try very hard to find space for performance hints, as this is
32 * counter-productive. */
33#define FILTER_CTL_SRCH_HINT_MAX 5
34
35enum efx_filter_table_id {
36 EFX_FILTER_TABLE_RX_IP = 0,
37 EFX_FILTER_TABLE_RX_MAC,
38 EFX_FILTER_TABLE_COUNT,
39};
40
41struct efx_filter_table {
42 enum efx_filter_table_id id;
43 u32 offset; /* address of table relative to BAR */
44 unsigned size; /* number of entries */
45 unsigned step; /* step between entries */
46 unsigned used; /* number currently used */
47 unsigned long *used_bitmap;
48 struct efx_filter_spec *spec;
49 unsigned search_depth[EFX_FILTER_TYPE_COUNT];
50};
51
52struct efx_filter_state {
53 spinlock_t lock;
54 struct efx_filter_table table[EFX_FILTER_TABLE_COUNT];
55#ifdef CONFIG_RFS_ACCEL
56 u32 *rps_flow_id;
57 unsigned rps_expire_index;
58#endif
59};
60
61/* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
62 * key derived from the n-tuple. The initial LFSR state is 0xffff. */
63static u16 efx_filter_hash(u32 key)
64{
65 u16 tmp;
66
67 /* First 16 rounds */
68 tmp = 0x1fff ^ key >> 16;
69 tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
70 tmp = tmp ^ tmp >> 9;
71 /* Last 16 rounds */
72 tmp = tmp ^ tmp << 13 ^ key;
73 tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
74 return tmp ^ tmp >> 9;
75}
76
77/* To allow for hash collisions, filter search continues at these
78 * increments from the first possible entry selected by the hash. */
79static u16 efx_filter_increment(u32 key)
80{
81 return key * 2 - 1;
82}
83
84static enum efx_filter_table_id
85efx_filter_spec_table_id(const struct efx_filter_spec *spec)
86{
87 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2));
88 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2));
89 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2));
90 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2));
91 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2));
92 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2));
93 EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC);
94 return spec->type >> 2;
95}
96
97static struct efx_filter_table *
98efx_filter_spec_table(struct efx_filter_state *state,
99 const struct efx_filter_spec *spec)
100{
101 if (spec->type == EFX_FILTER_UNSPEC)
102 return NULL;
103 else
104 return &state->table[efx_filter_spec_table_id(spec)];
105}
106
107static void efx_filter_table_reset_search_depth(struct efx_filter_table *table)
108{
109 memset(table->search_depth, 0, sizeof(table->search_depth));
110}
111
112static void efx_filter_push_rx_limits(struct efx_nic *efx)
113{
114 struct efx_filter_state *state = efx->filter_state;
115 struct efx_filter_table *table;
116 efx_oword_t filter_ctl;
117
118 efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
119
120 table = &state->table[EFX_FILTER_TABLE_RX_IP];
121 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT,
122 table->search_depth[EFX_FILTER_TCP_FULL] +
123 FILTER_CTL_SRCH_FUDGE_FULL);
124 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT,
125 table->search_depth[EFX_FILTER_TCP_WILD] +
126 FILTER_CTL_SRCH_FUDGE_WILD);
127 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT,
128 table->search_depth[EFX_FILTER_UDP_FULL] +
129 FILTER_CTL_SRCH_FUDGE_FULL);
130 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT,
131 table->search_depth[EFX_FILTER_UDP_WILD] +
132 FILTER_CTL_SRCH_FUDGE_WILD);
133
134 table = &state->table[EFX_FILTER_TABLE_RX_MAC];
135 if (table->size) {
136 EFX_SET_OWORD_FIELD(
137 filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
138 table->search_depth[EFX_FILTER_MAC_FULL] +
139 FILTER_CTL_SRCH_FUDGE_FULL);
140 EFX_SET_OWORD_FIELD(
141 filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
142 table->search_depth[EFX_FILTER_MAC_WILD] +
143 FILTER_CTL_SRCH_FUDGE_WILD);
144 }
145
146 efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
147}
148
149static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec,
150 __be32 host1, __be16 port1,
151 __be32 host2, __be16 port2)
152{
153 spec->data[0] = ntohl(host1) << 16 | ntohs(port1);
154 spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16;
155 spec->data[2] = ntohl(host2);
156}
157
158/**
159 * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
160 * @spec: Specification to initialise
161 * @proto: Transport layer protocol number
162 * @host: Local host address (network byte order)
163 * @port: Local port (network byte order)
164 */
165int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto,
166 __be32 host, __be16 port)
167{
168 __be32 host1;
169 __be16 port1;
170
171 EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
172
173 /* This cannot currently be combined with other filtering */
174 if (spec->type != EFX_FILTER_UNSPEC)
175 return -EPROTONOSUPPORT;
176
177 if (port == 0)
178 return -EINVAL;
179
180 switch (proto) {
181 case IPPROTO_TCP:
182 spec->type = EFX_FILTER_TCP_WILD;
183 break;
184 case IPPROTO_UDP:
185 spec->type = EFX_FILTER_UDP_WILD;
186 break;
187 default:
188 return -EPROTONOSUPPORT;
189 }
190
191 /* Filter is constructed in terms of source and destination,
192 * with the odd wrinkle that the ports are swapped in a UDP
193 * wildcard filter. We need to convert from local and remote
194 * (= zero for wildcard) addresses.
195 */
196 host1 = 0;
197 if (proto != IPPROTO_UDP) {
198 port1 = 0;
199 } else {
200 port1 = port;
201 port = 0;
202 }
203
204 __efx_filter_set_ipv4(spec, host1, port1, host, port);
205 return 0;
206}
207
208/**
209 * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
210 * @spec: Specification to initialise
211 * @proto: Transport layer protocol number
212 * @host: Local host address (network byte order)
213 * @port: Local port (network byte order)
214 * @rhost: Remote host address (network byte order)
215 * @rport: Remote port (network byte order)
216 */
217int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto,
218 __be32 host, __be16 port,
219 __be32 rhost, __be16 rport)
220{
221 EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
222
223 /* This cannot currently be combined with other filtering */
224 if (spec->type != EFX_FILTER_UNSPEC)
225 return -EPROTONOSUPPORT;
226
227 if (port == 0 || rport == 0)
228 return -EINVAL;
229
230 switch (proto) {
231 case IPPROTO_TCP:
232 spec->type = EFX_FILTER_TCP_FULL;
233 break;
234 case IPPROTO_UDP:
235 spec->type = EFX_FILTER_UDP_FULL;
236 break;
237 default:
238 return -EPROTONOSUPPORT;
239 }
240
241 __efx_filter_set_ipv4(spec, rhost, rport, host, port);
242 return 0;
243}
244
245/**
246 * efx_filter_set_eth_local - specify local Ethernet address and optional VID
247 * @spec: Specification to initialise
248 * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
249 * @addr: Local Ethernet MAC address
250 */
251int efx_filter_set_eth_local(struct efx_filter_spec *spec,
252 u16 vid, const u8 *addr)
253{
254 EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
255
256 /* This cannot currently be combined with other filtering */
257 if (spec->type != EFX_FILTER_UNSPEC)
258 return -EPROTONOSUPPORT;
259
260 if (vid == EFX_FILTER_VID_UNSPEC) {
261 spec->type = EFX_FILTER_MAC_WILD;
262 spec->data[0] = 0;
263 } else {
264 spec->type = EFX_FILTER_MAC_FULL;
265 spec->data[0] = vid;
266 }
267
268 spec->data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5];
269 spec->data[2] = addr[0] << 8 | addr[1];
270 return 0;
271}
272
273/* Build a filter entry and return its n-tuple key. */
274static u32 efx_filter_build(efx_oword_t *filter, struct efx_filter_spec *spec)
275{
276 u32 data3;
277
278 switch (efx_filter_spec_table_id(spec)) {
279 case EFX_FILTER_TABLE_RX_IP: {
280 bool is_udp = (spec->type == EFX_FILTER_UDP_FULL ||
281 spec->type == EFX_FILTER_UDP_WILD);
282 EFX_POPULATE_OWORD_7(
283 *filter,
284 FRF_BZ_RSS_EN,
285 !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
286 FRF_BZ_SCATTER_EN,
287 !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
288 FRF_BZ_TCP_UDP, is_udp,
289 FRF_BZ_RXQ_ID, spec->dmaq_id,
290 EFX_DWORD_2, spec->data[2],
291 EFX_DWORD_1, spec->data[1],
292 EFX_DWORD_0, spec->data[0]);
293 data3 = is_udp;
294 break;
295 }
296
297 case EFX_FILTER_TABLE_RX_MAC: {
298 bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
299 EFX_POPULATE_OWORD_8(
300 *filter,
301 FRF_CZ_RMFT_RSS_EN,
302 !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
303 FRF_CZ_RMFT_SCATTER_EN,
304 !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
305 FRF_CZ_RMFT_IP_OVERRIDE,
306 !!(spec->flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP),
307 FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id,
308 FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
309 FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2],
310 FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1],
311 FRF_CZ_RMFT_VLAN_ID, spec->data[0]);
312 data3 = is_wild;
313 break;
314 }
315
316 default:
317 BUG();
318 }
319
320 return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3;
321}
322
323static bool efx_filter_equal(const struct efx_filter_spec *left,
324 const struct efx_filter_spec *right)
325{
326 if (left->type != right->type ||
327 memcmp(left->data, right->data, sizeof(left->data)))
328 return false;
329
330 return true;
331}
332
333static int efx_filter_search(struct efx_filter_table *table,
334 struct efx_filter_spec *spec, u32 key,
335 bool for_insert, int *depth_required)
336{
337 unsigned hash, incr, filter_idx, depth, depth_max;
338
339 hash = efx_filter_hash(key);
340 incr = efx_filter_increment(key);
341
342 filter_idx = hash & (table->size - 1);
343 depth = 1;
344 depth_max = (for_insert ?
345 (spec->priority <= EFX_FILTER_PRI_HINT ?
346 FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX) :
347 table->search_depth[spec->type]);
348
349 for (;;) {
350 /* Return success if entry is used and matches this spec
351 * or entry is unused and we are trying to insert.
352 */
353 if (test_bit(filter_idx, table->used_bitmap) ?
354 efx_filter_equal(spec, &table->spec[filter_idx]) :
355 for_insert) {
356 *depth_required = depth;
357 return filter_idx;
358 }
359
360 /* Return failure if we reached the maximum search depth */
361 if (depth == depth_max)
362 return for_insert ? -EBUSY : -ENOENT;
363
364 filter_idx = (filter_idx + incr) & (table->size - 1);
365 ++depth;
366 }
367}
368
369/* Construct/deconstruct external filter IDs */
370
371static inline int
372efx_filter_make_id(enum efx_filter_table_id table_id, unsigned index)
373{
374 return table_id << 16 | index;
375}
376
377/**
378 * efx_filter_insert_filter - add or replace a filter
379 * @efx: NIC in which to insert the filter
380 * @spec: Specification for the filter
381 * @replace: Flag for whether the specified filter may replace a filter
382 * with an identical match expression and equal or lower priority
383 *
384 * On success, return the filter ID.
385 * On failure, return a negative error code.
386 */
387int efx_filter_insert_filter(struct efx_nic *efx, struct efx_filter_spec *spec,
388 bool replace)
389{
390 struct efx_filter_state *state = efx->filter_state;
391 struct efx_filter_table *table = efx_filter_spec_table(state, spec);
392 struct efx_filter_spec *saved_spec;
393 efx_oword_t filter;
394 int filter_idx, depth;
395 u32 key;
396 int rc;
397
398 if (!table || table->size == 0)
399 return -EINVAL;
400
401 key = efx_filter_build(&filter, spec);
402
403 netif_vdbg(efx, hw, efx->net_dev,
404 "%s: type %d search_depth=%d", __func__, spec->type,
405 table->search_depth[spec->type]);
406
407 spin_lock_bh(&state->lock);
408
409 rc = efx_filter_search(table, spec, key, true, &depth);
410 if (rc < 0)
411 goto out;
412 filter_idx = rc;
413 BUG_ON(filter_idx >= table->size);
414 saved_spec = &table->spec[filter_idx];
415
416 if (test_bit(filter_idx, table->used_bitmap)) {
417 /* Should we replace the existing filter? */
418 if (!replace) {
419 rc = -EEXIST;
420 goto out;
421 }
422 if (spec->priority < saved_spec->priority) {
423 rc = -EPERM;
424 goto out;
425 }
426 } else {
427 __set_bit(filter_idx, table->used_bitmap);
428 ++table->used;
429 }
430 *saved_spec = *spec;
431
432 if (table->search_depth[spec->type] < depth) {
433 table->search_depth[spec->type] = depth;
434 efx_filter_push_rx_limits(efx);
435 }
436
437 efx_writeo(efx, &filter, table->offset + table->step * filter_idx);
438
439 netif_vdbg(efx, hw, efx->net_dev,
440 "%s: filter type %d index %d rxq %u set",
441 __func__, spec->type, filter_idx, spec->dmaq_id);
442 rc = efx_filter_make_id(table->id, filter_idx);
443
444out:
445 spin_unlock_bh(&state->lock);
446 return rc;
447}
448
449static void efx_filter_table_clear_entry(struct efx_nic *efx,
450 struct efx_filter_table *table,
451 int filter_idx)
452{
453 static efx_oword_t filter;
454
455 if (test_bit(filter_idx, table->used_bitmap)) {
456 __clear_bit(filter_idx, table->used_bitmap);
457 --table->used;
458 memset(&table->spec[filter_idx], 0, sizeof(table->spec[0]));
459
460 efx_writeo(efx, &filter,
461 table->offset + table->step * filter_idx);
462 }
463}
464
465/**
466 * efx_filter_remove_filter - remove a filter by specification
467 * @efx: NIC from which to remove the filter
468 * @spec: Specification for the filter
469 *
470 * On success, return zero.
471 * On failure, return a negative error code.
472 */
473int efx_filter_remove_filter(struct efx_nic *efx, struct efx_filter_spec *spec)
474{
475 struct efx_filter_state *state = efx->filter_state;
476 struct efx_filter_table *table = efx_filter_spec_table(state, spec);
477 struct efx_filter_spec *saved_spec;
478 efx_oword_t filter;
479 int filter_idx, depth;
480 u32 key;
481 int rc;
482
483 if (!table)
484 return -EINVAL;
485
486 key = efx_filter_build(&filter, spec);
487
488 spin_lock_bh(&state->lock);
489
490 rc = efx_filter_search(table, spec, key, false, &depth);
491 if (rc < 0)
492 goto out;
493 filter_idx = rc;
494 saved_spec = &table->spec[filter_idx];
495
496 if (spec->priority < saved_spec->priority) {
497 rc = -EPERM;
498 goto out;
499 }
500
501 efx_filter_table_clear_entry(efx, table, filter_idx);
502 if (table->used == 0)
503 efx_filter_table_reset_search_depth(table);
504 rc = 0;
505
506out:
507 spin_unlock_bh(&state->lock);
508 return rc;
509}
510
511static void efx_filter_table_clear(struct efx_nic *efx,
512 enum efx_filter_table_id table_id,
513 enum efx_filter_priority priority)
514{
515 struct efx_filter_state *state = efx->filter_state;
516 struct efx_filter_table *table = &state->table[table_id];
517 int filter_idx;
518
519 spin_lock_bh(&state->lock);
520
521 for (filter_idx = 0; filter_idx < table->size; ++filter_idx)
522 if (table->spec[filter_idx].priority <= priority)
523 efx_filter_table_clear_entry(efx, table, filter_idx);
524 if (table->used == 0)
525 efx_filter_table_reset_search_depth(table);
526
527 spin_unlock_bh(&state->lock);
528}
529
530/**
531 * efx_filter_clear_rx - remove RX filters by priority
532 * @efx: NIC from which to remove the filters
533 * @priority: Maximum priority to remove
534 */
535void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority)
536{
537 efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority);
538 efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority);
539}
540
541/* Restore filter stater after reset */
542void efx_restore_filters(struct efx_nic *efx)
543{
544 struct efx_filter_state *state = efx->filter_state;
545 enum efx_filter_table_id table_id;
546 struct efx_filter_table *table;
547 efx_oword_t filter;
548 int filter_idx;
549
550 spin_lock_bh(&state->lock);
551
552 for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
553 table = &state->table[table_id];
554 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
555 if (!test_bit(filter_idx, table->used_bitmap))
556 continue;
557 efx_filter_build(&filter, &table->spec[filter_idx]);
558 efx_writeo(efx, &filter,
559 table->offset + table->step * filter_idx);
560 }
561 }
562
563 efx_filter_push_rx_limits(efx);
564
565 spin_unlock_bh(&state->lock);
566}
567
568int efx_probe_filters(struct efx_nic *efx)
569{
570 struct efx_filter_state *state;
571 struct efx_filter_table *table;
572 unsigned table_id;
573
574 state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL);
575 if (!state)
576 return -ENOMEM;
577 efx->filter_state = state;
578
579 spin_lock_init(&state->lock);
580
581 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
582#ifdef CONFIG_RFS_ACCEL
583 state->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS,
584 sizeof(*state->rps_flow_id),
585 GFP_KERNEL);
586 if (!state->rps_flow_id)
587 goto fail;
588#endif
589 table = &state->table[EFX_FILTER_TABLE_RX_IP];
590 table->id = EFX_FILTER_TABLE_RX_IP;
591 table->offset = FR_BZ_RX_FILTER_TBL0;
592 table->size = FR_BZ_RX_FILTER_TBL0_ROWS;
593 table->step = FR_BZ_RX_FILTER_TBL0_STEP;
594 }
595
596 if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
597 table = &state->table[EFX_FILTER_TABLE_RX_MAC];
598 table->id = EFX_FILTER_TABLE_RX_MAC;
599 table->offset = FR_CZ_RX_MAC_FILTER_TBL0;
600 table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
601 table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP;
602 }
603
604 for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
605 table = &state->table[table_id];
606 if (table->size == 0)
607 continue;
608 table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size),
609 sizeof(unsigned long),
610 GFP_KERNEL);
611 if (!table->used_bitmap)
612 goto fail;
613 table->spec = vzalloc(table->size * sizeof(*table->spec));
614 if (!table->spec)
615 goto fail;
616 }
617
618 return 0;
619
620fail:
621 efx_remove_filters(efx);
622 return -ENOMEM;
623}
624
625void efx_remove_filters(struct efx_nic *efx)
626{
627 struct efx_filter_state *state = efx->filter_state;
628 enum efx_filter_table_id table_id;
629
630 for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
631 kfree(state->table[table_id].used_bitmap);
632 vfree(state->table[table_id].spec);
633 }
634#ifdef CONFIG_RFS_ACCEL
635 kfree(state->rps_flow_id);
636#endif
637 kfree(state);
638}
639
640#ifdef CONFIG_RFS_ACCEL
641
642int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
643 u16 rxq_index, u32 flow_id)
644{
645 struct efx_nic *efx = netdev_priv(net_dev);
646 struct efx_channel *channel;
647 struct efx_filter_state *state = efx->filter_state;
648 struct efx_filter_spec spec;
649 const struct iphdr *ip;
650 const __be16 *ports;
651 int nhoff;
652 int rc;
653
654 nhoff = skb_network_offset(skb);
655
656 if (skb->protocol != htons(ETH_P_IP))
657 return -EPROTONOSUPPORT;
658
659 /* RFS must validate the IP header length before calling us */
660 EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip));
661 ip = (const struct iphdr *)(skb->data + nhoff);
662 if (ip_is_fragment(ip))
663 return -EPROTONOSUPPORT;
664 EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + 4 * ip->ihl + 4);
665 ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl);
666
667 efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, 0, rxq_index);
668 rc = efx_filter_set_ipv4_full(&spec, ip->protocol,
669 ip->daddr, ports[1], ip->saddr, ports[0]);
670 if (rc)
671 return rc;
672
673 rc = efx_filter_insert_filter(efx, &spec, true);
674 if (rc < 0)
675 return rc;
676
677 /* Remember this so we can check whether to expire the filter later */
678 state->rps_flow_id[rc] = flow_id;
679 channel = efx_get_channel(efx, skb_get_rx_queue(skb));
680 ++channel->rfs_filters_added;
681
682 netif_info(efx, rx_status, efx->net_dev,
683 "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
684 (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP",
685 &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]),
686 rxq_index, flow_id, rc);
687
688 return rc;
689}
690
691bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota)
692{
693 struct efx_filter_state *state = efx->filter_state;
694 struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_IP];
695 unsigned mask = table->size - 1;
696 unsigned index;
697 unsigned stop;
698
699 if (!spin_trylock_bh(&state->lock))
700 return false;
701
702 index = state->rps_expire_index;
703 stop = (index + quota) & mask;
704
705 while (index != stop) {
706 if (test_bit(index, table->used_bitmap) &&
707 table->spec[index].priority == EFX_FILTER_PRI_HINT &&
708 rps_may_expire_flow(efx->net_dev,
709 table->spec[index].dmaq_id,
710 state->rps_flow_id[index], index)) {
711 netif_info(efx, rx_status, efx->net_dev,
712 "expiring filter %d [flow %u]\n",
713 index, state->rps_flow_id[index]);
714 efx_filter_table_clear_entry(efx, table, index);
715 }
716 index = (index + 1) & mask;
717 }
718
719 state->rps_expire_index = stop;
720 if (table->used == 0)
721 efx_filter_table_reset_search_depth(table);
722
723 spin_unlock_bh(&state->lock);
724 return true;
725}
726
727#endif /* CONFIG_RFS_ACCEL */