Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/****************************************************************************
3 * Driver for Solarflare network controllers and boards
4 * Copyright 2019 Solarflare Communications Inc.
5 * Copyright 2020-2022 Xilinx Inc.
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 as published
9 * by the Free Software Foundation, incorporated herein by reference.
10 */
11
12#include <net/pkt_cls.h>
13#include <net/vxlan.h>
14#include <net/geneve.h>
15#include <net/tc_act/tc_ct.h>
16#include "tc.h"
17#include "tc_bindings.h"
18#include "tc_encap_actions.h"
19#include "tc_conntrack.h"
20#include "mae.h"
21#include "ef100_rep.h"
22#include "efx.h"
23
24enum efx_encap_type efx_tc_indr_netdev_type(struct net_device *net_dev)
25{
26 if (netif_is_vxlan(net_dev))
27 return EFX_ENCAP_TYPE_VXLAN;
28 if (netif_is_geneve(net_dev))
29 return EFX_ENCAP_TYPE_GENEVE;
30
31 return EFX_ENCAP_TYPE_NONE;
32}
33
34#define EFX_TC_HDR_TYPE_TTL_MASK ((u32)0xff)
35/* Hoplimit is stored in the most significant byte in the pedit ipv6 header action */
36#define EFX_TC_HDR_TYPE_HLIMIT_MASK ~((u32)0xff000000)
37#define EFX_EFV_PF NULL
38/* Look up the representor information (efv) for a device.
39 * May return NULL for the PF (us), or an error pointer for a device that
40 * isn't supported as a TC offload endpoint
41 */
42struct efx_rep *efx_tc_flower_lookup_efv(struct efx_nic *efx,
43 struct net_device *dev)
44{
45 struct efx_rep *efv;
46
47 if (!dev)
48 return ERR_PTR(-EOPNOTSUPP);
49 /* Is it us (the PF)? */
50 if (dev == efx->net_dev)
51 return EFX_EFV_PF;
52 /* Is it an efx vfrep at all? */
53 if (dev->netdev_ops != &efx_ef100_rep_netdev_ops)
54 return ERR_PTR(-EOPNOTSUPP);
55 /* Is it ours? We don't support TC rules that include another
56 * EF100's netdevices (not even on another port of the same NIC).
57 */
58 efv = netdev_priv(dev);
59 if (efv->parent != efx)
60 return ERR_PTR(-EOPNOTSUPP);
61 return efv;
62}
63
64/* Convert a driver-internal vport ID into an internal device (PF or VF) */
65static s64 efx_tc_flower_internal_mport(struct efx_nic *efx, struct efx_rep *efv)
66{
67 u32 mport;
68
69 if (IS_ERR(efv))
70 return PTR_ERR(efv);
71 if (!efv) /* device is PF (us) */
72 efx_mae_mport_uplink(efx, &mport);
73 else /* device is repr */
74 efx_mae_mport_mport(efx, efv->mport, &mport);
75 return mport;
76}
77
78/* Convert a driver-internal vport ID into an external device (wire or VF) */
79s64 efx_tc_flower_external_mport(struct efx_nic *efx, struct efx_rep *efv)
80{
81 u32 mport;
82
83 if (IS_ERR(efv))
84 return PTR_ERR(efv);
85 if (!efv) /* device is PF (us) */
86 efx_mae_mport_wire(efx, &mport);
87 else /* device is repr */
88 efx_mae_mport_mport(efx, efv->mport, &mport);
89 return mport;
90}
91
92static const struct rhashtable_params efx_tc_mac_ht_params = {
93 .key_len = offsetofend(struct efx_tc_mac_pedit_action, h_addr),
94 .key_offset = 0,
95 .head_offset = offsetof(struct efx_tc_mac_pedit_action, linkage),
96};
97
98static const struct rhashtable_params efx_tc_encap_match_ht_params = {
99 .key_len = offsetof(struct efx_tc_encap_match, linkage),
100 .key_offset = 0,
101 .head_offset = offsetof(struct efx_tc_encap_match, linkage),
102};
103
104static const struct rhashtable_params efx_tc_match_action_ht_params = {
105 .key_len = sizeof(unsigned long),
106 .key_offset = offsetof(struct efx_tc_flow_rule, cookie),
107 .head_offset = offsetof(struct efx_tc_flow_rule, linkage),
108};
109
110static const struct rhashtable_params efx_tc_lhs_rule_ht_params = {
111 .key_len = sizeof(unsigned long),
112 .key_offset = offsetof(struct efx_tc_lhs_rule, cookie),
113 .head_offset = offsetof(struct efx_tc_lhs_rule, linkage),
114};
115
116static const struct rhashtable_params efx_tc_recirc_ht_params = {
117 .key_len = offsetof(struct efx_tc_recirc_id, linkage),
118 .key_offset = 0,
119 .head_offset = offsetof(struct efx_tc_recirc_id, linkage),
120};
121
122static struct efx_tc_mac_pedit_action *efx_tc_flower_get_mac(struct efx_nic *efx,
123 unsigned char h_addr[ETH_ALEN],
124 struct netlink_ext_ack *extack)
125{
126 struct efx_tc_mac_pedit_action *ped, *old;
127 int rc;
128
129 ped = kzalloc(sizeof(*ped), GFP_USER);
130 if (!ped)
131 return ERR_PTR(-ENOMEM);
132 memcpy(ped->h_addr, h_addr, ETH_ALEN);
133 old = rhashtable_lookup_get_insert_fast(&efx->tc->mac_ht,
134 &ped->linkage,
135 efx_tc_mac_ht_params);
136 if (old) {
137 /* don't need our new entry */
138 kfree(ped);
139 if (IS_ERR(old)) /* oh dear, it's actually an error */
140 return ERR_CAST(old);
141 if (!refcount_inc_not_zero(&old->ref))
142 return ERR_PTR(-EAGAIN);
143 /* existing entry found, ref taken */
144 return old;
145 }
146
147 rc = efx_mae_allocate_pedit_mac(efx, ped);
148 if (rc < 0) {
149 NL_SET_ERR_MSG_MOD(extack, "Failed to store pedit MAC address in hw");
150 goto out_remove;
151 }
152
153 /* ref and return */
154 refcount_set(&ped->ref, 1);
155 return ped;
156out_remove:
157 rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage,
158 efx_tc_mac_ht_params);
159 kfree(ped);
160 return ERR_PTR(rc);
161}
162
163static void efx_tc_flower_put_mac(struct efx_nic *efx,
164 struct efx_tc_mac_pedit_action *ped)
165{
166 if (!refcount_dec_and_test(&ped->ref))
167 return; /* still in use */
168 rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage,
169 efx_tc_mac_ht_params);
170 efx_mae_free_pedit_mac(efx, ped);
171 kfree(ped);
172}
173
174static void efx_tc_free_action_set(struct efx_nic *efx,
175 struct efx_tc_action_set *act, bool in_hw)
176{
177 /* Failure paths calling this on the 'cursor' action set in_hw=false,
178 * because if the alloc had succeeded we'd've put it in acts.list and
179 * not still have it in act.
180 */
181 if (in_hw) {
182 efx_mae_free_action_set(efx, act->fw_id);
183 /* in_hw is true iff we are on an acts.list; make sure to
184 * remove ourselves from that list before we are freed.
185 */
186 list_del(&act->list);
187 }
188 if (act->count) {
189 spin_lock_bh(&act->count->cnt->lock);
190 if (!list_empty(&act->count_user))
191 list_del(&act->count_user);
192 spin_unlock_bh(&act->count->cnt->lock);
193 efx_tc_flower_put_counter_index(efx, act->count);
194 }
195 if (act->encap_md) {
196 list_del(&act->encap_user);
197 efx_tc_flower_release_encap_md(efx, act->encap_md);
198 }
199 if (act->src_mac)
200 efx_tc_flower_put_mac(efx, act->src_mac);
201 if (act->dst_mac)
202 efx_tc_flower_put_mac(efx, act->dst_mac);
203 kfree(act);
204}
205
206static void efx_tc_free_action_set_list(struct efx_nic *efx,
207 struct efx_tc_action_set_list *acts,
208 bool in_hw)
209{
210 struct efx_tc_action_set *act, *next;
211
212 /* Failure paths set in_hw=false, because usually the acts didn't get
213 * to efx_mae_alloc_action_set_list(); if they did, the failure tree
214 * has a separate efx_mae_free_action_set_list() before calling us.
215 */
216 if (in_hw)
217 efx_mae_free_action_set_list(efx, acts);
218 /* Any act that's on the list will be in_hw even if the list isn't */
219 list_for_each_entry_safe(act, next, &acts->list, list)
220 efx_tc_free_action_set(efx, act, true);
221 /* Don't kfree, as acts is embedded inside a struct efx_tc_flow_rule */
222}
223
224/* Boilerplate for the simple 'copy a field' cases */
225#define _MAP_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field) \
226if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_##_name)) { \
227 struct flow_match_##_type fm; \
228 \
229 flow_rule_match_##_tcget(rule, &fm); \
230 match->value._field = fm.key->_tcfield; \
231 match->mask._field = fm.mask->_tcfield; \
232}
233#define MAP_KEY_AND_MASK(_name, _type, _tcfield, _field) \
234 _MAP_KEY_AND_MASK(_name, _type, _type, _tcfield, _field)
235#define MAP_ENC_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field) \
236 _MAP_KEY_AND_MASK(ENC_##_name, _type, _tcget, _tcfield, _field)
237
238static int efx_tc_flower_parse_match(struct efx_nic *efx,
239 struct flow_rule *rule,
240 struct efx_tc_match *match,
241 struct netlink_ext_ack *extack)
242{
243 struct flow_dissector *dissector = rule->match.dissector;
244 unsigned char ipv = 0;
245
246 /* Owing to internal TC infelicities, the IPV6_ADDRS key might be set
247 * even on IPv4 filters; so rather than relying on dissector->used_keys
248 * we check the addr_type in the CONTROL key. If we don't find it (or
249 * it's masked, which should never happen), we treat both IPV4_ADDRS
250 * and IPV6_ADDRS as absent.
251 */
252 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
253 struct flow_match_control fm;
254
255 flow_rule_match_control(rule, &fm);
256 if (IS_ALL_ONES(fm.mask->addr_type))
257 switch (fm.key->addr_type) {
258 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
259 ipv = 4;
260 break;
261 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
262 ipv = 6;
263 break;
264 default:
265 break;
266 }
267
268 if (fm.mask->flags & FLOW_DIS_IS_FRAGMENT) {
269 match->value.ip_frag = fm.key->flags & FLOW_DIS_IS_FRAGMENT;
270 match->mask.ip_frag = true;
271 }
272 if (fm.mask->flags & FLOW_DIS_FIRST_FRAG) {
273 match->value.ip_firstfrag = fm.key->flags & FLOW_DIS_FIRST_FRAG;
274 match->mask.ip_firstfrag = true;
275 }
276 if (!flow_rule_is_supp_control_flags(FLOW_DIS_IS_FRAGMENT |
277 FLOW_DIS_FIRST_FRAG,
278 fm.mask->flags, extack))
279 return -EOPNOTSUPP;
280 }
281 if (dissector->used_keys &
282 ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
283 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
284 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
285 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
286 BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN) |
287 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
288 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
289 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
290 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) |
291 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
292 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
293 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) |
294 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS) |
295 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) |
296 BIT_ULL(FLOW_DISSECTOR_KEY_CT) |
297 BIT_ULL(FLOW_DISSECTOR_KEY_TCP) |
298 BIT_ULL(FLOW_DISSECTOR_KEY_IP))) {
299 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported flower keys %#llx",
300 dissector->used_keys);
301 return -EOPNOTSUPP;
302 }
303
304 MAP_KEY_AND_MASK(BASIC, basic, n_proto, eth_proto);
305 /* Make sure we're IP if any L3/L4 keys used. */
306 if (!IS_ALL_ONES(match->mask.eth_proto) ||
307 !(match->value.eth_proto == htons(ETH_P_IP) ||
308 match->value.eth_proto == htons(ETH_P_IPV6)))
309 if (dissector->used_keys &
310 (BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
311 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
312 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
313 BIT_ULL(FLOW_DISSECTOR_KEY_IP) |
314 BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) {
315 NL_SET_ERR_MSG_FMT_MOD(extack,
316 "L3/L4 flower keys %#llx require protocol ipv[46]",
317 dissector->used_keys);
318 return -EINVAL;
319 }
320
321 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
322 struct flow_match_vlan fm;
323
324 flow_rule_match_vlan(rule, &fm);
325 if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) {
326 match->value.vlan_proto[0] = fm.key->vlan_tpid;
327 match->mask.vlan_proto[0] = fm.mask->vlan_tpid;
328 match->value.vlan_tci[0] = cpu_to_be16(fm.key->vlan_priority << 13 |
329 fm.key->vlan_id);
330 match->mask.vlan_tci[0] = cpu_to_be16(fm.mask->vlan_priority << 13 |
331 fm.mask->vlan_id);
332 }
333 }
334
335 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)) {
336 struct flow_match_vlan fm;
337
338 flow_rule_match_cvlan(rule, &fm);
339 if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) {
340 match->value.vlan_proto[1] = fm.key->vlan_tpid;
341 match->mask.vlan_proto[1] = fm.mask->vlan_tpid;
342 match->value.vlan_tci[1] = cpu_to_be16(fm.key->vlan_priority << 13 |
343 fm.key->vlan_id);
344 match->mask.vlan_tci[1] = cpu_to_be16(fm.mask->vlan_priority << 13 |
345 fm.mask->vlan_id);
346 }
347 }
348
349 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
350 struct flow_match_eth_addrs fm;
351
352 flow_rule_match_eth_addrs(rule, &fm);
353 ether_addr_copy(match->value.eth_saddr, fm.key->src);
354 ether_addr_copy(match->value.eth_daddr, fm.key->dst);
355 ether_addr_copy(match->mask.eth_saddr, fm.mask->src);
356 ether_addr_copy(match->mask.eth_daddr, fm.mask->dst);
357 }
358
359 MAP_KEY_AND_MASK(BASIC, basic, ip_proto, ip_proto);
360 /* Make sure we're TCP/UDP if any L4 keys used. */
361 if ((match->value.ip_proto != IPPROTO_UDP &&
362 match->value.ip_proto != IPPROTO_TCP) || !IS_ALL_ONES(match->mask.ip_proto))
363 if (dissector->used_keys &
364 (BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
365 BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) {
366 NL_SET_ERR_MSG_FMT_MOD(extack,
367 "L4 flower keys %#llx require ipproto udp or tcp",
368 dissector->used_keys);
369 return -EINVAL;
370 }
371 MAP_KEY_AND_MASK(IP, ip, tos, ip_tos);
372 MAP_KEY_AND_MASK(IP, ip, ttl, ip_ttl);
373 if (ipv == 4) {
374 MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, src, src_ip);
375 MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, dst, dst_ip);
376 }
377#ifdef CONFIG_IPV6
378 else if (ipv == 6) {
379 MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, src, src_ip6);
380 MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, dst, dst_ip6);
381 }
382#endif
383 MAP_KEY_AND_MASK(PORTS, ports, src, l4_sport);
384 MAP_KEY_AND_MASK(PORTS, ports, dst, l4_dport);
385 MAP_KEY_AND_MASK(TCP, tcp, flags, tcp_flags);
386 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
387 struct flow_match_control fm;
388
389 flow_rule_match_enc_control(rule, &fm);
390 if (flow_rule_has_enc_control_flags(fm.mask->flags, extack))
391 return -EOPNOTSUPP;
392 if (!IS_ALL_ONES(fm.mask->addr_type)) {
393 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported enc addr_type mask %u (key %u)",
394 fm.mask->addr_type,
395 fm.key->addr_type);
396 return -EOPNOTSUPP;
397 }
398 switch (fm.key->addr_type) {
399 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
400 MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs,
401 src, enc_src_ip);
402 MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs,
403 dst, enc_dst_ip);
404 break;
405#ifdef CONFIG_IPV6
406 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
407 MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs,
408 src, enc_src_ip6);
409 MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs,
410 dst, enc_dst_ip6);
411 break;
412#endif
413 default:
414 NL_SET_ERR_MSG_FMT_MOD(extack,
415 "Unsupported enc addr_type %u (supported are IPv4, IPv6)",
416 fm.key->addr_type);
417 return -EOPNOTSUPP;
418 }
419 MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, tos, enc_ip_tos);
420 MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, ttl, enc_ip_ttl);
421 MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, src, enc_sport);
422 MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, dst, enc_dport);
423 MAP_ENC_KEY_AND_MASK(KEYID, enc_keyid, enc_keyid, keyid, enc_keyid);
424 } else if (dissector->used_keys &
425 (BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) |
426 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
427 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
428 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) |
429 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS))) {
430 NL_SET_ERR_MSG_FMT_MOD(extack,
431 "Flower enc keys require enc_control (keys: %#llx)",
432 dissector->used_keys);
433 return -EOPNOTSUPP;
434 }
435 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CT)) {
436 struct flow_match_ct fm;
437
438 flow_rule_match_ct(rule, &fm);
439 match->value.ct_state_trk = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED);
440 match->mask.ct_state_trk = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED);
441 match->value.ct_state_est = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED);
442 match->mask.ct_state_est = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED);
443 if (fm.mask->ct_state & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
444 TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)) {
445 NL_SET_ERR_MSG_FMT_MOD(extack,
446 "Unsupported ct_state match %#x",
447 fm.mask->ct_state);
448 return -EOPNOTSUPP;
449 }
450 match->value.ct_mark = fm.key->ct_mark;
451 match->mask.ct_mark = fm.mask->ct_mark;
452 match->value.ct_zone = fm.key->ct_zone;
453 match->mask.ct_zone = fm.mask->ct_zone;
454
455 if (memchr_inv(fm.mask->ct_labels, 0, sizeof(fm.mask->ct_labels))) {
456 NL_SET_ERR_MSG_MOD(extack, "Matching on ct_label not supported");
457 return -EOPNOTSUPP;
458 }
459 }
460
461 return 0;
462}
463
464static void efx_tc_flower_release_encap_match(struct efx_nic *efx,
465 struct efx_tc_encap_match *encap)
466{
467 int rc;
468
469 if (!refcount_dec_and_test(&encap->ref))
470 return; /* still in use */
471
472 if (encap->type == EFX_TC_EM_DIRECT) {
473 rc = efx_mae_unregister_encap_match(efx, encap);
474 if (rc)
475 /* Display message but carry on and remove entry from our
476 * SW tables, because there's not much we can do about it.
477 */
478 netif_err(efx, drv, efx->net_dev,
479 "Failed to release encap match %#x, rc %d\n",
480 encap->fw_id, rc);
481 }
482 rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage,
483 efx_tc_encap_match_ht_params);
484 if (encap->pseudo)
485 efx_tc_flower_release_encap_match(efx, encap->pseudo);
486 kfree(encap);
487}
488
489static int efx_tc_flower_record_encap_match(struct efx_nic *efx,
490 struct efx_tc_match *match,
491 enum efx_encap_type type,
492 enum efx_tc_em_pseudo_type em_type,
493 u8 child_ip_tos_mask,
494 __be16 child_udp_sport_mask,
495 struct netlink_ext_ack *extack)
496{
497 struct efx_tc_encap_match *encap, *old, *pseudo = NULL;
498 bool ipv6 = false;
499 int rc;
500
501 /* We require that the socket-defining fields (IP addrs and UDP dest
502 * port) are present and exact-match. Other fields may only be used
503 * if the field-set (and any masks) are the same for all encap
504 * matches on the same <sip,dip,dport> tuple; this is enforced by
505 * pseudo encap matches.
506 */
507 if (match->mask.enc_dst_ip | match->mask.enc_src_ip) {
508 if (!IS_ALL_ONES(match->mask.enc_dst_ip)) {
509 NL_SET_ERR_MSG_MOD(extack,
510 "Egress encap match is not exact on dst IP address");
511 return -EOPNOTSUPP;
512 }
513 if (!IS_ALL_ONES(match->mask.enc_src_ip)) {
514 NL_SET_ERR_MSG_MOD(extack,
515 "Egress encap match is not exact on src IP address");
516 return -EOPNOTSUPP;
517 }
518#ifdef CONFIG_IPV6
519 if (!ipv6_addr_any(&match->mask.enc_dst_ip6) ||
520 !ipv6_addr_any(&match->mask.enc_src_ip6)) {
521 NL_SET_ERR_MSG_MOD(extack,
522 "Egress encap match on both IPv4 and IPv6, don't understand");
523 return -EOPNOTSUPP;
524 }
525 } else {
526 ipv6 = true;
527 if (!efx_ipv6_addr_all_ones(&match->mask.enc_dst_ip6)) {
528 NL_SET_ERR_MSG_MOD(extack,
529 "Egress encap match is not exact on dst IP address");
530 return -EOPNOTSUPP;
531 }
532 if (!efx_ipv6_addr_all_ones(&match->mask.enc_src_ip6)) {
533 NL_SET_ERR_MSG_MOD(extack,
534 "Egress encap match is not exact on src IP address");
535 return -EOPNOTSUPP;
536 }
537#endif
538 }
539 if (!IS_ALL_ONES(match->mask.enc_dport)) {
540 NL_SET_ERR_MSG_MOD(extack, "Egress encap match is not exact on dst UDP port");
541 return -EOPNOTSUPP;
542 }
543 if (match->mask.enc_sport || match->mask.enc_ip_tos) {
544 struct efx_tc_match pmatch = *match;
545
546 if (em_type == EFX_TC_EM_PSEUDO_MASK) { /* can't happen */
547 NL_SET_ERR_MSG_MOD(extack, "Bad recursion in egress encap match handler");
548 return -EOPNOTSUPP;
549 }
550 pmatch.value.enc_ip_tos = 0;
551 pmatch.mask.enc_ip_tos = 0;
552 pmatch.value.enc_sport = 0;
553 pmatch.mask.enc_sport = 0;
554 rc = efx_tc_flower_record_encap_match(efx, &pmatch, type,
555 EFX_TC_EM_PSEUDO_MASK,
556 match->mask.enc_ip_tos,
557 match->mask.enc_sport,
558 extack);
559 if (rc)
560 return rc;
561 pseudo = pmatch.encap;
562 }
563 if (match->mask.enc_ip_ttl) {
564 NL_SET_ERR_MSG_MOD(extack, "Egress encap match on IP TTL not supported");
565 rc = -EOPNOTSUPP;
566 goto fail_pseudo;
567 }
568
569 rc = efx_mae_check_encap_match_caps(efx, ipv6, match->mask.enc_ip_tos,
570 match->mask.enc_sport, extack);
571 if (rc)
572 goto fail_pseudo;
573
574 encap = kzalloc(sizeof(*encap), GFP_USER);
575 if (!encap) {
576 rc = -ENOMEM;
577 goto fail_pseudo;
578 }
579 encap->src_ip = match->value.enc_src_ip;
580 encap->dst_ip = match->value.enc_dst_ip;
581#ifdef CONFIG_IPV6
582 encap->src_ip6 = match->value.enc_src_ip6;
583 encap->dst_ip6 = match->value.enc_dst_ip6;
584#endif
585 encap->udp_dport = match->value.enc_dport;
586 encap->tun_type = type;
587 encap->ip_tos = match->value.enc_ip_tos;
588 encap->ip_tos_mask = match->mask.enc_ip_tos;
589 encap->child_ip_tos_mask = child_ip_tos_mask;
590 encap->udp_sport = match->value.enc_sport;
591 encap->udp_sport_mask = match->mask.enc_sport;
592 encap->child_udp_sport_mask = child_udp_sport_mask;
593 encap->type = em_type;
594 encap->pseudo = pseudo;
595 old = rhashtable_lookup_get_insert_fast(&efx->tc->encap_match_ht,
596 &encap->linkage,
597 efx_tc_encap_match_ht_params);
598 if (old) {
599 /* don't need our new entry */
600 kfree(encap);
601 if (pseudo) /* don't need our new pseudo either */
602 efx_tc_flower_release_encap_match(efx, pseudo);
603 if (IS_ERR(old)) /* oh dear, it's actually an error */
604 return PTR_ERR(old);
605 /* check old and new em_types are compatible */
606 switch (old->type) {
607 case EFX_TC_EM_DIRECT:
608 /* old EM is in hardware, so mustn't overlap with a
609 * pseudo, but may be shared with another direct EM
610 */
611 if (em_type == EFX_TC_EM_DIRECT)
612 break;
613 NL_SET_ERR_MSG_MOD(extack, "Pseudo encap match conflicts with existing direct entry");
614 return -EEXIST;
615 case EFX_TC_EM_PSEUDO_MASK:
616 /* old EM is protecting a ToS- or src port-qualified
617 * filter, so may only be shared with another pseudo
618 * for the same ToS and src port masks.
619 */
620 if (em_type != EFX_TC_EM_PSEUDO_MASK) {
621 NL_SET_ERR_MSG_FMT_MOD(extack,
622 "%s encap match conflicts with existing pseudo(MASK) entry",
623 em_type ? "Pseudo" : "Direct");
624 return -EEXIST;
625 }
626 if (child_ip_tos_mask != old->child_ip_tos_mask) {
627 NL_SET_ERR_MSG_FMT_MOD(extack,
628 "Pseudo encap match for TOS mask %#04x conflicts with existing mask %#04x",
629 child_ip_tos_mask,
630 old->child_ip_tos_mask);
631 return -EEXIST;
632 }
633 if (child_udp_sport_mask != old->child_udp_sport_mask) {
634 NL_SET_ERR_MSG_FMT_MOD(extack,
635 "Pseudo encap match for UDP src port mask %#x conflicts with existing mask %#x",
636 child_udp_sport_mask,
637 old->child_udp_sport_mask);
638 return -EEXIST;
639 }
640 break;
641 case EFX_TC_EM_PSEUDO_OR:
642 /* old EM corresponds to an OR that has to be unique
643 * (it must not overlap with any other OR, whether
644 * direct-EM or pseudo).
645 */
646 NL_SET_ERR_MSG_FMT_MOD(extack,
647 "%s encap match conflicts with existing pseudo(OR) entry",
648 em_type ? "Pseudo" : "Direct");
649 return -EEXIST;
650 default: /* Unrecognised pseudo-type. Just say no */
651 NL_SET_ERR_MSG_FMT_MOD(extack,
652 "%s encap match conflicts with existing pseudo(%d) entry",
653 em_type ? "Pseudo" : "Direct",
654 old->type);
655 return -EEXIST;
656 }
657 /* check old and new tun_types are compatible */
658 if (old->tun_type != type) {
659 NL_SET_ERR_MSG_FMT_MOD(extack,
660 "Egress encap match with conflicting tun_type %u != %u",
661 old->tun_type, type);
662 return -EEXIST;
663 }
664 if (!refcount_inc_not_zero(&old->ref))
665 return -EAGAIN;
666 /* existing entry found */
667 encap = old;
668 } else {
669 if (em_type == EFX_TC_EM_DIRECT) {
670 rc = efx_mae_register_encap_match(efx, encap);
671 if (rc) {
672 NL_SET_ERR_MSG_MOD(extack, "Failed to record egress encap match in HW");
673 goto fail;
674 }
675 }
676 refcount_set(&encap->ref, 1);
677 }
678 match->encap = encap;
679 return 0;
680fail:
681 rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage,
682 efx_tc_encap_match_ht_params);
683 kfree(encap);
684fail_pseudo:
685 if (pseudo)
686 efx_tc_flower_release_encap_match(efx, pseudo);
687 return rc;
688}
689
690static struct efx_tc_recirc_id *efx_tc_get_recirc_id(struct efx_nic *efx,
691 u32 chain_index,
692 struct net_device *net_dev)
693{
694 struct efx_tc_recirc_id *rid, *old;
695 int rc;
696
697 rid = kzalloc(sizeof(*rid), GFP_USER);
698 if (!rid)
699 return ERR_PTR(-ENOMEM);
700 rid->chain_index = chain_index;
701 /* We don't take a reference here, because it's implied - if there's
702 * a rule on the net_dev that's been offloaded to us, then the net_dev
703 * can't go away until the rule has been deoffloaded.
704 */
705 rid->net_dev = net_dev;
706 old = rhashtable_lookup_get_insert_fast(&efx->tc->recirc_ht,
707 &rid->linkage,
708 efx_tc_recirc_ht_params);
709 if (old) {
710 /* don't need our new entry */
711 kfree(rid);
712 if (IS_ERR(old)) /* oh dear, it's actually an error */
713 return ERR_CAST(old);
714 if (!refcount_inc_not_zero(&old->ref))
715 return ERR_PTR(-EAGAIN);
716 /* existing entry found */
717 rid = old;
718 } else {
719 rc = ida_alloc_range(&efx->tc->recirc_ida, 1, U8_MAX, GFP_USER);
720 if (rc < 0) {
721 rhashtable_remove_fast(&efx->tc->recirc_ht,
722 &rid->linkage,
723 efx_tc_recirc_ht_params);
724 kfree(rid);
725 return ERR_PTR(rc);
726 }
727 rid->fw_id = rc;
728 refcount_set(&rid->ref, 1);
729 }
730 return rid;
731}
732
733static void efx_tc_put_recirc_id(struct efx_nic *efx, struct efx_tc_recirc_id *rid)
734{
735 if (!refcount_dec_and_test(&rid->ref))
736 return; /* still in use */
737 rhashtable_remove_fast(&efx->tc->recirc_ht, &rid->linkage,
738 efx_tc_recirc_ht_params);
739 ida_free(&efx->tc->recirc_ida, rid->fw_id);
740 kfree(rid);
741}
742
743static void efx_tc_delete_rule(struct efx_nic *efx, struct efx_tc_flow_rule *rule)
744{
745 efx_mae_delete_rule(efx, rule->fw_id);
746
747 /* Release entries in subsidiary tables */
748 efx_tc_free_action_set_list(efx, &rule->acts, true);
749 if (rule->match.rid)
750 efx_tc_put_recirc_id(efx, rule->match.rid);
751 if (rule->match.encap)
752 efx_tc_flower_release_encap_match(efx, rule->match.encap);
753 rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
754}
755
756static const char *efx_tc_encap_type_name(enum efx_encap_type typ)
757{
758 switch (typ) {
759 case EFX_ENCAP_TYPE_NONE:
760 return "none";
761 case EFX_ENCAP_TYPE_VXLAN:
762 return "vxlan";
763 case EFX_ENCAP_TYPE_GENEVE:
764 return "geneve";
765 default:
766 pr_warn_once("Unknown efx_encap_type %d encountered\n", typ);
767 return "unknown";
768 }
769}
770
771/* For details of action order constraints refer to SF-123102-TC-1§12.6.1 */
772enum efx_tc_action_order {
773 EFX_TC_AO_DECAP,
774 EFX_TC_AO_DEC_TTL,
775 EFX_TC_AO_PEDIT_MAC_ADDRS,
776 EFX_TC_AO_VLAN_POP,
777 EFX_TC_AO_VLAN_PUSH,
778 EFX_TC_AO_COUNT,
779 EFX_TC_AO_ENCAP,
780 EFX_TC_AO_DELIVER
781};
782/* Determine whether we can add @new action without violating order */
783static bool efx_tc_flower_action_order_ok(const struct efx_tc_action_set *act,
784 enum efx_tc_action_order new)
785{
786 switch (new) {
787 case EFX_TC_AO_DECAP:
788 if (act->decap)
789 return false;
790 /* PEDIT_MAC_ADDRS must not happen before DECAP, though it
791 * can wait until much later
792 */
793 if (act->dst_mac || act->src_mac)
794 return false;
795
796 /* Decrementing ttl must not happen before DECAP */
797 if (act->do_ttl_dec)
798 return false;
799 fallthrough;
800 case EFX_TC_AO_VLAN_POP:
801 if (act->vlan_pop >= 2)
802 return false;
803 /* If we've already pushed a VLAN, we can't then pop it;
804 * the hardware would instead try to pop an existing VLAN
805 * before pushing the new one.
806 */
807 if (act->vlan_push)
808 return false;
809 fallthrough;
810 case EFX_TC_AO_VLAN_PUSH:
811 if (act->vlan_push >= 2)
812 return false;
813 fallthrough;
814 case EFX_TC_AO_COUNT:
815 if (act->count)
816 return false;
817 fallthrough;
818 case EFX_TC_AO_PEDIT_MAC_ADDRS:
819 case EFX_TC_AO_ENCAP:
820 if (act->encap_md)
821 return false;
822 fallthrough;
823 case EFX_TC_AO_DELIVER:
824 return !act->deliver;
825 case EFX_TC_AO_DEC_TTL:
826 if (act->encap_md)
827 return false;
828 return !act->do_ttl_dec;
829 default:
830 /* Bad caller. Whatever they wanted to do, say they can't. */
831 WARN_ON_ONCE(1);
832 return false;
833 }
834}
835
836/**
837 * DOC: TC conntrack sequences
838 *
839 * The MAE hardware can handle at most two rounds of action rule matching,
840 * consequently we support conntrack through the notion of a "left-hand side
841 * rule". This is a rule which typically contains only the actions "ct" and
842 * "goto chain N", and corresponds to one or more "right-hand side rules" in
843 * chain N, which typically match on +trk+est, and may perform ct(nat) actions.
844 * RHS rules go in the Action Rule table as normal but with a nonzero recirc_id
845 * (the hardware equivalent of chain_index), while LHS rules may go in either
846 * the Action Rule or the Outer Rule table, the latter being preferred for
847 * performance reasons, and set both DO_CT and a recirc_id in their response.
848 *
849 * Besides the RHS rules, there are often also similar rules matching on
850 * +trk+new which perform the ct(commit) action. These are not offloaded.
851 */
852
853static bool efx_tc_rule_is_lhs_rule(struct flow_rule *fr,
854 struct efx_tc_match *match)
855{
856 const struct flow_action_entry *fa;
857 int i;
858
859 flow_action_for_each(i, fa, &fr->action) {
860 switch (fa->id) {
861 case FLOW_ACTION_GOTO:
862 return true;
863 case FLOW_ACTION_CT:
864 /* If rule is -trk, or doesn't mention trk at all, then
865 * a CT action implies a conntrack lookup (hence it's an
866 * LHS rule). If rule is +trk, then a CT action could
867 * just be ct(nat) or even ct(commit) (though the latter
868 * can't be offloaded).
869 */
870 if (!match->mask.ct_state_trk || !match->value.ct_state_trk)
871 return true;
872 break;
873 default:
874 break;
875 }
876 }
877 return false;
878}
879
880/* A foreign LHS rule has matches on enc_ keys at the TC layer (including an
881 * implied match on enc_ip_proto UDP). Translate these into non-enc_ keys,
882 * so that we can use the same MAE machinery as local LHS rules (and so that
883 * the lhs_rules entries have uniform semantics). It may seem odd to do it
884 * this way round, given that the corresponding fields in the MAE MCDIs are
885 * all ENC_, but (a) we don't have enc_L2 or enc_ip_proto in struct
886 * efx_tc_match_fields and (b) semantically an LHS rule doesn't have inner
887 * fields so it's just matching on *the* header rather than the outer header.
888 * Make sure that the non-enc_ keys were not already being matched on, as that
889 * would imply a rule that needed a triple lookup. (Hardware can do that,
890 * with OR-AR-CT-AR, but it halves packet rate so we avoid it where possible;
891 * see efx_tc_flower_flhs_needs_ar().)
892 */
893static int efx_tc_flower_translate_flhs_match(struct efx_tc_match *match)
894{
895 int rc = 0;
896
897#define COPY_MASK_AND_VALUE(_key, _ekey) ({ \
898 if (match->mask._key) { \
899 rc = -EOPNOTSUPP; \
900 } else { \
901 match->mask._key = match->mask._ekey; \
902 match->mask._ekey = 0; \
903 match->value._key = match->value._ekey; \
904 match->value._ekey = 0; \
905 } \
906 rc; \
907})
908#define COPY_FROM_ENC(_key) COPY_MASK_AND_VALUE(_key, enc_##_key)
909 if (match->mask.ip_proto)
910 return -EOPNOTSUPP;
911 match->mask.ip_proto = ~0;
912 match->value.ip_proto = IPPROTO_UDP;
913 if (COPY_FROM_ENC(src_ip) || COPY_FROM_ENC(dst_ip))
914 return rc;
915#ifdef CONFIG_IPV6
916 if (!ipv6_addr_any(&match->mask.src_ip6))
917 return -EOPNOTSUPP;
918 match->mask.src_ip6 = match->mask.enc_src_ip6;
919 memset(&match->mask.enc_src_ip6, 0, sizeof(struct in6_addr));
920 if (!ipv6_addr_any(&match->mask.dst_ip6))
921 return -EOPNOTSUPP;
922 match->mask.dst_ip6 = match->mask.enc_dst_ip6;
923 memset(&match->mask.enc_dst_ip6, 0, sizeof(struct in6_addr));
924#endif
925 if (COPY_FROM_ENC(ip_tos) || COPY_FROM_ENC(ip_ttl))
926 return rc;
927 /* should really copy enc_ip_frag but we don't have that in
928 * parse_match yet
929 */
930 if (COPY_MASK_AND_VALUE(l4_sport, enc_sport) ||
931 COPY_MASK_AND_VALUE(l4_dport, enc_dport))
932 return rc;
933 return 0;
934#undef COPY_FROM_ENC
935#undef COPY_MASK_AND_VALUE
936}
937
938/* If a foreign LHS rule wants to match on keys that are only available after
939 * encap header identification and parsing, then it can't be done in the Outer
940 * Rule lookup, because that lookup determines the encap type used to parse
941 * beyond the outer headers. Thus, such rules must use the OR-AR-CT-AR lookup
942 * sequence, with an EM (struct efx_tc_encap_match) in the OR step.
943 * Return true iff the passed match requires this.
944 */
945static bool efx_tc_flower_flhs_needs_ar(struct efx_tc_match *match)
946{
947 /* matches on inner-header keys can't be done in OR */
948 return match->mask.eth_proto ||
949 match->mask.vlan_tci[0] || match->mask.vlan_tci[1] ||
950 match->mask.vlan_proto[0] || match->mask.vlan_proto[1] ||
951 memchr_inv(match->mask.eth_saddr, 0, ETH_ALEN) ||
952 memchr_inv(match->mask.eth_daddr, 0, ETH_ALEN) ||
953 match->mask.ip_proto ||
954 match->mask.ip_tos || match->mask.ip_ttl ||
955 match->mask.src_ip || match->mask.dst_ip ||
956#ifdef CONFIG_IPV6
957 !ipv6_addr_any(&match->mask.src_ip6) ||
958 !ipv6_addr_any(&match->mask.dst_ip6) ||
959#endif
960 match->mask.ip_frag || match->mask.ip_firstfrag ||
961 match->mask.l4_sport || match->mask.l4_dport ||
962 match->mask.tcp_flags ||
963 /* nor can VNI */
964 match->mask.enc_keyid;
965}
966
967static int efx_tc_flower_handle_lhs_actions(struct efx_nic *efx,
968 struct flow_cls_offload *tc,
969 struct flow_rule *fr,
970 struct net_device *net_dev,
971 struct efx_tc_lhs_rule *rule)
972
973{
974 struct netlink_ext_ack *extack = tc->common.extack;
975 struct efx_tc_lhs_action *act = &rule->lhs_act;
976 const struct flow_action_entry *fa;
977 enum efx_tc_counter_type ctype;
978 bool pipe = true;
979 int i;
980
981 ctype = rule->is_ar ? EFX_TC_COUNTER_TYPE_AR : EFX_TC_COUNTER_TYPE_OR;
982
983 flow_action_for_each(i, fa, &fr->action) {
984 struct efx_tc_ct_zone *ct_zone;
985 struct efx_tc_recirc_id *rid;
986
987 if (!pipe) {
988 /* more actions after a non-pipe action */
989 NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action");
990 return -EINVAL;
991 }
992 switch (fa->id) {
993 case FLOW_ACTION_GOTO:
994 if (!fa->chain_index) {
995 NL_SET_ERR_MSG_MOD(extack, "Can't goto chain 0, no looping in hw");
996 return -EOPNOTSUPP;
997 }
998 rid = efx_tc_get_recirc_id(efx, fa->chain_index,
999 net_dev);
1000 if (IS_ERR(rid)) {
1001 NL_SET_ERR_MSG_MOD(extack, "Failed to allocate a hardware recirculation ID for this chain_index");
1002 return PTR_ERR(rid);
1003 }
1004 act->rid = rid;
1005 if (fa->hw_stats) {
1006 struct efx_tc_counter_index *cnt;
1007
1008 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
1009 NL_SET_ERR_MSG_FMT_MOD(extack,
1010 "hw_stats_type %u not supported (only 'delayed')",
1011 fa->hw_stats);
1012 return -EOPNOTSUPP;
1013 }
1014 cnt = efx_tc_flower_get_counter_index(efx, tc->cookie,
1015 ctype);
1016 if (IS_ERR(cnt)) {
1017 NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
1018 return PTR_ERR(cnt);
1019 }
1020 WARN_ON(act->count); /* can't happen */
1021 act->count = cnt;
1022 }
1023 pipe = false;
1024 break;
1025 case FLOW_ACTION_CT:
1026 if (act->zone) {
1027 NL_SET_ERR_MSG_MOD(extack, "Can't offload multiple ct actions");
1028 return -EOPNOTSUPP;
1029 }
1030 if (fa->ct.action & (TCA_CT_ACT_COMMIT |
1031 TCA_CT_ACT_FORCE)) {
1032 NL_SET_ERR_MSG_MOD(extack, "Can't offload ct commit/force");
1033 return -EOPNOTSUPP;
1034 }
1035 if (fa->ct.action & TCA_CT_ACT_CLEAR) {
1036 NL_SET_ERR_MSG_MOD(extack, "Can't clear ct in LHS rule");
1037 return -EOPNOTSUPP;
1038 }
1039 if (fa->ct.action & (TCA_CT_ACT_NAT |
1040 TCA_CT_ACT_NAT_SRC |
1041 TCA_CT_ACT_NAT_DST)) {
1042 NL_SET_ERR_MSG_MOD(extack, "Can't perform NAT in LHS rule - packet isn't conntracked yet");
1043 return -EOPNOTSUPP;
1044 }
1045 if (fa->ct.action) {
1046 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled ct.action %u for LHS rule\n",
1047 fa->ct.action);
1048 return -EOPNOTSUPP;
1049 }
1050 ct_zone = efx_tc_ct_register_zone(efx, fa->ct.zone,
1051 fa->ct.flow_table);
1052 if (IS_ERR(ct_zone)) {
1053 NL_SET_ERR_MSG_MOD(extack, "Failed to register for CT updates");
1054 return PTR_ERR(ct_zone);
1055 }
1056 act->zone = ct_zone;
1057 break;
1058 default:
1059 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u for LHS rule\n",
1060 fa->id);
1061 return -EOPNOTSUPP;
1062 }
1063 }
1064
1065 if (pipe) {
1066 NL_SET_ERR_MSG_MOD(extack, "Missing goto chain in LHS rule");
1067 return -EOPNOTSUPP;
1068 }
1069 return 0;
1070}
1071
1072static void efx_tc_flower_release_lhs_actions(struct efx_nic *efx,
1073 struct efx_tc_lhs_action *act)
1074{
1075 if (act->rid)
1076 efx_tc_put_recirc_id(efx, act->rid);
1077 if (act->zone)
1078 efx_tc_ct_unregister_zone(efx, act->zone);
1079 if (act->count)
1080 efx_tc_flower_put_counter_index(efx, act->count);
1081}
1082
1083/**
1084 * struct efx_tc_mangler_state - accumulates 32-bit pedits into fields
1085 *
1086 * @dst_mac_32: dst_mac[0:3] has been populated
1087 * @dst_mac_16: dst_mac[4:5] has been populated
1088 * @src_mac_16: src_mac[0:1] has been populated
1089 * @src_mac_32: src_mac[2:5] has been populated
1090 * @dst_mac: h_dest field of ethhdr
1091 * @src_mac: h_source field of ethhdr
1092 *
1093 * Since FLOW_ACTION_MANGLE comes in 32-bit chunks that do not
1094 * necessarily equate to whole fields of the packet header, this
1095 * structure is used to hold the cumulative effect of the partial
1096 * field pedits that have been processed so far.
1097 */
1098struct efx_tc_mangler_state {
1099 u8 dst_mac_32:1; /* eth->h_dest[0:3] */
1100 u8 dst_mac_16:1; /* eth->h_dest[4:5] */
1101 u8 src_mac_16:1; /* eth->h_source[0:1] */
1102 u8 src_mac_32:1; /* eth->h_source[2:5] */
1103 unsigned char dst_mac[ETH_ALEN];
1104 unsigned char src_mac[ETH_ALEN];
1105};
1106
1107/** efx_tc_complete_mac_mangle() - pull complete field pedits out of @mung
1108 * @efx: NIC we're installing a flow rule on
1109 * @act: action set (cursor) to update
1110 * @mung: accumulated partial mangles
1111 * @extack: netlink extended ack for reporting errors
1112 *
1113 * Check @mung to find any combinations of partial mangles that can be
1114 * combined into a complete packet field edit, add that edit to @act,
1115 * and consume the partial mangles from @mung.
1116 */
1117
1118static int efx_tc_complete_mac_mangle(struct efx_nic *efx,
1119 struct efx_tc_action_set *act,
1120 struct efx_tc_mangler_state *mung,
1121 struct netlink_ext_ack *extack)
1122{
1123 struct efx_tc_mac_pedit_action *ped;
1124
1125 if (mung->dst_mac_32 && mung->dst_mac_16) {
1126 ped = efx_tc_flower_get_mac(efx, mung->dst_mac, extack);
1127 if (IS_ERR(ped))
1128 return PTR_ERR(ped);
1129
1130 /* Check that we have not already populated dst_mac */
1131 if (act->dst_mac)
1132 efx_tc_flower_put_mac(efx, act->dst_mac);
1133
1134 act->dst_mac = ped;
1135
1136 /* consume the incomplete state */
1137 mung->dst_mac_32 = 0;
1138 mung->dst_mac_16 = 0;
1139 }
1140 if (mung->src_mac_16 && mung->src_mac_32) {
1141 ped = efx_tc_flower_get_mac(efx, mung->src_mac, extack);
1142 if (IS_ERR(ped))
1143 return PTR_ERR(ped);
1144
1145 /* Check that we have not already populated src_mac */
1146 if (act->src_mac)
1147 efx_tc_flower_put_mac(efx, act->src_mac);
1148
1149 act->src_mac = ped;
1150
1151 /* consume the incomplete state */
1152 mung->src_mac_32 = 0;
1153 mung->src_mac_16 = 0;
1154 }
1155 return 0;
1156}
1157
1158static int efx_tc_pedit_add(struct efx_nic *efx, struct efx_tc_action_set *act,
1159 const struct flow_action_entry *fa,
1160 struct netlink_ext_ack *extack)
1161{
1162 switch (fa->mangle.htype) {
1163 case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
1164 switch (fa->mangle.offset) {
1165 case offsetof(struct iphdr, ttl):
1166 /* check that pedit applies to ttl only */
1167 if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK)
1168 break;
1169
1170 /* Adding 0xff is equivalent to decrementing the ttl.
1171 * Other added values are not supported.
1172 */
1173 if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) != U8_MAX)
1174 break;
1175
1176 /* check that we do not decrement ttl twice */
1177 if (!efx_tc_flower_action_order_ok(act,
1178 EFX_TC_AO_DEC_TTL)) {
1179 NL_SET_ERR_MSG_MOD(extack, "multiple dec ttl are not supported");
1180 return -EOPNOTSUPP;
1181 }
1182 act->do_ttl_dec = 1;
1183 return 0;
1184 default:
1185 break;
1186 }
1187 break;
1188 case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
1189 switch (fa->mangle.offset) {
1190 case round_down(offsetof(struct ipv6hdr, hop_limit), 4):
1191 /* check that pedit applies to hoplimit only */
1192 if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK)
1193 break;
1194
1195 /* Adding 0xff is equivalent to decrementing the hoplimit.
1196 * Other added values are not supported.
1197 */
1198 if ((fa->mangle.val >> 24) != U8_MAX)
1199 break;
1200
1201 /* check that we do not decrement hoplimit twice */
1202 if (!efx_tc_flower_action_order_ok(act,
1203 EFX_TC_AO_DEC_TTL)) {
1204 NL_SET_ERR_MSG_MOD(extack, "multiple dec ttl are not supported");
1205 return -EOPNOTSUPP;
1206 }
1207 act->do_ttl_dec = 1;
1208 return 0;
1209 default:
1210 break;
1211 }
1212 break;
1213 default:
1214 break;
1215 }
1216
1217 NL_SET_ERR_MSG_FMT_MOD(extack,
1218 "ttl add action type %x %x %x/%x is not supported",
1219 fa->mangle.htype, fa->mangle.offset,
1220 fa->mangle.val, fa->mangle.mask);
1221 return -EOPNOTSUPP;
1222}
1223
1224/**
1225 * efx_tc_mangle() - handle a single 32-bit (or less) pedit
1226 * @efx: NIC we're installing a flow rule on
1227 * @act: action set (cursor) to update
1228 * @fa: FLOW_ACTION_MANGLE action metadata
1229 * @mung: accumulator for partial mangles
1230 * @extack: netlink extended ack for reporting errors
1231 * @match: original match used along with the mangle action
1232 *
1233 * Identify the fields written by a FLOW_ACTION_MANGLE, and record
1234 * the partial mangle state in @mung. If this mangle completes an
1235 * earlier partial mangle, consume and apply to @act by calling
1236 * efx_tc_complete_mac_mangle().
1237 */
1238
1239static int efx_tc_mangle(struct efx_nic *efx, struct efx_tc_action_set *act,
1240 const struct flow_action_entry *fa,
1241 struct efx_tc_mangler_state *mung,
1242 struct netlink_ext_ack *extack,
1243 struct efx_tc_match *match)
1244{
1245 __le32 mac32;
1246 __le16 mac16;
1247 u8 tr_ttl;
1248
1249 switch (fa->mangle.htype) {
1250 case FLOW_ACT_MANGLE_HDR_TYPE_ETH:
1251 BUILD_BUG_ON(offsetof(struct ethhdr, h_dest) != 0);
1252 BUILD_BUG_ON(offsetof(struct ethhdr, h_source) != 6);
1253 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_PEDIT_MAC_ADDRS)) {
1254 NL_SET_ERR_MSG_MOD(extack,
1255 "Pedit mangle mac action violates action order");
1256 return -EOPNOTSUPP;
1257 }
1258 switch (fa->mangle.offset) {
1259 case 0:
1260 if (fa->mangle.mask) {
1261 NL_SET_ERR_MSG_FMT_MOD(extack,
1262 "mask (%#x) of eth.dst32 mangle is not supported",
1263 fa->mangle.mask);
1264 return -EOPNOTSUPP;
1265 }
1266 /* Ethernet address is little-endian */
1267 mac32 = cpu_to_le32(fa->mangle.val);
1268 memcpy(mung->dst_mac, &mac32, sizeof(mac32));
1269 mung->dst_mac_32 = 1;
1270 return efx_tc_complete_mac_mangle(efx, act, mung, extack);
1271 case 4:
1272 if (fa->mangle.mask == 0xffff) {
1273 mac16 = cpu_to_le16(fa->mangle.val >> 16);
1274 memcpy(mung->src_mac, &mac16, sizeof(mac16));
1275 mung->src_mac_16 = 1;
1276 } else if (fa->mangle.mask == 0xffff0000) {
1277 mac16 = cpu_to_le16((u16)fa->mangle.val);
1278 memcpy(mung->dst_mac + 4, &mac16, sizeof(mac16));
1279 mung->dst_mac_16 = 1;
1280 } else {
1281 NL_SET_ERR_MSG_FMT_MOD(extack,
1282 "mask (%#x) of eth+4 mangle is not high or low 16b",
1283 fa->mangle.mask);
1284 return -EOPNOTSUPP;
1285 }
1286 return efx_tc_complete_mac_mangle(efx, act, mung, extack);
1287 case 8:
1288 if (fa->mangle.mask) {
1289 NL_SET_ERR_MSG_FMT_MOD(extack,
1290 "mask (%#x) of eth.src32 mangle is not supported",
1291 fa->mangle.mask);
1292 return -EOPNOTSUPP;
1293 }
1294 mac32 = cpu_to_le32(fa->mangle.val);
1295 memcpy(mung->src_mac + 2, &mac32, sizeof(mac32));
1296 mung->src_mac_32 = 1;
1297 return efx_tc_complete_mac_mangle(efx, act, mung, extack);
1298 default:
1299 NL_SET_ERR_MSG_FMT_MOD(extack, "mangle eth+%u %x/%x is not supported",
1300 fa->mangle.offset, fa->mangle.val, fa->mangle.mask);
1301 return -EOPNOTSUPP;
1302 }
1303 break;
1304 case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
1305 switch (fa->mangle.offset) {
1306 case offsetof(struct iphdr, ttl):
1307 /* we currently only support pedit IP4 when it applies
1308 * to TTL and then only when it can be achieved with a
1309 * decrement ttl action
1310 */
1311
1312 /* check that pedit applies to ttl only */
1313 if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK) {
1314 NL_SET_ERR_MSG_FMT_MOD(extack,
1315 "mask (%#x) out of range, only support mangle action on ipv4.ttl",
1316 fa->mangle.mask);
1317 return -EOPNOTSUPP;
1318 }
1319
1320 /* we can only convert to a dec ttl when we have an
1321 * exact match on the ttl field
1322 */
1323 if (match->mask.ip_ttl != U8_MAX) {
1324 NL_SET_ERR_MSG_FMT_MOD(extack,
1325 "only support mangle ttl when we have an exact match, current mask (%#x)",
1326 match->mask.ip_ttl);
1327 return -EOPNOTSUPP;
1328 }
1329
1330 /* check that we don't try to decrement 0, which equates
1331 * to setting the ttl to 0xff
1332 */
1333 if (match->value.ip_ttl == 0) {
1334 NL_SET_ERR_MSG_MOD(extack,
1335 "decrement ttl past 0 is not supported");
1336 return -EOPNOTSUPP;
1337 }
1338
1339 /* check that we do not decrement ttl twice */
1340 if (!efx_tc_flower_action_order_ok(act,
1341 EFX_TC_AO_DEC_TTL)) {
1342 NL_SET_ERR_MSG_MOD(extack,
1343 "multiple dec ttl is not supported");
1344 return -EOPNOTSUPP;
1345 }
1346
1347 /* check pedit can be achieved with decrement action */
1348 tr_ttl = match->value.ip_ttl - 1;
1349 if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) == tr_ttl) {
1350 act->do_ttl_dec = 1;
1351 return 0;
1352 }
1353
1354 fallthrough;
1355 default:
1356 NL_SET_ERR_MSG_FMT_MOD(extack,
1357 "only support mangle on the ttl field (offset is %u)",
1358 fa->mangle.offset);
1359 return -EOPNOTSUPP;
1360 }
1361 break;
1362 case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
1363 switch (fa->mangle.offset) {
1364 case round_down(offsetof(struct ipv6hdr, hop_limit), 4):
1365 /* we currently only support pedit IP6 when it applies
1366 * to the hoplimit and then only when it can be achieved
1367 * with a decrement hoplimit action
1368 */
1369
1370 /* check that pedit applies to ttl only */
1371 if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK) {
1372 NL_SET_ERR_MSG_FMT_MOD(extack,
1373 "mask (%#x) out of range, only support mangle action on ipv6.hop_limit",
1374 fa->mangle.mask);
1375
1376 return -EOPNOTSUPP;
1377 }
1378
1379 /* we can only convert to a dec ttl when we have an
1380 * exact match on the ttl field
1381 */
1382 if (match->mask.ip_ttl != U8_MAX) {
1383 NL_SET_ERR_MSG_FMT_MOD(extack,
1384 "only support hop_limit when we have an exact match, current mask (%#x)",
1385 match->mask.ip_ttl);
1386 return -EOPNOTSUPP;
1387 }
1388
1389 /* check that we don't try to decrement 0, which equates
1390 * to setting the ttl to 0xff
1391 */
1392 if (match->value.ip_ttl == 0) {
1393 NL_SET_ERR_MSG_MOD(extack,
1394 "decrementing hop_limit past 0 is not supported");
1395 return -EOPNOTSUPP;
1396 }
1397
1398 /* check that we do not decrement hoplimit twice */
1399 if (!efx_tc_flower_action_order_ok(act,
1400 EFX_TC_AO_DEC_TTL)) {
1401 NL_SET_ERR_MSG_MOD(extack,
1402 "multiple dec ttl is not supported");
1403 return -EOPNOTSUPP;
1404 }
1405
1406 /* check pedit can be achieved with decrement action */
1407 tr_ttl = match->value.ip_ttl - 1;
1408 if ((fa->mangle.val >> 24) == tr_ttl) {
1409 act->do_ttl_dec = 1;
1410 return 0;
1411 }
1412
1413 fallthrough;
1414 default:
1415 NL_SET_ERR_MSG_FMT_MOD(extack,
1416 "only support mangle on the hop_limit field");
1417 return -EOPNOTSUPP;
1418 }
1419 default:
1420 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled mangle htype %u for action rule",
1421 fa->mangle.htype);
1422 return -EOPNOTSUPP;
1423 }
1424 return 0;
1425}
1426
1427/**
1428 * efx_tc_incomplete_mangle() - check for leftover partial pedits
1429 * @mung: accumulator for partial mangles
1430 * @extack: netlink extended ack for reporting errors
1431 *
1432 * Since the MAE can only overwrite whole fields, any partial
1433 * field mangle left over on reaching packet delivery (mirred or
1434 * end of TC actions) cannot be offloaded. Check for any such
1435 * and reject them with -%EOPNOTSUPP.
1436 */
1437
1438static int efx_tc_incomplete_mangle(struct efx_tc_mangler_state *mung,
1439 struct netlink_ext_ack *extack)
1440{
1441 if (mung->dst_mac_32 || mung->dst_mac_16) {
1442 NL_SET_ERR_MSG_MOD(extack, "Incomplete pedit of destination MAC address");
1443 return -EOPNOTSUPP;
1444 }
1445 if (mung->src_mac_16 || mung->src_mac_32) {
1446 NL_SET_ERR_MSG_MOD(extack, "Incomplete pedit of source MAC address");
1447 return -EOPNOTSUPP;
1448 }
1449 return 0;
1450}
1451
1452static int efx_tc_flower_replace_foreign_lhs_ar(struct efx_nic *efx,
1453 struct flow_cls_offload *tc,
1454 struct flow_rule *fr,
1455 struct efx_tc_match *match,
1456 struct net_device *net_dev)
1457{
1458 struct netlink_ext_ack *extack = tc->common.extack;
1459 struct efx_tc_lhs_rule *rule, *old;
1460 enum efx_encap_type type;
1461 int rc;
1462
1463 type = efx_tc_indr_netdev_type(net_dev);
1464 if (type == EFX_ENCAP_TYPE_NONE) {
1465 NL_SET_ERR_MSG_MOD(extack, "Egress encap match on unsupported tunnel device");
1466 return -EOPNOTSUPP;
1467 }
1468
1469 rc = efx_mae_check_encap_type_supported(efx, type);
1470 if (rc) {
1471 NL_SET_ERR_MSG_FMT_MOD(extack,
1472 "Firmware reports no support for %s encap match",
1473 efx_tc_encap_type_name(type));
1474 return rc;
1475 }
1476 /* This is an Action Rule, so it needs a separate Encap Match in the
1477 * Outer Rule table. Insert that now.
1478 */
1479 rc = efx_tc_flower_record_encap_match(efx, match, type,
1480 EFX_TC_EM_DIRECT, 0, 0, extack);
1481 if (rc)
1482 return rc;
1483
1484 match->mask.recirc_id = 0xff;
1485 if (match->mask.ct_state_trk && match->value.ct_state_trk) {
1486 NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");
1487 rc = -EOPNOTSUPP;
1488 goto release_encap_match;
1489 }
1490 /* LHS rules are always -trk, so we don't need to match on that */
1491 match->mask.ct_state_trk = 0;
1492 match->value.ct_state_trk = 0;
1493 /* We must inhibit match on TCP SYN/FIN/RST, so that SW can see
1494 * the packet and update the conntrack table.
1495 * Outer Rules will do that with CT_TCP_FLAGS_INHIBIT, but Action
1496 * Rules don't have that; instead they support matching on
1497 * TCP_SYN_FIN_RST (aka TCP_INTERESTING_FLAGS), so use that.
1498 * This is only strictly needed if there will be a DO_CT action,
1499 * which we don't know yet, but typically there will be and it's
1500 * simpler not to bother checking here.
1501 */
1502 match->mask.tcp_syn_fin_rst = true;
1503
1504 rc = efx_mae_match_check_caps(efx, &match->mask, extack);
1505 if (rc)
1506 goto release_encap_match;
1507
1508 rule = kzalloc(sizeof(*rule), GFP_USER);
1509 if (!rule) {
1510 rc = -ENOMEM;
1511 goto release_encap_match;
1512 }
1513 rule->cookie = tc->cookie;
1514 rule->is_ar = true;
1515 old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,
1516 &rule->linkage,
1517 efx_tc_lhs_rule_ht_params);
1518 if (old) {
1519 netif_dbg(efx, drv, efx->net_dev,
1520 "Already offloaded rule (cookie %lx)\n", tc->cookie);
1521 rc = -EEXIST;
1522 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
1523 goto release;
1524 }
1525
1526 /* Parse actions */
1527 rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);
1528 if (rc)
1529 goto release;
1530
1531 rule->match = *match;
1532 rule->lhs_act.tun_type = type;
1533
1534 rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);
1535 if (rc) {
1536 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
1537 goto release;
1538 }
1539 netif_dbg(efx, drv, efx->net_dev,
1540 "Successfully parsed lhs rule (cookie %lx)\n",
1541 tc->cookie);
1542 return 0;
1543
1544release:
1545 efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);
1546 if (!old)
1547 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,
1548 efx_tc_lhs_rule_ht_params);
1549 kfree(rule);
1550release_encap_match:
1551 if (match->encap)
1552 efx_tc_flower_release_encap_match(efx, match->encap);
1553 return rc;
1554}
1555
1556static int efx_tc_flower_replace_foreign_lhs(struct efx_nic *efx,
1557 struct flow_cls_offload *tc,
1558 struct flow_rule *fr,
1559 struct efx_tc_match *match,
1560 struct net_device *net_dev)
1561{
1562 struct netlink_ext_ack *extack = tc->common.extack;
1563 struct efx_tc_lhs_rule *rule, *old;
1564 enum efx_encap_type type;
1565 int rc;
1566
1567 if (tc->common.chain_index) {
1568 NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0");
1569 return -EOPNOTSUPP;
1570 }
1571
1572 if (!efx_tc_match_is_encap(&match->mask)) {
1573 /* This is not a tunnel decap rule, ignore it */
1574 netif_dbg(efx, drv, efx->net_dev, "Ignoring foreign LHS filter without encap match\n");
1575 return -EOPNOTSUPP;
1576 }
1577
1578 if (efx_tc_flower_flhs_needs_ar(match))
1579 return efx_tc_flower_replace_foreign_lhs_ar(efx, tc, fr, match,
1580 net_dev);
1581
1582 type = efx_tc_indr_netdev_type(net_dev);
1583 if (type == EFX_ENCAP_TYPE_NONE) {
1584 NL_SET_ERR_MSG_MOD(extack, "Egress encap match on unsupported tunnel device\n");
1585 return -EOPNOTSUPP;
1586 }
1587
1588 rc = efx_mae_check_encap_type_supported(efx, type);
1589 if (rc) {
1590 NL_SET_ERR_MSG_FMT_MOD(extack,
1591 "Firmware reports no support for %s encap match",
1592 efx_tc_encap_type_name(type));
1593 return rc;
1594 }
1595 /* Reserve the outer tuple with a pseudo Encap Match */
1596 rc = efx_tc_flower_record_encap_match(efx, match, type,
1597 EFX_TC_EM_PSEUDO_OR, 0, 0,
1598 extack);
1599 if (rc)
1600 return rc;
1601
1602 if (match->mask.ct_state_trk && match->value.ct_state_trk) {
1603 NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");
1604 rc = -EOPNOTSUPP;
1605 goto release_encap_match;
1606 }
1607 /* LHS rules are always -trk, so we don't need to match on that */
1608 match->mask.ct_state_trk = 0;
1609 match->value.ct_state_trk = 0;
1610
1611 rc = efx_tc_flower_translate_flhs_match(match);
1612 if (rc) {
1613 NL_SET_ERR_MSG_MOD(extack, "LHS rule cannot match on inner fields");
1614 goto release_encap_match;
1615 }
1616
1617 rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack);
1618 if (rc)
1619 goto release_encap_match;
1620
1621 rule = kzalloc(sizeof(*rule), GFP_USER);
1622 if (!rule) {
1623 rc = -ENOMEM;
1624 goto release_encap_match;
1625 }
1626 rule->cookie = tc->cookie;
1627 old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,
1628 &rule->linkage,
1629 efx_tc_lhs_rule_ht_params);
1630 if (old) {
1631 netif_dbg(efx, drv, efx->net_dev,
1632 "Already offloaded rule (cookie %lx)\n", tc->cookie);
1633 rc = -EEXIST;
1634 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
1635 goto release;
1636 }
1637
1638 /* Parse actions */
1639 rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);
1640 if (rc)
1641 goto release;
1642
1643 rule->match = *match;
1644 rule->lhs_act.tun_type = type;
1645
1646 rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);
1647 if (rc) {
1648 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
1649 goto release;
1650 }
1651 netif_dbg(efx, drv, efx->net_dev,
1652 "Successfully parsed lhs rule (cookie %lx)\n",
1653 tc->cookie);
1654 return 0;
1655
1656release:
1657 efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);
1658 if (!old)
1659 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,
1660 efx_tc_lhs_rule_ht_params);
1661 kfree(rule);
1662release_encap_match:
1663 if (match->encap)
1664 efx_tc_flower_release_encap_match(efx, match->encap);
1665 return rc;
1666}
1667
1668static int efx_tc_flower_replace_foreign(struct efx_nic *efx,
1669 struct net_device *net_dev,
1670 struct flow_cls_offload *tc)
1671{
1672 struct flow_rule *fr = flow_cls_offload_flow_rule(tc);
1673 struct netlink_ext_ack *extack = tc->common.extack;
1674 struct efx_tc_flow_rule *rule = NULL, *old = NULL;
1675 struct efx_tc_action_set *act = NULL;
1676 bool found = false, uplinked = false;
1677 const struct flow_action_entry *fa;
1678 struct efx_tc_match match;
1679 struct efx_rep *to_efv;
1680 s64 rc;
1681 int i;
1682
1683 /* Parse match */
1684 memset(&match, 0, sizeof(match));
1685 rc = efx_tc_flower_parse_match(efx, fr, &match, extack);
1686 if (rc)
1687 return rc;
1688 /* The rule as given to us doesn't specify a source netdevice.
1689 * But, determining whether packets from a VF should match it is
1690 * complicated, so leave those to the software slowpath: qualify
1691 * the filter with source m-port == wire.
1692 */
1693 rc = efx_tc_flower_external_mport(efx, EFX_EFV_PF);
1694 if (rc < 0) {
1695 NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port for foreign filter");
1696 return rc;
1697 }
1698 match.value.ingress_port = rc;
1699 match.mask.ingress_port = ~0;
1700
1701 if (efx_tc_rule_is_lhs_rule(fr, &match))
1702 return efx_tc_flower_replace_foreign_lhs(efx, tc, fr, &match,
1703 net_dev);
1704
1705 if (tc->common.chain_index) {
1706 struct efx_tc_recirc_id *rid;
1707
1708 rid = efx_tc_get_recirc_id(efx, tc->common.chain_index, net_dev);
1709 if (IS_ERR(rid)) {
1710 NL_SET_ERR_MSG_FMT_MOD(extack,
1711 "Failed to allocate a hardware recirculation ID for chain_index %u",
1712 tc->common.chain_index);
1713 return PTR_ERR(rid);
1714 }
1715 match.rid = rid;
1716 match.value.recirc_id = rid->fw_id;
1717 }
1718 match.mask.recirc_id = 0xff;
1719
1720 /* AR table can't match on DO_CT (+trk). But a commonly used pattern is
1721 * +trk+est, which is strictly implied by +est, so rewrite it to that.
1722 */
1723 if (match.mask.ct_state_trk && match.value.ct_state_trk &&
1724 match.mask.ct_state_est && match.value.ct_state_est)
1725 match.mask.ct_state_trk = 0;
1726 /* Thanks to CT_TCP_FLAGS_INHIBIT, packets with interesting flags could
1727 * match +trk-est (CT_HIT=0) despite being on an established connection.
1728 * So make -est imply -tcp_syn_fin_rst match to ensure these packets
1729 * still hit the software path.
1730 */
1731 if (match.mask.ct_state_est && !match.value.ct_state_est) {
1732 if (match.value.tcp_syn_fin_rst) {
1733 /* Can't offload this combination */
1734 NL_SET_ERR_MSG_MOD(extack, "TCP flags and -est conflict for offload");
1735 rc = -EOPNOTSUPP;
1736 goto release;
1737 }
1738 match.mask.tcp_syn_fin_rst = true;
1739 }
1740
1741 flow_action_for_each(i, fa, &fr->action) {
1742 switch (fa->id) {
1743 case FLOW_ACTION_REDIRECT:
1744 case FLOW_ACTION_MIRRED: /* mirred means mirror here */
1745 to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
1746 if (IS_ERR(to_efv))
1747 continue;
1748 found = true;
1749 break;
1750 default:
1751 break;
1752 }
1753 }
1754 if (!found) { /* We don't care. */
1755 netif_dbg(efx, drv, efx->net_dev,
1756 "Ignoring foreign filter that doesn't egdev us\n");
1757 rc = -EOPNOTSUPP;
1758 goto release;
1759 }
1760
1761 rc = efx_mae_match_check_caps(efx, &match.mask, extack);
1762 if (rc)
1763 goto release;
1764
1765 if (efx_tc_match_is_encap(&match.mask)) {
1766 enum efx_encap_type type;
1767
1768 type = efx_tc_indr_netdev_type(net_dev);
1769 if (type == EFX_ENCAP_TYPE_NONE) {
1770 NL_SET_ERR_MSG_MOD(extack,
1771 "Egress encap match on unsupported tunnel device");
1772 rc = -EOPNOTSUPP;
1773 goto release;
1774 }
1775
1776 rc = efx_mae_check_encap_type_supported(efx, type);
1777 if (rc) {
1778 NL_SET_ERR_MSG_FMT_MOD(extack,
1779 "Firmware reports no support for %s encap match",
1780 efx_tc_encap_type_name(type));
1781 goto release;
1782 }
1783
1784 rc = efx_tc_flower_record_encap_match(efx, &match, type,
1785 EFX_TC_EM_DIRECT, 0, 0,
1786 extack);
1787 if (rc)
1788 goto release;
1789 } else if (!tc->common.chain_index) {
1790 /* This is not a tunnel decap rule, ignore it */
1791 netif_dbg(efx, drv, efx->net_dev,
1792 "Ignoring foreign filter without encap match\n");
1793 rc = -EOPNOTSUPP;
1794 goto release;
1795 }
1796
1797 rule = kzalloc(sizeof(*rule), GFP_USER);
1798 if (!rule) {
1799 rc = -ENOMEM;
1800 goto release;
1801 }
1802 INIT_LIST_HEAD(&rule->acts.list);
1803 rule->cookie = tc->cookie;
1804 old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht,
1805 &rule->linkage,
1806 efx_tc_match_action_ht_params);
1807 if (IS_ERR(old)) {
1808 rc = PTR_ERR(old);
1809 goto release;
1810 } else if (old) {
1811 netif_dbg(efx, drv, efx->net_dev,
1812 "Ignoring already-offloaded rule (cookie %lx)\n",
1813 tc->cookie);
1814 rc = -EEXIST;
1815 goto release;
1816 }
1817
1818 act = kzalloc(sizeof(*act), GFP_USER);
1819 if (!act) {
1820 rc = -ENOMEM;
1821 goto release;
1822 }
1823
1824 /* Parse actions. For foreign rules we only support decap & redirect.
1825 * See corresponding code in efx_tc_flower_replace() for theory of
1826 * operation & how 'act' cursor is used.
1827 */
1828 flow_action_for_each(i, fa, &fr->action) {
1829 struct efx_tc_action_set save;
1830
1831 switch (fa->id) {
1832 case FLOW_ACTION_REDIRECT:
1833 case FLOW_ACTION_MIRRED:
1834 /* See corresponding code in efx_tc_flower_replace() for
1835 * long explanations of what's going on here.
1836 */
1837 save = *act;
1838 if (fa->hw_stats) {
1839 struct efx_tc_counter_index *ctr;
1840
1841 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
1842 NL_SET_ERR_MSG_FMT_MOD(extack,
1843 "hw_stats_type %u not supported (only 'delayed')",
1844 fa->hw_stats);
1845 rc = -EOPNOTSUPP;
1846 goto release;
1847 }
1848 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) {
1849 NL_SET_ERR_MSG_MOD(extack, "Count action violates action order (can't happen)");
1850 rc = -EOPNOTSUPP;
1851 goto release;
1852 }
1853
1854 ctr = efx_tc_flower_get_counter_index(efx,
1855 tc->cookie,
1856 EFX_TC_COUNTER_TYPE_AR);
1857 if (IS_ERR(ctr)) {
1858 rc = PTR_ERR(ctr);
1859 NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
1860 goto release;
1861 }
1862 act->count = ctr;
1863 INIT_LIST_HEAD(&act->count_user);
1864 }
1865
1866 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) {
1867 /* can't happen */
1868 rc = -EOPNOTSUPP;
1869 NL_SET_ERR_MSG_MOD(extack,
1870 "Deliver action violates action order (can't happen)");
1871 goto release;
1872 }
1873 to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
1874 /* PF implies egdev is us, in which case we really
1875 * want to deliver to the uplink (because this is an
1876 * ingress filter). If we don't recognise the egdev
1877 * at all, then we'd better trap so SW can handle it.
1878 */
1879 if (IS_ERR(to_efv))
1880 to_efv = EFX_EFV_PF;
1881 if (to_efv == EFX_EFV_PF) {
1882 if (uplinked)
1883 break;
1884 uplinked = true;
1885 }
1886 rc = efx_tc_flower_internal_mport(efx, to_efv);
1887 if (rc < 0) {
1888 NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port");
1889 goto release;
1890 }
1891 act->dest_mport = rc;
1892 act->deliver = 1;
1893 rc = efx_mae_alloc_action_set(efx, act);
1894 if (rc) {
1895 NL_SET_ERR_MSG_MOD(extack,
1896 "Failed to write action set to hw (mirred)");
1897 goto release;
1898 }
1899 list_add_tail(&act->list, &rule->acts.list);
1900 act = NULL;
1901 if (fa->id == FLOW_ACTION_REDIRECT)
1902 break; /* end of the line */
1903 /* Mirror, so continue on with saved act */
1904 act = kzalloc(sizeof(*act), GFP_USER);
1905 if (!act) {
1906 rc = -ENOMEM;
1907 goto release;
1908 }
1909 *act = save;
1910 break;
1911 case FLOW_ACTION_TUNNEL_DECAP:
1912 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DECAP)) {
1913 rc = -EINVAL;
1914 NL_SET_ERR_MSG_MOD(extack, "Decap action violates action order");
1915 goto release;
1916 }
1917 act->decap = 1;
1918 /* If we previously delivered/trapped to uplink, now
1919 * that we've decapped we'll want another copy if we
1920 * try to deliver/trap to uplink again.
1921 */
1922 uplinked = false;
1923 break;
1924 default:
1925 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u",
1926 fa->id);
1927 rc = -EOPNOTSUPP;
1928 goto release;
1929 }
1930 }
1931
1932 if (act) {
1933 if (!uplinked) {
1934 /* Not shot/redirected, so deliver to default dest (which is
1935 * the uplink, as this is an ingress filter)
1936 */
1937 efx_mae_mport_uplink(efx, &act->dest_mport);
1938 act->deliver = 1;
1939 }
1940 rc = efx_mae_alloc_action_set(efx, act);
1941 if (rc) {
1942 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)");
1943 goto release;
1944 }
1945 list_add_tail(&act->list, &rule->acts.list);
1946 act = NULL; /* Prevent double-free in error path */
1947 }
1948
1949 rule->match = match;
1950
1951 netif_dbg(efx, drv, efx->net_dev,
1952 "Successfully parsed foreign filter (cookie %lx)\n",
1953 tc->cookie);
1954
1955 rc = efx_mae_alloc_action_set_list(efx, &rule->acts);
1956 if (rc) {
1957 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw");
1958 goto release;
1959 }
1960 rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,
1961 rule->acts.fw_id, &rule->fw_id);
1962 if (rc) {
1963 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
1964 goto release_acts;
1965 }
1966 return 0;
1967
1968release_acts:
1969 efx_mae_free_action_set_list(efx, &rule->acts);
1970release:
1971 /* We failed to insert the rule, so free up any entries we created in
1972 * subsidiary tables.
1973 */
1974 if (match.rid)
1975 efx_tc_put_recirc_id(efx, match.rid);
1976 if (act)
1977 efx_tc_free_action_set(efx, act, false);
1978 if (rule) {
1979 if (!old)
1980 rhashtable_remove_fast(&efx->tc->match_action_ht,
1981 &rule->linkage,
1982 efx_tc_match_action_ht_params);
1983 efx_tc_free_action_set_list(efx, &rule->acts, false);
1984 }
1985 kfree(rule);
1986 if (match.encap)
1987 efx_tc_flower_release_encap_match(efx, match.encap);
1988 return rc;
1989}
1990
1991static int efx_tc_flower_replace_lhs(struct efx_nic *efx,
1992 struct flow_cls_offload *tc,
1993 struct flow_rule *fr,
1994 struct efx_tc_match *match,
1995 struct efx_rep *efv,
1996 struct net_device *net_dev)
1997{
1998 struct netlink_ext_ack *extack = tc->common.extack;
1999 struct efx_tc_lhs_rule *rule, *old;
2000 int rc;
2001
2002 if (tc->common.chain_index) {
2003 NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0");
2004 return -EOPNOTSUPP;
2005 }
2006
2007 if (match->mask.ct_state_trk && match->value.ct_state_trk) {
2008 NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");
2009 return -EOPNOTSUPP;
2010 }
2011 /* LHS rules are always -trk, so we don't need to match on that */
2012 match->mask.ct_state_trk = 0;
2013 match->value.ct_state_trk = 0;
2014
2015 rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack);
2016 if (rc)
2017 return rc;
2018
2019 rule = kzalloc(sizeof(*rule), GFP_USER);
2020 if (!rule)
2021 return -ENOMEM;
2022 rule->cookie = tc->cookie;
2023 old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,
2024 &rule->linkage,
2025 efx_tc_lhs_rule_ht_params);
2026 if (IS_ERR(old)) {
2027 rc = PTR_ERR(old);
2028 goto release;
2029 } else if (old) {
2030 netif_dbg(efx, drv, efx->net_dev,
2031 "Already offloaded rule (cookie %lx)\n", tc->cookie);
2032 rc = -EEXIST;
2033 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
2034 goto release;
2035 }
2036
2037 /* Parse actions */
2038 /* See note in efx_tc_flower_replace() regarding passed net_dev
2039 * (used for efx_tc_get_recirc_id()).
2040 */
2041 rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, efx->net_dev, rule);
2042 if (rc)
2043 goto release;
2044
2045 rule->match = *match;
2046
2047 rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);
2048 if (rc) {
2049 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
2050 goto release;
2051 }
2052 netif_dbg(efx, drv, efx->net_dev,
2053 "Successfully parsed lhs rule (cookie %lx)\n",
2054 tc->cookie);
2055 return 0;
2056
2057release:
2058 efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);
2059 if (!old)
2060 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,
2061 efx_tc_lhs_rule_ht_params);
2062 kfree(rule);
2063 return rc;
2064}
2065
2066static int efx_tc_flower_replace(struct efx_nic *efx,
2067 struct net_device *net_dev,
2068 struct flow_cls_offload *tc,
2069 struct efx_rep *efv)
2070{
2071 struct flow_rule *fr = flow_cls_offload_flow_rule(tc);
2072 struct netlink_ext_ack *extack = tc->common.extack;
2073 const struct ip_tunnel_info *encap_info = NULL;
2074 struct efx_tc_flow_rule *rule = NULL, *old;
2075 struct efx_tc_mangler_state mung = {};
2076 struct efx_tc_action_set *act = NULL;
2077 const struct flow_action_entry *fa;
2078 struct efx_rep *from_efv, *to_efv;
2079 struct efx_tc_match match;
2080 u32 acts_id;
2081 s64 rc;
2082 int i;
2083
2084 if (!tc_can_offload_extack(efx->net_dev, extack))
2085 return -EOPNOTSUPP;
2086 if (WARN_ON(!efx->tc))
2087 return -ENETDOWN;
2088 if (WARN_ON(!efx->tc->up))
2089 return -ENETDOWN;
2090
2091 from_efv = efx_tc_flower_lookup_efv(efx, net_dev);
2092 if (IS_ERR(from_efv)) {
2093 /* Not from our PF or representors, so probably a tunnel dev */
2094 return efx_tc_flower_replace_foreign(efx, net_dev, tc);
2095 }
2096
2097 if (efv != from_efv) {
2098 /* can't happen */
2099 NL_SET_ERR_MSG_FMT_MOD(extack, "for %s efv is %snull but from_efv is %snull (can't happen)",
2100 netdev_name(net_dev), efv ? "non-" : "",
2101 from_efv ? "non-" : "");
2102 return -EINVAL;
2103 }
2104
2105 /* Parse match */
2106 memset(&match, 0, sizeof(match));
2107 rc = efx_tc_flower_external_mport(efx, from_efv);
2108 if (rc < 0) {
2109 NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port");
2110 return rc;
2111 }
2112 match.value.ingress_port = rc;
2113 match.mask.ingress_port = ~0;
2114 rc = efx_tc_flower_parse_match(efx, fr, &match, extack);
2115 if (rc)
2116 return rc;
2117 if (efx_tc_match_is_encap(&match.mask)) {
2118 NL_SET_ERR_MSG_MOD(extack, "Ingress enc_key matches not supported");
2119 return -EOPNOTSUPP;
2120 }
2121
2122 if (efx_tc_rule_is_lhs_rule(fr, &match))
2123 return efx_tc_flower_replace_lhs(efx, tc, fr, &match, efv,
2124 net_dev);
2125
2126 /* chain_index 0 is always recirc_id 0 (and does not appear in recirc_ht).
2127 * Conveniently, match.rid == NULL and match.value.recirc_id == 0 owing
2128 * to the initial memset(), so we don't need to do anything in that case.
2129 */
2130 if (tc->common.chain_index) {
2131 struct efx_tc_recirc_id *rid;
2132
2133 /* Note regarding passed net_dev:
2134 * VFreps and PF can share chain namespace, as they have
2135 * distinct ingress_mports. So we don't need to burn an
2136 * extra recirc_id if both use the same chain_index.
2137 * (Strictly speaking, we could give each VFrep its own
2138 * recirc_id namespace that doesn't take IDs away from the
2139 * PF, but that would require a bunch of additional IDAs -
2140 * one for each representor - and that's not likely to be
2141 * the main cause of recirc_id exhaustion anyway.)
2142 */
2143 rid = efx_tc_get_recirc_id(efx, tc->common.chain_index,
2144 efx->net_dev);
2145 if (IS_ERR(rid)) {
2146 NL_SET_ERR_MSG_FMT_MOD(extack,
2147 "Failed to allocate a hardware recirculation ID for chain_index %u",
2148 tc->common.chain_index);
2149 return PTR_ERR(rid);
2150 }
2151 match.rid = rid;
2152 match.value.recirc_id = rid->fw_id;
2153 }
2154 match.mask.recirc_id = 0xff;
2155
2156 /* AR table can't match on DO_CT (+trk). But a commonly used pattern is
2157 * +trk+est, which is strictly implied by +est, so rewrite it to that.
2158 */
2159 if (match.mask.ct_state_trk && match.value.ct_state_trk &&
2160 match.mask.ct_state_est && match.value.ct_state_est)
2161 match.mask.ct_state_trk = 0;
2162 /* Thanks to CT_TCP_FLAGS_INHIBIT, packets with interesting flags could
2163 * match +trk-est (CT_HIT=0) despite being on an established connection.
2164 * So make -est imply -tcp_syn_fin_rst match to ensure these packets
2165 * still hit the software path.
2166 */
2167 if (match.mask.ct_state_est && !match.value.ct_state_est) {
2168 if (match.value.tcp_syn_fin_rst) {
2169 /* Can't offload this combination */
2170 rc = -EOPNOTSUPP;
2171 goto release;
2172 }
2173 match.mask.tcp_syn_fin_rst = true;
2174 }
2175
2176 rc = efx_mae_match_check_caps(efx, &match.mask, extack);
2177 if (rc)
2178 goto release;
2179
2180 rule = kzalloc(sizeof(*rule), GFP_USER);
2181 if (!rule) {
2182 rc = -ENOMEM;
2183 goto release;
2184 }
2185 INIT_LIST_HEAD(&rule->acts.list);
2186 rule->cookie = tc->cookie;
2187 old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht,
2188 &rule->linkage,
2189 efx_tc_match_action_ht_params);
2190 if (IS_ERR(old)) {
2191 rc = PTR_ERR(old);
2192 goto release;
2193 } else if (old) {
2194 netif_dbg(efx, drv, efx->net_dev,
2195 "Already offloaded rule (cookie %lx)\n", tc->cookie);
2196 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
2197 rc = -EEXIST;
2198 goto release;
2199 }
2200
2201 /* Parse actions */
2202 act = kzalloc(sizeof(*act), GFP_USER);
2203 if (!act) {
2204 rc = -ENOMEM;
2205 goto release;
2206 }
2207
2208 /**
2209 * DOC: TC action translation
2210 *
2211 * Actions in TC are sequential and cumulative, with delivery actions
2212 * potentially anywhere in the order. The EF100 MAE, however, takes
2213 * an 'action set list' consisting of 'action sets', each of which is
2214 * applied to the _original_ packet, and consists of a set of optional
2215 * actions in a fixed order with delivery at the end.
2216 * To translate between these two models, we maintain a 'cursor', @act,
2217 * which describes the cumulative effect of all the packet-mutating
2218 * actions encountered so far; on handling a delivery (mirred or drop)
2219 * action, once the action-set has been inserted into hardware, we
2220 * append @act to the action-set list (@rule->acts); if this is a pipe
2221 * action (mirred mirror) we then allocate a new @act with a copy of
2222 * the cursor state _before_ the delivery action, otherwise we set @act
2223 * to %NULL.
2224 * This ensures that every allocated action-set is either attached to
2225 * @rule->acts or pointed to by @act (and never both), and that only
2226 * those action-sets in @rule->acts exist in hardware. Consequently,
2227 * in the failure path, @act only needs to be freed in memory, whereas
2228 * for @rule->acts we remove each action-set from hardware before
2229 * freeing it (efx_tc_free_action_set_list()), even if the action-set
2230 * list itself is not in hardware.
2231 */
2232 flow_action_for_each(i, fa, &fr->action) {
2233 struct efx_tc_action_set save;
2234 u16 tci;
2235
2236 if (!act) {
2237 /* more actions after a non-pipe action */
2238 NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action");
2239 rc = -EINVAL;
2240 goto release;
2241 }
2242
2243 if ((fa->id == FLOW_ACTION_REDIRECT ||
2244 fa->id == FLOW_ACTION_MIRRED ||
2245 fa->id == FLOW_ACTION_DROP) && fa->hw_stats) {
2246 struct efx_tc_counter_index *ctr;
2247
2248 /* Currently the only actions that want stats are
2249 * mirred and gact (ok, shot, trap, goto-chain), which
2250 * means we want stats just before delivery. Also,
2251 * note that tunnel_key set shouldn't change the length
2252 * — it's only the subsequent mirred that does that,
2253 * and the stats are taken _before_ the mirred action
2254 * happens.
2255 */
2256 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) {
2257 /* All supported actions that count either steal
2258 * (gact shot, mirred redirect) or clone act
2259 * (mirred mirror), so we should never get two
2260 * count actions on one action_set.
2261 */
2262 NL_SET_ERR_MSG_MOD(extack, "Count-action conflict (can't happen)");
2263 rc = -EOPNOTSUPP;
2264 goto release;
2265 }
2266
2267 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
2268 NL_SET_ERR_MSG_FMT_MOD(extack, "hw_stats_type %u not supported (only 'delayed')",
2269 fa->hw_stats);
2270 rc = -EOPNOTSUPP;
2271 goto release;
2272 }
2273
2274 ctr = efx_tc_flower_get_counter_index(efx, tc->cookie,
2275 EFX_TC_COUNTER_TYPE_AR);
2276 if (IS_ERR(ctr)) {
2277 rc = PTR_ERR(ctr);
2278 NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
2279 goto release;
2280 }
2281 act->count = ctr;
2282 INIT_LIST_HEAD(&act->count_user);
2283 }
2284
2285 switch (fa->id) {
2286 case FLOW_ACTION_DROP:
2287 rc = efx_mae_alloc_action_set(efx, act);
2288 if (rc) {
2289 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (drop)");
2290 goto release;
2291 }
2292 list_add_tail(&act->list, &rule->acts.list);
2293 act = NULL; /* end of the line */
2294 break;
2295 case FLOW_ACTION_REDIRECT:
2296 case FLOW_ACTION_MIRRED:
2297 save = *act;
2298
2299 if (encap_info) {
2300 struct efx_tc_encap_action *encap;
2301
2302 if (!efx_tc_flower_action_order_ok(act,
2303 EFX_TC_AO_ENCAP)) {
2304 rc = -EOPNOTSUPP;
2305 NL_SET_ERR_MSG_MOD(extack, "Encap action violates action order");
2306 goto release;
2307 }
2308 encap = efx_tc_flower_create_encap_md(
2309 efx, encap_info, fa->dev, extack);
2310 if (IS_ERR_OR_NULL(encap)) {
2311 rc = PTR_ERR(encap);
2312 if (!rc)
2313 rc = -EIO; /* arbitrary */
2314 goto release;
2315 }
2316 act->encap_md = encap;
2317 list_add_tail(&act->encap_user, &encap->users);
2318 act->dest_mport = encap->dest_mport;
2319 act->deliver = 1;
2320 if (act->count && !WARN_ON(!act->count->cnt)) {
2321 /* This counter is used by an encap
2322 * action, which needs a reference back
2323 * so it can prod neighbouring whenever
2324 * traffic is seen.
2325 */
2326 spin_lock_bh(&act->count->cnt->lock);
2327 list_add_tail(&act->count_user,
2328 &act->count->cnt->users);
2329 spin_unlock_bh(&act->count->cnt->lock);
2330 }
2331 rc = efx_mae_alloc_action_set(efx, act);
2332 if (rc) {
2333 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (encap)");
2334 goto release;
2335 }
2336 list_add_tail(&act->list, &rule->acts.list);
2337 act->user = &rule->acts;
2338 act = NULL;
2339 if (fa->id == FLOW_ACTION_REDIRECT)
2340 break; /* end of the line */
2341 /* Mirror, so continue on with saved act */
2342 save.count = NULL;
2343 act = kzalloc(sizeof(*act), GFP_USER);
2344 if (!act) {
2345 rc = -ENOMEM;
2346 goto release;
2347 }
2348 *act = save;
2349 break;
2350 }
2351
2352 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) {
2353 /* can't happen */
2354 rc = -EOPNOTSUPP;
2355 NL_SET_ERR_MSG_MOD(extack, "Deliver action violates action order (can't happen)");
2356 goto release;
2357 }
2358
2359 to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
2360 if (IS_ERR(to_efv)) {
2361 NL_SET_ERR_MSG_MOD(extack, "Mirred egress device not on switch");
2362 rc = PTR_ERR(to_efv);
2363 goto release;
2364 }
2365 rc = efx_tc_flower_external_mport(efx, to_efv);
2366 if (rc < 0) {
2367 NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port");
2368 goto release;
2369 }
2370 act->dest_mport = rc;
2371 act->deliver = 1;
2372 rc = efx_mae_alloc_action_set(efx, act);
2373 if (rc) {
2374 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (mirred)");
2375 goto release;
2376 }
2377 list_add_tail(&act->list, &rule->acts.list);
2378 act = NULL;
2379 if (fa->id == FLOW_ACTION_REDIRECT)
2380 break; /* end of the line */
2381 /* Mirror, so continue on with saved act */
2382 save.count = NULL;
2383 act = kzalloc(sizeof(*act), GFP_USER);
2384 if (!act) {
2385 rc = -ENOMEM;
2386 goto release;
2387 }
2388 *act = save;
2389 break;
2390 case FLOW_ACTION_VLAN_POP:
2391 if (act->vlan_push) {
2392 act->vlan_push--;
2393 } else if (efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_POP)) {
2394 act->vlan_pop++;
2395 } else {
2396 NL_SET_ERR_MSG_MOD(extack,
2397 "More than two VLAN pops, or action order violated");
2398 rc = -EINVAL;
2399 goto release;
2400 }
2401 break;
2402 case FLOW_ACTION_VLAN_PUSH:
2403 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_PUSH)) {
2404 rc = -EINVAL;
2405 NL_SET_ERR_MSG_MOD(extack,
2406 "More than two VLAN pushes, or action order violated");
2407 goto release;
2408 }
2409 tci = fa->vlan.vid & VLAN_VID_MASK;
2410 tci |= fa->vlan.prio << VLAN_PRIO_SHIFT;
2411 act->vlan_tci[act->vlan_push] = cpu_to_be16(tci);
2412 act->vlan_proto[act->vlan_push] = fa->vlan.proto;
2413 act->vlan_push++;
2414 break;
2415 case FLOW_ACTION_ADD:
2416 rc = efx_tc_pedit_add(efx, act, fa, extack);
2417 if (rc < 0)
2418 goto release;
2419 break;
2420 case FLOW_ACTION_MANGLE:
2421 rc = efx_tc_mangle(efx, act, fa, &mung, extack, &match);
2422 if (rc < 0)
2423 goto release;
2424 break;
2425 case FLOW_ACTION_TUNNEL_ENCAP:
2426 if (encap_info) {
2427 /* Can't specify encap multiple times.
2428 * If you want to overwrite an existing
2429 * encap_info, use an intervening
2430 * FLOW_ACTION_TUNNEL_DECAP to clear it.
2431 */
2432 NL_SET_ERR_MSG_MOD(extack, "Tunnel key set when already set");
2433 rc = -EINVAL;
2434 goto release;
2435 }
2436 if (!fa->tunnel) {
2437 NL_SET_ERR_MSG_MOD(extack, "Tunnel key set is missing key");
2438 rc = -EOPNOTSUPP;
2439 goto release;
2440 }
2441 encap_info = fa->tunnel;
2442 break;
2443 case FLOW_ACTION_TUNNEL_DECAP:
2444 if (encap_info) {
2445 encap_info = NULL;
2446 break;
2447 }
2448 /* Since we don't support enc_key matches on ingress
2449 * (and if we did there'd be no tunnel-device to give
2450 * us a type), we can't offload a decap that's not
2451 * just undoing a previous encap action.
2452 */
2453 NL_SET_ERR_MSG_MOD(extack, "Cannot offload tunnel decap action without tunnel device");
2454 rc = -EOPNOTSUPP;
2455 goto release;
2456 case FLOW_ACTION_CT:
2457 if (fa->ct.action != TCA_CT_ACT_NAT) {
2458 rc = -EOPNOTSUPP;
2459 NL_SET_ERR_MSG_FMT_MOD(extack, "Can only offload CT 'nat' action in RHS rules, not %d", fa->ct.action);
2460 goto release;
2461 }
2462 act->do_nat = 1;
2463 break;
2464 default:
2465 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u",
2466 fa->id);
2467 rc = -EOPNOTSUPP;
2468 goto release;
2469 }
2470 }
2471
2472 rc = efx_tc_incomplete_mangle(&mung, extack);
2473 if (rc < 0)
2474 goto release;
2475 if (act) {
2476 /* Not shot/redirected, so deliver to default dest */
2477 if (from_efv == EFX_EFV_PF)
2478 /* Rule applies to traffic from the wire,
2479 * and default dest is thus the PF
2480 */
2481 efx_mae_mport_uplink(efx, &act->dest_mport);
2482 else
2483 /* Representor, so rule applies to traffic from
2484 * representee, and default dest is thus the rep.
2485 * All reps use the same mport for delivery
2486 */
2487 efx_mae_mport_mport(efx, efx->tc->reps_mport_id,
2488 &act->dest_mport);
2489 act->deliver = 1;
2490 rc = efx_mae_alloc_action_set(efx, act);
2491 if (rc) {
2492 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)");
2493 goto release;
2494 }
2495 list_add_tail(&act->list, &rule->acts.list);
2496 act = NULL; /* Prevent double-free in error path */
2497 }
2498
2499 netif_dbg(efx, drv, efx->net_dev,
2500 "Successfully parsed filter (cookie %lx)\n",
2501 tc->cookie);
2502
2503 rule->match = match;
2504
2505 rc = efx_mae_alloc_action_set_list(efx, &rule->acts);
2506 if (rc) {
2507 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw");
2508 goto release;
2509 }
2510 if (from_efv == EFX_EFV_PF)
2511 /* PF netdev, so rule applies to traffic from wire */
2512 rule->fallback = &efx->tc->facts.pf;
2513 else
2514 /* repdev, so rule applies to traffic from representee */
2515 rule->fallback = &efx->tc->facts.reps;
2516 if (!efx_tc_check_ready(efx, rule)) {
2517 netif_dbg(efx, drv, efx->net_dev, "action not ready for hw\n");
2518 acts_id = rule->fallback->fw_id;
2519 } else {
2520 netif_dbg(efx, drv, efx->net_dev, "ready for hw\n");
2521 acts_id = rule->acts.fw_id;
2522 }
2523 rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,
2524 acts_id, &rule->fw_id);
2525 if (rc) {
2526 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
2527 goto release_acts;
2528 }
2529 return 0;
2530
2531release_acts:
2532 efx_mae_free_action_set_list(efx, &rule->acts);
2533release:
2534 /* We failed to insert the rule, so free up any entries we created in
2535 * subsidiary tables.
2536 */
2537 if (match.rid)
2538 efx_tc_put_recirc_id(efx, match.rid);
2539 if (act)
2540 efx_tc_free_action_set(efx, act, false);
2541 if (rule) {
2542 if (!old)
2543 rhashtable_remove_fast(&efx->tc->match_action_ht,
2544 &rule->linkage,
2545 efx_tc_match_action_ht_params);
2546 efx_tc_free_action_set_list(efx, &rule->acts, false);
2547 }
2548 kfree(rule);
2549 return rc;
2550}
2551
2552static int efx_tc_flower_destroy(struct efx_nic *efx,
2553 struct net_device *net_dev,
2554 struct flow_cls_offload *tc)
2555{
2556 struct netlink_ext_ack *extack = tc->common.extack;
2557 struct efx_tc_lhs_rule *lhs_rule;
2558 struct efx_tc_flow_rule *rule;
2559
2560 lhs_rule = rhashtable_lookup_fast(&efx->tc->lhs_rule_ht, &tc->cookie,
2561 efx_tc_lhs_rule_ht_params);
2562 if (lhs_rule) {
2563 /* Remove it from HW */
2564 efx_mae_remove_lhs_rule(efx, lhs_rule);
2565 /* Delete it from SW */
2566 efx_tc_flower_release_lhs_actions(efx, &lhs_rule->lhs_act);
2567 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &lhs_rule->linkage,
2568 efx_tc_lhs_rule_ht_params);
2569 if (lhs_rule->match.encap)
2570 efx_tc_flower_release_encap_match(efx, lhs_rule->match.encap);
2571 netif_dbg(efx, drv, efx->net_dev, "Removed (lhs) filter %lx\n",
2572 lhs_rule->cookie);
2573 kfree(lhs_rule);
2574 return 0;
2575 }
2576
2577 rule = rhashtable_lookup_fast(&efx->tc->match_action_ht, &tc->cookie,
2578 efx_tc_match_action_ht_params);
2579 if (!rule) {
2580 /* Only log a message if we're the ingress device. Otherwise
2581 * it's a foreign filter and we might just not have been
2582 * interested (e.g. we might not have been the egress device
2583 * either).
2584 */
2585 if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev)))
2586 netif_warn(efx, drv, efx->net_dev,
2587 "Filter %lx not found to remove\n", tc->cookie);
2588 NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules");
2589 return -ENOENT;
2590 }
2591
2592 /* Remove it from HW */
2593 efx_tc_delete_rule(efx, rule);
2594 /* Delete it from SW */
2595 rhashtable_remove_fast(&efx->tc->match_action_ht, &rule->linkage,
2596 efx_tc_match_action_ht_params);
2597 netif_dbg(efx, drv, efx->net_dev, "Removed filter %lx\n", rule->cookie);
2598 kfree(rule);
2599 return 0;
2600}
2601
2602static int efx_tc_flower_stats(struct efx_nic *efx, struct net_device *net_dev,
2603 struct flow_cls_offload *tc)
2604{
2605 struct netlink_ext_ack *extack = tc->common.extack;
2606 struct efx_tc_counter_index *ctr;
2607 struct efx_tc_counter *cnt;
2608 u64 packets, bytes;
2609
2610 ctr = efx_tc_flower_find_counter_index(efx, tc->cookie);
2611 if (!ctr) {
2612 /* See comment in efx_tc_flower_destroy() */
2613 if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev)))
2614 if (net_ratelimit())
2615 netif_warn(efx, drv, efx->net_dev,
2616 "Filter %lx not found for stats\n",
2617 tc->cookie);
2618 NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules");
2619 return -ENOENT;
2620 }
2621 if (WARN_ON(!ctr->cnt)) /* can't happen */
2622 return -EIO;
2623 cnt = ctr->cnt;
2624
2625 spin_lock_bh(&cnt->lock);
2626 /* Report only new pkts/bytes since last time TC asked */
2627 packets = cnt->packets;
2628 bytes = cnt->bytes;
2629 flow_stats_update(&tc->stats, bytes - cnt->old_bytes,
2630 packets - cnt->old_packets, 0, cnt->touched,
2631 FLOW_ACTION_HW_STATS_DELAYED);
2632 cnt->old_packets = packets;
2633 cnt->old_bytes = bytes;
2634 spin_unlock_bh(&cnt->lock);
2635 return 0;
2636}
2637
2638int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev,
2639 struct flow_cls_offload *tc, struct efx_rep *efv)
2640{
2641 int rc;
2642
2643 if (!efx->tc)
2644 return -EOPNOTSUPP;
2645
2646 mutex_lock(&efx->tc->mutex);
2647 switch (tc->command) {
2648 case FLOW_CLS_REPLACE:
2649 rc = efx_tc_flower_replace(efx, net_dev, tc, efv);
2650 break;
2651 case FLOW_CLS_DESTROY:
2652 rc = efx_tc_flower_destroy(efx, net_dev, tc);
2653 break;
2654 case FLOW_CLS_STATS:
2655 rc = efx_tc_flower_stats(efx, net_dev, tc);
2656 break;
2657 default:
2658 rc = -EOPNOTSUPP;
2659 break;
2660 }
2661 mutex_unlock(&efx->tc->mutex);
2662 return rc;
2663}
2664
2665static int efx_tc_configure_default_rule(struct efx_nic *efx, u32 ing_port,
2666 u32 eg_port, struct efx_tc_flow_rule *rule)
2667{
2668 struct efx_tc_action_set_list *acts = &rule->acts;
2669 struct efx_tc_match *match = &rule->match;
2670 struct efx_tc_action_set *act;
2671 int rc;
2672
2673 match->value.ingress_port = ing_port;
2674 match->mask.ingress_port = ~0;
2675 act = kzalloc(sizeof(*act), GFP_KERNEL);
2676 if (!act)
2677 return -ENOMEM;
2678 act->deliver = 1;
2679 act->dest_mport = eg_port;
2680 rc = efx_mae_alloc_action_set(efx, act);
2681 if (rc)
2682 goto fail1;
2683 EFX_WARN_ON_PARANOID(!list_empty(&acts->list));
2684 list_add_tail(&act->list, &acts->list);
2685 rc = efx_mae_alloc_action_set_list(efx, acts);
2686 if (rc)
2687 goto fail2;
2688 rc = efx_mae_insert_rule(efx, match, EFX_TC_PRIO_DFLT,
2689 acts->fw_id, &rule->fw_id);
2690 if (rc)
2691 goto fail3;
2692 return 0;
2693fail3:
2694 efx_mae_free_action_set_list(efx, acts);
2695fail2:
2696 list_del(&act->list);
2697 efx_mae_free_action_set(efx, act->fw_id);
2698fail1:
2699 kfree(act);
2700 return rc;
2701}
2702
2703static int efx_tc_configure_default_rule_pf(struct efx_nic *efx)
2704{
2705 struct efx_tc_flow_rule *rule = &efx->tc->dflt.pf;
2706 u32 ing_port, eg_port;
2707
2708 efx_mae_mport_uplink(efx, &ing_port);
2709 efx_mae_mport_wire(efx, &eg_port);
2710 return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);
2711}
2712
2713static int efx_tc_configure_default_rule_wire(struct efx_nic *efx)
2714{
2715 struct efx_tc_flow_rule *rule = &efx->tc->dflt.wire;
2716 u32 ing_port, eg_port;
2717
2718 efx_mae_mport_wire(efx, &ing_port);
2719 efx_mae_mport_uplink(efx, &eg_port);
2720 return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);
2721}
2722
2723int efx_tc_configure_default_rule_rep(struct efx_rep *efv)
2724{
2725 struct efx_tc_flow_rule *rule = &efv->dflt;
2726 struct efx_nic *efx = efv->parent;
2727 u32 ing_port, eg_port;
2728
2729 efx_mae_mport_mport(efx, efv->mport, &ing_port);
2730 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port);
2731 return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);
2732}
2733
2734void efx_tc_deconfigure_default_rule(struct efx_nic *efx,
2735 struct efx_tc_flow_rule *rule)
2736{
2737 if (rule->fw_id != MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL)
2738 efx_tc_delete_rule(efx, rule);
2739 rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
2740}
2741
2742static int efx_tc_configure_fallback_acts(struct efx_nic *efx, u32 eg_port,
2743 struct efx_tc_action_set_list *acts)
2744{
2745 struct efx_tc_action_set *act;
2746 int rc;
2747
2748 act = kzalloc(sizeof(*act), GFP_KERNEL);
2749 if (!act)
2750 return -ENOMEM;
2751 act->deliver = 1;
2752 act->dest_mport = eg_port;
2753 rc = efx_mae_alloc_action_set(efx, act);
2754 if (rc)
2755 goto fail1;
2756 EFX_WARN_ON_PARANOID(!list_empty(&acts->list));
2757 list_add_tail(&act->list, &acts->list);
2758 rc = efx_mae_alloc_action_set_list(efx, acts);
2759 if (rc)
2760 goto fail2;
2761 return 0;
2762fail2:
2763 list_del(&act->list);
2764 efx_mae_free_action_set(efx, act->fw_id);
2765fail1:
2766 kfree(act);
2767 return rc;
2768}
2769
2770static int efx_tc_configure_fallback_acts_pf(struct efx_nic *efx)
2771{
2772 struct efx_tc_action_set_list *acts = &efx->tc->facts.pf;
2773 u32 eg_port;
2774
2775 efx_mae_mport_uplink(efx, &eg_port);
2776 return efx_tc_configure_fallback_acts(efx, eg_port, acts);
2777}
2778
2779static int efx_tc_configure_fallback_acts_reps(struct efx_nic *efx)
2780{
2781 struct efx_tc_action_set_list *acts = &efx->tc->facts.reps;
2782 u32 eg_port;
2783
2784 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port);
2785 return efx_tc_configure_fallback_acts(efx, eg_port, acts);
2786}
2787
2788static void efx_tc_deconfigure_fallback_acts(struct efx_nic *efx,
2789 struct efx_tc_action_set_list *acts)
2790{
2791 efx_tc_free_action_set_list(efx, acts, true);
2792}
2793
2794static int efx_tc_configure_rep_mport(struct efx_nic *efx)
2795{
2796 u32 rep_mport_label;
2797 int rc;
2798
2799 rc = efx_mae_allocate_mport(efx, &efx->tc->reps_mport_id, &rep_mport_label);
2800 if (rc)
2801 return rc;
2802 pci_dbg(efx->pci_dev, "created rep mport 0x%08x (0x%04x)\n",
2803 efx->tc->reps_mport_id, rep_mport_label);
2804 /* Use mport *selector* as vport ID */
2805 efx_mae_mport_mport(efx, efx->tc->reps_mport_id,
2806 &efx->tc->reps_mport_vport_id);
2807 return 0;
2808}
2809
2810static void efx_tc_deconfigure_rep_mport(struct efx_nic *efx)
2811{
2812 efx_mae_free_mport(efx, efx->tc->reps_mport_id);
2813 efx->tc->reps_mport_id = MAE_MPORT_SELECTOR_NULL;
2814}
2815
2816int efx_tc_insert_rep_filters(struct efx_nic *efx)
2817{
2818 struct efx_filter_spec promisc, allmulti;
2819 int rc;
2820
2821 if (efx->type->is_vf)
2822 return 0;
2823 if (!efx->tc)
2824 return 0;
2825 efx_filter_init_rx(&promisc, EFX_FILTER_PRI_REQUIRED, 0, 0);
2826 efx_filter_set_uc_def(&promisc);
2827 efx_filter_set_vport_id(&promisc, efx->tc->reps_mport_vport_id);
2828 rc = efx_filter_insert_filter(efx, &promisc, false);
2829 if (rc < 0)
2830 return rc;
2831 efx->tc->reps_filter_uc = rc;
2832 efx_filter_init_rx(&allmulti, EFX_FILTER_PRI_REQUIRED, 0, 0);
2833 efx_filter_set_mc_def(&allmulti);
2834 efx_filter_set_vport_id(&allmulti, efx->tc->reps_mport_vport_id);
2835 rc = efx_filter_insert_filter(efx, &allmulti, false);
2836 if (rc < 0)
2837 return rc;
2838 efx->tc->reps_filter_mc = rc;
2839 return 0;
2840}
2841
2842void efx_tc_remove_rep_filters(struct efx_nic *efx)
2843{
2844 if (efx->type->is_vf)
2845 return;
2846 if (!efx->tc)
2847 return;
2848 if (efx->tc->reps_filter_mc >= 0)
2849 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_mc);
2850 efx->tc->reps_filter_mc = -1;
2851 if (efx->tc->reps_filter_uc >= 0)
2852 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_uc);
2853 efx->tc->reps_filter_uc = -1;
2854}
2855
2856int efx_init_tc(struct efx_nic *efx)
2857{
2858 int rc;
2859
2860 rc = efx_mae_get_caps(efx, efx->tc->caps);
2861 if (rc)
2862 return rc;
2863 if (efx->tc->caps->match_field_count > MAE_NUM_FIELDS)
2864 /* Firmware supports some match fields the driver doesn't know
2865 * about. Not fatal, unless any of those fields are required
2866 * (MAE_FIELD_SUPPORTED_MATCH_ALWAYS) but if so we don't know.
2867 */
2868 netif_warn(efx, probe, efx->net_dev,
2869 "FW reports additional match fields %u\n",
2870 efx->tc->caps->match_field_count);
2871 if (efx->tc->caps->action_prios < EFX_TC_PRIO__NUM) {
2872 netif_err(efx, probe, efx->net_dev,
2873 "Too few action prios supported (have %u, need %u)\n",
2874 efx->tc->caps->action_prios, EFX_TC_PRIO__NUM);
2875 return -EIO;
2876 }
2877 rc = efx_tc_configure_default_rule_pf(efx);
2878 if (rc)
2879 return rc;
2880 rc = efx_tc_configure_default_rule_wire(efx);
2881 if (rc)
2882 return rc;
2883 rc = efx_tc_configure_rep_mport(efx);
2884 if (rc)
2885 return rc;
2886 rc = efx_tc_configure_fallback_acts_pf(efx);
2887 if (rc)
2888 return rc;
2889 rc = efx_tc_configure_fallback_acts_reps(efx);
2890 if (rc)
2891 return rc;
2892 rc = efx_mae_get_tables(efx);
2893 if (rc)
2894 return rc;
2895 rc = flow_indr_dev_register(efx_tc_indr_setup_cb, efx);
2896 if (rc)
2897 goto out_free;
2898 efx->tc->up = true;
2899 return 0;
2900out_free:
2901 efx_mae_free_tables(efx);
2902 return rc;
2903}
2904
2905void efx_fini_tc(struct efx_nic *efx)
2906{
2907 /* We can get called even if efx_init_struct_tc() failed */
2908 if (!efx->tc)
2909 return;
2910 if (efx->tc->up)
2911 flow_indr_dev_unregister(efx_tc_indr_setup_cb, efx, efx_tc_block_unbind);
2912 efx_tc_deconfigure_rep_mport(efx);
2913 efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.pf);
2914 efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.wire);
2915 efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.pf);
2916 efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.reps);
2917 efx->tc->up = false;
2918 efx_mae_free_tables(efx);
2919}
2920
2921/* At teardown time, all TC filter rules (and thus all resources they created)
2922 * should already have been removed. If we find any in our hashtables, make a
2923 * cursory attempt to clean up the software side.
2924 */
2925static void efx_tc_encap_match_free(void *ptr, void *__unused)
2926{
2927 struct efx_tc_encap_match *encap = ptr;
2928
2929 WARN_ON(refcount_read(&encap->ref));
2930 kfree(encap);
2931}
2932
2933static void efx_tc_recirc_free(void *ptr, void *arg)
2934{
2935 struct efx_tc_recirc_id *rid = ptr;
2936 struct efx_nic *efx = arg;
2937
2938 WARN_ON(refcount_read(&rid->ref));
2939 ida_free(&efx->tc->recirc_ida, rid->fw_id);
2940 kfree(rid);
2941}
2942
2943static void efx_tc_lhs_free(void *ptr, void *arg)
2944{
2945 struct efx_tc_lhs_rule *rule = ptr;
2946 struct efx_nic *efx = arg;
2947
2948 netif_err(efx, drv, efx->net_dev,
2949 "tc lhs_rule %lx still present at teardown, removing\n",
2950 rule->cookie);
2951
2952 if (rule->lhs_act.zone)
2953 efx_tc_ct_unregister_zone(efx, rule->lhs_act.zone);
2954 if (rule->lhs_act.count)
2955 efx_tc_flower_put_counter_index(efx, rule->lhs_act.count);
2956 efx_mae_remove_lhs_rule(efx, rule);
2957
2958 kfree(rule);
2959}
2960
2961static void efx_tc_mac_free(void *ptr, void *__unused)
2962{
2963 struct efx_tc_mac_pedit_action *ped = ptr;
2964
2965 WARN_ON(refcount_read(&ped->ref));
2966 kfree(ped);
2967}
2968
2969static void efx_tc_flow_free(void *ptr, void *arg)
2970{
2971 struct efx_tc_flow_rule *rule = ptr;
2972 struct efx_nic *efx = arg;
2973
2974 netif_err(efx, drv, efx->net_dev,
2975 "tc rule %lx still present at teardown, removing\n",
2976 rule->cookie);
2977
2978 /* Also releases entries in subsidiary tables */
2979 efx_tc_delete_rule(efx, rule);
2980
2981 kfree(rule);
2982}
2983
2984int efx_init_struct_tc(struct efx_nic *efx)
2985{
2986 int rc;
2987
2988 if (efx->type->is_vf)
2989 return 0;
2990
2991 efx->tc = kzalloc(sizeof(*efx->tc), GFP_KERNEL);
2992 if (!efx->tc)
2993 return -ENOMEM;
2994 efx->tc->caps = kzalloc(sizeof(struct mae_caps), GFP_KERNEL);
2995 if (!efx->tc->caps) {
2996 rc = -ENOMEM;
2997 goto fail_alloc_caps;
2998 }
2999 INIT_LIST_HEAD(&efx->tc->block_list);
3000
3001 mutex_init(&efx->tc->mutex);
3002 init_waitqueue_head(&efx->tc->flush_wq);
3003 rc = efx_tc_init_encap_actions(efx);
3004 if (rc < 0)
3005 goto fail_encap_actions;
3006 rc = efx_tc_init_counters(efx);
3007 if (rc < 0)
3008 goto fail_counters;
3009 rc = rhashtable_init(&efx->tc->mac_ht, &efx_tc_mac_ht_params);
3010 if (rc < 0)
3011 goto fail_mac_ht;
3012 rc = rhashtable_init(&efx->tc->encap_match_ht, &efx_tc_encap_match_ht_params);
3013 if (rc < 0)
3014 goto fail_encap_match_ht;
3015 rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params);
3016 if (rc < 0)
3017 goto fail_match_action_ht;
3018 rc = rhashtable_init(&efx->tc->lhs_rule_ht, &efx_tc_lhs_rule_ht_params);
3019 if (rc < 0)
3020 goto fail_lhs_rule_ht;
3021 rc = efx_tc_init_conntrack(efx);
3022 if (rc < 0)
3023 goto fail_conntrack;
3024 rc = rhashtable_init(&efx->tc->recirc_ht, &efx_tc_recirc_ht_params);
3025 if (rc < 0)
3026 goto fail_recirc_ht;
3027 ida_init(&efx->tc->recirc_ida);
3028 efx->tc->reps_filter_uc = -1;
3029 efx->tc->reps_filter_mc = -1;
3030 INIT_LIST_HEAD(&efx->tc->dflt.pf.acts.list);
3031 efx->tc->dflt.pf.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
3032 INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list);
3033 efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
3034 INIT_LIST_HEAD(&efx->tc->facts.pf.list);
3035 efx->tc->facts.pf.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL;
3036 INIT_LIST_HEAD(&efx->tc->facts.reps.list);
3037 efx->tc->facts.reps.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL;
3038 efx->extra_channel_type[EFX_EXTRA_CHANNEL_TC] = &efx_tc_channel_type;
3039 return 0;
3040fail_recirc_ht:
3041 efx_tc_destroy_conntrack(efx);
3042fail_conntrack:
3043 rhashtable_destroy(&efx->tc->lhs_rule_ht);
3044fail_lhs_rule_ht:
3045 rhashtable_destroy(&efx->tc->match_action_ht);
3046fail_match_action_ht:
3047 rhashtable_destroy(&efx->tc->encap_match_ht);
3048fail_encap_match_ht:
3049 rhashtable_destroy(&efx->tc->mac_ht);
3050fail_mac_ht:
3051 efx_tc_destroy_counters(efx);
3052fail_counters:
3053 efx_tc_destroy_encap_actions(efx);
3054fail_encap_actions:
3055 mutex_destroy(&efx->tc->mutex);
3056 kfree(efx->tc->caps);
3057fail_alloc_caps:
3058 kfree(efx->tc);
3059 efx->tc = NULL;
3060 return rc;
3061}
3062
3063void efx_fini_struct_tc(struct efx_nic *efx)
3064{
3065 if (!efx->tc)
3066 return;
3067
3068 mutex_lock(&efx->tc->mutex);
3069 EFX_WARN_ON_PARANOID(efx->tc->dflt.pf.fw_id !=
3070 MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);
3071 EFX_WARN_ON_PARANOID(efx->tc->dflt.wire.fw_id !=
3072 MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);
3073 EFX_WARN_ON_PARANOID(efx->tc->facts.pf.fw_id !=
3074 MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL);
3075 EFX_WARN_ON_PARANOID(efx->tc->facts.reps.fw_id !=
3076 MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL);
3077 rhashtable_free_and_destroy(&efx->tc->lhs_rule_ht, efx_tc_lhs_free, efx);
3078 rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free,
3079 efx);
3080 rhashtable_free_and_destroy(&efx->tc->encap_match_ht,
3081 efx_tc_encap_match_free, NULL);
3082 efx_tc_fini_conntrack(efx);
3083 rhashtable_free_and_destroy(&efx->tc->recirc_ht, efx_tc_recirc_free, efx);
3084 WARN_ON(!ida_is_empty(&efx->tc->recirc_ida));
3085 ida_destroy(&efx->tc->recirc_ida);
3086 rhashtable_free_and_destroy(&efx->tc->mac_ht, efx_tc_mac_free, NULL);
3087 efx_tc_fini_counters(efx);
3088 efx_tc_fini_encap_actions(efx);
3089 mutex_unlock(&efx->tc->mutex);
3090 mutex_destroy(&efx->tc->mutex);
3091 kfree(efx->tc->caps);
3092 kfree(efx->tc);
3093 efx->tc = NULL;
3094}
1// SPDX-License-Identifier: GPL-2.0-only
2/****************************************************************************
3 * Driver for Solarflare network controllers and boards
4 * Copyright 2019 Solarflare Communications Inc.
5 * Copyright 2020-2022 Xilinx Inc.
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 as published
9 * by the Free Software Foundation, incorporated herein by reference.
10 */
11
12#include <net/pkt_cls.h>
13#include <net/vxlan.h>
14#include <net/geneve.h>
15#include <net/tc_act/tc_ct.h>
16#include "tc.h"
17#include "tc_bindings.h"
18#include "tc_encap_actions.h"
19#include "tc_conntrack.h"
20#include "mae.h"
21#include "ef100_rep.h"
22#include "efx.h"
23
24enum efx_encap_type efx_tc_indr_netdev_type(struct net_device *net_dev)
25{
26 if (netif_is_vxlan(net_dev))
27 return EFX_ENCAP_TYPE_VXLAN;
28 if (netif_is_geneve(net_dev))
29 return EFX_ENCAP_TYPE_GENEVE;
30
31 return EFX_ENCAP_TYPE_NONE;
32}
33
34#define EFX_TC_HDR_TYPE_TTL_MASK ((u32)0xff)
35/* Hoplimit is stored in the most significant byte in the pedit ipv6 header action */
36#define EFX_TC_HDR_TYPE_HLIMIT_MASK ~((u32)0xff000000)
37#define EFX_EFV_PF NULL
38/* Look up the representor information (efv) for a device.
39 * May return NULL for the PF (us), or an error pointer for a device that
40 * isn't supported as a TC offload endpoint
41 */
42struct efx_rep *efx_tc_flower_lookup_efv(struct efx_nic *efx,
43 struct net_device *dev)
44{
45 struct efx_rep *efv;
46
47 if (!dev)
48 return ERR_PTR(-EOPNOTSUPP);
49 /* Is it us (the PF)? */
50 if (dev == efx->net_dev)
51 return EFX_EFV_PF;
52 /* Is it an efx vfrep at all? */
53 if (dev->netdev_ops != &efx_ef100_rep_netdev_ops)
54 return ERR_PTR(-EOPNOTSUPP);
55 /* Is it ours? We don't support TC rules that include another
56 * EF100's netdevices (not even on another port of the same NIC).
57 */
58 efv = netdev_priv(dev);
59 if (efv->parent != efx)
60 return ERR_PTR(-EOPNOTSUPP);
61 return efv;
62}
63
64/* Convert a driver-internal vport ID into an internal device (PF or VF) */
65static s64 efx_tc_flower_internal_mport(struct efx_nic *efx, struct efx_rep *efv)
66{
67 u32 mport;
68
69 if (IS_ERR(efv))
70 return PTR_ERR(efv);
71 if (!efv) /* device is PF (us) */
72 efx_mae_mport_uplink(efx, &mport);
73 else /* device is repr */
74 efx_mae_mport_mport(efx, efv->mport, &mport);
75 return mport;
76}
77
78/* Convert a driver-internal vport ID into an external device (wire or VF) */
79s64 efx_tc_flower_external_mport(struct efx_nic *efx, struct efx_rep *efv)
80{
81 u32 mport;
82
83 if (IS_ERR(efv))
84 return PTR_ERR(efv);
85 if (!efv) /* device is PF (us) */
86 efx_mae_mport_wire(efx, &mport);
87 else /* device is repr */
88 efx_mae_mport_mport(efx, efv->mport, &mport);
89 return mport;
90}
91
92static const struct rhashtable_params efx_tc_mac_ht_params = {
93 .key_len = offsetofend(struct efx_tc_mac_pedit_action, h_addr),
94 .key_offset = 0,
95 .head_offset = offsetof(struct efx_tc_mac_pedit_action, linkage),
96};
97
98static const struct rhashtable_params efx_tc_encap_match_ht_params = {
99 .key_len = offsetof(struct efx_tc_encap_match, linkage),
100 .key_offset = 0,
101 .head_offset = offsetof(struct efx_tc_encap_match, linkage),
102};
103
104static const struct rhashtable_params efx_tc_match_action_ht_params = {
105 .key_len = sizeof(unsigned long),
106 .key_offset = offsetof(struct efx_tc_flow_rule, cookie),
107 .head_offset = offsetof(struct efx_tc_flow_rule, linkage),
108};
109
110static const struct rhashtable_params efx_tc_lhs_rule_ht_params = {
111 .key_len = sizeof(unsigned long),
112 .key_offset = offsetof(struct efx_tc_lhs_rule, cookie),
113 .head_offset = offsetof(struct efx_tc_lhs_rule, linkage),
114};
115
116static const struct rhashtable_params efx_tc_recirc_ht_params = {
117 .key_len = offsetof(struct efx_tc_recirc_id, linkage),
118 .key_offset = 0,
119 .head_offset = offsetof(struct efx_tc_recirc_id, linkage),
120};
121
122static struct efx_tc_mac_pedit_action *efx_tc_flower_get_mac(struct efx_nic *efx,
123 unsigned char h_addr[ETH_ALEN],
124 struct netlink_ext_ack *extack)
125{
126 struct efx_tc_mac_pedit_action *ped, *old;
127 int rc;
128
129 ped = kzalloc(sizeof(*ped), GFP_USER);
130 if (!ped)
131 return ERR_PTR(-ENOMEM);
132 memcpy(ped->h_addr, h_addr, ETH_ALEN);
133 old = rhashtable_lookup_get_insert_fast(&efx->tc->mac_ht,
134 &ped->linkage,
135 efx_tc_mac_ht_params);
136 if (old) {
137 /* don't need our new entry */
138 kfree(ped);
139 if (IS_ERR(old)) /* oh dear, it's actually an error */
140 return ERR_CAST(old);
141 if (!refcount_inc_not_zero(&old->ref))
142 return ERR_PTR(-EAGAIN);
143 /* existing entry found, ref taken */
144 return old;
145 }
146
147 rc = efx_mae_allocate_pedit_mac(efx, ped);
148 if (rc < 0) {
149 NL_SET_ERR_MSG_MOD(extack, "Failed to store pedit MAC address in hw");
150 goto out_remove;
151 }
152
153 /* ref and return */
154 refcount_set(&ped->ref, 1);
155 return ped;
156out_remove:
157 rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage,
158 efx_tc_mac_ht_params);
159 kfree(ped);
160 return ERR_PTR(rc);
161}
162
163static void efx_tc_flower_put_mac(struct efx_nic *efx,
164 struct efx_tc_mac_pedit_action *ped)
165{
166 if (!refcount_dec_and_test(&ped->ref))
167 return; /* still in use */
168 rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage,
169 efx_tc_mac_ht_params);
170 efx_mae_free_pedit_mac(efx, ped);
171 kfree(ped);
172}
173
174static void efx_tc_free_action_set(struct efx_nic *efx,
175 struct efx_tc_action_set *act, bool in_hw)
176{
177 /* Failure paths calling this on the 'cursor' action set in_hw=false,
178 * because if the alloc had succeeded we'd've put it in acts.list and
179 * not still have it in act.
180 */
181 if (in_hw) {
182 efx_mae_free_action_set(efx, act->fw_id);
183 /* in_hw is true iff we are on an acts.list; make sure to
184 * remove ourselves from that list before we are freed.
185 */
186 list_del(&act->list);
187 }
188 if (act->count) {
189 spin_lock_bh(&act->count->cnt->lock);
190 if (!list_empty(&act->count_user))
191 list_del(&act->count_user);
192 spin_unlock_bh(&act->count->cnt->lock);
193 efx_tc_flower_put_counter_index(efx, act->count);
194 }
195 if (act->encap_md) {
196 list_del(&act->encap_user);
197 efx_tc_flower_release_encap_md(efx, act->encap_md);
198 }
199 if (act->src_mac)
200 efx_tc_flower_put_mac(efx, act->src_mac);
201 if (act->dst_mac)
202 efx_tc_flower_put_mac(efx, act->dst_mac);
203 kfree(act);
204}
205
206static void efx_tc_free_action_set_list(struct efx_nic *efx,
207 struct efx_tc_action_set_list *acts,
208 bool in_hw)
209{
210 struct efx_tc_action_set *act, *next;
211
212 /* Failure paths set in_hw=false, because usually the acts didn't get
213 * to efx_mae_alloc_action_set_list(); if they did, the failure tree
214 * has a separate efx_mae_free_action_set_list() before calling us.
215 */
216 if (in_hw)
217 efx_mae_free_action_set_list(efx, acts);
218 /* Any act that's on the list will be in_hw even if the list isn't */
219 list_for_each_entry_safe(act, next, &acts->list, list)
220 efx_tc_free_action_set(efx, act, true);
221 /* Don't kfree, as acts is embedded inside a struct efx_tc_flow_rule */
222}
223
224/* Boilerplate for the simple 'copy a field' cases */
225#define _MAP_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field) \
226if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_##_name)) { \
227 struct flow_match_##_type fm; \
228 \
229 flow_rule_match_##_tcget(rule, &fm); \
230 match->value._field = fm.key->_tcfield; \
231 match->mask._field = fm.mask->_tcfield; \
232}
233#define MAP_KEY_AND_MASK(_name, _type, _tcfield, _field) \
234 _MAP_KEY_AND_MASK(_name, _type, _type, _tcfield, _field)
235#define MAP_ENC_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field) \
236 _MAP_KEY_AND_MASK(ENC_##_name, _type, _tcget, _tcfield, _field)
237
238static int efx_tc_flower_parse_match(struct efx_nic *efx,
239 struct flow_rule *rule,
240 struct efx_tc_match *match,
241 struct netlink_ext_ack *extack)
242{
243 struct flow_dissector *dissector = rule->match.dissector;
244 unsigned char ipv = 0;
245
246 /* Owing to internal TC infelicities, the IPV6_ADDRS key might be set
247 * even on IPv4 filters; so rather than relying on dissector->used_keys
248 * we check the addr_type in the CONTROL key. If we don't find it (or
249 * it's masked, which should never happen), we treat both IPV4_ADDRS
250 * and IPV6_ADDRS as absent.
251 */
252 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
253 struct flow_match_control fm;
254
255 flow_rule_match_control(rule, &fm);
256 if (IS_ALL_ONES(fm.mask->addr_type))
257 switch (fm.key->addr_type) {
258 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
259 ipv = 4;
260 break;
261 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
262 ipv = 6;
263 break;
264 default:
265 break;
266 }
267
268 if (fm.mask->flags & FLOW_DIS_IS_FRAGMENT) {
269 match->value.ip_frag = fm.key->flags & FLOW_DIS_IS_FRAGMENT;
270 match->mask.ip_frag = true;
271 }
272 if (fm.mask->flags & FLOW_DIS_FIRST_FRAG) {
273 match->value.ip_firstfrag = fm.key->flags & FLOW_DIS_FIRST_FRAG;
274 match->mask.ip_firstfrag = true;
275 }
276 if (fm.mask->flags & ~(FLOW_DIS_IS_FRAGMENT | FLOW_DIS_FIRST_FRAG)) {
277 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported match on control.flags %#x",
278 fm.mask->flags);
279 return -EOPNOTSUPP;
280 }
281 }
282 if (dissector->used_keys &
283 ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
284 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
285 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
286 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
287 BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN) |
288 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
289 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
290 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
291 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) |
292 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
293 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
294 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) |
295 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS) |
296 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) |
297 BIT_ULL(FLOW_DISSECTOR_KEY_CT) |
298 BIT_ULL(FLOW_DISSECTOR_KEY_TCP) |
299 BIT_ULL(FLOW_DISSECTOR_KEY_IP))) {
300 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported flower keys %#llx",
301 dissector->used_keys);
302 return -EOPNOTSUPP;
303 }
304
305 MAP_KEY_AND_MASK(BASIC, basic, n_proto, eth_proto);
306 /* Make sure we're IP if any L3/L4 keys used. */
307 if (!IS_ALL_ONES(match->mask.eth_proto) ||
308 !(match->value.eth_proto == htons(ETH_P_IP) ||
309 match->value.eth_proto == htons(ETH_P_IPV6)))
310 if (dissector->used_keys &
311 (BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
312 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
313 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
314 BIT_ULL(FLOW_DISSECTOR_KEY_IP) |
315 BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) {
316 NL_SET_ERR_MSG_FMT_MOD(extack,
317 "L3/L4 flower keys %#llx require protocol ipv[46]",
318 dissector->used_keys);
319 return -EINVAL;
320 }
321
322 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
323 struct flow_match_vlan fm;
324
325 flow_rule_match_vlan(rule, &fm);
326 if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) {
327 match->value.vlan_proto[0] = fm.key->vlan_tpid;
328 match->mask.vlan_proto[0] = fm.mask->vlan_tpid;
329 match->value.vlan_tci[0] = cpu_to_be16(fm.key->vlan_priority << 13 |
330 fm.key->vlan_id);
331 match->mask.vlan_tci[0] = cpu_to_be16(fm.mask->vlan_priority << 13 |
332 fm.mask->vlan_id);
333 }
334 }
335
336 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)) {
337 struct flow_match_vlan fm;
338
339 flow_rule_match_cvlan(rule, &fm);
340 if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) {
341 match->value.vlan_proto[1] = fm.key->vlan_tpid;
342 match->mask.vlan_proto[1] = fm.mask->vlan_tpid;
343 match->value.vlan_tci[1] = cpu_to_be16(fm.key->vlan_priority << 13 |
344 fm.key->vlan_id);
345 match->mask.vlan_tci[1] = cpu_to_be16(fm.mask->vlan_priority << 13 |
346 fm.mask->vlan_id);
347 }
348 }
349
350 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
351 struct flow_match_eth_addrs fm;
352
353 flow_rule_match_eth_addrs(rule, &fm);
354 ether_addr_copy(match->value.eth_saddr, fm.key->src);
355 ether_addr_copy(match->value.eth_daddr, fm.key->dst);
356 ether_addr_copy(match->mask.eth_saddr, fm.mask->src);
357 ether_addr_copy(match->mask.eth_daddr, fm.mask->dst);
358 }
359
360 MAP_KEY_AND_MASK(BASIC, basic, ip_proto, ip_proto);
361 /* Make sure we're TCP/UDP if any L4 keys used. */
362 if ((match->value.ip_proto != IPPROTO_UDP &&
363 match->value.ip_proto != IPPROTO_TCP) || !IS_ALL_ONES(match->mask.ip_proto))
364 if (dissector->used_keys &
365 (BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
366 BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) {
367 NL_SET_ERR_MSG_FMT_MOD(extack,
368 "L4 flower keys %#llx require ipproto udp or tcp",
369 dissector->used_keys);
370 return -EINVAL;
371 }
372 MAP_KEY_AND_MASK(IP, ip, tos, ip_tos);
373 MAP_KEY_AND_MASK(IP, ip, ttl, ip_ttl);
374 if (ipv == 4) {
375 MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, src, src_ip);
376 MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, dst, dst_ip);
377 }
378#ifdef CONFIG_IPV6
379 else if (ipv == 6) {
380 MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, src, src_ip6);
381 MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, dst, dst_ip6);
382 }
383#endif
384 MAP_KEY_AND_MASK(PORTS, ports, src, l4_sport);
385 MAP_KEY_AND_MASK(PORTS, ports, dst, l4_dport);
386 MAP_KEY_AND_MASK(TCP, tcp, flags, tcp_flags);
387 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
388 struct flow_match_control fm;
389
390 flow_rule_match_enc_control(rule, &fm);
391 if (fm.mask->flags) {
392 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported match on enc_control.flags %#x",
393 fm.mask->flags);
394 return -EOPNOTSUPP;
395 }
396 if (!IS_ALL_ONES(fm.mask->addr_type)) {
397 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported enc addr_type mask %u (key %u)",
398 fm.mask->addr_type,
399 fm.key->addr_type);
400 return -EOPNOTSUPP;
401 }
402 switch (fm.key->addr_type) {
403 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
404 MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs,
405 src, enc_src_ip);
406 MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs,
407 dst, enc_dst_ip);
408 break;
409#ifdef CONFIG_IPV6
410 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
411 MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs,
412 src, enc_src_ip6);
413 MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs,
414 dst, enc_dst_ip6);
415 break;
416#endif
417 default:
418 NL_SET_ERR_MSG_FMT_MOD(extack,
419 "Unsupported enc addr_type %u (supported are IPv4, IPv6)",
420 fm.key->addr_type);
421 return -EOPNOTSUPP;
422 }
423 MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, tos, enc_ip_tos);
424 MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, ttl, enc_ip_ttl);
425 MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, src, enc_sport);
426 MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, dst, enc_dport);
427 MAP_ENC_KEY_AND_MASK(KEYID, enc_keyid, enc_keyid, keyid, enc_keyid);
428 } else if (dissector->used_keys &
429 (BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) |
430 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
431 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
432 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) |
433 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS))) {
434 NL_SET_ERR_MSG_FMT_MOD(extack,
435 "Flower enc keys require enc_control (keys: %#llx)",
436 dissector->used_keys);
437 return -EOPNOTSUPP;
438 }
439 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CT)) {
440 struct flow_match_ct fm;
441
442 flow_rule_match_ct(rule, &fm);
443 match->value.ct_state_trk = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED);
444 match->mask.ct_state_trk = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED);
445 match->value.ct_state_est = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED);
446 match->mask.ct_state_est = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED);
447 if (fm.mask->ct_state & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
448 TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)) {
449 NL_SET_ERR_MSG_FMT_MOD(extack,
450 "Unsupported ct_state match %#x",
451 fm.mask->ct_state);
452 return -EOPNOTSUPP;
453 }
454 match->value.ct_mark = fm.key->ct_mark;
455 match->mask.ct_mark = fm.mask->ct_mark;
456 match->value.ct_zone = fm.key->ct_zone;
457 match->mask.ct_zone = fm.mask->ct_zone;
458
459 if (memchr_inv(fm.mask->ct_labels, 0, sizeof(fm.mask->ct_labels))) {
460 NL_SET_ERR_MSG_MOD(extack, "Matching on ct_label not supported");
461 return -EOPNOTSUPP;
462 }
463 }
464
465 return 0;
466}
467
468static void efx_tc_flower_release_encap_match(struct efx_nic *efx,
469 struct efx_tc_encap_match *encap)
470{
471 int rc;
472
473 if (!refcount_dec_and_test(&encap->ref))
474 return; /* still in use */
475
476 if (encap->type == EFX_TC_EM_DIRECT) {
477 rc = efx_mae_unregister_encap_match(efx, encap);
478 if (rc)
479 /* Display message but carry on and remove entry from our
480 * SW tables, because there's not much we can do about it.
481 */
482 netif_err(efx, drv, efx->net_dev,
483 "Failed to release encap match %#x, rc %d\n",
484 encap->fw_id, rc);
485 }
486 rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage,
487 efx_tc_encap_match_ht_params);
488 if (encap->pseudo)
489 efx_tc_flower_release_encap_match(efx, encap->pseudo);
490 kfree(encap);
491}
492
493static int efx_tc_flower_record_encap_match(struct efx_nic *efx,
494 struct efx_tc_match *match,
495 enum efx_encap_type type,
496 enum efx_tc_em_pseudo_type em_type,
497 u8 child_ip_tos_mask,
498 __be16 child_udp_sport_mask,
499 struct netlink_ext_ack *extack)
500{
501 struct efx_tc_encap_match *encap, *old, *pseudo = NULL;
502 bool ipv6 = false;
503 int rc;
504
505 /* We require that the socket-defining fields (IP addrs and UDP dest
506 * port) are present and exact-match. Other fields may only be used
507 * if the field-set (and any masks) are the same for all encap
508 * matches on the same <sip,dip,dport> tuple; this is enforced by
509 * pseudo encap matches.
510 */
511 if (match->mask.enc_dst_ip | match->mask.enc_src_ip) {
512 if (!IS_ALL_ONES(match->mask.enc_dst_ip)) {
513 NL_SET_ERR_MSG_MOD(extack,
514 "Egress encap match is not exact on dst IP address");
515 return -EOPNOTSUPP;
516 }
517 if (!IS_ALL_ONES(match->mask.enc_src_ip)) {
518 NL_SET_ERR_MSG_MOD(extack,
519 "Egress encap match is not exact on src IP address");
520 return -EOPNOTSUPP;
521 }
522#ifdef CONFIG_IPV6
523 if (!ipv6_addr_any(&match->mask.enc_dst_ip6) ||
524 !ipv6_addr_any(&match->mask.enc_src_ip6)) {
525 NL_SET_ERR_MSG_MOD(extack,
526 "Egress encap match on both IPv4 and IPv6, don't understand");
527 return -EOPNOTSUPP;
528 }
529 } else {
530 ipv6 = true;
531 if (!efx_ipv6_addr_all_ones(&match->mask.enc_dst_ip6)) {
532 NL_SET_ERR_MSG_MOD(extack,
533 "Egress encap match is not exact on dst IP address");
534 return -EOPNOTSUPP;
535 }
536 if (!efx_ipv6_addr_all_ones(&match->mask.enc_src_ip6)) {
537 NL_SET_ERR_MSG_MOD(extack,
538 "Egress encap match is not exact on src IP address");
539 return -EOPNOTSUPP;
540 }
541#endif
542 }
543 if (!IS_ALL_ONES(match->mask.enc_dport)) {
544 NL_SET_ERR_MSG_MOD(extack, "Egress encap match is not exact on dst UDP port");
545 return -EOPNOTSUPP;
546 }
547 if (match->mask.enc_sport || match->mask.enc_ip_tos) {
548 struct efx_tc_match pmatch = *match;
549
550 if (em_type == EFX_TC_EM_PSEUDO_MASK) { /* can't happen */
551 NL_SET_ERR_MSG_MOD(extack, "Bad recursion in egress encap match handler");
552 return -EOPNOTSUPP;
553 }
554 pmatch.value.enc_ip_tos = 0;
555 pmatch.mask.enc_ip_tos = 0;
556 pmatch.value.enc_sport = 0;
557 pmatch.mask.enc_sport = 0;
558 rc = efx_tc_flower_record_encap_match(efx, &pmatch, type,
559 EFX_TC_EM_PSEUDO_MASK,
560 match->mask.enc_ip_tos,
561 match->mask.enc_sport,
562 extack);
563 if (rc)
564 return rc;
565 pseudo = pmatch.encap;
566 }
567 if (match->mask.enc_ip_ttl) {
568 NL_SET_ERR_MSG_MOD(extack, "Egress encap match on IP TTL not supported");
569 rc = -EOPNOTSUPP;
570 goto fail_pseudo;
571 }
572
573 rc = efx_mae_check_encap_match_caps(efx, ipv6, match->mask.enc_ip_tos,
574 match->mask.enc_sport, extack);
575 if (rc)
576 goto fail_pseudo;
577
578 encap = kzalloc(sizeof(*encap), GFP_USER);
579 if (!encap) {
580 rc = -ENOMEM;
581 goto fail_pseudo;
582 }
583 encap->src_ip = match->value.enc_src_ip;
584 encap->dst_ip = match->value.enc_dst_ip;
585#ifdef CONFIG_IPV6
586 encap->src_ip6 = match->value.enc_src_ip6;
587 encap->dst_ip6 = match->value.enc_dst_ip6;
588#endif
589 encap->udp_dport = match->value.enc_dport;
590 encap->tun_type = type;
591 encap->ip_tos = match->value.enc_ip_tos;
592 encap->ip_tos_mask = match->mask.enc_ip_tos;
593 encap->child_ip_tos_mask = child_ip_tos_mask;
594 encap->udp_sport = match->value.enc_sport;
595 encap->udp_sport_mask = match->mask.enc_sport;
596 encap->child_udp_sport_mask = child_udp_sport_mask;
597 encap->type = em_type;
598 encap->pseudo = pseudo;
599 old = rhashtable_lookup_get_insert_fast(&efx->tc->encap_match_ht,
600 &encap->linkage,
601 efx_tc_encap_match_ht_params);
602 if (old) {
603 /* don't need our new entry */
604 kfree(encap);
605 if (pseudo) /* don't need our new pseudo either */
606 efx_tc_flower_release_encap_match(efx, pseudo);
607 if (IS_ERR(old)) /* oh dear, it's actually an error */
608 return PTR_ERR(old);
609 /* check old and new em_types are compatible */
610 switch (old->type) {
611 case EFX_TC_EM_DIRECT:
612 /* old EM is in hardware, so mustn't overlap with a
613 * pseudo, but may be shared with another direct EM
614 */
615 if (em_type == EFX_TC_EM_DIRECT)
616 break;
617 NL_SET_ERR_MSG_MOD(extack, "Pseudo encap match conflicts with existing direct entry");
618 return -EEXIST;
619 case EFX_TC_EM_PSEUDO_MASK:
620 /* old EM is protecting a ToS- or src port-qualified
621 * filter, so may only be shared with another pseudo
622 * for the same ToS and src port masks.
623 */
624 if (em_type != EFX_TC_EM_PSEUDO_MASK) {
625 NL_SET_ERR_MSG_FMT_MOD(extack,
626 "%s encap match conflicts with existing pseudo(MASK) entry",
627 em_type ? "Pseudo" : "Direct");
628 return -EEXIST;
629 }
630 if (child_ip_tos_mask != old->child_ip_tos_mask) {
631 NL_SET_ERR_MSG_FMT_MOD(extack,
632 "Pseudo encap match for TOS mask %#04x conflicts with existing mask %#04x",
633 child_ip_tos_mask,
634 old->child_ip_tos_mask);
635 return -EEXIST;
636 }
637 if (child_udp_sport_mask != old->child_udp_sport_mask) {
638 NL_SET_ERR_MSG_FMT_MOD(extack,
639 "Pseudo encap match for UDP src port mask %#x conflicts with existing mask %#x",
640 child_udp_sport_mask,
641 old->child_udp_sport_mask);
642 return -EEXIST;
643 }
644 break;
645 case EFX_TC_EM_PSEUDO_OR:
646 /* old EM corresponds to an OR that has to be unique
647 * (it must not overlap with any other OR, whether
648 * direct-EM or pseudo).
649 */
650 NL_SET_ERR_MSG_FMT_MOD(extack,
651 "%s encap match conflicts with existing pseudo(OR) entry",
652 em_type ? "Pseudo" : "Direct");
653 return -EEXIST;
654 default: /* Unrecognised pseudo-type. Just say no */
655 NL_SET_ERR_MSG_FMT_MOD(extack,
656 "%s encap match conflicts with existing pseudo(%d) entry",
657 em_type ? "Pseudo" : "Direct",
658 old->type);
659 return -EEXIST;
660 }
661 /* check old and new tun_types are compatible */
662 if (old->tun_type != type) {
663 NL_SET_ERR_MSG_FMT_MOD(extack,
664 "Egress encap match with conflicting tun_type %u != %u",
665 old->tun_type, type);
666 return -EEXIST;
667 }
668 if (!refcount_inc_not_zero(&old->ref))
669 return -EAGAIN;
670 /* existing entry found */
671 encap = old;
672 } else {
673 if (em_type == EFX_TC_EM_DIRECT) {
674 rc = efx_mae_register_encap_match(efx, encap);
675 if (rc) {
676 NL_SET_ERR_MSG_MOD(extack, "Failed to record egress encap match in HW");
677 goto fail;
678 }
679 }
680 refcount_set(&encap->ref, 1);
681 }
682 match->encap = encap;
683 return 0;
684fail:
685 rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage,
686 efx_tc_encap_match_ht_params);
687 kfree(encap);
688fail_pseudo:
689 if (pseudo)
690 efx_tc_flower_release_encap_match(efx, pseudo);
691 return rc;
692}
693
694static struct efx_tc_recirc_id *efx_tc_get_recirc_id(struct efx_nic *efx,
695 u32 chain_index,
696 struct net_device *net_dev)
697{
698 struct efx_tc_recirc_id *rid, *old;
699 int rc;
700
701 rid = kzalloc(sizeof(*rid), GFP_USER);
702 if (!rid)
703 return ERR_PTR(-ENOMEM);
704 rid->chain_index = chain_index;
705 /* We don't take a reference here, because it's implied - if there's
706 * a rule on the net_dev that's been offloaded to us, then the net_dev
707 * can't go away until the rule has been deoffloaded.
708 */
709 rid->net_dev = net_dev;
710 old = rhashtable_lookup_get_insert_fast(&efx->tc->recirc_ht,
711 &rid->linkage,
712 efx_tc_recirc_ht_params);
713 if (old) {
714 /* don't need our new entry */
715 kfree(rid);
716 if (IS_ERR(old)) /* oh dear, it's actually an error */
717 return ERR_CAST(old);
718 if (!refcount_inc_not_zero(&old->ref))
719 return ERR_PTR(-EAGAIN);
720 /* existing entry found */
721 rid = old;
722 } else {
723 rc = ida_alloc_range(&efx->tc->recirc_ida, 1, U8_MAX, GFP_USER);
724 if (rc < 0) {
725 rhashtable_remove_fast(&efx->tc->recirc_ht,
726 &rid->linkage,
727 efx_tc_recirc_ht_params);
728 kfree(rid);
729 return ERR_PTR(rc);
730 }
731 rid->fw_id = rc;
732 refcount_set(&rid->ref, 1);
733 }
734 return rid;
735}
736
737static void efx_tc_put_recirc_id(struct efx_nic *efx, struct efx_tc_recirc_id *rid)
738{
739 if (!refcount_dec_and_test(&rid->ref))
740 return; /* still in use */
741 rhashtable_remove_fast(&efx->tc->recirc_ht, &rid->linkage,
742 efx_tc_recirc_ht_params);
743 ida_free(&efx->tc->recirc_ida, rid->fw_id);
744 kfree(rid);
745}
746
747static void efx_tc_delete_rule(struct efx_nic *efx, struct efx_tc_flow_rule *rule)
748{
749 efx_mae_delete_rule(efx, rule->fw_id);
750
751 /* Release entries in subsidiary tables */
752 efx_tc_free_action_set_list(efx, &rule->acts, true);
753 if (rule->match.rid)
754 efx_tc_put_recirc_id(efx, rule->match.rid);
755 if (rule->match.encap)
756 efx_tc_flower_release_encap_match(efx, rule->match.encap);
757 rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
758}
759
760static const char *efx_tc_encap_type_name(enum efx_encap_type typ)
761{
762 switch (typ) {
763 case EFX_ENCAP_TYPE_NONE:
764 return "none";
765 case EFX_ENCAP_TYPE_VXLAN:
766 return "vxlan";
767 case EFX_ENCAP_TYPE_GENEVE:
768 return "geneve";
769 default:
770 pr_warn_once("Unknown efx_encap_type %d encountered\n", typ);
771 return "unknown";
772 }
773}
774
775/* For details of action order constraints refer to SF-123102-TC-1§12.6.1 */
776enum efx_tc_action_order {
777 EFX_TC_AO_DECAP,
778 EFX_TC_AO_DEC_TTL,
779 EFX_TC_AO_PEDIT_MAC_ADDRS,
780 EFX_TC_AO_VLAN_POP,
781 EFX_TC_AO_VLAN_PUSH,
782 EFX_TC_AO_COUNT,
783 EFX_TC_AO_ENCAP,
784 EFX_TC_AO_DELIVER
785};
786/* Determine whether we can add @new action without violating order */
787static bool efx_tc_flower_action_order_ok(const struct efx_tc_action_set *act,
788 enum efx_tc_action_order new)
789{
790 switch (new) {
791 case EFX_TC_AO_DECAP:
792 if (act->decap)
793 return false;
794 /* PEDIT_MAC_ADDRS must not happen before DECAP, though it
795 * can wait until much later
796 */
797 if (act->dst_mac || act->src_mac)
798 return false;
799
800 /* Decrementing ttl must not happen before DECAP */
801 if (act->do_ttl_dec)
802 return false;
803 fallthrough;
804 case EFX_TC_AO_VLAN_POP:
805 if (act->vlan_pop >= 2)
806 return false;
807 /* If we've already pushed a VLAN, we can't then pop it;
808 * the hardware would instead try to pop an existing VLAN
809 * before pushing the new one.
810 */
811 if (act->vlan_push)
812 return false;
813 fallthrough;
814 case EFX_TC_AO_VLAN_PUSH:
815 if (act->vlan_push >= 2)
816 return false;
817 fallthrough;
818 case EFX_TC_AO_COUNT:
819 if (act->count)
820 return false;
821 fallthrough;
822 case EFX_TC_AO_PEDIT_MAC_ADDRS:
823 case EFX_TC_AO_ENCAP:
824 if (act->encap_md)
825 return false;
826 fallthrough;
827 case EFX_TC_AO_DELIVER:
828 return !act->deliver;
829 case EFX_TC_AO_DEC_TTL:
830 if (act->encap_md)
831 return false;
832 return !act->do_ttl_dec;
833 default:
834 /* Bad caller. Whatever they wanted to do, say they can't. */
835 WARN_ON_ONCE(1);
836 return false;
837 }
838}
839
840/**
841 * DOC: TC conntrack sequences
842 *
843 * The MAE hardware can handle at most two rounds of action rule matching,
844 * consequently we support conntrack through the notion of a "left-hand side
845 * rule". This is a rule which typically contains only the actions "ct" and
846 * "goto chain N", and corresponds to one or more "right-hand side rules" in
847 * chain N, which typically match on +trk+est, and may perform ct(nat) actions.
848 * RHS rules go in the Action Rule table as normal but with a nonzero recirc_id
849 * (the hardware equivalent of chain_index), while LHS rules may go in either
850 * the Action Rule or the Outer Rule table, the latter being preferred for
851 * performance reasons, and set both DO_CT and a recirc_id in their response.
852 *
853 * Besides the RHS rules, there are often also similar rules matching on
854 * +trk+new which perform the ct(commit) action. These are not offloaded.
855 */
856
857static bool efx_tc_rule_is_lhs_rule(struct flow_rule *fr,
858 struct efx_tc_match *match)
859{
860 const struct flow_action_entry *fa;
861 int i;
862
863 flow_action_for_each(i, fa, &fr->action) {
864 switch (fa->id) {
865 case FLOW_ACTION_GOTO:
866 return true;
867 case FLOW_ACTION_CT:
868 /* If rule is -trk, or doesn't mention trk at all, then
869 * a CT action implies a conntrack lookup (hence it's an
870 * LHS rule). If rule is +trk, then a CT action could
871 * just be ct(nat) or even ct(commit) (though the latter
872 * can't be offloaded).
873 */
874 if (!match->mask.ct_state_trk || !match->value.ct_state_trk)
875 return true;
876 break;
877 default:
878 break;
879 }
880 }
881 return false;
882}
883
884/* A foreign LHS rule has matches on enc_ keys at the TC layer (including an
885 * implied match on enc_ip_proto UDP). Translate these into non-enc_ keys,
886 * so that we can use the same MAE machinery as local LHS rules (and so that
887 * the lhs_rules entries have uniform semantics). It may seem odd to do it
888 * this way round, given that the corresponding fields in the MAE MCDIs are
889 * all ENC_, but (a) we don't have enc_L2 or enc_ip_proto in struct
890 * efx_tc_match_fields and (b) semantically an LHS rule doesn't have inner
891 * fields so it's just matching on *the* header rather than the outer header.
892 * Make sure that the non-enc_ keys were not already being matched on, as that
893 * would imply a rule that needed a triple lookup. (Hardware can do that,
894 * with OR-AR-CT-AR, but it halves packet rate so we avoid it where possible;
895 * see efx_tc_flower_flhs_needs_ar().)
896 */
897static int efx_tc_flower_translate_flhs_match(struct efx_tc_match *match)
898{
899 int rc = 0;
900
901#define COPY_MASK_AND_VALUE(_key, _ekey) ({ \
902 if (match->mask._key) { \
903 rc = -EOPNOTSUPP; \
904 } else { \
905 match->mask._key = match->mask._ekey; \
906 match->mask._ekey = 0; \
907 match->value._key = match->value._ekey; \
908 match->value._ekey = 0; \
909 } \
910 rc; \
911})
912#define COPY_FROM_ENC(_key) COPY_MASK_AND_VALUE(_key, enc_##_key)
913 if (match->mask.ip_proto)
914 return -EOPNOTSUPP;
915 match->mask.ip_proto = ~0;
916 match->value.ip_proto = IPPROTO_UDP;
917 if (COPY_FROM_ENC(src_ip) || COPY_FROM_ENC(dst_ip))
918 return rc;
919#ifdef CONFIG_IPV6
920 if (!ipv6_addr_any(&match->mask.src_ip6))
921 return -EOPNOTSUPP;
922 match->mask.src_ip6 = match->mask.enc_src_ip6;
923 memset(&match->mask.enc_src_ip6, 0, sizeof(struct in6_addr));
924 if (!ipv6_addr_any(&match->mask.dst_ip6))
925 return -EOPNOTSUPP;
926 match->mask.dst_ip6 = match->mask.enc_dst_ip6;
927 memset(&match->mask.enc_dst_ip6, 0, sizeof(struct in6_addr));
928#endif
929 if (COPY_FROM_ENC(ip_tos) || COPY_FROM_ENC(ip_ttl))
930 return rc;
931 /* should really copy enc_ip_frag but we don't have that in
932 * parse_match yet
933 */
934 if (COPY_MASK_AND_VALUE(l4_sport, enc_sport) ||
935 COPY_MASK_AND_VALUE(l4_dport, enc_dport))
936 return rc;
937 return 0;
938#undef COPY_FROM_ENC
939#undef COPY_MASK_AND_VALUE
940}
941
942/* If a foreign LHS rule wants to match on keys that are only available after
943 * encap header identification and parsing, then it can't be done in the Outer
944 * Rule lookup, because that lookup determines the encap type used to parse
945 * beyond the outer headers. Thus, such rules must use the OR-AR-CT-AR lookup
946 * sequence, with an EM (struct efx_tc_encap_match) in the OR step.
947 * Return true iff the passed match requires this.
948 */
949static bool efx_tc_flower_flhs_needs_ar(struct efx_tc_match *match)
950{
951 /* matches on inner-header keys can't be done in OR */
952 return match->mask.eth_proto ||
953 match->mask.vlan_tci[0] || match->mask.vlan_tci[1] ||
954 match->mask.vlan_proto[0] || match->mask.vlan_proto[1] ||
955 memchr_inv(match->mask.eth_saddr, 0, ETH_ALEN) ||
956 memchr_inv(match->mask.eth_daddr, 0, ETH_ALEN) ||
957 match->mask.ip_proto ||
958 match->mask.ip_tos || match->mask.ip_ttl ||
959 match->mask.src_ip || match->mask.dst_ip ||
960#ifdef CONFIG_IPV6
961 !ipv6_addr_any(&match->mask.src_ip6) ||
962 !ipv6_addr_any(&match->mask.dst_ip6) ||
963#endif
964 match->mask.ip_frag || match->mask.ip_firstfrag ||
965 match->mask.l4_sport || match->mask.l4_dport ||
966 match->mask.tcp_flags ||
967 /* nor can VNI */
968 match->mask.enc_keyid;
969}
970
971static int efx_tc_flower_handle_lhs_actions(struct efx_nic *efx,
972 struct flow_cls_offload *tc,
973 struct flow_rule *fr,
974 struct net_device *net_dev,
975 struct efx_tc_lhs_rule *rule)
976
977{
978 struct netlink_ext_ack *extack = tc->common.extack;
979 struct efx_tc_lhs_action *act = &rule->lhs_act;
980 const struct flow_action_entry *fa;
981 enum efx_tc_counter_type ctype;
982 bool pipe = true;
983 int i;
984
985 ctype = rule->is_ar ? EFX_TC_COUNTER_TYPE_AR : EFX_TC_COUNTER_TYPE_OR;
986
987 flow_action_for_each(i, fa, &fr->action) {
988 struct efx_tc_ct_zone *ct_zone;
989 struct efx_tc_recirc_id *rid;
990
991 if (!pipe) {
992 /* more actions after a non-pipe action */
993 NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action");
994 return -EINVAL;
995 }
996 switch (fa->id) {
997 case FLOW_ACTION_GOTO:
998 if (!fa->chain_index) {
999 NL_SET_ERR_MSG_MOD(extack, "Can't goto chain 0, no looping in hw");
1000 return -EOPNOTSUPP;
1001 }
1002 rid = efx_tc_get_recirc_id(efx, fa->chain_index,
1003 net_dev);
1004 if (IS_ERR(rid)) {
1005 NL_SET_ERR_MSG_MOD(extack, "Failed to allocate a hardware recirculation ID for this chain_index");
1006 return PTR_ERR(rid);
1007 }
1008 act->rid = rid;
1009 if (fa->hw_stats) {
1010 struct efx_tc_counter_index *cnt;
1011
1012 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
1013 NL_SET_ERR_MSG_FMT_MOD(extack,
1014 "hw_stats_type %u not supported (only 'delayed')",
1015 fa->hw_stats);
1016 return -EOPNOTSUPP;
1017 }
1018 cnt = efx_tc_flower_get_counter_index(efx, tc->cookie,
1019 ctype);
1020 if (IS_ERR(cnt)) {
1021 NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
1022 return PTR_ERR(cnt);
1023 }
1024 WARN_ON(act->count); /* can't happen */
1025 act->count = cnt;
1026 }
1027 pipe = false;
1028 break;
1029 case FLOW_ACTION_CT:
1030 if (act->zone) {
1031 NL_SET_ERR_MSG_MOD(extack, "Can't offload multiple ct actions");
1032 return -EOPNOTSUPP;
1033 }
1034 if (fa->ct.action & (TCA_CT_ACT_COMMIT |
1035 TCA_CT_ACT_FORCE)) {
1036 NL_SET_ERR_MSG_MOD(extack, "Can't offload ct commit/force");
1037 return -EOPNOTSUPP;
1038 }
1039 if (fa->ct.action & TCA_CT_ACT_CLEAR) {
1040 NL_SET_ERR_MSG_MOD(extack, "Can't clear ct in LHS rule");
1041 return -EOPNOTSUPP;
1042 }
1043 if (fa->ct.action & (TCA_CT_ACT_NAT |
1044 TCA_CT_ACT_NAT_SRC |
1045 TCA_CT_ACT_NAT_DST)) {
1046 NL_SET_ERR_MSG_MOD(extack, "Can't perform NAT in LHS rule - packet isn't conntracked yet");
1047 return -EOPNOTSUPP;
1048 }
1049 if (fa->ct.action) {
1050 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled ct.action %u for LHS rule\n",
1051 fa->ct.action);
1052 return -EOPNOTSUPP;
1053 }
1054 ct_zone = efx_tc_ct_register_zone(efx, fa->ct.zone,
1055 fa->ct.flow_table);
1056 if (IS_ERR(ct_zone)) {
1057 NL_SET_ERR_MSG_MOD(extack, "Failed to register for CT updates");
1058 return PTR_ERR(ct_zone);
1059 }
1060 act->zone = ct_zone;
1061 break;
1062 default:
1063 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u for LHS rule\n",
1064 fa->id);
1065 return -EOPNOTSUPP;
1066 }
1067 }
1068
1069 if (pipe) {
1070 NL_SET_ERR_MSG_MOD(extack, "Missing goto chain in LHS rule");
1071 return -EOPNOTSUPP;
1072 }
1073 return 0;
1074}
1075
1076static void efx_tc_flower_release_lhs_actions(struct efx_nic *efx,
1077 struct efx_tc_lhs_action *act)
1078{
1079 if (act->rid)
1080 efx_tc_put_recirc_id(efx, act->rid);
1081 if (act->zone)
1082 efx_tc_ct_unregister_zone(efx, act->zone);
1083 if (act->count)
1084 efx_tc_flower_put_counter_index(efx, act->count);
1085}
1086
1087/**
1088 * struct efx_tc_mangler_state - accumulates 32-bit pedits into fields
1089 *
1090 * @dst_mac_32: dst_mac[0:3] has been populated
1091 * @dst_mac_16: dst_mac[4:5] has been populated
1092 * @src_mac_16: src_mac[0:1] has been populated
1093 * @src_mac_32: src_mac[2:5] has been populated
1094 * @dst_mac: h_dest field of ethhdr
1095 * @src_mac: h_source field of ethhdr
1096 *
1097 * Since FLOW_ACTION_MANGLE comes in 32-bit chunks that do not
1098 * necessarily equate to whole fields of the packet header, this
1099 * structure is used to hold the cumulative effect of the partial
1100 * field pedits that have been processed so far.
1101 */
1102struct efx_tc_mangler_state {
1103 u8 dst_mac_32:1; /* eth->h_dest[0:3] */
1104 u8 dst_mac_16:1; /* eth->h_dest[4:5] */
1105 u8 src_mac_16:1; /* eth->h_source[0:1] */
1106 u8 src_mac_32:1; /* eth->h_source[2:5] */
1107 unsigned char dst_mac[ETH_ALEN];
1108 unsigned char src_mac[ETH_ALEN];
1109};
1110
1111/** efx_tc_complete_mac_mangle() - pull complete field pedits out of @mung
1112 * @efx: NIC we're installing a flow rule on
1113 * @act: action set (cursor) to update
1114 * @mung: accumulated partial mangles
1115 * @extack: netlink extended ack for reporting errors
1116 *
1117 * Check @mung to find any combinations of partial mangles that can be
1118 * combined into a complete packet field edit, add that edit to @act,
1119 * and consume the partial mangles from @mung.
1120 */
1121
1122static int efx_tc_complete_mac_mangle(struct efx_nic *efx,
1123 struct efx_tc_action_set *act,
1124 struct efx_tc_mangler_state *mung,
1125 struct netlink_ext_ack *extack)
1126{
1127 struct efx_tc_mac_pedit_action *ped;
1128
1129 if (mung->dst_mac_32 && mung->dst_mac_16) {
1130 ped = efx_tc_flower_get_mac(efx, mung->dst_mac, extack);
1131 if (IS_ERR(ped))
1132 return PTR_ERR(ped);
1133
1134 /* Check that we have not already populated dst_mac */
1135 if (act->dst_mac)
1136 efx_tc_flower_put_mac(efx, act->dst_mac);
1137
1138 act->dst_mac = ped;
1139
1140 /* consume the incomplete state */
1141 mung->dst_mac_32 = 0;
1142 mung->dst_mac_16 = 0;
1143 }
1144 if (mung->src_mac_16 && mung->src_mac_32) {
1145 ped = efx_tc_flower_get_mac(efx, mung->src_mac, extack);
1146 if (IS_ERR(ped))
1147 return PTR_ERR(ped);
1148
1149 /* Check that we have not already populated src_mac */
1150 if (act->src_mac)
1151 efx_tc_flower_put_mac(efx, act->src_mac);
1152
1153 act->src_mac = ped;
1154
1155 /* consume the incomplete state */
1156 mung->src_mac_32 = 0;
1157 mung->src_mac_16 = 0;
1158 }
1159 return 0;
1160}
1161
1162static int efx_tc_pedit_add(struct efx_nic *efx, struct efx_tc_action_set *act,
1163 const struct flow_action_entry *fa,
1164 struct netlink_ext_ack *extack)
1165{
1166 switch (fa->mangle.htype) {
1167 case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
1168 switch (fa->mangle.offset) {
1169 case offsetof(struct iphdr, ttl):
1170 /* check that pedit applies to ttl only */
1171 if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK)
1172 break;
1173
1174 /* Adding 0xff is equivalent to decrementing the ttl.
1175 * Other added values are not supported.
1176 */
1177 if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) != U8_MAX)
1178 break;
1179
1180 /* check that we do not decrement ttl twice */
1181 if (!efx_tc_flower_action_order_ok(act,
1182 EFX_TC_AO_DEC_TTL)) {
1183 NL_SET_ERR_MSG_MOD(extack, "multiple dec ttl are not supported");
1184 return -EOPNOTSUPP;
1185 }
1186 act->do_ttl_dec = 1;
1187 return 0;
1188 default:
1189 break;
1190 }
1191 break;
1192 case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
1193 switch (fa->mangle.offset) {
1194 case round_down(offsetof(struct ipv6hdr, hop_limit), 4):
1195 /* check that pedit applies to hoplimit only */
1196 if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK)
1197 break;
1198
1199 /* Adding 0xff is equivalent to decrementing the hoplimit.
1200 * Other added values are not supported.
1201 */
1202 if ((fa->mangle.val >> 24) != U8_MAX)
1203 break;
1204
1205 /* check that we do not decrement hoplimit twice */
1206 if (!efx_tc_flower_action_order_ok(act,
1207 EFX_TC_AO_DEC_TTL)) {
1208 NL_SET_ERR_MSG_MOD(extack, "multiple dec ttl are not supported");
1209 return -EOPNOTSUPP;
1210 }
1211 act->do_ttl_dec = 1;
1212 return 0;
1213 default:
1214 break;
1215 }
1216 break;
1217 default:
1218 break;
1219 }
1220
1221 NL_SET_ERR_MSG_FMT_MOD(extack,
1222 "ttl add action type %x %x %x/%x is not supported",
1223 fa->mangle.htype, fa->mangle.offset,
1224 fa->mangle.val, fa->mangle.mask);
1225 return -EOPNOTSUPP;
1226}
1227
1228/**
1229 * efx_tc_mangle() - handle a single 32-bit (or less) pedit
1230 * @efx: NIC we're installing a flow rule on
1231 * @act: action set (cursor) to update
1232 * @fa: FLOW_ACTION_MANGLE action metadata
1233 * @mung: accumulator for partial mangles
1234 * @extack: netlink extended ack for reporting errors
1235 * @match: original match used along with the mangle action
1236 *
1237 * Identify the fields written by a FLOW_ACTION_MANGLE, and record
1238 * the partial mangle state in @mung. If this mangle completes an
1239 * earlier partial mangle, consume and apply to @act by calling
1240 * efx_tc_complete_mac_mangle().
1241 */
1242
1243static int efx_tc_mangle(struct efx_nic *efx, struct efx_tc_action_set *act,
1244 const struct flow_action_entry *fa,
1245 struct efx_tc_mangler_state *mung,
1246 struct netlink_ext_ack *extack,
1247 struct efx_tc_match *match)
1248{
1249 __le32 mac32;
1250 __le16 mac16;
1251 u8 tr_ttl;
1252
1253 switch (fa->mangle.htype) {
1254 case FLOW_ACT_MANGLE_HDR_TYPE_ETH:
1255 BUILD_BUG_ON(offsetof(struct ethhdr, h_dest) != 0);
1256 BUILD_BUG_ON(offsetof(struct ethhdr, h_source) != 6);
1257 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_PEDIT_MAC_ADDRS)) {
1258 NL_SET_ERR_MSG_MOD(extack,
1259 "Pedit mangle mac action violates action order");
1260 return -EOPNOTSUPP;
1261 }
1262 switch (fa->mangle.offset) {
1263 case 0:
1264 if (fa->mangle.mask) {
1265 NL_SET_ERR_MSG_FMT_MOD(extack,
1266 "mask (%#x) of eth.dst32 mangle is not supported",
1267 fa->mangle.mask);
1268 return -EOPNOTSUPP;
1269 }
1270 /* Ethernet address is little-endian */
1271 mac32 = cpu_to_le32(fa->mangle.val);
1272 memcpy(mung->dst_mac, &mac32, sizeof(mac32));
1273 mung->dst_mac_32 = 1;
1274 return efx_tc_complete_mac_mangle(efx, act, mung, extack);
1275 case 4:
1276 if (fa->mangle.mask == 0xffff) {
1277 mac16 = cpu_to_le16(fa->mangle.val >> 16);
1278 memcpy(mung->src_mac, &mac16, sizeof(mac16));
1279 mung->src_mac_16 = 1;
1280 } else if (fa->mangle.mask == 0xffff0000) {
1281 mac16 = cpu_to_le16((u16)fa->mangle.val);
1282 memcpy(mung->dst_mac + 4, &mac16, sizeof(mac16));
1283 mung->dst_mac_16 = 1;
1284 } else {
1285 NL_SET_ERR_MSG_FMT_MOD(extack,
1286 "mask (%#x) of eth+4 mangle is not high or low 16b",
1287 fa->mangle.mask);
1288 return -EOPNOTSUPP;
1289 }
1290 return efx_tc_complete_mac_mangle(efx, act, mung, extack);
1291 case 8:
1292 if (fa->mangle.mask) {
1293 NL_SET_ERR_MSG_FMT_MOD(extack,
1294 "mask (%#x) of eth.src32 mangle is not supported",
1295 fa->mangle.mask);
1296 return -EOPNOTSUPP;
1297 }
1298 mac32 = cpu_to_le32(fa->mangle.val);
1299 memcpy(mung->src_mac + 2, &mac32, sizeof(mac32));
1300 mung->src_mac_32 = 1;
1301 return efx_tc_complete_mac_mangle(efx, act, mung, extack);
1302 default:
1303 NL_SET_ERR_MSG_FMT_MOD(extack, "mangle eth+%u %x/%x is not supported",
1304 fa->mangle.offset, fa->mangle.val, fa->mangle.mask);
1305 return -EOPNOTSUPP;
1306 }
1307 break;
1308 case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
1309 switch (fa->mangle.offset) {
1310 case offsetof(struct iphdr, ttl):
1311 /* we currently only support pedit IP4 when it applies
1312 * to TTL and then only when it can be achieved with a
1313 * decrement ttl action
1314 */
1315
1316 /* check that pedit applies to ttl only */
1317 if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK) {
1318 NL_SET_ERR_MSG_FMT_MOD(extack,
1319 "mask (%#x) out of range, only support mangle action on ipv4.ttl",
1320 fa->mangle.mask);
1321 return -EOPNOTSUPP;
1322 }
1323
1324 /* we can only convert to a dec ttl when we have an
1325 * exact match on the ttl field
1326 */
1327 if (match->mask.ip_ttl != U8_MAX) {
1328 NL_SET_ERR_MSG_FMT_MOD(extack,
1329 "only support mangle ttl when we have an exact match, current mask (%#x)",
1330 match->mask.ip_ttl);
1331 return -EOPNOTSUPP;
1332 }
1333
1334 /* check that we don't try to decrement 0, which equates
1335 * to setting the ttl to 0xff
1336 */
1337 if (match->value.ip_ttl == 0) {
1338 NL_SET_ERR_MSG_MOD(extack,
1339 "decrement ttl past 0 is not supported");
1340 return -EOPNOTSUPP;
1341 }
1342
1343 /* check that we do not decrement ttl twice */
1344 if (!efx_tc_flower_action_order_ok(act,
1345 EFX_TC_AO_DEC_TTL)) {
1346 NL_SET_ERR_MSG_MOD(extack,
1347 "multiple dec ttl is not supported");
1348 return -EOPNOTSUPP;
1349 }
1350
1351 /* check pedit can be achieved with decrement action */
1352 tr_ttl = match->value.ip_ttl - 1;
1353 if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) == tr_ttl) {
1354 act->do_ttl_dec = 1;
1355 return 0;
1356 }
1357
1358 fallthrough;
1359 default:
1360 NL_SET_ERR_MSG_FMT_MOD(extack,
1361 "only support mangle on the ttl field (offset is %u)",
1362 fa->mangle.offset);
1363 return -EOPNOTSUPP;
1364 }
1365 break;
1366 case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
1367 switch (fa->mangle.offset) {
1368 case round_down(offsetof(struct ipv6hdr, hop_limit), 4):
1369 /* we currently only support pedit IP6 when it applies
1370 * to the hoplimit and then only when it can be achieved
1371 * with a decrement hoplimit action
1372 */
1373
1374 /* check that pedit applies to ttl only */
1375 if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK) {
1376 NL_SET_ERR_MSG_FMT_MOD(extack,
1377 "mask (%#x) out of range, only support mangle action on ipv6.hop_limit",
1378 fa->mangle.mask);
1379
1380 return -EOPNOTSUPP;
1381 }
1382
1383 /* we can only convert to a dec ttl when we have an
1384 * exact match on the ttl field
1385 */
1386 if (match->mask.ip_ttl != U8_MAX) {
1387 NL_SET_ERR_MSG_FMT_MOD(extack,
1388 "only support hop_limit when we have an exact match, current mask (%#x)",
1389 match->mask.ip_ttl);
1390 return -EOPNOTSUPP;
1391 }
1392
1393 /* check that we don't try to decrement 0, which equates
1394 * to setting the ttl to 0xff
1395 */
1396 if (match->value.ip_ttl == 0) {
1397 NL_SET_ERR_MSG_MOD(extack,
1398 "decrementing hop_limit past 0 is not supported");
1399 return -EOPNOTSUPP;
1400 }
1401
1402 /* check that we do not decrement hoplimit twice */
1403 if (!efx_tc_flower_action_order_ok(act,
1404 EFX_TC_AO_DEC_TTL)) {
1405 NL_SET_ERR_MSG_MOD(extack,
1406 "multiple dec ttl is not supported");
1407 return -EOPNOTSUPP;
1408 }
1409
1410 /* check pedit can be achieved with decrement action */
1411 tr_ttl = match->value.ip_ttl - 1;
1412 if ((fa->mangle.val >> 24) == tr_ttl) {
1413 act->do_ttl_dec = 1;
1414 return 0;
1415 }
1416
1417 fallthrough;
1418 default:
1419 NL_SET_ERR_MSG_FMT_MOD(extack,
1420 "only support mangle on the hop_limit field");
1421 return -EOPNOTSUPP;
1422 }
1423 default:
1424 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled mangle htype %u for action rule",
1425 fa->mangle.htype);
1426 return -EOPNOTSUPP;
1427 }
1428 return 0;
1429}
1430
1431/**
1432 * efx_tc_incomplete_mangle() - check for leftover partial pedits
1433 * @mung: accumulator for partial mangles
1434 * @extack: netlink extended ack for reporting errors
1435 *
1436 * Since the MAE can only overwrite whole fields, any partial
1437 * field mangle left over on reaching packet delivery (mirred or
1438 * end of TC actions) cannot be offloaded. Check for any such
1439 * and reject them with -%EOPNOTSUPP.
1440 */
1441
1442static int efx_tc_incomplete_mangle(struct efx_tc_mangler_state *mung,
1443 struct netlink_ext_ack *extack)
1444{
1445 if (mung->dst_mac_32 || mung->dst_mac_16) {
1446 NL_SET_ERR_MSG_MOD(extack, "Incomplete pedit of destination MAC address");
1447 return -EOPNOTSUPP;
1448 }
1449 if (mung->src_mac_16 || mung->src_mac_32) {
1450 NL_SET_ERR_MSG_MOD(extack, "Incomplete pedit of source MAC address");
1451 return -EOPNOTSUPP;
1452 }
1453 return 0;
1454}
1455
1456static int efx_tc_flower_replace_foreign_lhs_ar(struct efx_nic *efx,
1457 struct flow_cls_offload *tc,
1458 struct flow_rule *fr,
1459 struct efx_tc_match *match,
1460 struct net_device *net_dev)
1461{
1462 struct netlink_ext_ack *extack = tc->common.extack;
1463 struct efx_tc_lhs_rule *rule, *old;
1464 enum efx_encap_type type;
1465 int rc;
1466
1467 type = efx_tc_indr_netdev_type(net_dev);
1468 if (type == EFX_ENCAP_TYPE_NONE) {
1469 NL_SET_ERR_MSG_MOD(extack, "Egress encap match on unsupported tunnel device");
1470 return -EOPNOTSUPP;
1471 }
1472
1473 rc = efx_mae_check_encap_type_supported(efx, type);
1474 if (rc) {
1475 NL_SET_ERR_MSG_FMT_MOD(extack,
1476 "Firmware reports no support for %s encap match",
1477 efx_tc_encap_type_name(type));
1478 return rc;
1479 }
1480 /* This is an Action Rule, so it needs a separate Encap Match in the
1481 * Outer Rule table. Insert that now.
1482 */
1483 rc = efx_tc_flower_record_encap_match(efx, match, type,
1484 EFX_TC_EM_DIRECT, 0, 0, extack);
1485 if (rc)
1486 return rc;
1487
1488 match->mask.recirc_id = 0xff;
1489 if (match->mask.ct_state_trk && match->value.ct_state_trk) {
1490 NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");
1491 rc = -EOPNOTSUPP;
1492 goto release_encap_match;
1493 }
1494 /* LHS rules are always -trk, so we don't need to match on that */
1495 match->mask.ct_state_trk = 0;
1496 match->value.ct_state_trk = 0;
1497 /* We must inhibit match on TCP SYN/FIN/RST, so that SW can see
1498 * the packet and update the conntrack table.
1499 * Outer Rules will do that with CT_TCP_FLAGS_INHIBIT, but Action
1500 * Rules don't have that; instead they support matching on
1501 * TCP_SYN_FIN_RST (aka TCP_INTERESTING_FLAGS), so use that.
1502 * This is only strictly needed if there will be a DO_CT action,
1503 * which we don't know yet, but typically there will be and it's
1504 * simpler not to bother checking here.
1505 */
1506 match->mask.tcp_syn_fin_rst = true;
1507
1508 rc = efx_mae_match_check_caps(efx, &match->mask, extack);
1509 if (rc)
1510 goto release_encap_match;
1511
1512 rule = kzalloc(sizeof(*rule), GFP_USER);
1513 if (!rule) {
1514 rc = -ENOMEM;
1515 goto release_encap_match;
1516 }
1517 rule->cookie = tc->cookie;
1518 rule->is_ar = true;
1519 old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,
1520 &rule->linkage,
1521 efx_tc_lhs_rule_ht_params);
1522 if (old) {
1523 netif_dbg(efx, drv, efx->net_dev,
1524 "Already offloaded rule (cookie %lx)\n", tc->cookie);
1525 rc = -EEXIST;
1526 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
1527 goto release;
1528 }
1529
1530 /* Parse actions */
1531 rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);
1532 if (rc)
1533 goto release;
1534
1535 rule->match = *match;
1536 rule->lhs_act.tun_type = type;
1537
1538 rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);
1539 if (rc) {
1540 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
1541 goto release;
1542 }
1543 netif_dbg(efx, drv, efx->net_dev,
1544 "Successfully parsed lhs rule (cookie %lx)\n",
1545 tc->cookie);
1546 return 0;
1547
1548release:
1549 efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);
1550 if (!old)
1551 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,
1552 efx_tc_lhs_rule_ht_params);
1553 kfree(rule);
1554release_encap_match:
1555 if (match->encap)
1556 efx_tc_flower_release_encap_match(efx, match->encap);
1557 return rc;
1558}
1559
1560static int efx_tc_flower_replace_foreign_lhs(struct efx_nic *efx,
1561 struct flow_cls_offload *tc,
1562 struct flow_rule *fr,
1563 struct efx_tc_match *match,
1564 struct net_device *net_dev)
1565{
1566 struct netlink_ext_ack *extack = tc->common.extack;
1567 struct efx_tc_lhs_rule *rule, *old;
1568 enum efx_encap_type type;
1569 int rc;
1570
1571 if (tc->common.chain_index) {
1572 NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0");
1573 return -EOPNOTSUPP;
1574 }
1575
1576 if (!efx_tc_match_is_encap(&match->mask)) {
1577 /* This is not a tunnel decap rule, ignore it */
1578 netif_dbg(efx, drv, efx->net_dev, "Ignoring foreign LHS filter without encap match\n");
1579 return -EOPNOTSUPP;
1580 }
1581
1582 if (efx_tc_flower_flhs_needs_ar(match))
1583 return efx_tc_flower_replace_foreign_lhs_ar(efx, tc, fr, match,
1584 net_dev);
1585
1586 type = efx_tc_indr_netdev_type(net_dev);
1587 if (type == EFX_ENCAP_TYPE_NONE) {
1588 NL_SET_ERR_MSG_MOD(extack, "Egress encap match on unsupported tunnel device\n");
1589 return -EOPNOTSUPP;
1590 }
1591
1592 rc = efx_mae_check_encap_type_supported(efx, type);
1593 if (rc) {
1594 NL_SET_ERR_MSG_FMT_MOD(extack,
1595 "Firmware reports no support for %s encap match",
1596 efx_tc_encap_type_name(type));
1597 return rc;
1598 }
1599 /* Reserve the outer tuple with a pseudo Encap Match */
1600 rc = efx_tc_flower_record_encap_match(efx, match, type,
1601 EFX_TC_EM_PSEUDO_OR, 0, 0,
1602 extack);
1603 if (rc)
1604 return rc;
1605
1606 if (match->mask.ct_state_trk && match->value.ct_state_trk) {
1607 NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");
1608 rc = -EOPNOTSUPP;
1609 goto release_encap_match;
1610 }
1611 /* LHS rules are always -trk, so we don't need to match on that */
1612 match->mask.ct_state_trk = 0;
1613 match->value.ct_state_trk = 0;
1614
1615 rc = efx_tc_flower_translate_flhs_match(match);
1616 if (rc) {
1617 NL_SET_ERR_MSG_MOD(extack, "LHS rule cannot match on inner fields");
1618 goto release_encap_match;
1619 }
1620
1621 rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack);
1622 if (rc)
1623 goto release_encap_match;
1624
1625 rule = kzalloc(sizeof(*rule), GFP_USER);
1626 if (!rule) {
1627 rc = -ENOMEM;
1628 goto release_encap_match;
1629 }
1630 rule->cookie = tc->cookie;
1631 old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,
1632 &rule->linkage,
1633 efx_tc_lhs_rule_ht_params);
1634 if (old) {
1635 netif_dbg(efx, drv, efx->net_dev,
1636 "Already offloaded rule (cookie %lx)\n", tc->cookie);
1637 rc = -EEXIST;
1638 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
1639 goto release;
1640 }
1641
1642 /* Parse actions */
1643 rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);
1644 if (rc)
1645 goto release;
1646
1647 rule->match = *match;
1648 rule->lhs_act.tun_type = type;
1649
1650 rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);
1651 if (rc) {
1652 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
1653 goto release;
1654 }
1655 netif_dbg(efx, drv, efx->net_dev,
1656 "Successfully parsed lhs rule (cookie %lx)\n",
1657 tc->cookie);
1658 return 0;
1659
1660release:
1661 efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);
1662 if (!old)
1663 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,
1664 efx_tc_lhs_rule_ht_params);
1665 kfree(rule);
1666release_encap_match:
1667 if (match->encap)
1668 efx_tc_flower_release_encap_match(efx, match->encap);
1669 return rc;
1670}
1671
1672static int efx_tc_flower_replace_foreign(struct efx_nic *efx,
1673 struct net_device *net_dev,
1674 struct flow_cls_offload *tc)
1675{
1676 struct flow_rule *fr = flow_cls_offload_flow_rule(tc);
1677 struct netlink_ext_ack *extack = tc->common.extack;
1678 struct efx_tc_flow_rule *rule = NULL, *old = NULL;
1679 struct efx_tc_action_set *act = NULL;
1680 bool found = false, uplinked = false;
1681 const struct flow_action_entry *fa;
1682 struct efx_tc_match match;
1683 struct efx_rep *to_efv;
1684 s64 rc;
1685 int i;
1686
1687 /* Parse match */
1688 memset(&match, 0, sizeof(match));
1689 rc = efx_tc_flower_parse_match(efx, fr, &match, extack);
1690 if (rc)
1691 return rc;
1692 /* The rule as given to us doesn't specify a source netdevice.
1693 * But, determining whether packets from a VF should match it is
1694 * complicated, so leave those to the software slowpath: qualify
1695 * the filter with source m-port == wire.
1696 */
1697 rc = efx_tc_flower_external_mport(efx, EFX_EFV_PF);
1698 if (rc < 0) {
1699 NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port for foreign filter");
1700 return rc;
1701 }
1702 match.value.ingress_port = rc;
1703 match.mask.ingress_port = ~0;
1704
1705 if (efx_tc_rule_is_lhs_rule(fr, &match))
1706 return efx_tc_flower_replace_foreign_lhs(efx, tc, fr, &match,
1707 net_dev);
1708
1709 if (tc->common.chain_index) {
1710 struct efx_tc_recirc_id *rid;
1711
1712 rid = efx_tc_get_recirc_id(efx, tc->common.chain_index, net_dev);
1713 if (IS_ERR(rid)) {
1714 NL_SET_ERR_MSG_FMT_MOD(extack,
1715 "Failed to allocate a hardware recirculation ID for chain_index %u",
1716 tc->common.chain_index);
1717 return PTR_ERR(rid);
1718 }
1719 match.rid = rid;
1720 match.value.recirc_id = rid->fw_id;
1721 }
1722 match.mask.recirc_id = 0xff;
1723
1724 /* AR table can't match on DO_CT (+trk). But a commonly used pattern is
1725 * +trk+est, which is strictly implied by +est, so rewrite it to that.
1726 */
1727 if (match.mask.ct_state_trk && match.value.ct_state_trk &&
1728 match.mask.ct_state_est && match.value.ct_state_est)
1729 match.mask.ct_state_trk = 0;
1730 /* Thanks to CT_TCP_FLAGS_INHIBIT, packets with interesting flags could
1731 * match +trk-est (CT_HIT=0) despite being on an established connection.
1732 * So make -est imply -tcp_syn_fin_rst match to ensure these packets
1733 * still hit the software path.
1734 */
1735 if (match.mask.ct_state_est && !match.value.ct_state_est) {
1736 if (match.value.tcp_syn_fin_rst) {
1737 /* Can't offload this combination */
1738 NL_SET_ERR_MSG_MOD(extack, "TCP flags and -est conflict for offload");
1739 rc = -EOPNOTSUPP;
1740 goto release;
1741 }
1742 match.mask.tcp_syn_fin_rst = true;
1743 }
1744
1745 flow_action_for_each(i, fa, &fr->action) {
1746 switch (fa->id) {
1747 case FLOW_ACTION_REDIRECT:
1748 case FLOW_ACTION_MIRRED: /* mirred means mirror here */
1749 to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
1750 if (IS_ERR(to_efv))
1751 continue;
1752 found = true;
1753 break;
1754 default:
1755 break;
1756 }
1757 }
1758 if (!found) { /* We don't care. */
1759 netif_dbg(efx, drv, efx->net_dev,
1760 "Ignoring foreign filter that doesn't egdev us\n");
1761 rc = -EOPNOTSUPP;
1762 goto release;
1763 }
1764
1765 rc = efx_mae_match_check_caps(efx, &match.mask, extack);
1766 if (rc)
1767 goto release;
1768
1769 if (efx_tc_match_is_encap(&match.mask)) {
1770 enum efx_encap_type type;
1771
1772 type = efx_tc_indr_netdev_type(net_dev);
1773 if (type == EFX_ENCAP_TYPE_NONE) {
1774 NL_SET_ERR_MSG_MOD(extack,
1775 "Egress encap match on unsupported tunnel device");
1776 rc = -EOPNOTSUPP;
1777 goto release;
1778 }
1779
1780 rc = efx_mae_check_encap_type_supported(efx, type);
1781 if (rc) {
1782 NL_SET_ERR_MSG_FMT_MOD(extack,
1783 "Firmware reports no support for %s encap match",
1784 efx_tc_encap_type_name(type));
1785 goto release;
1786 }
1787
1788 rc = efx_tc_flower_record_encap_match(efx, &match, type,
1789 EFX_TC_EM_DIRECT, 0, 0,
1790 extack);
1791 if (rc)
1792 goto release;
1793 } else if (!tc->common.chain_index) {
1794 /* This is not a tunnel decap rule, ignore it */
1795 netif_dbg(efx, drv, efx->net_dev,
1796 "Ignoring foreign filter without encap match\n");
1797 rc = -EOPNOTSUPP;
1798 goto release;
1799 }
1800
1801 rule = kzalloc(sizeof(*rule), GFP_USER);
1802 if (!rule) {
1803 rc = -ENOMEM;
1804 goto release;
1805 }
1806 INIT_LIST_HEAD(&rule->acts.list);
1807 rule->cookie = tc->cookie;
1808 old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht,
1809 &rule->linkage,
1810 efx_tc_match_action_ht_params);
1811 if (IS_ERR(old)) {
1812 rc = PTR_ERR(old);
1813 goto release;
1814 } else if (old) {
1815 netif_dbg(efx, drv, efx->net_dev,
1816 "Ignoring already-offloaded rule (cookie %lx)\n",
1817 tc->cookie);
1818 rc = -EEXIST;
1819 goto release;
1820 }
1821
1822 act = kzalloc(sizeof(*act), GFP_USER);
1823 if (!act) {
1824 rc = -ENOMEM;
1825 goto release;
1826 }
1827
1828 /* Parse actions. For foreign rules we only support decap & redirect.
1829 * See corresponding code in efx_tc_flower_replace() for theory of
1830 * operation & how 'act' cursor is used.
1831 */
1832 flow_action_for_each(i, fa, &fr->action) {
1833 struct efx_tc_action_set save;
1834
1835 switch (fa->id) {
1836 case FLOW_ACTION_REDIRECT:
1837 case FLOW_ACTION_MIRRED:
1838 /* See corresponding code in efx_tc_flower_replace() for
1839 * long explanations of what's going on here.
1840 */
1841 save = *act;
1842 if (fa->hw_stats) {
1843 struct efx_tc_counter_index *ctr;
1844
1845 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
1846 NL_SET_ERR_MSG_FMT_MOD(extack,
1847 "hw_stats_type %u not supported (only 'delayed')",
1848 fa->hw_stats);
1849 rc = -EOPNOTSUPP;
1850 goto release;
1851 }
1852 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) {
1853 NL_SET_ERR_MSG_MOD(extack, "Count action violates action order (can't happen)");
1854 rc = -EOPNOTSUPP;
1855 goto release;
1856 }
1857
1858 ctr = efx_tc_flower_get_counter_index(efx,
1859 tc->cookie,
1860 EFX_TC_COUNTER_TYPE_AR);
1861 if (IS_ERR(ctr)) {
1862 rc = PTR_ERR(ctr);
1863 NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
1864 goto release;
1865 }
1866 act->count = ctr;
1867 INIT_LIST_HEAD(&act->count_user);
1868 }
1869
1870 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) {
1871 /* can't happen */
1872 rc = -EOPNOTSUPP;
1873 NL_SET_ERR_MSG_MOD(extack,
1874 "Deliver action violates action order (can't happen)");
1875 goto release;
1876 }
1877 to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
1878 /* PF implies egdev is us, in which case we really
1879 * want to deliver to the uplink (because this is an
1880 * ingress filter). If we don't recognise the egdev
1881 * at all, then we'd better trap so SW can handle it.
1882 */
1883 if (IS_ERR(to_efv))
1884 to_efv = EFX_EFV_PF;
1885 if (to_efv == EFX_EFV_PF) {
1886 if (uplinked)
1887 break;
1888 uplinked = true;
1889 }
1890 rc = efx_tc_flower_internal_mport(efx, to_efv);
1891 if (rc < 0) {
1892 NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port");
1893 goto release;
1894 }
1895 act->dest_mport = rc;
1896 act->deliver = 1;
1897 rc = efx_mae_alloc_action_set(efx, act);
1898 if (rc) {
1899 NL_SET_ERR_MSG_MOD(extack,
1900 "Failed to write action set to hw (mirred)");
1901 goto release;
1902 }
1903 list_add_tail(&act->list, &rule->acts.list);
1904 act = NULL;
1905 if (fa->id == FLOW_ACTION_REDIRECT)
1906 break; /* end of the line */
1907 /* Mirror, so continue on with saved act */
1908 act = kzalloc(sizeof(*act), GFP_USER);
1909 if (!act) {
1910 rc = -ENOMEM;
1911 goto release;
1912 }
1913 *act = save;
1914 break;
1915 case FLOW_ACTION_TUNNEL_DECAP:
1916 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DECAP)) {
1917 rc = -EINVAL;
1918 NL_SET_ERR_MSG_MOD(extack, "Decap action violates action order");
1919 goto release;
1920 }
1921 act->decap = 1;
1922 /* If we previously delivered/trapped to uplink, now
1923 * that we've decapped we'll want another copy if we
1924 * try to deliver/trap to uplink again.
1925 */
1926 uplinked = false;
1927 break;
1928 default:
1929 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u",
1930 fa->id);
1931 rc = -EOPNOTSUPP;
1932 goto release;
1933 }
1934 }
1935
1936 if (act) {
1937 if (!uplinked) {
1938 /* Not shot/redirected, so deliver to default dest (which is
1939 * the uplink, as this is an ingress filter)
1940 */
1941 efx_mae_mport_uplink(efx, &act->dest_mport);
1942 act->deliver = 1;
1943 }
1944 rc = efx_mae_alloc_action_set(efx, act);
1945 if (rc) {
1946 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)");
1947 goto release;
1948 }
1949 list_add_tail(&act->list, &rule->acts.list);
1950 act = NULL; /* Prevent double-free in error path */
1951 }
1952
1953 rule->match = match;
1954
1955 netif_dbg(efx, drv, efx->net_dev,
1956 "Successfully parsed foreign filter (cookie %lx)\n",
1957 tc->cookie);
1958
1959 rc = efx_mae_alloc_action_set_list(efx, &rule->acts);
1960 if (rc) {
1961 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw");
1962 goto release;
1963 }
1964 rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,
1965 rule->acts.fw_id, &rule->fw_id);
1966 if (rc) {
1967 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
1968 goto release_acts;
1969 }
1970 return 0;
1971
1972release_acts:
1973 efx_mae_free_action_set_list(efx, &rule->acts);
1974release:
1975 /* We failed to insert the rule, so free up any entries we created in
1976 * subsidiary tables.
1977 */
1978 if (match.rid)
1979 efx_tc_put_recirc_id(efx, match.rid);
1980 if (act)
1981 efx_tc_free_action_set(efx, act, false);
1982 if (rule) {
1983 if (!old)
1984 rhashtable_remove_fast(&efx->tc->match_action_ht,
1985 &rule->linkage,
1986 efx_tc_match_action_ht_params);
1987 efx_tc_free_action_set_list(efx, &rule->acts, false);
1988 }
1989 kfree(rule);
1990 if (match.encap)
1991 efx_tc_flower_release_encap_match(efx, match.encap);
1992 return rc;
1993}
1994
1995static int efx_tc_flower_replace_lhs(struct efx_nic *efx,
1996 struct flow_cls_offload *tc,
1997 struct flow_rule *fr,
1998 struct efx_tc_match *match,
1999 struct efx_rep *efv,
2000 struct net_device *net_dev)
2001{
2002 struct netlink_ext_ack *extack = tc->common.extack;
2003 struct efx_tc_lhs_rule *rule, *old;
2004 int rc;
2005
2006 if (tc->common.chain_index) {
2007 NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0");
2008 return -EOPNOTSUPP;
2009 }
2010
2011 if (match->mask.ct_state_trk && match->value.ct_state_trk) {
2012 NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");
2013 return -EOPNOTSUPP;
2014 }
2015 /* LHS rules are always -trk, so we don't need to match on that */
2016 match->mask.ct_state_trk = 0;
2017 match->value.ct_state_trk = 0;
2018
2019 rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack);
2020 if (rc)
2021 return rc;
2022
2023 rule = kzalloc(sizeof(*rule), GFP_USER);
2024 if (!rule)
2025 return -ENOMEM;
2026 rule->cookie = tc->cookie;
2027 old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,
2028 &rule->linkage,
2029 efx_tc_lhs_rule_ht_params);
2030 if (IS_ERR(old)) {
2031 rc = PTR_ERR(old);
2032 goto release;
2033 } else if (old) {
2034 netif_dbg(efx, drv, efx->net_dev,
2035 "Already offloaded rule (cookie %lx)\n", tc->cookie);
2036 rc = -EEXIST;
2037 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
2038 goto release;
2039 }
2040
2041 /* Parse actions */
2042 /* See note in efx_tc_flower_replace() regarding passed net_dev
2043 * (used for efx_tc_get_recirc_id()).
2044 */
2045 rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, efx->net_dev, rule);
2046 if (rc)
2047 goto release;
2048
2049 rule->match = *match;
2050
2051 rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);
2052 if (rc) {
2053 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
2054 goto release;
2055 }
2056 netif_dbg(efx, drv, efx->net_dev,
2057 "Successfully parsed lhs rule (cookie %lx)\n",
2058 tc->cookie);
2059 return 0;
2060
2061release:
2062 efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);
2063 if (!old)
2064 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,
2065 efx_tc_lhs_rule_ht_params);
2066 kfree(rule);
2067 return rc;
2068}
2069
2070static int efx_tc_flower_replace(struct efx_nic *efx,
2071 struct net_device *net_dev,
2072 struct flow_cls_offload *tc,
2073 struct efx_rep *efv)
2074{
2075 struct flow_rule *fr = flow_cls_offload_flow_rule(tc);
2076 struct netlink_ext_ack *extack = tc->common.extack;
2077 const struct ip_tunnel_info *encap_info = NULL;
2078 struct efx_tc_flow_rule *rule = NULL, *old;
2079 struct efx_tc_mangler_state mung = {};
2080 struct efx_tc_action_set *act = NULL;
2081 const struct flow_action_entry *fa;
2082 struct efx_rep *from_efv, *to_efv;
2083 struct efx_tc_match match;
2084 u32 acts_id;
2085 s64 rc;
2086 int i;
2087
2088 if (!tc_can_offload_extack(efx->net_dev, extack))
2089 return -EOPNOTSUPP;
2090 if (WARN_ON(!efx->tc))
2091 return -ENETDOWN;
2092 if (WARN_ON(!efx->tc->up))
2093 return -ENETDOWN;
2094
2095 from_efv = efx_tc_flower_lookup_efv(efx, net_dev);
2096 if (IS_ERR(from_efv)) {
2097 /* Not from our PF or representors, so probably a tunnel dev */
2098 return efx_tc_flower_replace_foreign(efx, net_dev, tc);
2099 }
2100
2101 if (efv != from_efv) {
2102 /* can't happen */
2103 NL_SET_ERR_MSG_FMT_MOD(extack, "for %s efv is %snull but from_efv is %snull (can't happen)",
2104 netdev_name(net_dev), efv ? "non-" : "",
2105 from_efv ? "non-" : "");
2106 return -EINVAL;
2107 }
2108
2109 /* Parse match */
2110 memset(&match, 0, sizeof(match));
2111 rc = efx_tc_flower_external_mport(efx, from_efv);
2112 if (rc < 0) {
2113 NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port");
2114 return rc;
2115 }
2116 match.value.ingress_port = rc;
2117 match.mask.ingress_port = ~0;
2118 rc = efx_tc_flower_parse_match(efx, fr, &match, extack);
2119 if (rc)
2120 return rc;
2121 if (efx_tc_match_is_encap(&match.mask)) {
2122 NL_SET_ERR_MSG_MOD(extack, "Ingress enc_key matches not supported");
2123 return -EOPNOTSUPP;
2124 }
2125
2126 if (efx_tc_rule_is_lhs_rule(fr, &match))
2127 return efx_tc_flower_replace_lhs(efx, tc, fr, &match, efv,
2128 net_dev);
2129
2130 /* chain_index 0 is always recirc_id 0 (and does not appear in recirc_ht).
2131 * Conveniently, match.rid == NULL and match.value.recirc_id == 0 owing
2132 * to the initial memset(), so we don't need to do anything in that case.
2133 */
2134 if (tc->common.chain_index) {
2135 struct efx_tc_recirc_id *rid;
2136
2137 /* Note regarding passed net_dev:
2138 * VFreps and PF can share chain namespace, as they have
2139 * distinct ingress_mports. So we don't need to burn an
2140 * extra recirc_id if both use the same chain_index.
2141 * (Strictly speaking, we could give each VFrep its own
2142 * recirc_id namespace that doesn't take IDs away from the
2143 * PF, but that would require a bunch of additional IDAs -
2144 * one for each representor - and that's not likely to be
2145 * the main cause of recirc_id exhaustion anyway.)
2146 */
2147 rid = efx_tc_get_recirc_id(efx, tc->common.chain_index,
2148 efx->net_dev);
2149 if (IS_ERR(rid)) {
2150 NL_SET_ERR_MSG_FMT_MOD(extack,
2151 "Failed to allocate a hardware recirculation ID for chain_index %u",
2152 tc->common.chain_index);
2153 return PTR_ERR(rid);
2154 }
2155 match.rid = rid;
2156 match.value.recirc_id = rid->fw_id;
2157 }
2158 match.mask.recirc_id = 0xff;
2159
2160 /* AR table can't match on DO_CT (+trk). But a commonly used pattern is
2161 * +trk+est, which is strictly implied by +est, so rewrite it to that.
2162 */
2163 if (match.mask.ct_state_trk && match.value.ct_state_trk &&
2164 match.mask.ct_state_est && match.value.ct_state_est)
2165 match.mask.ct_state_trk = 0;
2166 /* Thanks to CT_TCP_FLAGS_INHIBIT, packets with interesting flags could
2167 * match +trk-est (CT_HIT=0) despite being on an established connection.
2168 * So make -est imply -tcp_syn_fin_rst match to ensure these packets
2169 * still hit the software path.
2170 */
2171 if (match.mask.ct_state_est && !match.value.ct_state_est) {
2172 if (match.value.tcp_syn_fin_rst) {
2173 /* Can't offload this combination */
2174 rc = -EOPNOTSUPP;
2175 goto release;
2176 }
2177 match.mask.tcp_syn_fin_rst = true;
2178 }
2179
2180 rc = efx_mae_match_check_caps(efx, &match.mask, extack);
2181 if (rc)
2182 goto release;
2183
2184 rule = kzalloc(sizeof(*rule), GFP_USER);
2185 if (!rule) {
2186 rc = -ENOMEM;
2187 goto release;
2188 }
2189 INIT_LIST_HEAD(&rule->acts.list);
2190 rule->cookie = tc->cookie;
2191 old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht,
2192 &rule->linkage,
2193 efx_tc_match_action_ht_params);
2194 if (IS_ERR(old)) {
2195 rc = PTR_ERR(old);
2196 goto release;
2197 } else if (old) {
2198 netif_dbg(efx, drv, efx->net_dev,
2199 "Already offloaded rule (cookie %lx)\n", tc->cookie);
2200 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
2201 rc = -EEXIST;
2202 goto release;
2203 }
2204
2205 /* Parse actions */
2206 act = kzalloc(sizeof(*act), GFP_USER);
2207 if (!act) {
2208 rc = -ENOMEM;
2209 goto release;
2210 }
2211
2212 /**
2213 * DOC: TC action translation
2214 *
2215 * Actions in TC are sequential and cumulative, with delivery actions
2216 * potentially anywhere in the order. The EF100 MAE, however, takes
2217 * an 'action set list' consisting of 'action sets', each of which is
2218 * applied to the _original_ packet, and consists of a set of optional
2219 * actions in a fixed order with delivery at the end.
2220 * To translate between these two models, we maintain a 'cursor', @act,
2221 * which describes the cumulative effect of all the packet-mutating
2222 * actions encountered so far; on handling a delivery (mirred or drop)
2223 * action, once the action-set has been inserted into hardware, we
2224 * append @act to the action-set list (@rule->acts); if this is a pipe
2225 * action (mirred mirror) we then allocate a new @act with a copy of
2226 * the cursor state _before_ the delivery action, otherwise we set @act
2227 * to %NULL.
2228 * This ensures that every allocated action-set is either attached to
2229 * @rule->acts or pointed to by @act (and never both), and that only
2230 * those action-sets in @rule->acts exist in hardware. Consequently,
2231 * in the failure path, @act only needs to be freed in memory, whereas
2232 * for @rule->acts we remove each action-set from hardware before
2233 * freeing it (efx_tc_free_action_set_list()), even if the action-set
2234 * list itself is not in hardware.
2235 */
2236 flow_action_for_each(i, fa, &fr->action) {
2237 struct efx_tc_action_set save;
2238 u16 tci;
2239
2240 if (!act) {
2241 /* more actions after a non-pipe action */
2242 NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action");
2243 rc = -EINVAL;
2244 goto release;
2245 }
2246
2247 if ((fa->id == FLOW_ACTION_REDIRECT ||
2248 fa->id == FLOW_ACTION_MIRRED ||
2249 fa->id == FLOW_ACTION_DROP) && fa->hw_stats) {
2250 struct efx_tc_counter_index *ctr;
2251
2252 /* Currently the only actions that want stats are
2253 * mirred and gact (ok, shot, trap, goto-chain), which
2254 * means we want stats just before delivery. Also,
2255 * note that tunnel_key set shouldn't change the length
2256 * — it's only the subsequent mirred that does that,
2257 * and the stats are taken _before_ the mirred action
2258 * happens.
2259 */
2260 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) {
2261 /* All supported actions that count either steal
2262 * (gact shot, mirred redirect) or clone act
2263 * (mirred mirror), so we should never get two
2264 * count actions on one action_set.
2265 */
2266 NL_SET_ERR_MSG_MOD(extack, "Count-action conflict (can't happen)");
2267 rc = -EOPNOTSUPP;
2268 goto release;
2269 }
2270
2271 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
2272 NL_SET_ERR_MSG_FMT_MOD(extack, "hw_stats_type %u not supported (only 'delayed')",
2273 fa->hw_stats);
2274 rc = -EOPNOTSUPP;
2275 goto release;
2276 }
2277
2278 ctr = efx_tc_flower_get_counter_index(efx, tc->cookie,
2279 EFX_TC_COUNTER_TYPE_AR);
2280 if (IS_ERR(ctr)) {
2281 rc = PTR_ERR(ctr);
2282 NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
2283 goto release;
2284 }
2285 act->count = ctr;
2286 INIT_LIST_HEAD(&act->count_user);
2287 }
2288
2289 switch (fa->id) {
2290 case FLOW_ACTION_DROP:
2291 rc = efx_mae_alloc_action_set(efx, act);
2292 if (rc) {
2293 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (drop)");
2294 goto release;
2295 }
2296 list_add_tail(&act->list, &rule->acts.list);
2297 act = NULL; /* end of the line */
2298 break;
2299 case FLOW_ACTION_REDIRECT:
2300 case FLOW_ACTION_MIRRED:
2301 save = *act;
2302
2303 if (encap_info) {
2304 struct efx_tc_encap_action *encap;
2305
2306 if (!efx_tc_flower_action_order_ok(act,
2307 EFX_TC_AO_ENCAP)) {
2308 rc = -EOPNOTSUPP;
2309 NL_SET_ERR_MSG_MOD(extack, "Encap action violates action order");
2310 goto release;
2311 }
2312 encap = efx_tc_flower_create_encap_md(
2313 efx, encap_info, fa->dev, extack);
2314 if (IS_ERR_OR_NULL(encap)) {
2315 rc = PTR_ERR(encap);
2316 if (!rc)
2317 rc = -EIO; /* arbitrary */
2318 goto release;
2319 }
2320 act->encap_md = encap;
2321 list_add_tail(&act->encap_user, &encap->users);
2322 act->dest_mport = encap->dest_mport;
2323 act->deliver = 1;
2324 if (act->count && !WARN_ON(!act->count->cnt)) {
2325 /* This counter is used by an encap
2326 * action, which needs a reference back
2327 * so it can prod neighbouring whenever
2328 * traffic is seen.
2329 */
2330 spin_lock_bh(&act->count->cnt->lock);
2331 list_add_tail(&act->count_user,
2332 &act->count->cnt->users);
2333 spin_unlock_bh(&act->count->cnt->lock);
2334 }
2335 rc = efx_mae_alloc_action_set(efx, act);
2336 if (rc) {
2337 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (encap)");
2338 goto release;
2339 }
2340 list_add_tail(&act->list, &rule->acts.list);
2341 act->user = &rule->acts;
2342 act = NULL;
2343 if (fa->id == FLOW_ACTION_REDIRECT)
2344 break; /* end of the line */
2345 /* Mirror, so continue on with saved act */
2346 save.count = NULL;
2347 act = kzalloc(sizeof(*act), GFP_USER);
2348 if (!act) {
2349 rc = -ENOMEM;
2350 goto release;
2351 }
2352 *act = save;
2353 break;
2354 }
2355
2356 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) {
2357 /* can't happen */
2358 rc = -EOPNOTSUPP;
2359 NL_SET_ERR_MSG_MOD(extack, "Deliver action violates action order (can't happen)");
2360 goto release;
2361 }
2362
2363 to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
2364 if (IS_ERR(to_efv)) {
2365 NL_SET_ERR_MSG_MOD(extack, "Mirred egress device not on switch");
2366 rc = PTR_ERR(to_efv);
2367 goto release;
2368 }
2369 rc = efx_tc_flower_external_mport(efx, to_efv);
2370 if (rc < 0) {
2371 NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port");
2372 goto release;
2373 }
2374 act->dest_mport = rc;
2375 act->deliver = 1;
2376 rc = efx_mae_alloc_action_set(efx, act);
2377 if (rc) {
2378 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (mirred)");
2379 goto release;
2380 }
2381 list_add_tail(&act->list, &rule->acts.list);
2382 act = NULL;
2383 if (fa->id == FLOW_ACTION_REDIRECT)
2384 break; /* end of the line */
2385 /* Mirror, so continue on with saved act */
2386 save.count = NULL;
2387 act = kzalloc(sizeof(*act), GFP_USER);
2388 if (!act) {
2389 rc = -ENOMEM;
2390 goto release;
2391 }
2392 *act = save;
2393 break;
2394 case FLOW_ACTION_VLAN_POP:
2395 if (act->vlan_push) {
2396 act->vlan_push--;
2397 } else if (efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_POP)) {
2398 act->vlan_pop++;
2399 } else {
2400 NL_SET_ERR_MSG_MOD(extack,
2401 "More than two VLAN pops, or action order violated");
2402 rc = -EINVAL;
2403 goto release;
2404 }
2405 break;
2406 case FLOW_ACTION_VLAN_PUSH:
2407 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_PUSH)) {
2408 rc = -EINVAL;
2409 NL_SET_ERR_MSG_MOD(extack,
2410 "More than two VLAN pushes, or action order violated");
2411 goto release;
2412 }
2413 tci = fa->vlan.vid & VLAN_VID_MASK;
2414 tci |= fa->vlan.prio << VLAN_PRIO_SHIFT;
2415 act->vlan_tci[act->vlan_push] = cpu_to_be16(tci);
2416 act->vlan_proto[act->vlan_push] = fa->vlan.proto;
2417 act->vlan_push++;
2418 break;
2419 case FLOW_ACTION_ADD:
2420 rc = efx_tc_pedit_add(efx, act, fa, extack);
2421 if (rc < 0)
2422 goto release;
2423 break;
2424 case FLOW_ACTION_MANGLE:
2425 rc = efx_tc_mangle(efx, act, fa, &mung, extack, &match);
2426 if (rc < 0)
2427 goto release;
2428 break;
2429 case FLOW_ACTION_TUNNEL_ENCAP:
2430 if (encap_info) {
2431 /* Can't specify encap multiple times.
2432 * If you want to overwrite an existing
2433 * encap_info, use an intervening
2434 * FLOW_ACTION_TUNNEL_DECAP to clear it.
2435 */
2436 NL_SET_ERR_MSG_MOD(extack, "Tunnel key set when already set");
2437 rc = -EINVAL;
2438 goto release;
2439 }
2440 if (!fa->tunnel) {
2441 NL_SET_ERR_MSG_MOD(extack, "Tunnel key set is missing key");
2442 rc = -EOPNOTSUPP;
2443 goto release;
2444 }
2445 encap_info = fa->tunnel;
2446 break;
2447 case FLOW_ACTION_TUNNEL_DECAP:
2448 if (encap_info) {
2449 encap_info = NULL;
2450 break;
2451 }
2452 /* Since we don't support enc_key matches on ingress
2453 * (and if we did there'd be no tunnel-device to give
2454 * us a type), we can't offload a decap that's not
2455 * just undoing a previous encap action.
2456 */
2457 NL_SET_ERR_MSG_MOD(extack, "Cannot offload tunnel decap action without tunnel device");
2458 rc = -EOPNOTSUPP;
2459 goto release;
2460 case FLOW_ACTION_CT:
2461 if (fa->ct.action != TCA_CT_ACT_NAT) {
2462 rc = -EOPNOTSUPP;
2463 NL_SET_ERR_MSG_FMT_MOD(extack, "Can only offload CT 'nat' action in RHS rules, not %d", fa->ct.action);
2464 goto release;
2465 }
2466 act->do_nat = 1;
2467 break;
2468 default:
2469 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u",
2470 fa->id);
2471 rc = -EOPNOTSUPP;
2472 goto release;
2473 }
2474 }
2475
2476 rc = efx_tc_incomplete_mangle(&mung, extack);
2477 if (rc < 0)
2478 goto release;
2479 if (act) {
2480 /* Not shot/redirected, so deliver to default dest */
2481 if (from_efv == EFX_EFV_PF)
2482 /* Rule applies to traffic from the wire,
2483 * and default dest is thus the PF
2484 */
2485 efx_mae_mport_uplink(efx, &act->dest_mport);
2486 else
2487 /* Representor, so rule applies to traffic from
2488 * representee, and default dest is thus the rep.
2489 * All reps use the same mport for delivery
2490 */
2491 efx_mae_mport_mport(efx, efx->tc->reps_mport_id,
2492 &act->dest_mport);
2493 act->deliver = 1;
2494 rc = efx_mae_alloc_action_set(efx, act);
2495 if (rc) {
2496 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)");
2497 goto release;
2498 }
2499 list_add_tail(&act->list, &rule->acts.list);
2500 act = NULL; /* Prevent double-free in error path */
2501 }
2502
2503 netif_dbg(efx, drv, efx->net_dev,
2504 "Successfully parsed filter (cookie %lx)\n",
2505 tc->cookie);
2506
2507 rule->match = match;
2508
2509 rc = efx_mae_alloc_action_set_list(efx, &rule->acts);
2510 if (rc) {
2511 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw");
2512 goto release;
2513 }
2514 if (from_efv == EFX_EFV_PF)
2515 /* PF netdev, so rule applies to traffic from wire */
2516 rule->fallback = &efx->tc->facts.pf;
2517 else
2518 /* repdev, so rule applies to traffic from representee */
2519 rule->fallback = &efx->tc->facts.reps;
2520 if (!efx_tc_check_ready(efx, rule)) {
2521 netif_dbg(efx, drv, efx->net_dev, "action not ready for hw\n");
2522 acts_id = rule->fallback->fw_id;
2523 } else {
2524 netif_dbg(efx, drv, efx->net_dev, "ready for hw\n");
2525 acts_id = rule->acts.fw_id;
2526 }
2527 rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,
2528 acts_id, &rule->fw_id);
2529 if (rc) {
2530 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
2531 goto release_acts;
2532 }
2533 return 0;
2534
2535release_acts:
2536 efx_mae_free_action_set_list(efx, &rule->acts);
2537release:
2538 /* We failed to insert the rule, so free up any entries we created in
2539 * subsidiary tables.
2540 */
2541 if (match.rid)
2542 efx_tc_put_recirc_id(efx, match.rid);
2543 if (act)
2544 efx_tc_free_action_set(efx, act, false);
2545 if (rule) {
2546 if (!old)
2547 rhashtable_remove_fast(&efx->tc->match_action_ht,
2548 &rule->linkage,
2549 efx_tc_match_action_ht_params);
2550 efx_tc_free_action_set_list(efx, &rule->acts, false);
2551 }
2552 kfree(rule);
2553 return rc;
2554}
2555
2556static int efx_tc_flower_destroy(struct efx_nic *efx,
2557 struct net_device *net_dev,
2558 struct flow_cls_offload *tc)
2559{
2560 struct netlink_ext_ack *extack = tc->common.extack;
2561 struct efx_tc_lhs_rule *lhs_rule;
2562 struct efx_tc_flow_rule *rule;
2563
2564 lhs_rule = rhashtable_lookup_fast(&efx->tc->lhs_rule_ht, &tc->cookie,
2565 efx_tc_lhs_rule_ht_params);
2566 if (lhs_rule) {
2567 /* Remove it from HW */
2568 efx_mae_remove_lhs_rule(efx, lhs_rule);
2569 /* Delete it from SW */
2570 efx_tc_flower_release_lhs_actions(efx, &lhs_rule->lhs_act);
2571 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &lhs_rule->linkage,
2572 efx_tc_lhs_rule_ht_params);
2573 if (lhs_rule->match.encap)
2574 efx_tc_flower_release_encap_match(efx, lhs_rule->match.encap);
2575 netif_dbg(efx, drv, efx->net_dev, "Removed (lhs) filter %lx\n",
2576 lhs_rule->cookie);
2577 kfree(lhs_rule);
2578 return 0;
2579 }
2580
2581 rule = rhashtable_lookup_fast(&efx->tc->match_action_ht, &tc->cookie,
2582 efx_tc_match_action_ht_params);
2583 if (!rule) {
2584 /* Only log a message if we're the ingress device. Otherwise
2585 * it's a foreign filter and we might just not have been
2586 * interested (e.g. we might not have been the egress device
2587 * either).
2588 */
2589 if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev)))
2590 netif_warn(efx, drv, efx->net_dev,
2591 "Filter %lx not found to remove\n", tc->cookie);
2592 NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules");
2593 return -ENOENT;
2594 }
2595
2596 /* Remove it from HW */
2597 efx_tc_delete_rule(efx, rule);
2598 /* Delete it from SW */
2599 rhashtable_remove_fast(&efx->tc->match_action_ht, &rule->linkage,
2600 efx_tc_match_action_ht_params);
2601 netif_dbg(efx, drv, efx->net_dev, "Removed filter %lx\n", rule->cookie);
2602 kfree(rule);
2603 return 0;
2604}
2605
2606static int efx_tc_flower_stats(struct efx_nic *efx, struct net_device *net_dev,
2607 struct flow_cls_offload *tc)
2608{
2609 struct netlink_ext_ack *extack = tc->common.extack;
2610 struct efx_tc_counter_index *ctr;
2611 struct efx_tc_counter *cnt;
2612 u64 packets, bytes;
2613
2614 ctr = efx_tc_flower_find_counter_index(efx, tc->cookie);
2615 if (!ctr) {
2616 /* See comment in efx_tc_flower_destroy() */
2617 if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev)))
2618 if (net_ratelimit())
2619 netif_warn(efx, drv, efx->net_dev,
2620 "Filter %lx not found for stats\n",
2621 tc->cookie);
2622 NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules");
2623 return -ENOENT;
2624 }
2625 if (WARN_ON(!ctr->cnt)) /* can't happen */
2626 return -EIO;
2627 cnt = ctr->cnt;
2628
2629 spin_lock_bh(&cnt->lock);
2630 /* Report only new pkts/bytes since last time TC asked */
2631 packets = cnt->packets;
2632 bytes = cnt->bytes;
2633 flow_stats_update(&tc->stats, bytes - cnt->old_bytes,
2634 packets - cnt->old_packets, 0, cnt->touched,
2635 FLOW_ACTION_HW_STATS_DELAYED);
2636 cnt->old_packets = packets;
2637 cnt->old_bytes = bytes;
2638 spin_unlock_bh(&cnt->lock);
2639 return 0;
2640}
2641
2642int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev,
2643 struct flow_cls_offload *tc, struct efx_rep *efv)
2644{
2645 int rc;
2646
2647 if (!efx->tc)
2648 return -EOPNOTSUPP;
2649
2650 mutex_lock(&efx->tc->mutex);
2651 switch (tc->command) {
2652 case FLOW_CLS_REPLACE:
2653 rc = efx_tc_flower_replace(efx, net_dev, tc, efv);
2654 break;
2655 case FLOW_CLS_DESTROY:
2656 rc = efx_tc_flower_destroy(efx, net_dev, tc);
2657 break;
2658 case FLOW_CLS_STATS:
2659 rc = efx_tc_flower_stats(efx, net_dev, tc);
2660 break;
2661 default:
2662 rc = -EOPNOTSUPP;
2663 break;
2664 }
2665 mutex_unlock(&efx->tc->mutex);
2666 return rc;
2667}
2668
2669static int efx_tc_configure_default_rule(struct efx_nic *efx, u32 ing_port,
2670 u32 eg_port, struct efx_tc_flow_rule *rule)
2671{
2672 struct efx_tc_action_set_list *acts = &rule->acts;
2673 struct efx_tc_match *match = &rule->match;
2674 struct efx_tc_action_set *act;
2675 int rc;
2676
2677 match->value.ingress_port = ing_port;
2678 match->mask.ingress_port = ~0;
2679 act = kzalloc(sizeof(*act), GFP_KERNEL);
2680 if (!act)
2681 return -ENOMEM;
2682 act->deliver = 1;
2683 act->dest_mport = eg_port;
2684 rc = efx_mae_alloc_action_set(efx, act);
2685 if (rc)
2686 goto fail1;
2687 EFX_WARN_ON_PARANOID(!list_empty(&acts->list));
2688 list_add_tail(&act->list, &acts->list);
2689 rc = efx_mae_alloc_action_set_list(efx, acts);
2690 if (rc)
2691 goto fail2;
2692 rc = efx_mae_insert_rule(efx, match, EFX_TC_PRIO_DFLT,
2693 acts->fw_id, &rule->fw_id);
2694 if (rc)
2695 goto fail3;
2696 return 0;
2697fail3:
2698 efx_mae_free_action_set_list(efx, acts);
2699fail2:
2700 list_del(&act->list);
2701 efx_mae_free_action_set(efx, act->fw_id);
2702fail1:
2703 kfree(act);
2704 return rc;
2705}
2706
2707static int efx_tc_configure_default_rule_pf(struct efx_nic *efx)
2708{
2709 struct efx_tc_flow_rule *rule = &efx->tc->dflt.pf;
2710 u32 ing_port, eg_port;
2711
2712 efx_mae_mport_uplink(efx, &ing_port);
2713 efx_mae_mport_wire(efx, &eg_port);
2714 return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);
2715}
2716
2717static int efx_tc_configure_default_rule_wire(struct efx_nic *efx)
2718{
2719 struct efx_tc_flow_rule *rule = &efx->tc->dflt.wire;
2720 u32 ing_port, eg_port;
2721
2722 efx_mae_mport_wire(efx, &ing_port);
2723 efx_mae_mport_uplink(efx, &eg_port);
2724 return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);
2725}
2726
2727int efx_tc_configure_default_rule_rep(struct efx_rep *efv)
2728{
2729 struct efx_tc_flow_rule *rule = &efv->dflt;
2730 struct efx_nic *efx = efv->parent;
2731 u32 ing_port, eg_port;
2732
2733 efx_mae_mport_mport(efx, efv->mport, &ing_port);
2734 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port);
2735 return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);
2736}
2737
2738void efx_tc_deconfigure_default_rule(struct efx_nic *efx,
2739 struct efx_tc_flow_rule *rule)
2740{
2741 if (rule->fw_id != MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL)
2742 efx_tc_delete_rule(efx, rule);
2743 rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
2744}
2745
2746static int efx_tc_configure_fallback_acts(struct efx_nic *efx, u32 eg_port,
2747 struct efx_tc_action_set_list *acts)
2748{
2749 struct efx_tc_action_set *act;
2750 int rc;
2751
2752 act = kzalloc(sizeof(*act), GFP_KERNEL);
2753 if (!act)
2754 return -ENOMEM;
2755 act->deliver = 1;
2756 act->dest_mport = eg_port;
2757 rc = efx_mae_alloc_action_set(efx, act);
2758 if (rc)
2759 goto fail1;
2760 EFX_WARN_ON_PARANOID(!list_empty(&acts->list));
2761 list_add_tail(&act->list, &acts->list);
2762 rc = efx_mae_alloc_action_set_list(efx, acts);
2763 if (rc)
2764 goto fail2;
2765 return 0;
2766fail2:
2767 list_del(&act->list);
2768 efx_mae_free_action_set(efx, act->fw_id);
2769fail1:
2770 kfree(act);
2771 return rc;
2772}
2773
2774static int efx_tc_configure_fallback_acts_pf(struct efx_nic *efx)
2775{
2776 struct efx_tc_action_set_list *acts = &efx->tc->facts.pf;
2777 u32 eg_port;
2778
2779 efx_mae_mport_uplink(efx, &eg_port);
2780 return efx_tc_configure_fallback_acts(efx, eg_port, acts);
2781}
2782
2783static int efx_tc_configure_fallback_acts_reps(struct efx_nic *efx)
2784{
2785 struct efx_tc_action_set_list *acts = &efx->tc->facts.reps;
2786 u32 eg_port;
2787
2788 efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port);
2789 return efx_tc_configure_fallback_acts(efx, eg_port, acts);
2790}
2791
2792static void efx_tc_deconfigure_fallback_acts(struct efx_nic *efx,
2793 struct efx_tc_action_set_list *acts)
2794{
2795 efx_tc_free_action_set_list(efx, acts, true);
2796}
2797
2798static int efx_tc_configure_rep_mport(struct efx_nic *efx)
2799{
2800 u32 rep_mport_label;
2801 int rc;
2802
2803 rc = efx_mae_allocate_mport(efx, &efx->tc->reps_mport_id, &rep_mport_label);
2804 if (rc)
2805 return rc;
2806 pci_dbg(efx->pci_dev, "created rep mport 0x%08x (0x%04x)\n",
2807 efx->tc->reps_mport_id, rep_mport_label);
2808 /* Use mport *selector* as vport ID */
2809 efx_mae_mport_mport(efx, efx->tc->reps_mport_id,
2810 &efx->tc->reps_mport_vport_id);
2811 return 0;
2812}
2813
2814static void efx_tc_deconfigure_rep_mport(struct efx_nic *efx)
2815{
2816 efx_mae_free_mport(efx, efx->tc->reps_mport_id);
2817 efx->tc->reps_mport_id = MAE_MPORT_SELECTOR_NULL;
2818}
2819
2820int efx_tc_insert_rep_filters(struct efx_nic *efx)
2821{
2822 struct efx_filter_spec promisc, allmulti;
2823 int rc;
2824
2825 if (efx->type->is_vf)
2826 return 0;
2827 if (!efx->tc)
2828 return 0;
2829 efx_filter_init_rx(&promisc, EFX_FILTER_PRI_REQUIRED, 0, 0);
2830 efx_filter_set_uc_def(&promisc);
2831 efx_filter_set_vport_id(&promisc, efx->tc->reps_mport_vport_id);
2832 rc = efx_filter_insert_filter(efx, &promisc, false);
2833 if (rc < 0)
2834 return rc;
2835 efx->tc->reps_filter_uc = rc;
2836 efx_filter_init_rx(&allmulti, EFX_FILTER_PRI_REQUIRED, 0, 0);
2837 efx_filter_set_mc_def(&allmulti);
2838 efx_filter_set_vport_id(&allmulti, efx->tc->reps_mport_vport_id);
2839 rc = efx_filter_insert_filter(efx, &allmulti, false);
2840 if (rc < 0)
2841 return rc;
2842 efx->tc->reps_filter_mc = rc;
2843 return 0;
2844}
2845
2846void efx_tc_remove_rep_filters(struct efx_nic *efx)
2847{
2848 if (efx->type->is_vf)
2849 return;
2850 if (!efx->tc)
2851 return;
2852 if (efx->tc->reps_filter_mc >= 0)
2853 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_mc);
2854 efx->tc->reps_filter_mc = -1;
2855 if (efx->tc->reps_filter_uc >= 0)
2856 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_uc);
2857 efx->tc->reps_filter_uc = -1;
2858}
2859
2860int efx_init_tc(struct efx_nic *efx)
2861{
2862 int rc;
2863
2864 rc = efx_mae_get_caps(efx, efx->tc->caps);
2865 if (rc)
2866 return rc;
2867 if (efx->tc->caps->match_field_count > MAE_NUM_FIELDS)
2868 /* Firmware supports some match fields the driver doesn't know
2869 * about. Not fatal, unless any of those fields are required
2870 * (MAE_FIELD_SUPPORTED_MATCH_ALWAYS) but if so we don't know.
2871 */
2872 netif_warn(efx, probe, efx->net_dev,
2873 "FW reports additional match fields %u\n",
2874 efx->tc->caps->match_field_count);
2875 if (efx->tc->caps->action_prios < EFX_TC_PRIO__NUM) {
2876 netif_err(efx, probe, efx->net_dev,
2877 "Too few action prios supported (have %u, need %u)\n",
2878 efx->tc->caps->action_prios, EFX_TC_PRIO__NUM);
2879 return -EIO;
2880 }
2881 rc = efx_tc_configure_default_rule_pf(efx);
2882 if (rc)
2883 return rc;
2884 rc = efx_tc_configure_default_rule_wire(efx);
2885 if (rc)
2886 return rc;
2887 rc = efx_tc_configure_rep_mport(efx);
2888 if (rc)
2889 return rc;
2890 rc = efx_tc_configure_fallback_acts_pf(efx);
2891 if (rc)
2892 return rc;
2893 rc = efx_tc_configure_fallback_acts_reps(efx);
2894 if (rc)
2895 return rc;
2896 rc = efx_mae_get_tables(efx);
2897 if (rc)
2898 return rc;
2899 rc = flow_indr_dev_register(efx_tc_indr_setup_cb, efx);
2900 if (rc)
2901 goto out_free;
2902 efx->tc->up = true;
2903 return 0;
2904out_free:
2905 efx_mae_free_tables(efx);
2906 return rc;
2907}
2908
2909void efx_fini_tc(struct efx_nic *efx)
2910{
2911 /* We can get called even if efx_init_struct_tc() failed */
2912 if (!efx->tc)
2913 return;
2914 if (efx->tc->up)
2915 flow_indr_dev_unregister(efx_tc_indr_setup_cb, efx, efx_tc_block_unbind);
2916 efx_tc_deconfigure_rep_mport(efx);
2917 efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.pf);
2918 efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.wire);
2919 efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.pf);
2920 efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.reps);
2921 efx->tc->up = false;
2922 efx_mae_free_tables(efx);
2923}
2924
2925/* At teardown time, all TC filter rules (and thus all resources they created)
2926 * should already have been removed. If we find any in our hashtables, make a
2927 * cursory attempt to clean up the software side.
2928 */
2929static void efx_tc_encap_match_free(void *ptr, void *__unused)
2930{
2931 struct efx_tc_encap_match *encap = ptr;
2932
2933 WARN_ON(refcount_read(&encap->ref));
2934 kfree(encap);
2935}
2936
2937static void efx_tc_recirc_free(void *ptr, void *arg)
2938{
2939 struct efx_tc_recirc_id *rid = ptr;
2940 struct efx_nic *efx = arg;
2941
2942 WARN_ON(refcount_read(&rid->ref));
2943 ida_free(&efx->tc->recirc_ida, rid->fw_id);
2944 kfree(rid);
2945}
2946
2947static void efx_tc_lhs_free(void *ptr, void *arg)
2948{
2949 struct efx_tc_lhs_rule *rule = ptr;
2950 struct efx_nic *efx = arg;
2951
2952 netif_err(efx, drv, efx->net_dev,
2953 "tc lhs_rule %lx still present at teardown, removing\n",
2954 rule->cookie);
2955
2956 if (rule->lhs_act.zone)
2957 efx_tc_ct_unregister_zone(efx, rule->lhs_act.zone);
2958 if (rule->lhs_act.count)
2959 efx_tc_flower_put_counter_index(efx, rule->lhs_act.count);
2960 efx_mae_remove_lhs_rule(efx, rule);
2961
2962 kfree(rule);
2963}
2964
2965static void efx_tc_mac_free(void *ptr, void *__unused)
2966{
2967 struct efx_tc_mac_pedit_action *ped = ptr;
2968
2969 WARN_ON(refcount_read(&ped->ref));
2970 kfree(ped);
2971}
2972
2973static void efx_tc_flow_free(void *ptr, void *arg)
2974{
2975 struct efx_tc_flow_rule *rule = ptr;
2976 struct efx_nic *efx = arg;
2977
2978 netif_err(efx, drv, efx->net_dev,
2979 "tc rule %lx still present at teardown, removing\n",
2980 rule->cookie);
2981
2982 /* Also releases entries in subsidiary tables */
2983 efx_tc_delete_rule(efx, rule);
2984
2985 kfree(rule);
2986}
2987
2988int efx_init_struct_tc(struct efx_nic *efx)
2989{
2990 int rc;
2991
2992 if (efx->type->is_vf)
2993 return 0;
2994
2995 efx->tc = kzalloc(sizeof(*efx->tc), GFP_KERNEL);
2996 if (!efx->tc)
2997 return -ENOMEM;
2998 efx->tc->caps = kzalloc(sizeof(struct mae_caps), GFP_KERNEL);
2999 if (!efx->tc->caps) {
3000 rc = -ENOMEM;
3001 goto fail_alloc_caps;
3002 }
3003 INIT_LIST_HEAD(&efx->tc->block_list);
3004
3005 mutex_init(&efx->tc->mutex);
3006 init_waitqueue_head(&efx->tc->flush_wq);
3007 rc = efx_tc_init_encap_actions(efx);
3008 if (rc < 0)
3009 goto fail_encap_actions;
3010 rc = efx_tc_init_counters(efx);
3011 if (rc < 0)
3012 goto fail_counters;
3013 rc = rhashtable_init(&efx->tc->mac_ht, &efx_tc_mac_ht_params);
3014 if (rc < 0)
3015 goto fail_mac_ht;
3016 rc = rhashtable_init(&efx->tc->encap_match_ht, &efx_tc_encap_match_ht_params);
3017 if (rc < 0)
3018 goto fail_encap_match_ht;
3019 rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params);
3020 if (rc < 0)
3021 goto fail_match_action_ht;
3022 rc = rhashtable_init(&efx->tc->lhs_rule_ht, &efx_tc_lhs_rule_ht_params);
3023 if (rc < 0)
3024 goto fail_lhs_rule_ht;
3025 rc = efx_tc_init_conntrack(efx);
3026 if (rc < 0)
3027 goto fail_conntrack;
3028 rc = rhashtable_init(&efx->tc->recirc_ht, &efx_tc_recirc_ht_params);
3029 if (rc < 0)
3030 goto fail_recirc_ht;
3031 ida_init(&efx->tc->recirc_ida);
3032 efx->tc->reps_filter_uc = -1;
3033 efx->tc->reps_filter_mc = -1;
3034 INIT_LIST_HEAD(&efx->tc->dflt.pf.acts.list);
3035 efx->tc->dflt.pf.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
3036 INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list);
3037 efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
3038 INIT_LIST_HEAD(&efx->tc->facts.pf.list);
3039 efx->tc->facts.pf.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL;
3040 INIT_LIST_HEAD(&efx->tc->facts.reps.list);
3041 efx->tc->facts.reps.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL;
3042 efx->extra_channel_type[EFX_EXTRA_CHANNEL_TC] = &efx_tc_channel_type;
3043 return 0;
3044fail_recirc_ht:
3045 efx_tc_destroy_conntrack(efx);
3046fail_conntrack:
3047 rhashtable_destroy(&efx->tc->lhs_rule_ht);
3048fail_lhs_rule_ht:
3049 rhashtable_destroy(&efx->tc->match_action_ht);
3050fail_match_action_ht:
3051 rhashtable_destroy(&efx->tc->encap_match_ht);
3052fail_encap_match_ht:
3053 rhashtable_destroy(&efx->tc->mac_ht);
3054fail_mac_ht:
3055 efx_tc_destroy_counters(efx);
3056fail_counters:
3057 efx_tc_destroy_encap_actions(efx);
3058fail_encap_actions:
3059 mutex_destroy(&efx->tc->mutex);
3060 kfree(efx->tc->caps);
3061fail_alloc_caps:
3062 kfree(efx->tc);
3063 efx->tc = NULL;
3064 return rc;
3065}
3066
3067void efx_fini_struct_tc(struct efx_nic *efx)
3068{
3069 if (!efx->tc)
3070 return;
3071
3072 mutex_lock(&efx->tc->mutex);
3073 EFX_WARN_ON_PARANOID(efx->tc->dflt.pf.fw_id !=
3074 MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);
3075 EFX_WARN_ON_PARANOID(efx->tc->dflt.wire.fw_id !=
3076 MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);
3077 EFX_WARN_ON_PARANOID(efx->tc->facts.pf.fw_id !=
3078 MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL);
3079 EFX_WARN_ON_PARANOID(efx->tc->facts.reps.fw_id !=
3080 MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL);
3081 rhashtable_free_and_destroy(&efx->tc->lhs_rule_ht, efx_tc_lhs_free, efx);
3082 rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free,
3083 efx);
3084 rhashtable_free_and_destroy(&efx->tc->encap_match_ht,
3085 efx_tc_encap_match_free, NULL);
3086 efx_tc_fini_conntrack(efx);
3087 rhashtable_free_and_destroy(&efx->tc->recirc_ht, efx_tc_recirc_free, efx);
3088 WARN_ON(!ida_is_empty(&efx->tc->recirc_ida));
3089 ida_destroy(&efx->tc->recirc_ida);
3090 rhashtable_free_and_destroy(&efx->tc->mac_ht, efx_tc_mac_free, NULL);
3091 efx_tc_fini_counters(efx);
3092 efx_tc_fini_encap_actions(efx);
3093 mutex_unlock(&efx->tc->mutex);
3094 mutex_destroy(&efx->tc->mutex);
3095 kfree(efx->tc->caps);
3096 kfree(efx->tc);
3097 efx->tc = NULL;
3098}