Loading...
1// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2/* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
3
4#include <linux/bitmap.h>
5#include <linux/errno.h>
6#include <linux/genalloc.h>
7#include <linux/gfp.h>
8#include <linux/kernel.h>
9#include <linux/list.h>
10#include <linux/mutex.h>
11#include <linux/objagg.h>
12#include <linux/rtnetlink.h>
13#include <linux/slab.h>
14
15#include "core.h"
16#include "reg.h"
17#include "spectrum.h"
18#include "spectrum_acl_tcam.h"
19
20/* gen_pool_alloc() returns 0 when allocation fails, so use an offset */
21#define MLXSW_SP_ACL_ERP_GENALLOC_OFFSET 0x100
22#define MLXSW_SP_ACL_ERP_MAX_PER_REGION 16
23
24struct mlxsw_sp_acl_erp_core {
25 unsigned int erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_MAX + 1];
26 struct gen_pool *erp_tables;
27 struct mlxsw_sp *mlxsw_sp;
28 struct mlxsw_sp_acl_bf *bf;
29 unsigned int num_erp_banks;
30};
31
32struct mlxsw_sp_acl_erp_key {
33 char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN];
34#define __MASK_LEN 0x38
35#define __MASK_IDX(i) (__MASK_LEN - (i) - 1)
36 bool ctcam;
37};
38
39struct mlxsw_sp_acl_erp {
40 struct mlxsw_sp_acl_erp_key key;
41 u8 id;
42 u8 index;
43 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
44 struct list_head list;
45 struct mlxsw_sp_acl_erp_table *erp_table;
46};
47
48struct mlxsw_sp_acl_erp_master_mask {
49 DECLARE_BITMAP(bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
50 unsigned int count[MLXSW_SP_ACL_TCAM_MASK_LEN];
51};
52
53struct mlxsw_sp_acl_erp_table {
54 struct mlxsw_sp_acl_erp_master_mask master_mask;
55 DECLARE_BITMAP(erp_id_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);
56 DECLARE_BITMAP(erp_index_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);
57 struct list_head atcam_erps_list;
58 struct mlxsw_sp_acl_erp_core *erp_core;
59 struct mlxsw_sp_acl_atcam_region *aregion;
60 const struct mlxsw_sp_acl_erp_table_ops *ops;
61 unsigned long base_index;
62 unsigned int num_atcam_erps;
63 unsigned int num_max_atcam_erps;
64 unsigned int num_ctcam_erps;
65 unsigned int num_deltas;
66 struct objagg *objagg;
67 struct mutex objagg_lock; /* guards objagg manipulation */
68};
69
70struct mlxsw_sp_acl_erp_table_ops {
71 struct mlxsw_sp_acl_erp *
72 (*erp_create)(struct mlxsw_sp_acl_erp_table *erp_table,
73 struct mlxsw_sp_acl_erp_key *key);
74 void (*erp_destroy)(struct mlxsw_sp_acl_erp_table *erp_table,
75 struct mlxsw_sp_acl_erp *erp);
76};
77
78static struct mlxsw_sp_acl_erp *
79mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
80 struct mlxsw_sp_acl_erp_key *key);
81static void
82mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
83 struct mlxsw_sp_acl_erp *erp);
84static struct mlxsw_sp_acl_erp *
85mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
86 struct mlxsw_sp_acl_erp_key *key);
87static void
88mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
89 struct mlxsw_sp_acl_erp *erp);
90static struct mlxsw_sp_acl_erp *
91mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
92 struct mlxsw_sp_acl_erp_key *key);
93static void
94mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
95 struct mlxsw_sp_acl_erp *erp);
96static void
97mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
98 struct mlxsw_sp_acl_erp *erp);
99
100static const struct mlxsw_sp_acl_erp_table_ops erp_multiple_masks_ops = {
101 .erp_create = mlxsw_sp_acl_erp_mask_create,
102 .erp_destroy = mlxsw_sp_acl_erp_mask_destroy,
103};
104
105static const struct mlxsw_sp_acl_erp_table_ops erp_two_masks_ops = {
106 .erp_create = mlxsw_sp_acl_erp_mask_create,
107 .erp_destroy = mlxsw_sp_acl_erp_second_mask_destroy,
108};
109
110static const struct mlxsw_sp_acl_erp_table_ops erp_single_mask_ops = {
111 .erp_create = mlxsw_sp_acl_erp_second_mask_create,
112 .erp_destroy = mlxsw_sp_acl_erp_first_mask_destroy,
113};
114
115static const struct mlxsw_sp_acl_erp_table_ops erp_no_mask_ops = {
116 .erp_create = mlxsw_sp_acl_erp_first_mask_create,
117 .erp_destroy = mlxsw_sp_acl_erp_no_mask_destroy,
118};
119
120static bool
121mlxsw_sp_acl_erp_table_is_used(const struct mlxsw_sp_acl_erp_table *erp_table)
122{
123 return erp_table->ops != &erp_single_mask_ops &&
124 erp_table->ops != &erp_no_mask_ops;
125}
126
127static unsigned int
128mlxsw_sp_acl_erp_bank_get(const struct mlxsw_sp_acl_erp *erp)
129{
130 return erp->index % erp->erp_table->erp_core->num_erp_banks;
131}
132
133static unsigned int
134mlxsw_sp_acl_erp_table_entry_size(const struct mlxsw_sp_acl_erp_table *erp_table)
135{
136 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
137 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
138
139 return erp_core->erpt_entries_size[aregion->type];
140}
141
142static int mlxsw_sp_acl_erp_id_get(struct mlxsw_sp_acl_erp_table *erp_table,
143 u8 *p_id)
144{
145 u8 id;
146
147 id = find_first_zero_bit(erp_table->erp_id_bitmap,
148 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
149 if (id < MLXSW_SP_ACL_ERP_MAX_PER_REGION) {
150 __set_bit(id, erp_table->erp_id_bitmap);
151 *p_id = id;
152 return 0;
153 }
154
155 return -ENOBUFS;
156}
157
158static void mlxsw_sp_acl_erp_id_put(struct mlxsw_sp_acl_erp_table *erp_table,
159 u8 id)
160{
161 __clear_bit(id, erp_table->erp_id_bitmap);
162}
163
164static void
165mlxsw_sp_acl_erp_master_mask_bit_set(unsigned long bit,
166 struct mlxsw_sp_acl_erp_master_mask *mask)
167{
168 if (mask->count[bit]++ == 0)
169 __set_bit(bit, mask->bitmap);
170}
171
172static void
173mlxsw_sp_acl_erp_master_mask_bit_clear(unsigned long bit,
174 struct mlxsw_sp_acl_erp_master_mask *mask)
175{
176 if (--mask->count[bit] == 0)
177 __clear_bit(bit, mask->bitmap);
178}
179
180static int
181mlxsw_sp_acl_erp_master_mask_update(struct mlxsw_sp_acl_erp_table *erp_table)
182{
183 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
184 struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;
185 char percr_pl[MLXSW_REG_PERCR_LEN];
186 char *master_mask;
187
188 mlxsw_reg_percr_pack(percr_pl, region->id);
189 master_mask = mlxsw_reg_percr_master_mask_data(percr_pl);
190 bitmap_to_arr32((u32 *) master_mask, erp_table->master_mask.bitmap,
191 MLXSW_SP_ACL_TCAM_MASK_LEN);
192
193 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl);
194}
195
196static int
197mlxsw_sp_acl_erp_master_mask_set(struct mlxsw_sp_acl_erp_table *erp_table,
198 struct mlxsw_sp_acl_erp_key *key)
199{
200 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
201 unsigned long bit;
202 int err;
203
204 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask,
205 MLXSW_SP_ACL_TCAM_MASK_LEN);
206 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
207 mlxsw_sp_acl_erp_master_mask_bit_set(bit,
208 &erp_table->master_mask);
209
210 err = mlxsw_sp_acl_erp_master_mask_update(erp_table);
211 if (err)
212 goto err_master_mask_update;
213
214 return 0;
215
216err_master_mask_update:
217 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
218 mlxsw_sp_acl_erp_master_mask_bit_clear(bit,
219 &erp_table->master_mask);
220 return err;
221}
222
223static int
224mlxsw_sp_acl_erp_master_mask_clear(struct mlxsw_sp_acl_erp_table *erp_table,
225 struct mlxsw_sp_acl_erp_key *key)
226{
227 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
228 unsigned long bit;
229 int err;
230
231 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask,
232 MLXSW_SP_ACL_TCAM_MASK_LEN);
233 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
234 mlxsw_sp_acl_erp_master_mask_bit_clear(bit,
235 &erp_table->master_mask);
236
237 err = mlxsw_sp_acl_erp_master_mask_update(erp_table);
238 if (err)
239 goto err_master_mask_update;
240
241 return 0;
242
243err_master_mask_update:
244 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
245 mlxsw_sp_acl_erp_master_mask_bit_set(bit,
246 &erp_table->master_mask);
247 return err;
248}
249
250static struct mlxsw_sp_acl_erp *
251mlxsw_sp_acl_erp_generic_create(struct mlxsw_sp_acl_erp_table *erp_table,
252 struct mlxsw_sp_acl_erp_key *key)
253{
254 struct mlxsw_sp_acl_erp *erp;
255 int err;
256
257 erp = kzalloc(sizeof(*erp), GFP_KERNEL);
258 if (!erp)
259 return ERR_PTR(-ENOMEM);
260
261 err = mlxsw_sp_acl_erp_id_get(erp_table, &erp->id);
262 if (err)
263 goto err_erp_id_get;
264
265 memcpy(&erp->key, key, sizeof(*key));
266 list_add(&erp->list, &erp_table->atcam_erps_list);
267 erp_table->num_atcam_erps++;
268 erp->erp_table = erp_table;
269
270 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key);
271 if (err)
272 goto err_master_mask_set;
273
274 return erp;
275
276err_master_mask_set:
277 erp_table->num_atcam_erps--;
278 list_del(&erp->list);
279 mlxsw_sp_acl_erp_id_put(erp_table, erp->id);
280err_erp_id_get:
281 kfree(erp);
282 return ERR_PTR(err);
283}
284
285static void
286mlxsw_sp_acl_erp_generic_destroy(struct mlxsw_sp_acl_erp *erp)
287{
288 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
289
290 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
291 erp_table->num_atcam_erps--;
292 list_del(&erp->list);
293 mlxsw_sp_acl_erp_id_put(erp_table, erp->id);
294 kfree(erp);
295}
296
297static int
298mlxsw_sp_acl_erp_table_alloc(struct mlxsw_sp_acl_erp_core *erp_core,
299 unsigned int num_erps,
300 enum mlxsw_sp_acl_atcam_region_type region_type,
301 unsigned long *p_index)
302{
303 unsigned int num_rows, entry_size;
304 unsigned long index;
305
306 /* We only allow allocations of entire rows */
307 if (num_erps % erp_core->num_erp_banks != 0)
308 return -EINVAL;
309
310 entry_size = erp_core->erpt_entries_size[region_type];
311 num_rows = num_erps / erp_core->num_erp_banks;
312
313 index = gen_pool_alloc(erp_core->erp_tables, num_rows * entry_size);
314 if (!index)
315 return -ENOBUFS;
316
317 *p_index = index - MLXSW_SP_ACL_ERP_GENALLOC_OFFSET;
318
319 return 0;
320}
321
322static void
323mlxsw_sp_acl_erp_table_free(struct mlxsw_sp_acl_erp_core *erp_core,
324 unsigned int num_erps,
325 enum mlxsw_sp_acl_atcam_region_type region_type,
326 unsigned long index)
327{
328 unsigned long base_index;
329 unsigned int entry_size;
330 size_t size;
331
332 entry_size = erp_core->erpt_entries_size[region_type];
333 base_index = index + MLXSW_SP_ACL_ERP_GENALLOC_OFFSET;
334 size = num_erps / erp_core->num_erp_banks * entry_size;
335 gen_pool_free(erp_core->erp_tables, base_index, size);
336}
337
338static struct mlxsw_sp_acl_erp *
339mlxsw_sp_acl_erp_table_master_rp(struct mlxsw_sp_acl_erp_table *erp_table)
340{
341 if (!list_is_singular(&erp_table->atcam_erps_list))
342 return NULL;
343
344 return list_first_entry(&erp_table->atcam_erps_list,
345 struct mlxsw_sp_acl_erp, list);
346}
347
348static int mlxsw_sp_acl_erp_index_get(struct mlxsw_sp_acl_erp_table *erp_table,
349 u8 *p_index)
350{
351 u8 index;
352
353 index = find_first_zero_bit(erp_table->erp_index_bitmap,
354 erp_table->num_max_atcam_erps);
355 if (index < erp_table->num_max_atcam_erps) {
356 __set_bit(index, erp_table->erp_index_bitmap);
357 *p_index = index;
358 return 0;
359 }
360
361 return -ENOBUFS;
362}
363
364static void mlxsw_sp_acl_erp_index_put(struct mlxsw_sp_acl_erp_table *erp_table,
365 u8 index)
366{
367 __clear_bit(index, erp_table->erp_index_bitmap);
368}
369
370static void
371mlxsw_sp_acl_erp_table_locate(const struct mlxsw_sp_acl_erp_table *erp_table,
372 const struct mlxsw_sp_acl_erp *erp,
373 u8 *p_erpt_bank, u8 *p_erpt_index)
374{
375 unsigned int entry_size = mlxsw_sp_acl_erp_table_entry_size(erp_table);
376 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
377 unsigned int row;
378
379 *p_erpt_bank = erp->index % erp_core->num_erp_banks;
380 row = erp->index / erp_core->num_erp_banks;
381 *p_erpt_index = erp_table->base_index + row * entry_size;
382}
383
384static int
385mlxsw_sp_acl_erp_table_erp_add(struct mlxsw_sp_acl_erp_table *erp_table,
386 struct mlxsw_sp_acl_erp *erp)
387{
388 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
389 enum mlxsw_reg_perpt_key_size key_size;
390 char perpt_pl[MLXSW_REG_PERPT_LEN];
391 u8 erpt_bank, erpt_index;
392
393 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index);
394 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type;
395 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id,
396 0, erp_table->base_index, erp->index,
397 erp->key.mask);
398 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap,
399 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
400 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, true);
401 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl);
402}
403
404static void mlxsw_sp_acl_erp_table_erp_del(struct mlxsw_sp_acl_erp *erp)
405{
406 char empty_mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 };
407 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
408 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
409 enum mlxsw_reg_perpt_key_size key_size;
410 char perpt_pl[MLXSW_REG_PERPT_LEN];
411 u8 erpt_bank, erpt_index;
412
413 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index);
414 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type;
415 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id,
416 0, erp_table->base_index, erp->index, empty_mask);
417 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap,
418 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
419 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, false);
420 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl);
421}
422
423static int
424mlxsw_sp_acl_erp_table_enable(struct mlxsw_sp_acl_erp_table *erp_table,
425 bool ctcam_le)
426{
427 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
428 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
429 char pererp_pl[MLXSW_REG_PERERP_LEN];
430
431 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
432 erp_table->base_index, 0);
433 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
434 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
435
436 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
437}
438
439static void
440mlxsw_sp_acl_erp_table_disable(struct mlxsw_sp_acl_erp_table *erp_table)
441{
442 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
443 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
444 char pererp_pl[MLXSW_REG_PERERP_LEN];
445 struct mlxsw_sp_acl_erp *master_rp;
446
447 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
448 /* It is possible we do not have a master RP when we disable the
449 * table when there are no rules in the A-TCAM and the last C-TCAM
450 * rule is deleted
451 */
452 mlxsw_reg_pererp_pack(pererp_pl, region->id, false, false, 0, 0,
453 master_rp ? master_rp->id : 0);
454 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
455}
456
457static int
458mlxsw_sp_acl_erp_table_relocate(struct mlxsw_sp_acl_erp_table *erp_table)
459{
460 struct mlxsw_sp_acl_erp *erp;
461 int err;
462
463 list_for_each_entry(erp, &erp_table->atcam_erps_list, list) {
464 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
465 if (err)
466 goto err_table_erp_add;
467 }
468
469 return 0;
470
471err_table_erp_add:
472 list_for_each_entry_continue_reverse(erp, &erp_table->atcam_erps_list,
473 list)
474 mlxsw_sp_acl_erp_table_erp_del(erp);
475 return err;
476}
477
478static int
479mlxsw_sp_acl_erp_table_expand(struct mlxsw_sp_acl_erp_table *erp_table)
480{
481 unsigned int num_erps, old_num_erps = erp_table->num_max_atcam_erps;
482 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
483 unsigned long old_base_index = erp_table->base_index;
484 bool ctcam_le = erp_table->num_ctcam_erps > 0;
485 int err;
486
487 if (erp_table->num_atcam_erps < erp_table->num_max_atcam_erps)
488 return 0;
489
490 if (erp_table->num_max_atcam_erps == MLXSW_SP_ACL_ERP_MAX_PER_REGION)
491 return -ENOBUFS;
492
493 num_erps = old_num_erps + erp_core->num_erp_banks;
494 err = mlxsw_sp_acl_erp_table_alloc(erp_core, num_erps,
495 erp_table->aregion->type,
496 &erp_table->base_index);
497 if (err)
498 return err;
499 erp_table->num_max_atcam_erps = num_erps;
500
501 err = mlxsw_sp_acl_erp_table_relocate(erp_table);
502 if (err)
503 goto err_table_relocate;
504
505 err = mlxsw_sp_acl_erp_table_enable(erp_table, ctcam_le);
506 if (err)
507 goto err_table_enable;
508
509 mlxsw_sp_acl_erp_table_free(erp_core, old_num_erps,
510 erp_table->aregion->type, old_base_index);
511
512 return 0;
513
514err_table_enable:
515err_table_relocate:
516 erp_table->num_max_atcam_erps = old_num_erps;
517 mlxsw_sp_acl_erp_table_free(erp_core, num_erps,
518 erp_table->aregion->type,
519 erp_table->base_index);
520 erp_table->base_index = old_base_index;
521 return err;
522}
523
524static int
525mlxsw_acl_erp_table_bf_add(struct mlxsw_sp_acl_erp_table *erp_table,
526 struct mlxsw_sp_acl_erp *erp)
527{
528 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
529 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
530 struct mlxsw_sp_acl_atcam_entry *aentry;
531 int err;
532
533 list_for_each_entry(aentry, &aregion->entries_list, list) {
534 err = mlxsw_sp_acl_bf_entry_add(aregion->region->mlxsw_sp,
535 erp_table->erp_core->bf,
536 aregion, erp_bank, aentry);
537 if (err)
538 goto bf_entry_add_err;
539 }
540
541 return 0;
542
543bf_entry_add_err:
544 list_for_each_entry_continue_reverse(aentry, &aregion->entries_list,
545 list)
546 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp,
547 erp_table->erp_core->bf,
548 aregion, erp_bank, aentry);
549 return err;
550}
551
552static void
553mlxsw_acl_erp_table_bf_del(struct mlxsw_sp_acl_erp_table *erp_table,
554 struct mlxsw_sp_acl_erp *erp)
555{
556 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
557 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
558 struct mlxsw_sp_acl_atcam_entry *aentry;
559
560 list_for_each_entry_reverse(aentry, &aregion->entries_list, list)
561 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp,
562 erp_table->erp_core->bf,
563 aregion, erp_bank, aentry);
564}
565
566static int
567mlxsw_sp_acl_erp_region_table_trans(struct mlxsw_sp_acl_erp_table *erp_table)
568{
569 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
570 struct mlxsw_sp_acl_erp *master_rp;
571 int err;
572
573 /* Initially, allocate a single eRP row. Expand later as needed */
574 err = mlxsw_sp_acl_erp_table_alloc(erp_core, erp_core->num_erp_banks,
575 erp_table->aregion->type,
576 &erp_table->base_index);
577 if (err)
578 return err;
579 erp_table->num_max_atcam_erps = erp_core->num_erp_banks;
580
581 /* Transition the sole RP currently configured (the master RP)
582 * to the eRP table
583 */
584 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
585 if (!master_rp) {
586 err = -EINVAL;
587 goto err_table_master_rp;
588 }
589
590 /* Make sure the master RP is using a valid index, as
591 * only a single eRP row is currently allocated.
592 */
593 master_rp->index = 0;
594 __set_bit(master_rp->index, erp_table->erp_index_bitmap);
595
596 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, master_rp);
597 if (err)
598 goto err_table_master_rp_add;
599
600 /* Update Bloom filter before enabling eRP table, as rules
601 * on the master RP were not set to Bloom filter up to this
602 * point.
603 */
604 err = mlxsw_acl_erp_table_bf_add(erp_table, master_rp);
605 if (err)
606 goto err_table_bf_add;
607
608 err = mlxsw_sp_acl_erp_table_enable(erp_table, false);
609 if (err)
610 goto err_table_enable;
611
612 return 0;
613
614err_table_enable:
615 mlxsw_acl_erp_table_bf_del(erp_table, master_rp);
616err_table_bf_add:
617 mlxsw_sp_acl_erp_table_erp_del(master_rp);
618err_table_master_rp_add:
619 __clear_bit(master_rp->index, erp_table->erp_index_bitmap);
620err_table_master_rp:
621 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps,
622 erp_table->aregion->type,
623 erp_table->base_index);
624 return err;
625}
626
627static void
628mlxsw_sp_acl_erp_region_master_mask_trans(struct mlxsw_sp_acl_erp_table *erp_table)
629{
630 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
631 struct mlxsw_sp_acl_erp *master_rp;
632
633 mlxsw_sp_acl_erp_table_disable(erp_table);
634 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
635 if (!master_rp)
636 return;
637 mlxsw_acl_erp_table_bf_del(erp_table, master_rp);
638 mlxsw_sp_acl_erp_table_erp_del(master_rp);
639 __clear_bit(master_rp->index, erp_table->erp_index_bitmap);
640 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps,
641 erp_table->aregion->type,
642 erp_table->base_index);
643}
644
645static int
646mlxsw_sp_acl_erp_region_erp_add(struct mlxsw_sp_acl_erp_table *erp_table,
647 struct mlxsw_sp_acl_erp *erp)
648{
649 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
650 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
651 bool ctcam_le = erp_table->num_ctcam_erps > 0;
652 char pererp_pl[MLXSW_REG_PERERP_LEN];
653
654 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
655 erp_table->base_index, 0);
656 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
657 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
658 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, true);
659
660 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
661}
662
663static void mlxsw_sp_acl_erp_region_erp_del(struct mlxsw_sp_acl_erp *erp)
664{
665 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
666 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
667 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
668 bool ctcam_le = erp_table->num_ctcam_erps > 0;
669 char pererp_pl[MLXSW_REG_PERERP_LEN];
670
671 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
672 erp_table->base_index, 0);
673 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
674 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
675 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, false);
676
677 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
678}
679
680static int
681mlxsw_sp_acl_erp_region_ctcam_enable(struct mlxsw_sp_acl_erp_table *erp_table)
682{
683 /* No need to re-enable lookup in the C-TCAM */
684 if (erp_table->num_ctcam_erps > 1)
685 return 0;
686
687 return mlxsw_sp_acl_erp_table_enable(erp_table, true);
688}
689
690static void
691mlxsw_sp_acl_erp_region_ctcam_disable(struct mlxsw_sp_acl_erp_table *erp_table)
692{
693 /* Only disable C-TCAM lookup when last C-TCAM eRP is deleted */
694 if (erp_table->num_ctcam_erps > 1)
695 return;
696
697 mlxsw_sp_acl_erp_table_enable(erp_table, false);
698}
699
700static int
701__mlxsw_sp_acl_erp_table_other_inc(struct mlxsw_sp_acl_erp_table *erp_table,
702 unsigned int *inc_num)
703{
704 int err;
705
706 /* If there are C-TCAM eRP or deltas in use we need to transition
707 * the region to use eRP table, if it is not already done
708 */
709 if (!mlxsw_sp_acl_erp_table_is_used(erp_table)) {
710 err = mlxsw_sp_acl_erp_region_table_trans(erp_table);
711 if (err)
712 return err;
713 }
714
715 /* When C-TCAM or deltas are used, the eRP table must be used */
716 if (erp_table->ops != &erp_multiple_masks_ops)
717 erp_table->ops = &erp_multiple_masks_ops;
718
719 (*inc_num)++;
720
721 return 0;
722}
723
724static int mlxsw_sp_acl_erp_ctcam_inc(struct mlxsw_sp_acl_erp_table *erp_table)
725{
726 return __mlxsw_sp_acl_erp_table_other_inc(erp_table,
727 &erp_table->num_ctcam_erps);
728}
729
730static int mlxsw_sp_acl_erp_delta_inc(struct mlxsw_sp_acl_erp_table *erp_table)
731{
732 return __mlxsw_sp_acl_erp_table_other_inc(erp_table,
733 &erp_table->num_deltas);
734}
735
736static void
737__mlxsw_sp_acl_erp_table_other_dec(struct mlxsw_sp_acl_erp_table *erp_table,
738 unsigned int *dec_num)
739{
740 (*dec_num)--;
741
742 /* If there are no C-TCAM eRP or deltas in use, the state we
743 * transition to depends on the number of A-TCAM eRPs currently
744 * in use.
745 */
746 if (erp_table->num_ctcam_erps > 0 || erp_table->num_deltas > 0)
747 return;
748
749 switch (erp_table->num_atcam_erps) {
750 case 2:
751 /* Keep using the eRP table, but correctly set the
752 * operations pointer so that when an A-TCAM eRP is
753 * deleted we will transition to use the master mask
754 */
755 erp_table->ops = &erp_two_masks_ops;
756 break;
757 case 1:
758 /* We only kept the eRP table because we had C-TCAM
759 * eRPs in use. Now that the last C-TCAM eRP is gone we
760 * can stop using the table and transition to use the
761 * master mask
762 */
763 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
764 erp_table->ops = &erp_single_mask_ops;
765 break;
766 case 0:
767 /* There are no more eRPs of any kind used by the region
768 * so free its eRP table and transition to initial state
769 */
770 mlxsw_sp_acl_erp_table_disable(erp_table);
771 mlxsw_sp_acl_erp_table_free(erp_table->erp_core,
772 erp_table->num_max_atcam_erps,
773 erp_table->aregion->type,
774 erp_table->base_index);
775 erp_table->ops = &erp_no_mask_ops;
776 break;
777 default:
778 break;
779 }
780}
781
782static void mlxsw_sp_acl_erp_ctcam_dec(struct mlxsw_sp_acl_erp_table *erp_table)
783{
784 __mlxsw_sp_acl_erp_table_other_dec(erp_table,
785 &erp_table->num_ctcam_erps);
786}
787
788static void mlxsw_sp_acl_erp_delta_dec(struct mlxsw_sp_acl_erp_table *erp_table)
789{
790 __mlxsw_sp_acl_erp_table_other_dec(erp_table,
791 &erp_table->num_deltas);
792}
793
794static struct mlxsw_sp_acl_erp *
795mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
796 struct mlxsw_sp_acl_erp_key *key)
797{
798 struct mlxsw_sp_acl_erp *erp;
799 int err;
800
801 erp = kzalloc(sizeof(*erp), GFP_KERNEL);
802 if (!erp)
803 return ERR_PTR(-ENOMEM);
804
805 memcpy(&erp->key, key, sizeof(*key));
806 bitmap_from_arr32(erp->mask_bitmap, (u32 *) key->mask,
807 MLXSW_SP_ACL_TCAM_MASK_LEN);
808
809 err = mlxsw_sp_acl_erp_ctcam_inc(erp_table);
810 if (err)
811 goto err_erp_ctcam_inc;
812
813 erp->erp_table = erp_table;
814
815 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key);
816 if (err)
817 goto err_master_mask_set;
818
819 err = mlxsw_sp_acl_erp_region_ctcam_enable(erp_table);
820 if (err)
821 goto err_erp_region_ctcam_enable;
822
823 return erp;
824
825err_erp_region_ctcam_enable:
826 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
827err_master_mask_set:
828 mlxsw_sp_acl_erp_ctcam_dec(erp_table);
829err_erp_ctcam_inc:
830 kfree(erp);
831 return ERR_PTR(err);
832}
833
834static void
835mlxsw_sp_acl_erp_ctcam_mask_destroy(struct mlxsw_sp_acl_erp *erp)
836{
837 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
838
839 mlxsw_sp_acl_erp_region_ctcam_disable(erp_table);
840 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
841 mlxsw_sp_acl_erp_ctcam_dec(erp_table);
842 kfree(erp);
843}
844
845static struct mlxsw_sp_acl_erp *
846mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
847 struct mlxsw_sp_acl_erp_key *key)
848{
849 struct mlxsw_sp_acl_erp *erp;
850 int err;
851
852 if (key->ctcam)
853 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
854
855 /* Expand the eRP table for the new eRP, if needed */
856 err = mlxsw_sp_acl_erp_table_expand(erp_table);
857 if (err)
858 return ERR_PTR(err);
859
860 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
861 if (IS_ERR(erp))
862 return erp;
863
864 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index);
865 if (err)
866 goto err_erp_index_get;
867
868 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
869 if (err)
870 goto err_table_erp_add;
871
872 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp);
873 if (err)
874 goto err_region_erp_add;
875
876 erp_table->ops = &erp_multiple_masks_ops;
877
878 return erp;
879
880err_region_erp_add:
881 mlxsw_sp_acl_erp_table_erp_del(erp);
882err_table_erp_add:
883 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
884err_erp_index_get:
885 mlxsw_sp_acl_erp_generic_destroy(erp);
886 return ERR_PTR(err);
887}
888
889static void
890mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
891 struct mlxsw_sp_acl_erp *erp)
892{
893 if (erp->key.ctcam)
894 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp);
895
896 mlxsw_sp_acl_erp_region_erp_del(erp);
897 mlxsw_sp_acl_erp_table_erp_del(erp);
898 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
899 mlxsw_sp_acl_erp_generic_destroy(erp);
900
901 if (erp_table->num_atcam_erps == 2 && erp_table->num_ctcam_erps == 0 &&
902 erp_table->num_deltas == 0)
903 erp_table->ops = &erp_two_masks_ops;
904}
905
906static struct mlxsw_sp_acl_erp *
907mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
908 struct mlxsw_sp_acl_erp_key *key)
909{
910 struct mlxsw_sp_acl_erp *erp;
911 int err;
912
913 if (key->ctcam)
914 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
915
916 /* Transition to use eRP table instead of master mask */
917 err = mlxsw_sp_acl_erp_region_table_trans(erp_table);
918 if (err)
919 return ERR_PTR(err);
920
921 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
922 if (IS_ERR(erp)) {
923 err = PTR_ERR(erp);
924 goto err_erp_create;
925 }
926
927 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index);
928 if (err)
929 goto err_erp_index_get;
930
931 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
932 if (err)
933 goto err_table_erp_add;
934
935 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp);
936 if (err)
937 goto err_region_erp_add;
938
939 erp_table->ops = &erp_two_masks_ops;
940
941 return erp;
942
943err_region_erp_add:
944 mlxsw_sp_acl_erp_table_erp_del(erp);
945err_table_erp_add:
946 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
947err_erp_index_get:
948 mlxsw_sp_acl_erp_generic_destroy(erp);
949err_erp_create:
950 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
951 return ERR_PTR(err);
952}
953
954static void
955mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
956 struct mlxsw_sp_acl_erp *erp)
957{
958 if (erp->key.ctcam)
959 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp);
960
961 mlxsw_sp_acl_erp_region_erp_del(erp);
962 mlxsw_sp_acl_erp_table_erp_del(erp);
963 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
964 mlxsw_sp_acl_erp_generic_destroy(erp);
965 /* Transition to use master mask instead of eRP table */
966 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
967
968 erp_table->ops = &erp_single_mask_ops;
969}
970
971static struct mlxsw_sp_acl_erp *
972mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
973 struct mlxsw_sp_acl_erp_key *key)
974{
975 struct mlxsw_sp_acl_erp *erp;
976
977 if (key->ctcam)
978 return ERR_PTR(-EINVAL);
979
980 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
981 if (IS_ERR(erp))
982 return erp;
983
984 erp_table->ops = &erp_single_mask_ops;
985
986 return erp;
987}
988
989static void
990mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
991 struct mlxsw_sp_acl_erp *erp)
992{
993 mlxsw_sp_acl_erp_generic_destroy(erp);
994 erp_table->ops = &erp_no_mask_ops;
995}
996
997static void
998mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
999 struct mlxsw_sp_acl_erp *erp)
1000{
1001 WARN_ON(1);
1002}
1003
1004struct mlxsw_sp_acl_erp_mask *
1005mlxsw_sp_acl_erp_mask_get(struct mlxsw_sp_acl_atcam_region *aregion,
1006 const char *mask, bool ctcam)
1007{
1008 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1009 struct mlxsw_sp_acl_erp_key key;
1010 struct objagg_obj *objagg_obj;
1011
1012 memcpy(key.mask, mask, MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN);
1013 key.ctcam = ctcam;
1014 mutex_lock(&erp_table->objagg_lock);
1015 objagg_obj = objagg_obj_get(erp_table->objagg, &key);
1016 mutex_unlock(&erp_table->objagg_lock);
1017 if (IS_ERR(objagg_obj))
1018 return ERR_CAST(objagg_obj);
1019 return (struct mlxsw_sp_acl_erp_mask *) objagg_obj;
1020}
1021
1022void mlxsw_sp_acl_erp_mask_put(struct mlxsw_sp_acl_atcam_region *aregion,
1023 struct mlxsw_sp_acl_erp_mask *erp_mask)
1024{
1025 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1026 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1027
1028 mutex_lock(&erp_table->objagg_lock);
1029 objagg_obj_put(erp_table->objagg, objagg_obj);
1030 mutex_unlock(&erp_table->objagg_lock);
1031}
1032
1033int mlxsw_sp_acl_erp_bf_insert(struct mlxsw_sp *mlxsw_sp,
1034 struct mlxsw_sp_acl_atcam_region *aregion,
1035 struct mlxsw_sp_acl_erp_mask *erp_mask,
1036 struct mlxsw_sp_acl_atcam_entry *aentry)
1037{
1038 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1039 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1040 unsigned int erp_bank;
1041
1042 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table))
1043 return 0;
1044
1045 erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
1046 return mlxsw_sp_acl_bf_entry_add(mlxsw_sp,
1047 erp->erp_table->erp_core->bf,
1048 aregion, erp_bank, aentry);
1049}
1050
1051void mlxsw_sp_acl_erp_bf_remove(struct mlxsw_sp *mlxsw_sp,
1052 struct mlxsw_sp_acl_atcam_region *aregion,
1053 struct mlxsw_sp_acl_erp_mask *erp_mask,
1054 struct mlxsw_sp_acl_atcam_entry *aentry)
1055{
1056 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1057 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1058 unsigned int erp_bank;
1059
1060 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table))
1061 return;
1062
1063 erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
1064 mlxsw_sp_acl_bf_entry_del(mlxsw_sp,
1065 erp->erp_table->erp_core->bf,
1066 aregion, erp_bank, aentry);
1067}
1068
1069bool
1070mlxsw_sp_acl_erp_mask_is_ctcam(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1071{
1072 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1073 const struct mlxsw_sp_acl_erp_key *key = objagg_obj_raw(objagg_obj);
1074
1075 return key->ctcam;
1076}
1077
1078u8 mlxsw_sp_acl_erp_mask_erp_id(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1079{
1080 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1081 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1082
1083 return erp->id;
1084}
1085
1086struct mlxsw_sp_acl_erp_delta {
1087 struct mlxsw_sp_acl_erp_key key;
1088 u16 start;
1089 u8 mask;
1090};
1091
1092u16 mlxsw_sp_acl_erp_delta_start(const struct mlxsw_sp_acl_erp_delta *delta)
1093{
1094 return delta->start;
1095}
1096
1097u8 mlxsw_sp_acl_erp_delta_mask(const struct mlxsw_sp_acl_erp_delta *delta)
1098{
1099 return delta->mask;
1100}
1101
1102u8 mlxsw_sp_acl_erp_delta_value(const struct mlxsw_sp_acl_erp_delta *delta,
1103 const char *enc_key)
1104{
1105 u16 start = delta->start;
1106 u8 mask = delta->mask;
1107 u16 tmp;
1108
1109 if (!mask)
1110 return 0;
1111
1112 tmp = (unsigned char) enc_key[__MASK_IDX(start / 8)];
1113 if (start / 8 + 1 < __MASK_LEN)
1114 tmp |= (unsigned char) enc_key[__MASK_IDX(start / 8 + 1)] << 8;
1115 tmp >>= start % 8;
1116 tmp &= mask;
1117 return tmp;
1118}
1119
1120void mlxsw_sp_acl_erp_delta_clear(const struct mlxsw_sp_acl_erp_delta *delta,
1121 const char *enc_key)
1122{
1123 u16 start = delta->start;
1124 u8 mask = delta->mask;
1125 unsigned char *byte;
1126 u16 tmp;
1127
1128 tmp = mask;
1129 tmp <<= start % 8;
1130 tmp = ~tmp;
1131
1132 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8)];
1133 *byte &= tmp & 0xff;
1134 if (start / 8 + 1 < __MASK_LEN) {
1135 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8 + 1)];
1136 *byte &= (tmp >> 8) & 0xff;
1137 }
1138}
1139
1140static const struct mlxsw_sp_acl_erp_delta
1141mlxsw_sp_acl_erp_delta_default = {};
1142
1143const struct mlxsw_sp_acl_erp_delta *
1144mlxsw_sp_acl_erp_delta(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1145{
1146 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1147 const struct mlxsw_sp_acl_erp_delta *delta;
1148
1149 delta = objagg_obj_delta_priv(objagg_obj);
1150 if (!delta)
1151 delta = &mlxsw_sp_acl_erp_delta_default;
1152 return delta;
1153}
1154
1155static int
1156mlxsw_sp_acl_erp_delta_fill(const struct mlxsw_sp_acl_erp_key *parent_key,
1157 const struct mlxsw_sp_acl_erp_key *key,
1158 u16 *delta_start, u8 *delta_mask)
1159{
1160 int offset = 0;
1161 int si = -1;
1162 u16 pmask;
1163 u16 mask;
1164 int i;
1165
1166 /* The difference between 2 masks can be up to 8 consecutive bits. */
1167 for (i = 0; i < __MASK_LEN; i++) {
1168 if (parent_key->mask[__MASK_IDX(i)] == key->mask[__MASK_IDX(i)])
1169 continue;
1170 if (si == -1)
1171 si = i;
1172 else if (si != i - 1)
1173 return -EINVAL;
1174 }
1175 if (si == -1) {
1176 /* The masks are the same, this can happen in case eRPs with
1177 * the same mask were created in both A-TCAM and C-TCAM.
1178 * The only possible condition under which this can happen
1179 * is identical rule insertion. Delta is not possible here.
1180 */
1181 return -EINVAL;
1182 }
1183 pmask = (unsigned char) parent_key->mask[__MASK_IDX(si)];
1184 mask = (unsigned char) key->mask[__MASK_IDX(si)];
1185 if (si + 1 < __MASK_LEN) {
1186 pmask |= (unsigned char) parent_key->mask[__MASK_IDX(si + 1)] << 8;
1187 mask |= (unsigned char) key->mask[__MASK_IDX(si + 1)] << 8;
1188 }
1189
1190 if ((pmask ^ mask) & pmask)
1191 return -EINVAL;
1192 mask &= ~pmask;
1193 while (!(mask & (1 << offset)))
1194 offset++;
1195 while (!(mask & 1))
1196 mask >>= 1;
1197 if (mask & 0xff00)
1198 return -EINVAL;
1199
1200 *delta_start = si * 8 + offset;
1201 *delta_mask = mask;
1202
1203 return 0;
1204}
1205
1206static bool mlxsw_sp_acl_erp_delta_check(void *priv, const void *parent_obj,
1207 const void *obj)
1208{
1209 const struct mlxsw_sp_acl_erp_key *parent_key = parent_obj;
1210 const struct mlxsw_sp_acl_erp_key *key = obj;
1211 u16 delta_start;
1212 u8 delta_mask;
1213 int err;
1214
1215 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key,
1216 &delta_start, &delta_mask);
1217 return err ? false : true;
1218}
1219
1220static int mlxsw_sp_acl_erp_hints_obj_cmp(const void *obj1, const void *obj2)
1221{
1222 const struct mlxsw_sp_acl_erp_key *key1 = obj1;
1223 const struct mlxsw_sp_acl_erp_key *key2 = obj2;
1224
1225 /* For hints purposes, two objects are considered equal
1226 * in case the masks are the same. Does not matter what
1227 * the "ctcam" value is.
1228 */
1229 return memcmp(key1->mask, key2->mask, sizeof(key1->mask));
1230}
1231
1232static void *mlxsw_sp_acl_erp_delta_create(void *priv, void *parent_obj,
1233 void *obj)
1234{
1235 struct mlxsw_sp_acl_erp_key *parent_key = parent_obj;
1236 struct mlxsw_sp_acl_atcam_region *aregion = priv;
1237 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1238 struct mlxsw_sp_acl_erp_key *key = obj;
1239 struct mlxsw_sp_acl_erp_delta *delta;
1240 u16 delta_start;
1241 u8 delta_mask;
1242 int err;
1243
1244 if (parent_key->ctcam || key->ctcam)
1245 return ERR_PTR(-EINVAL);
1246 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key,
1247 &delta_start, &delta_mask);
1248 if (err)
1249 return ERR_PTR(-EINVAL);
1250
1251 delta = kzalloc(sizeof(*delta), GFP_KERNEL);
1252 if (!delta)
1253 return ERR_PTR(-ENOMEM);
1254 delta->start = delta_start;
1255 delta->mask = delta_mask;
1256
1257 err = mlxsw_sp_acl_erp_delta_inc(erp_table);
1258 if (err)
1259 goto err_erp_delta_inc;
1260
1261 memcpy(&delta->key, key, sizeof(*key));
1262 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &delta->key);
1263 if (err)
1264 goto err_master_mask_set;
1265
1266 return delta;
1267
1268err_master_mask_set:
1269 mlxsw_sp_acl_erp_delta_dec(erp_table);
1270err_erp_delta_inc:
1271 kfree(delta);
1272 return ERR_PTR(err);
1273}
1274
1275static void mlxsw_sp_acl_erp_delta_destroy(void *priv, void *delta_priv)
1276{
1277 struct mlxsw_sp_acl_erp_delta *delta = delta_priv;
1278 struct mlxsw_sp_acl_atcam_region *aregion = priv;
1279 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1280
1281 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &delta->key);
1282 mlxsw_sp_acl_erp_delta_dec(erp_table);
1283 kfree(delta);
1284}
1285
1286static void *mlxsw_sp_acl_erp_root_create(void *priv, void *obj,
1287 unsigned int root_id)
1288{
1289 struct mlxsw_sp_acl_atcam_region *aregion = priv;
1290 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1291 struct mlxsw_sp_acl_erp_key *key = obj;
1292
1293 if (!key->ctcam &&
1294 root_id != OBJAGG_OBJ_ROOT_ID_INVALID &&
1295 root_id >= MLXSW_SP_ACL_ERP_MAX_PER_REGION)
1296 return ERR_PTR(-ENOBUFS);
1297 return erp_table->ops->erp_create(erp_table, key);
1298}
1299
1300static void mlxsw_sp_acl_erp_root_destroy(void *priv, void *root_priv)
1301{
1302 struct mlxsw_sp_acl_atcam_region *aregion = priv;
1303 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1304
1305 erp_table->ops->erp_destroy(erp_table, root_priv);
1306}
1307
1308static const struct objagg_ops mlxsw_sp_acl_erp_objagg_ops = {
1309 .obj_size = sizeof(struct mlxsw_sp_acl_erp_key),
1310 .delta_check = mlxsw_sp_acl_erp_delta_check,
1311 .hints_obj_cmp = mlxsw_sp_acl_erp_hints_obj_cmp,
1312 .delta_create = mlxsw_sp_acl_erp_delta_create,
1313 .delta_destroy = mlxsw_sp_acl_erp_delta_destroy,
1314 .root_create = mlxsw_sp_acl_erp_root_create,
1315 .root_destroy = mlxsw_sp_acl_erp_root_destroy,
1316};
1317
1318static struct mlxsw_sp_acl_erp_table *
1319mlxsw_sp_acl_erp_table_create(struct mlxsw_sp_acl_atcam_region *aregion,
1320 struct objagg_hints *hints)
1321{
1322 struct mlxsw_sp_acl_erp_table *erp_table;
1323 int err;
1324
1325 erp_table = kzalloc(sizeof(*erp_table), GFP_KERNEL);
1326 if (!erp_table)
1327 return ERR_PTR(-ENOMEM);
1328
1329 erp_table->objagg = objagg_create(&mlxsw_sp_acl_erp_objagg_ops,
1330 hints, aregion);
1331 if (IS_ERR(erp_table->objagg)) {
1332 err = PTR_ERR(erp_table->objagg);
1333 goto err_objagg_create;
1334 }
1335
1336 erp_table->erp_core = aregion->atcam->erp_core;
1337 erp_table->ops = &erp_no_mask_ops;
1338 INIT_LIST_HEAD(&erp_table->atcam_erps_list);
1339 erp_table->aregion = aregion;
1340 mutex_init(&erp_table->objagg_lock);
1341
1342 return erp_table;
1343
1344err_objagg_create:
1345 kfree(erp_table);
1346 return ERR_PTR(err);
1347}
1348
1349static void
1350mlxsw_sp_acl_erp_table_destroy(struct mlxsw_sp_acl_erp_table *erp_table)
1351{
1352 WARN_ON(!list_empty(&erp_table->atcam_erps_list));
1353 mutex_destroy(&erp_table->objagg_lock);
1354 objagg_destroy(erp_table->objagg);
1355 kfree(erp_table);
1356}
1357
1358static int
1359mlxsw_sp_acl_erp_master_mask_init(struct mlxsw_sp_acl_atcam_region *aregion)
1360{
1361 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1362 char percr_pl[MLXSW_REG_PERCR_LEN];
1363
1364 mlxsw_reg_percr_pack(percr_pl, aregion->region->id);
1365 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl);
1366}
1367
1368static int
1369mlxsw_sp_acl_erp_region_param_init(struct mlxsw_sp_acl_atcam_region *aregion)
1370{
1371 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1372 char pererp_pl[MLXSW_REG_PERERP_LEN];
1373
1374 mlxsw_reg_pererp_pack(pererp_pl, aregion->region->id, false, false, 0,
1375 0, 0);
1376 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
1377}
1378
1379static int
1380mlxsw_sp_acl_erp_hints_check(struct mlxsw_sp *mlxsw_sp,
1381 struct mlxsw_sp_acl_atcam_region *aregion,
1382 struct objagg_hints *hints, bool *p_rehash_needed)
1383{
1384 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1385 const struct objagg_stats *ostats;
1386 const struct objagg_stats *hstats;
1387 int err;
1388
1389 *p_rehash_needed = false;
1390
1391 mutex_lock(&erp_table->objagg_lock);
1392 ostats = objagg_stats_get(erp_table->objagg);
1393 mutex_unlock(&erp_table->objagg_lock);
1394 if (IS_ERR(ostats)) {
1395 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP stats\n");
1396 return PTR_ERR(ostats);
1397 }
1398
1399 hstats = objagg_hints_stats_get(hints);
1400 if (IS_ERR(hstats)) {
1401 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP hints stats\n");
1402 err = PTR_ERR(hstats);
1403 goto err_hints_stats_get;
1404 }
1405
1406 /* Very basic criterion for now. */
1407 if (hstats->root_count < ostats->root_count)
1408 *p_rehash_needed = true;
1409
1410 err = 0;
1411
1412 objagg_stats_put(hstats);
1413err_hints_stats_get:
1414 objagg_stats_put(ostats);
1415 return err;
1416}
1417
1418void *
1419mlxsw_sp_acl_erp_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion)
1420{
1421 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1422 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1423 struct objagg_hints *hints;
1424 bool rehash_needed;
1425 int err;
1426
1427 mutex_lock(&erp_table->objagg_lock);
1428 hints = objagg_hints_get(erp_table->objagg,
1429 OBJAGG_OPT_ALGO_SIMPLE_GREEDY);
1430 mutex_unlock(&erp_table->objagg_lock);
1431 if (IS_ERR(hints)) {
1432 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to create ERP hints\n");
1433 return ERR_CAST(hints);
1434 }
1435 err = mlxsw_sp_acl_erp_hints_check(mlxsw_sp, aregion, hints,
1436 &rehash_needed);
1437 if (err)
1438 goto errout;
1439
1440 if (!rehash_needed) {
1441 err = -EAGAIN;
1442 goto errout;
1443 }
1444 return hints;
1445
1446errout:
1447 objagg_hints_put(hints);
1448 return ERR_PTR(err);
1449}
1450
1451void mlxsw_sp_acl_erp_rehash_hints_put(void *hints_priv)
1452{
1453 struct objagg_hints *hints = hints_priv;
1454
1455 objagg_hints_put(hints);
1456}
1457
1458int mlxsw_sp_acl_erp_region_init(struct mlxsw_sp_acl_atcam_region *aregion,
1459 void *hints_priv)
1460{
1461 struct mlxsw_sp_acl_erp_table *erp_table;
1462 struct objagg_hints *hints = hints_priv;
1463 int err;
1464
1465 erp_table = mlxsw_sp_acl_erp_table_create(aregion, hints);
1466 if (IS_ERR(erp_table))
1467 return PTR_ERR(erp_table);
1468 aregion->erp_table = erp_table;
1469
1470 /* Initialize the region's master mask to all zeroes */
1471 err = mlxsw_sp_acl_erp_master_mask_init(aregion);
1472 if (err)
1473 goto err_erp_master_mask_init;
1474
1475 /* Initialize the region to not use the eRP table */
1476 err = mlxsw_sp_acl_erp_region_param_init(aregion);
1477 if (err)
1478 goto err_erp_region_param_init;
1479
1480 return 0;
1481
1482err_erp_region_param_init:
1483err_erp_master_mask_init:
1484 mlxsw_sp_acl_erp_table_destroy(erp_table);
1485 return err;
1486}
1487
1488void mlxsw_sp_acl_erp_region_fini(struct mlxsw_sp_acl_atcam_region *aregion)
1489{
1490 mlxsw_sp_acl_erp_table_destroy(aregion->erp_table);
1491}
1492
1493static int
1494mlxsw_sp_acl_erp_tables_sizes_query(struct mlxsw_sp *mlxsw_sp,
1495 struct mlxsw_sp_acl_erp_core *erp_core)
1496{
1497 unsigned int size;
1498
1499 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB) ||
1500 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB) ||
1501 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB) ||
1502 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB))
1503 return -EIO;
1504
1505 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB);
1506 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB] = size;
1507
1508 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB);
1509 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB] = size;
1510
1511 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB);
1512 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB] = size;
1513
1514 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB);
1515 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB] = size;
1516
1517 return 0;
1518}
1519
1520static int mlxsw_sp_acl_erp_tables_init(struct mlxsw_sp *mlxsw_sp,
1521 struct mlxsw_sp_acl_erp_core *erp_core)
1522{
1523 unsigned int erpt_bank_size;
1524 int err;
1525
1526 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANK_SIZE) ||
1527 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANKS))
1528 return -EIO;
1529 erpt_bank_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1530 ACL_MAX_ERPT_BANK_SIZE);
1531 erp_core->num_erp_banks = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1532 ACL_MAX_ERPT_BANKS);
1533
1534 erp_core->erp_tables = gen_pool_create(0, -1);
1535 if (!erp_core->erp_tables)
1536 return -ENOMEM;
1537 gen_pool_set_algo(erp_core->erp_tables, gen_pool_best_fit, NULL);
1538
1539 err = gen_pool_add(erp_core->erp_tables,
1540 MLXSW_SP_ACL_ERP_GENALLOC_OFFSET, erpt_bank_size,
1541 -1);
1542 if (err)
1543 goto err_gen_pool_add;
1544
1545 erp_core->bf = mlxsw_sp_acl_bf_init(mlxsw_sp, erp_core->num_erp_banks);
1546 if (IS_ERR(erp_core->bf)) {
1547 err = PTR_ERR(erp_core->bf);
1548 goto err_bf_init;
1549 }
1550
1551 /* Different regions require masks of different sizes */
1552 err = mlxsw_sp_acl_erp_tables_sizes_query(mlxsw_sp, erp_core);
1553 if (err)
1554 goto err_erp_tables_sizes_query;
1555
1556 return 0;
1557
1558err_erp_tables_sizes_query:
1559 mlxsw_sp_acl_bf_fini(erp_core->bf);
1560err_bf_init:
1561err_gen_pool_add:
1562 gen_pool_destroy(erp_core->erp_tables);
1563 return err;
1564}
1565
1566static void mlxsw_sp_acl_erp_tables_fini(struct mlxsw_sp *mlxsw_sp,
1567 struct mlxsw_sp_acl_erp_core *erp_core)
1568{
1569 mlxsw_sp_acl_bf_fini(erp_core->bf);
1570 gen_pool_destroy(erp_core->erp_tables);
1571}
1572
1573int mlxsw_sp_acl_erps_init(struct mlxsw_sp *mlxsw_sp,
1574 struct mlxsw_sp_acl_atcam *atcam)
1575{
1576 struct mlxsw_sp_acl_erp_core *erp_core;
1577 int err;
1578
1579 erp_core = kzalloc(sizeof(*erp_core), GFP_KERNEL);
1580 if (!erp_core)
1581 return -ENOMEM;
1582 erp_core->mlxsw_sp = mlxsw_sp;
1583 atcam->erp_core = erp_core;
1584
1585 err = mlxsw_sp_acl_erp_tables_init(mlxsw_sp, erp_core);
1586 if (err)
1587 goto err_erp_tables_init;
1588
1589 return 0;
1590
1591err_erp_tables_init:
1592 kfree(erp_core);
1593 return err;
1594}
1595
1596void mlxsw_sp_acl_erps_fini(struct mlxsw_sp *mlxsw_sp,
1597 struct mlxsw_sp_acl_atcam *atcam)
1598{
1599 mlxsw_sp_acl_erp_tables_fini(mlxsw_sp, atcam->erp_core);
1600 kfree(atcam->erp_core);
1601}
1// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2/* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
3
4#include <linux/bitmap.h>
5#include <linux/errno.h>
6#include <linux/genalloc.h>
7#include <linux/gfp.h>
8#include <linux/kernel.h>
9#include <linux/list.h>
10#include <linux/mutex.h>
11#include <linux/objagg.h>
12#include <linux/rtnetlink.h>
13#include <linux/slab.h>
14
15#include "core.h"
16#include "reg.h"
17#include "spectrum.h"
18#include "spectrum_acl_tcam.h"
19
20/* gen_pool_alloc() returns 0 when allocation fails, so use an offset */
21#define MLXSW_SP_ACL_ERP_GENALLOC_OFFSET 0x100
22#define MLXSW_SP_ACL_ERP_MAX_PER_REGION 16
23
24struct mlxsw_sp_acl_erp_core {
25 unsigned int erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_MAX + 1];
26 struct gen_pool *erp_tables;
27 struct mlxsw_sp *mlxsw_sp;
28 struct mlxsw_sp_acl_bf *bf;
29 unsigned int num_erp_banks;
30};
31
32struct mlxsw_sp_acl_erp_key {
33 char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN];
34#define __MASK_LEN 0x38
35#define __MASK_IDX(i) (__MASK_LEN - (i) - 1)
36 bool ctcam;
37};
38
39struct mlxsw_sp_acl_erp {
40 struct mlxsw_sp_acl_erp_key key;
41 u8 id;
42 u8 index;
43 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
44 struct list_head list;
45 struct mlxsw_sp_acl_erp_table *erp_table;
46};
47
48struct mlxsw_sp_acl_erp_master_mask {
49 DECLARE_BITMAP(bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
50 unsigned int count[MLXSW_SP_ACL_TCAM_MASK_LEN];
51};
52
53struct mlxsw_sp_acl_erp_table {
54 struct mlxsw_sp_acl_erp_master_mask master_mask;
55 DECLARE_BITMAP(erp_id_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);
56 DECLARE_BITMAP(erp_index_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);
57 struct list_head atcam_erps_list;
58 struct mlxsw_sp_acl_erp_core *erp_core;
59 struct mlxsw_sp_acl_atcam_region *aregion;
60 const struct mlxsw_sp_acl_erp_table_ops *ops;
61 unsigned long base_index;
62 unsigned int num_atcam_erps;
63 unsigned int num_max_atcam_erps;
64 unsigned int num_ctcam_erps;
65 unsigned int num_deltas;
66 struct objagg *objagg;
67 struct mutex objagg_lock; /* guards objagg manipulation */
68};
69
70struct mlxsw_sp_acl_erp_table_ops {
71 struct mlxsw_sp_acl_erp *
72 (*erp_create)(struct mlxsw_sp_acl_erp_table *erp_table,
73 struct mlxsw_sp_acl_erp_key *key);
74 void (*erp_destroy)(struct mlxsw_sp_acl_erp_table *erp_table,
75 struct mlxsw_sp_acl_erp *erp);
76};
77
78static struct mlxsw_sp_acl_erp *
79mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
80 struct mlxsw_sp_acl_erp_key *key);
81static void
82mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
83 struct mlxsw_sp_acl_erp *erp);
84static struct mlxsw_sp_acl_erp *
85mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
86 struct mlxsw_sp_acl_erp_key *key);
87static void
88mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
89 struct mlxsw_sp_acl_erp *erp);
90static struct mlxsw_sp_acl_erp *
91mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
92 struct mlxsw_sp_acl_erp_key *key);
93static void
94mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
95 struct mlxsw_sp_acl_erp *erp);
96static void
97mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
98 struct mlxsw_sp_acl_erp *erp);
99
100static const struct mlxsw_sp_acl_erp_table_ops erp_multiple_masks_ops = {
101 .erp_create = mlxsw_sp_acl_erp_mask_create,
102 .erp_destroy = mlxsw_sp_acl_erp_mask_destroy,
103};
104
105static const struct mlxsw_sp_acl_erp_table_ops erp_two_masks_ops = {
106 .erp_create = mlxsw_sp_acl_erp_mask_create,
107 .erp_destroy = mlxsw_sp_acl_erp_second_mask_destroy,
108};
109
110static const struct mlxsw_sp_acl_erp_table_ops erp_single_mask_ops = {
111 .erp_create = mlxsw_sp_acl_erp_second_mask_create,
112 .erp_destroy = mlxsw_sp_acl_erp_first_mask_destroy,
113};
114
115static const struct mlxsw_sp_acl_erp_table_ops erp_no_mask_ops = {
116 .erp_create = mlxsw_sp_acl_erp_first_mask_create,
117 .erp_destroy = mlxsw_sp_acl_erp_no_mask_destroy,
118};
119
120static bool
121mlxsw_sp_acl_erp_table_is_used(const struct mlxsw_sp_acl_erp_table *erp_table)
122{
123 return erp_table->ops != &erp_single_mask_ops &&
124 erp_table->ops != &erp_no_mask_ops;
125}
126
127static unsigned int
128mlxsw_sp_acl_erp_bank_get(const struct mlxsw_sp_acl_erp *erp)
129{
130 return erp->index % erp->erp_table->erp_core->num_erp_banks;
131}
132
133static unsigned int
134mlxsw_sp_acl_erp_table_entry_size(const struct mlxsw_sp_acl_erp_table *erp_table)
135{
136 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
137 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
138
139 return erp_core->erpt_entries_size[aregion->type];
140}
141
142static int mlxsw_sp_acl_erp_id_get(struct mlxsw_sp_acl_erp_table *erp_table,
143 u8 *p_id)
144{
145 u8 id;
146
147 id = find_first_zero_bit(erp_table->erp_id_bitmap,
148 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
149 if (id < MLXSW_SP_ACL_ERP_MAX_PER_REGION) {
150 __set_bit(id, erp_table->erp_id_bitmap);
151 *p_id = id;
152 return 0;
153 }
154
155 return -ENOBUFS;
156}
157
158static void mlxsw_sp_acl_erp_id_put(struct mlxsw_sp_acl_erp_table *erp_table,
159 u8 id)
160{
161 __clear_bit(id, erp_table->erp_id_bitmap);
162}
163
164static void
165mlxsw_sp_acl_erp_master_mask_bit_set(unsigned long bit,
166 struct mlxsw_sp_acl_erp_master_mask *mask)
167{
168 if (mask->count[bit]++ == 0)
169 __set_bit(bit, mask->bitmap);
170}
171
172static void
173mlxsw_sp_acl_erp_master_mask_bit_clear(unsigned long bit,
174 struct mlxsw_sp_acl_erp_master_mask *mask)
175{
176 if (--mask->count[bit] == 0)
177 __clear_bit(bit, mask->bitmap);
178}
179
180static int
181mlxsw_sp_acl_erp_master_mask_update(struct mlxsw_sp_acl_erp_table *erp_table)
182{
183 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
184 struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;
185 char percr_pl[MLXSW_REG_PERCR_LEN];
186 char *master_mask;
187
188 mlxsw_reg_percr_pack(percr_pl, region->id);
189 master_mask = mlxsw_reg_percr_master_mask_data(percr_pl);
190 bitmap_to_arr32((u32 *) master_mask, erp_table->master_mask.bitmap,
191 MLXSW_SP_ACL_TCAM_MASK_LEN);
192
193 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl);
194}
195
196static int
197mlxsw_sp_acl_erp_master_mask_set(struct mlxsw_sp_acl_erp_table *erp_table,
198 struct mlxsw_sp_acl_erp_key *key)
199{
200 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
201 unsigned long bit;
202 int err;
203
204 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask,
205 MLXSW_SP_ACL_TCAM_MASK_LEN);
206 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
207 mlxsw_sp_acl_erp_master_mask_bit_set(bit,
208 &erp_table->master_mask);
209
210 err = mlxsw_sp_acl_erp_master_mask_update(erp_table);
211 if (err)
212 goto err_master_mask_update;
213
214 return 0;
215
216err_master_mask_update:
217 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
218 mlxsw_sp_acl_erp_master_mask_bit_clear(bit,
219 &erp_table->master_mask);
220 return err;
221}
222
223static int
224mlxsw_sp_acl_erp_master_mask_clear(struct mlxsw_sp_acl_erp_table *erp_table,
225 struct mlxsw_sp_acl_erp_key *key)
226{
227 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
228 unsigned long bit;
229 int err;
230
231 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask,
232 MLXSW_SP_ACL_TCAM_MASK_LEN);
233 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
234 mlxsw_sp_acl_erp_master_mask_bit_clear(bit,
235 &erp_table->master_mask);
236
237 err = mlxsw_sp_acl_erp_master_mask_update(erp_table);
238 if (err)
239 goto err_master_mask_update;
240
241 return 0;
242
243err_master_mask_update:
244 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
245 mlxsw_sp_acl_erp_master_mask_bit_set(bit,
246 &erp_table->master_mask);
247 return err;
248}
249
250static struct mlxsw_sp_acl_erp *
251mlxsw_sp_acl_erp_generic_create(struct mlxsw_sp_acl_erp_table *erp_table,
252 struct mlxsw_sp_acl_erp_key *key)
253{
254 struct mlxsw_sp_acl_erp *erp;
255 int err;
256
257 erp = kzalloc(sizeof(*erp), GFP_KERNEL);
258 if (!erp)
259 return ERR_PTR(-ENOMEM);
260
261 err = mlxsw_sp_acl_erp_id_get(erp_table, &erp->id);
262 if (err)
263 goto err_erp_id_get;
264
265 memcpy(&erp->key, key, sizeof(*key));
266 list_add(&erp->list, &erp_table->atcam_erps_list);
267 erp_table->num_atcam_erps++;
268 erp->erp_table = erp_table;
269
270 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key);
271 if (err)
272 goto err_master_mask_set;
273
274 return erp;
275
276err_master_mask_set:
277 erp_table->num_atcam_erps--;
278 list_del(&erp->list);
279 mlxsw_sp_acl_erp_id_put(erp_table, erp->id);
280err_erp_id_get:
281 kfree(erp);
282 return ERR_PTR(err);
283}
284
285static void
286mlxsw_sp_acl_erp_generic_destroy(struct mlxsw_sp_acl_erp *erp)
287{
288 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
289
290 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
291 erp_table->num_atcam_erps--;
292 list_del(&erp->list);
293 mlxsw_sp_acl_erp_id_put(erp_table, erp->id);
294 kfree(erp);
295}
296
297static int
298mlxsw_sp_acl_erp_table_alloc(struct mlxsw_sp_acl_erp_core *erp_core,
299 unsigned int num_erps,
300 enum mlxsw_sp_acl_atcam_region_type region_type,
301 unsigned long *p_index)
302{
303 unsigned int num_rows, entry_size;
304
305 /* We only allow allocations of entire rows */
306 if (num_erps % erp_core->num_erp_banks != 0)
307 return -EINVAL;
308
309 entry_size = erp_core->erpt_entries_size[region_type];
310 num_rows = num_erps / erp_core->num_erp_banks;
311
312 *p_index = gen_pool_alloc(erp_core->erp_tables, num_rows * entry_size);
313 if (*p_index == 0)
314 return -ENOBUFS;
315 *p_index -= MLXSW_SP_ACL_ERP_GENALLOC_OFFSET;
316
317 return 0;
318}
319
320static void
321mlxsw_sp_acl_erp_table_free(struct mlxsw_sp_acl_erp_core *erp_core,
322 unsigned int num_erps,
323 enum mlxsw_sp_acl_atcam_region_type region_type,
324 unsigned long index)
325{
326 unsigned long base_index;
327 unsigned int entry_size;
328 size_t size;
329
330 entry_size = erp_core->erpt_entries_size[region_type];
331 base_index = index + MLXSW_SP_ACL_ERP_GENALLOC_OFFSET;
332 size = num_erps / erp_core->num_erp_banks * entry_size;
333 gen_pool_free(erp_core->erp_tables, base_index, size);
334}
335
336static struct mlxsw_sp_acl_erp *
337mlxsw_sp_acl_erp_table_master_rp(struct mlxsw_sp_acl_erp_table *erp_table)
338{
339 if (!list_is_singular(&erp_table->atcam_erps_list))
340 return NULL;
341
342 return list_first_entry(&erp_table->atcam_erps_list,
343 struct mlxsw_sp_acl_erp, list);
344}
345
346static int mlxsw_sp_acl_erp_index_get(struct mlxsw_sp_acl_erp_table *erp_table,
347 u8 *p_index)
348{
349 u8 index;
350
351 index = find_first_zero_bit(erp_table->erp_index_bitmap,
352 erp_table->num_max_atcam_erps);
353 if (index < erp_table->num_max_atcam_erps) {
354 __set_bit(index, erp_table->erp_index_bitmap);
355 *p_index = index;
356 return 0;
357 }
358
359 return -ENOBUFS;
360}
361
362static void mlxsw_sp_acl_erp_index_put(struct mlxsw_sp_acl_erp_table *erp_table,
363 u8 index)
364{
365 __clear_bit(index, erp_table->erp_index_bitmap);
366}
367
368static void
369mlxsw_sp_acl_erp_table_locate(const struct mlxsw_sp_acl_erp_table *erp_table,
370 const struct mlxsw_sp_acl_erp *erp,
371 u8 *p_erpt_bank, u8 *p_erpt_index)
372{
373 unsigned int entry_size = mlxsw_sp_acl_erp_table_entry_size(erp_table);
374 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
375 unsigned int row;
376
377 *p_erpt_bank = erp->index % erp_core->num_erp_banks;
378 row = erp->index / erp_core->num_erp_banks;
379 *p_erpt_index = erp_table->base_index + row * entry_size;
380}
381
382static int
383mlxsw_sp_acl_erp_table_erp_add(struct mlxsw_sp_acl_erp_table *erp_table,
384 struct mlxsw_sp_acl_erp *erp)
385{
386 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
387 enum mlxsw_reg_perpt_key_size key_size;
388 char perpt_pl[MLXSW_REG_PERPT_LEN];
389 u8 erpt_bank, erpt_index;
390
391 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index);
392 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type;
393 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id,
394 0, erp_table->base_index, erp->index,
395 erp->key.mask);
396 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap,
397 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
398 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, true);
399 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl);
400}
401
402static void mlxsw_sp_acl_erp_table_erp_del(struct mlxsw_sp_acl_erp *erp)
403{
404 char empty_mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 };
405 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
406 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
407 enum mlxsw_reg_perpt_key_size key_size;
408 char perpt_pl[MLXSW_REG_PERPT_LEN];
409 u8 erpt_bank, erpt_index;
410
411 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index);
412 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type;
413 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id,
414 0, erp_table->base_index, erp->index, empty_mask);
415 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap,
416 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
417 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, false);
418 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl);
419}
420
421static int
422mlxsw_sp_acl_erp_table_enable(struct mlxsw_sp_acl_erp_table *erp_table,
423 bool ctcam_le)
424{
425 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
426 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
427 char pererp_pl[MLXSW_REG_PERERP_LEN];
428
429 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
430 erp_table->base_index, 0);
431 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
432 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
433
434 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
435}
436
437static void
438mlxsw_sp_acl_erp_table_disable(struct mlxsw_sp_acl_erp_table *erp_table)
439{
440 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
441 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
442 char pererp_pl[MLXSW_REG_PERERP_LEN];
443 struct mlxsw_sp_acl_erp *master_rp;
444
445 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
446 /* It is possible we do not have a master RP when we disable the
447 * table when there are no rules in the A-TCAM and the last C-TCAM
448 * rule is deleted
449 */
450 mlxsw_reg_pererp_pack(pererp_pl, region->id, false, false, 0, 0,
451 master_rp ? master_rp->id : 0);
452 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
453}
454
455static int
456mlxsw_sp_acl_erp_table_relocate(struct mlxsw_sp_acl_erp_table *erp_table)
457{
458 struct mlxsw_sp_acl_erp *erp;
459 int err;
460
461 list_for_each_entry(erp, &erp_table->atcam_erps_list, list) {
462 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
463 if (err)
464 goto err_table_erp_add;
465 }
466
467 return 0;
468
469err_table_erp_add:
470 list_for_each_entry_continue_reverse(erp, &erp_table->atcam_erps_list,
471 list)
472 mlxsw_sp_acl_erp_table_erp_del(erp);
473 return err;
474}
475
476static int
477mlxsw_sp_acl_erp_table_expand(struct mlxsw_sp_acl_erp_table *erp_table)
478{
479 unsigned int num_erps, old_num_erps = erp_table->num_max_atcam_erps;
480 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
481 unsigned long old_base_index = erp_table->base_index;
482 bool ctcam_le = erp_table->num_ctcam_erps > 0;
483 int err;
484
485 if (erp_table->num_atcam_erps < erp_table->num_max_atcam_erps)
486 return 0;
487
488 if (erp_table->num_max_atcam_erps == MLXSW_SP_ACL_ERP_MAX_PER_REGION)
489 return -ENOBUFS;
490
491 num_erps = old_num_erps + erp_core->num_erp_banks;
492 err = mlxsw_sp_acl_erp_table_alloc(erp_core, num_erps,
493 erp_table->aregion->type,
494 &erp_table->base_index);
495 if (err)
496 return err;
497 erp_table->num_max_atcam_erps = num_erps;
498
499 err = mlxsw_sp_acl_erp_table_relocate(erp_table);
500 if (err)
501 goto err_table_relocate;
502
503 err = mlxsw_sp_acl_erp_table_enable(erp_table, ctcam_le);
504 if (err)
505 goto err_table_enable;
506
507 mlxsw_sp_acl_erp_table_free(erp_core, old_num_erps,
508 erp_table->aregion->type, old_base_index);
509
510 return 0;
511
512err_table_enable:
513err_table_relocate:
514 erp_table->num_max_atcam_erps = old_num_erps;
515 mlxsw_sp_acl_erp_table_free(erp_core, num_erps,
516 erp_table->aregion->type,
517 erp_table->base_index);
518 erp_table->base_index = old_base_index;
519 return err;
520}
521
522static int
523mlxsw_acl_erp_table_bf_add(struct mlxsw_sp_acl_erp_table *erp_table,
524 struct mlxsw_sp_acl_erp *erp)
525{
526 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
527 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
528 struct mlxsw_sp_acl_atcam_entry *aentry;
529 int err;
530
531 list_for_each_entry(aentry, &aregion->entries_list, list) {
532 err = mlxsw_sp_acl_bf_entry_add(aregion->region->mlxsw_sp,
533 erp_table->erp_core->bf,
534 aregion, erp_bank, aentry);
535 if (err)
536 goto bf_entry_add_err;
537 }
538
539 return 0;
540
541bf_entry_add_err:
542 list_for_each_entry_continue_reverse(aentry, &aregion->entries_list,
543 list)
544 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp,
545 erp_table->erp_core->bf,
546 aregion, erp_bank, aentry);
547 return err;
548}
549
550static void
551mlxsw_acl_erp_table_bf_del(struct mlxsw_sp_acl_erp_table *erp_table,
552 struct mlxsw_sp_acl_erp *erp)
553{
554 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
555 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
556 struct mlxsw_sp_acl_atcam_entry *aentry;
557
558 list_for_each_entry_reverse(aentry, &aregion->entries_list, list)
559 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp,
560 erp_table->erp_core->bf,
561 aregion, erp_bank, aentry);
562}
563
564static int
565mlxsw_sp_acl_erp_region_table_trans(struct mlxsw_sp_acl_erp_table *erp_table)
566{
567 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
568 struct mlxsw_sp_acl_erp *master_rp;
569 int err;
570
571 /* Initially, allocate a single eRP row. Expand later as needed */
572 err = mlxsw_sp_acl_erp_table_alloc(erp_core, erp_core->num_erp_banks,
573 erp_table->aregion->type,
574 &erp_table->base_index);
575 if (err)
576 return err;
577 erp_table->num_max_atcam_erps = erp_core->num_erp_banks;
578
579 /* Transition the sole RP currently configured (the master RP)
580 * to the eRP table
581 */
582 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
583 if (!master_rp) {
584 err = -EINVAL;
585 goto err_table_master_rp;
586 }
587
588 /* Make sure the master RP is using a valid index, as
589 * only a single eRP row is currently allocated.
590 */
591 master_rp->index = 0;
592 __set_bit(master_rp->index, erp_table->erp_index_bitmap);
593
594 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, master_rp);
595 if (err)
596 goto err_table_master_rp_add;
597
598 /* Update Bloom filter before enabling eRP table, as rules
599 * on the master RP were not set to Bloom filter up to this
600 * point.
601 */
602 err = mlxsw_acl_erp_table_bf_add(erp_table, master_rp);
603 if (err)
604 goto err_table_bf_add;
605
606 err = mlxsw_sp_acl_erp_table_enable(erp_table, false);
607 if (err)
608 goto err_table_enable;
609
610 return 0;
611
612err_table_enable:
613 mlxsw_acl_erp_table_bf_del(erp_table, master_rp);
614err_table_bf_add:
615 mlxsw_sp_acl_erp_table_erp_del(master_rp);
616err_table_master_rp_add:
617 __clear_bit(master_rp->index, erp_table->erp_index_bitmap);
618err_table_master_rp:
619 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps,
620 erp_table->aregion->type,
621 erp_table->base_index);
622 return err;
623}
624
625static void
626mlxsw_sp_acl_erp_region_master_mask_trans(struct mlxsw_sp_acl_erp_table *erp_table)
627{
628 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
629 struct mlxsw_sp_acl_erp *master_rp;
630
631 mlxsw_sp_acl_erp_table_disable(erp_table);
632 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
633 if (!master_rp)
634 return;
635 mlxsw_acl_erp_table_bf_del(erp_table, master_rp);
636 mlxsw_sp_acl_erp_table_erp_del(master_rp);
637 __clear_bit(master_rp->index, erp_table->erp_index_bitmap);
638 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps,
639 erp_table->aregion->type,
640 erp_table->base_index);
641}
642
643static int
644mlxsw_sp_acl_erp_region_erp_add(struct mlxsw_sp_acl_erp_table *erp_table,
645 struct mlxsw_sp_acl_erp *erp)
646{
647 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
648 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
649 bool ctcam_le = erp_table->num_ctcam_erps > 0;
650 char pererp_pl[MLXSW_REG_PERERP_LEN];
651
652 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
653 erp_table->base_index, 0);
654 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
655 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
656 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, true);
657
658 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
659}
660
661static void mlxsw_sp_acl_erp_region_erp_del(struct mlxsw_sp_acl_erp *erp)
662{
663 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
664 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
665 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
666 bool ctcam_le = erp_table->num_ctcam_erps > 0;
667 char pererp_pl[MLXSW_REG_PERERP_LEN];
668
669 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
670 erp_table->base_index, 0);
671 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
672 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
673 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, false);
674
675 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
676}
677
678static int
679mlxsw_sp_acl_erp_region_ctcam_enable(struct mlxsw_sp_acl_erp_table *erp_table)
680{
681 /* No need to re-enable lookup in the C-TCAM */
682 if (erp_table->num_ctcam_erps > 1)
683 return 0;
684
685 return mlxsw_sp_acl_erp_table_enable(erp_table, true);
686}
687
688static void
689mlxsw_sp_acl_erp_region_ctcam_disable(struct mlxsw_sp_acl_erp_table *erp_table)
690{
691 /* Only disable C-TCAM lookup when last C-TCAM eRP is deleted */
692 if (erp_table->num_ctcam_erps > 1)
693 return;
694
695 mlxsw_sp_acl_erp_table_enable(erp_table, false);
696}
697
698static int
699__mlxsw_sp_acl_erp_table_other_inc(struct mlxsw_sp_acl_erp_table *erp_table,
700 unsigned int *inc_num)
701{
702 int err;
703
704 /* If there are C-TCAM eRP or deltas in use we need to transition
705 * the region to use eRP table, if it is not already done
706 */
707 if (!mlxsw_sp_acl_erp_table_is_used(erp_table)) {
708 err = mlxsw_sp_acl_erp_region_table_trans(erp_table);
709 if (err)
710 return err;
711 }
712
713 /* When C-TCAM or deltas are used, the eRP table must be used */
714 if (erp_table->ops != &erp_multiple_masks_ops)
715 erp_table->ops = &erp_multiple_masks_ops;
716
717 (*inc_num)++;
718
719 return 0;
720}
721
722static int mlxsw_sp_acl_erp_ctcam_inc(struct mlxsw_sp_acl_erp_table *erp_table)
723{
724 return __mlxsw_sp_acl_erp_table_other_inc(erp_table,
725 &erp_table->num_ctcam_erps);
726}
727
728static int mlxsw_sp_acl_erp_delta_inc(struct mlxsw_sp_acl_erp_table *erp_table)
729{
730 return __mlxsw_sp_acl_erp_table_other_inc(erp_table,
731 &erp_table->num_deltas);
732}
733
734static void
735__mlxsw_sp_acl_erp_table_other_dec(struct mlxsw_sp_acl_erp_table *erp_table,
736 unsigned int *dec_num)
737{
738 (*dec_num)--;
739
740 /* If there are no C-TCAM eRP or deltas in use, the state we
741 * transition to depends on the number of A-TCAM eRPs currently
742 * in use.
743 */
744 if (erp_table->num_ctcam_erps > 0 || erp_table->num_deltas > 0)
745 return;
746
747 switch (erp_table->num_atcam_erps) {
748 case 2:
749 /* Keep using the eRP table, but correctly set the
750 * operations pointer so that when an A-TCAM eRP is
751 * deleted we will transition to use the master mask
752 */
753 erp_table->ops = &erp_two_masks_ops;
754 break;
755 case 1:
756 /* We only kept the eRP table because we had C-TCAM
757 * eRPs in use. Now that the last C-TCAM eRP is gone we
758 * can stop using the table and transition to use the
759 * master mask
760 */
761 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
762 erp_table->ops = &erp_single_mask_ops;
763 break;
764 case 0:
765 /* There are no more eRPs of any kind used by the region
766 * so free its eRP table and transition to initial state
767 */
768 mlxsw_sp_acl_erp_table_disable(erp_table);
769 mlxsw_sp_acl_erp_table_free(erp_table->erp_core,
770 erp_table->num_max_atcam_erps,
771 erp_table->aregion->type,
772 erp_table->base_index);
773 erp_table->ops = &erp_no_mask_ops;
774 break;
775 default:
776 break;
777 }
778}
779
780static void mlxsw_sp_acl_erp_ctcam_dec(struct mlxsw_sp_acl_erp_table *erp_table)
781{
782 __mlxsw_sp_acl_erp_table_other_dec(erp_table,
783 &erp_table->num_ctcam_erps);
784}
785
786static void mlxsw_sp_acl_erp_delta_dec(struct mlxsw_sp_acl_erp_table *erp_table)
787{
788 __mlxsw_sp_acl_erp_table_other_dec(erp_table,
789 &erp_table->num_deltas);
790}
791
792static struct mlxsw_sp_acl_erp *
793mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
794 struct mlxsw_sp_acl_erp_key *key)
795{
796 struct mlxsw_sp_acl_erp *erp;
797 int err;
798
799 erp = kzalloc(sizeof(*erp), GFP_KERNEL);
800 if (!erp)
801 return ERR_PTR(-ENOMEM);
802
803 memcpy(&erp->key, key, sizeof(*key));
804 bitmap_from_arr32(erp->mask_bitmap, (u32 *) key->mask,
805 MLXSW_SP_ACL_TCAM_MASK_LEN);
806
807 err = mlxsw_sp_acl_erp_ctcam_inc(erp_table);
808 if (err)
809 goto err_erp_ctcam_inc;
810
811 erp->erp_table = erp_table;
812
813 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key);
814 if (err)
815 goto err_master_mask_set;
816
817 err = mlxsw_sp_acl_erp_region_ctcam_enable(erp_table);
818 if (err)
819 goto err_erp_region_ctcam_enable;
820
821 return erp;
822
823err_erp_region_ctcam_enable:
824 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
825err_master_mask_set:
826 mlxsw_sp_acl_erp_ctcam_dec(erp_table);
827err_erp_ctcam_inc:
828 kfree(erp);
829 return ERR_PTR(err);
830}
831
832static void
833mlxsw_sp_acl_erp_ctcam_mask_destroy(struct mlxsw_sp_acl_erp *erp)
834{
835 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
836
837 mlxsw_sp_acl_erp_region_ctcam_disable(erp_table);
838 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
839 mlxsw_sp_acl_erp_ctcam_dec(erp_table);
840 kfree(erp);
841}
842
843static struct mlxsw_sp_acl_erp *
844mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
845 struct mlxsw_sp_acl_erp_key *key)
846{
847 struct mlxsw_sp_acl_erp *erp;
848 int err;
849
850 if (key->ctcam)
851 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
852
853 /* Expand the eRP table for the new eRP, if needed */
854 err = mlxsw_sp_acl_erp_table_expand(erp_table);
855 if (err)
856 return ERR_PTR(err);
857
858 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
859 if (IS_ERR(erp))
860 return erp;
861
862 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index);
863 if (err)
864 goto err_erp_index_get;
865
866 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
867 if (err)
868 goto err_table_erp_add;
869
870 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp);
871 if (err)
872 goto err_region_erp_add;
873
874 erp_table->ops = &erp_multiple_masks_ops;
875
876 return erp;
877
878err_region_erp_add:
879 mlxsw_sp_acl_erp_table_erp_del(erp);
880err_table_erp_add:
881 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
882err_erp_index_get:
883 mlxsw_sp_acl_erp_generic_destroy(erp);
884 return ERR_PTR(err);
885}
886
887static void
888mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
889 struct mlxsw_sp_acl_erp *erp)
890{
891 if (erp->key.ctcam)
892 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp);
893
894 mlxsw_sp_acl_erp_region_erp_del(erp);
895 mlxsw_sp_acl_erp_table_erp_del(erp);
896 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
897 mlxsw_sp_acl_erp_generic_destroy(erp);
898
899 if (erp_table->num_atcam_erps == 2 && erp_table->num_ctcam_erps == 0 &&
900 erp_table->num_deltas == 0)
901 erp_table->ops = &erp_two_masks_ops;
902}
903
904static struct mlxsw_sp_acl_erp *
905mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
906 struct mlxsw_sp_acl_erp_key *key)
907{
908 struct mlxsw_sp_acl_erp *erp;
909 int err;
910
911 if (key->ctcam)
912 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
913
914 /* Transition to use eRP table instead of master mask */
915 err = mlxsw_sp_acl_erp_region_table_trans(erp_table);
916 if (err)
917 return ERR_PTR(err);
918
919 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
920 if (IS_ERR(erp)) {
921 err = PTR_ERR(erp);
922 goto err_erp_create;
923 }
924
925 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index);
926 if (err)
927 goto err_erp_index_get;
928
929 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
930 if (err)
931 goto err_table_erp_add;
932
933 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp);
934 if (err)
935 goto err_region_erp_add;
936
937 erp_table->ops = &erp_two_masks_ops;
938
939 return erp;
940
941err_region_erp_add:
942 mlxsw_sp_acl_erp_table_erp_del(erp);
943err_table_erp_add:
944 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
945err_erp_index_get:
946 mlxsw_sp_acl_erp_generic_destroy(erp);
947err_erp_create:
948 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
949 return ERR_PTR(err);
950}
951
952static void
953mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
954 struct mlxsw_sp_acl_erp *erp)
955{
956 if (erp->key.ctcam)
957 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp);
958
959 mlxsw_sp_acl_erp_region_erp_del(erp);
960 mlxsw_sp_acl_erp_table_erp_del(erp);
961 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
962 mlxsw_sp_acl_erp_generic_destroy(erp);
963 /* Transition to use master mask instead of eRP table */
964 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
965
966 erp_table->ops = &erp_single_mask_ops;
967}
968
969static struct mlxsw_sp_acl_erp *
970mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
971 struct mlxsw_sp_acl_erp_key *key)
972{
973 struct mlxsw_sp_acl_erp *erp;
974
975 if (key->ctcam)
976 return ERR_PTR(-EINVAL);
977
978 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
979 if (IS_ERR(erp))
980 return erp;
981
982 erp_table->ops = &erp_single_mask_ops;
983
984 return erp;
985}
986
987static void
988mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
989 struct mlxsw_sp_acl_erp *erp)
990{
991 mlxsw_sp_acl_erp_generic_destroy(erp);
992 erp_table->ops = &erp_no_mask_ops;
993}
994
995static void
996mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
997 struct mlxsw_sp_acl_erp *erp)
998{
999 WARN_ON(1);
1000}
1001
1002struct mlxsw_sp_acl_erp_mask *
1003mlxsw_sp_acl_erp_mask_get(struct mlxsw_sp_acl_atcam_region *aregion,
1004 const char *mask, bool ctcam)
1005{
1006 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1007 struct mlxsw_sp_acl_erp_key key;
1008 struct objagg_obj *objagg_obj;
1009
1010 memcpy(key.mask, mask, MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN);
1011 key.ctcam = ctcam;
1012 mutex_lock(&erp_table->objagg_lock);
1013 objagg_obj = objagg_obj_get(erp_table->objagg, &key);
1014 mutex_unlock(&erp_table->objagg_lock);
1015 if (IS_ERR(objagg_obj))
1016 return ERR_CAST(objagg_obj);
1017 return (struct mlxsw_sp_acl_erp_mask *) objagg_obj;
1018}
1019
1020void mlxsw_sp_acl_erp_mask_put(struct mlxsw_sp_acl_atcam_region *aregion,
1021 struct mlxsw_sp_acl_erp_mask *erp_mask)
1022{
1023 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1024 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1025
1026 mutex_lock(&erp_table->objagg_lock);
1027 objagg_obj_put(erp_table->objagg, objagg_obj);
1028 mutex_unlock(&erp_table->objagg_lock);
1029}
1030
1031int mlxsw_sp_acl_erp_bf_insert(struct mlxsw_sp *mlxsw_sp,
1032 struct mlxsw_sp_acl_atcam_region *aregion,
1033 struct mlxsw_sp_acl_erp_mask *erp_mask,
1034 struct mlxsw_sp_acl_atcam_entry *aentry)
1035{
1036 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1037 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1038 unsigned int erp_bank;
1039
1040 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table))
1041 return 0;
1042
1043 erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
1044 return mlxsw_sp_acl_bf_entry_add(mlxsw_sp,
1045 erp->erp_table->erp_core->bf,
1046 aregion, erp_bank, aentry);
1047}
1048
1049void mlxsw_sp_acl_erp_bf_remove(struct mlxsw_sp *mlxsw_sp,
1050 struct mlxsw_sp_acl_atcam_region *aregion,
1051 struct mlxsw_sp_acl_erp_mask *erp_mask,
1052 struct mlxsw_sp_acl_atcam_entry *aentry)
1053{
1054 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1055 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1056 unsigned int erp_bank;
1057
1058 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table))
1059 return;
1060
1061 erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
1062 mlxsw_sp_acl_bf_entry_del(mlxsw_sp,
1063 erp->erp_table->erp_core->bf,
1064 aregion, erp_bank, aentry);
1065}
1066
1067bool
1068mlxsw_sp_acl_erp_mask_is_ctcam(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1069{
1070 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1071 const struct mlxsw_sp_acl_erp_key *key = objagg_obj_raw(objagg_obj);
1072
1073 return key->ctcam;
1074}
1075
1076u8 mlxsw_sp_acl_erp_mask_erp_id(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1077{
1078 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1079 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1080
1081 return erp->id;
1082}
1083
1084struct mlxsw_sp_acl_erp_delta {
1085 struct mlxsw_sp_acl_erp_key key;
1086 u16 start;
1087 u8 mask;
1088};
1089
1090u16 mlxsw_sp_acl_erp_delta_start(const struct mlxsw_sp_acl_erp_delta *delta)
1091{
1092 return delta->start;
1093}
1094
1095u8 mlxsw_sp_acl_erp_delta_mask(const struct mlxsw_sp_acl_erp_delta *delta)
1096{
1097 return delta->mask;
1098}
1099
1100u8 mlxsw_sp_acl_erp_delta_value(const struct mlxsw_sp_acl_erp_delta *delta,
1101 const char *enc_key)
1102{
1103 u16 start = delta->start;
1104 u8 mask = delta->mask;
1105 u16 tmp;
1106
1107 if (!mask)
1108 return 0;
1109
1110 tmp = (unsigned char) enc_key[__MASK_IDX(start / 8)];
1111 if (start / 8 + 1 < __MASK_LEN)
1112 tmp |= (unsigned char) enc_key[__MASK_IDX(start / 8 + 1)] << 8;
1113 tmp >>= start % 8;
1114 tmp &= mask;
1115 return tmp;
1116}
1117
1118void mlxsw_sp_acl_erp_delta_clear(const struct mlxsw_sp_acl_erp_delta *delta,
1119 const char *enc_key)
1120{
1121 u16 start = delta->start;
1122 u8 mask = delta->mask;
1123 unsigned char *byte;
1124 u16 tmp;
1125
1126 tmp = mask;
1127 tmp <<= start % 8;
1128 tmp = ~tmp;
1129
1130 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8)];
1131 *byte &= tmp & 0xff;
1132 if (start / 8 + 1 < __MASK_LEN) {
1133 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8 + 1)];
1134 *byte &= (tmp >> 8) & 0xff;
1135 }
1136}
1137
1138static const struct mlxsw_sp_acl_erp_delta
1139mlxsw_sp_acl_erp_delta_default = {};
1140
1141const struct mlxsw_sp_acl_erp_delta *
1142mlxsw_sp_acl_erp_delta(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1143{
1144 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1145 const struct mlxsw_sp_acl_erp_delta *delta;
1146
1147 delta = objagg_obj_delta_priv(objagg_obj);
1148 if (!delta)
1149 delta = &mlxsw_sp_acl_erp_delta_default;
1150 return delta;
1151}
1152
1153static int
1154mlxsw_sp_acl_erp_delta_fill(const struct mlxsw_sp_acl_erp_key *parent_key,
1155 const struct mlxsw_sp_acl_erp_key *key,
1156 u16 *delta_start, u8 *delta_mask)
1157{
1158 int offset = 0;
1159 int si = -1;
1160 u16 pmask;
1161 u16 mask;
1162 int i;
1163
1164 /* The difference between 2 masks can be up to 8 consecutive bits. */
1165 for (i = 0; i < __MASK_LEN; i++) {
1166 if (parent_key->mask[__MASK_IDX(i)] == key->mask[__MASK_IDX(i)])
1167 continue;
1168 if (si == -1)
1169 si = i;
1170 else if (si != i - 1)
1171 return -EINVAL;
1172 }
1173 if (si == -1) {
1174 /* The masks are the same, this can happen in case eRPs with
1175 * the same mask were created in both A-TCAM and C-TCAM.
1176 * The only possible condition under which this can happen
1177 * is identical rule insertion. Delta is not possible here.
1178 */
1179 return -EINVAL;
1180 }
1181 pmask = (unsigned char) parent_key->mask[__MASK_IDX(si)];
1182 mask = (unsigned char) key->mask[__MASK_IDX(si)];
1183 if (si + 1 < __MASK_LEN) {
1184 pmask |= (unsigned char) parent_key->mask[__MASK_IDX(si + 1)] << 8;
1185 mask |= (unsigned char) key->mask[__MASK_IDX(si + 1)] << 8;
1186 }
1187
1188 if ((pmask ^ mask) & pmask)
1189 return -EINVAL;
1190 mask &= ~pmask;
1191 while (!(mask & (1 << offset)))
1192 offset++;
1193 while (!(mask & 1))
1194 mask >>= 1;
1195 if (mask & 0xff00)
1196 return -EINVAL;
1197
1198 *delta_start = si * 8 + offset;
1199 *delta_mask = mask;
1200
1201 return 0;
1202}
1203
1204static bool mlxsw_sp_acl_erp_delta_check(void *priv, const void *parent_obj,
1205 const void *obj)
1206{
1207 const struct mlxsw_sp_acl_erp_key *parent_key = parent_obj;
1208 const struct mlxsw_sp_acl_erp_key *key = obj;
1209 u16 delta_start;
1210 u8 delta_mask;
1211 int err;
1212
1213 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key,
1214 &delta_start, &delta_mask);
1215 return err ? false : true;
1216}
1217
1218static int mlxsw_sp_acl_erp_hints_obj_cmp(const void *obj1, const void *obj2)
1219{
1220 const struct mlxsw_sp_acl_erp_key *key1 = obj1;
1221 const struct mlxsw_sp_acl_erp_key *key2 = obj2;
1222
1223 /* For hints purposes, two objects are considered equal
1224 * in case the masks are the same. Does not matter what
1225 * the "ctcam" value is.
1226 */
1227 return memcmp(key1->mask, key2->mask, sizeof(key1->mask));
1228}
1229
1230static void *mlxsw_sp_acl_erp_delta_create(void *priv, void *parent_obj,
1231 void *obj)
1232{
1233 struct mlxsw_sp_acl_erp_key *parent_key = parent_obj;
1234 struct mlxsw_sp_acl_atcam_region *aregion = priv;
1235 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1236 struct mlxsw_sp_acl_erp_key *key = obj;
1237 struct mlxsw_sp_acl_erp_delta *delta;
1238 u16 delta_start;
1239 u8 delta_mask;
1240 int err;
1241
1242 if (parent_key->ctcam || key->ctcam)
1243 return ERR_PTR(-EINVAL);
1244 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key,
1245 &delta_start, &delta_mask);
1246 if (err)
1247 return ERR_PTR(-EINVAL);
1248
1249 delta = kzalloc(sizeof(*delta), GFP_KERNEL);
1250 if (!delta)
1251 return ERR_PTR(-ENOMEM);
1252 delta->start = delta_start;
1253 delta->mask = delta_mask;
1254
1255 err = mlxsw_sp_acl_erp_delta_inc(erp_table);
1256 if (err)
1257 goto err_erp_delta_inc;
1258
1259 memcpy(&delta->key, key, sizeof(*key));
1260 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &delta->key);
1261 if (err)
1262 goto err_master_mask_set;
1263
1264 return delta;
1265
1266err_master_mask_set:
1267 mlxsw_sp_acl_erp_delta_dec(erp_table);
1268err_erp_delta_inc:
1269 kfree(delta);
1270 return ERR_PTR(err);
1271}
1272
1273static void mlxsw_sp_acl_erp_delta_destroy(void *priv, void *delta_priv)
1274{
1275 struct mlxsw_sp_acl_erp_delta *delta = delta_priv;
1276 struct mlxsw_sp_acl_atcam_region *aregion = priv;
1277 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1278
1279 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &delta->key);
1280 mlxsw_sp_acl_erp_delta_dec(erp_table);
1281 kfree(delta);
1282}
1283
1284static void *mlxsw_sp_acl_erp_root_create(void *priv, void *obj,
1285 unsigned int root_id)
1286{
1287 struct mlxsw_sp_acl_atcam_region *aregion = priv;
1288 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1289 struct mlxsw_sp_acl_erp_key *key = obj;
1290
1291 if (!key->ctcam &&
1292 root_id != OBJAGG_OBJ_ROOT_ID_INVALID &&
1293 root_id >= MLXSW_SP_ACL_ERP_MAX_PER_REGION)
1294 return ERR_PTR(-ENOBUFS);
1295 return erp_table->ops->erp_create(erp_table, key);
1296}
1297
1298static void mlxsw_sp_acl_erp_root_destroy(void *priv, void *root_priv)
1299{
1300 struct mlxsw_sp_acl_atcam_region *aregion = priv;
1301 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1302
1303 erp_table->ops->erp_destroy(erp_table, root_priv);
1304}
1305
1306static const struct objagg_ops mlxsw_sp_acl_erp_objagg_ops = {
1307 .obj_size = sizeof(struct mlxsw_sp_acl_erp_key),
1308 .delta_check = mlxsw_sp_acl_erp_delta_check,
1309 .hints_obj_cmp = mlxsw_sp_acl_erp_hints_obj_cmp,
1310 .delta_create = mlxsw_sp_acl_erp_delta_create,
1311 .delta_destroy = mlxsw_sp_acl_erp_delta_destroy,
1312 .root_create = mlxsw_sp_acl_erp_root_create,
1313 .root_destroy = mlxsw_sp_acl_erp_root_destroy,
1314};
1315
1316static struct mlxsw_sp_acl_erp_table *
1317mlxsw_sp_acl_erp_table_create(struct mlxsw_sp_acl_atcam_region *aregion,
1318 struct objagg_hints *hints)
1319{
1320 struct mlxsw_sp_acl_erp_table *erp_table;
1321 int err;
1322
1323 erp_table = kzalloc(sizeof(*erp_table), GFP_KERNEL);
1324 if (!erp_table)
1325 return ERR_PTR(-ENOMEM);
1326
1327 erp_table->objagg = objagg_create(&mlxsw_sp_acl_erp_objagg_ops,
1328 hints, aregion);
1329 if (IS_ERR(erp_table->objagg)) {
1330 err = PTR_ERR(erp_table->objagg);
1331 goto err_objagg_create;
1332 }
1333
1334 erp_table->erp_core = aregion->atcam->erp_core;
1335 erp_table->ops = &erp_no_mask_ops;
1336 INIT_LIST_HEAD(&erp_table->atcam_erps_list);
1337 erp_table->aregion = aregion;
1338 mutex_init(&erp_table->objagg_lock);
1339
1340 return erp_table;
1341
1342err_objagg_create:
1343 kfree(erp_table);
1344 return ERR_PTR(err);
1345}
1346
1347static void
1348mlxsw_sp_acl_erp_table_destroy(struct mlxsw_sp_acl_erp_table *erp_table)
1349{
1350 WARN_ON(!list_empty(&erp_table->atcam_erps_list));
1351 mutex_destroy(&erp_table->objagg_lock);
1352 objagg_destroy(erp_table->objagg);
1353 kfree(erp_table);
1354}
1355
1356static int
1357mlxsw_sp_acl_erp_master_mask_init(struct mlxsw_sp_acl_atcam_region *aregion)
1358{
1359 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1360 char percr_pl[MLXSW_REG_PERCR_LEN];
1361
1362 mlxsw_reg_percr_pack(percr_pl, aregion->region->id);
1363 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl);
1364}
1365
1366static int
1367mlxsw_sp_acl_erp_region_param_init(struct mlxsw_sp_acl_atcam_region *aregion)
1368{
1369 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1370 char pererp_pl[MLXSW_REG_PERERP_LEN];
1371
1372 mlxsw_reg_pererp_pack(pererp_pl, aregion->region->id, false, false, 0,
1373 0, 0);
1374 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
1375}
1376
1377static int
1378mlxsw_sp_acl_erp_hints_check(struct mlxsw_sp *mlxsw_sp,
1379 struct mlxsw_sp_acl_atcam_region *aregion,
1380 struct objagg_hints *hints, bool *p_rehash_needed)
1381{
1382 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1383 const struct objagg_stats *ostats;
1384 const struct objagg_stats *hstats;
1385 int err;
1386
1387 *p_rehash_needed = false;
1388
1389 mutex_lock(&erp_table->objagg_lock);
1390 ostats = objagg_stats_get(erp_table->objagg);
1391 mutex_unlock(&erp_table->objagg_lock);
1392 if (IS_ERR(ostats)) {
1393 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP stats\n");
1394 return PTR_ERR(ostats);
1395 }
1396
1397 hstats = objagg_hints_stats_get(hints);
1398 if (IS_ERR(hstats)) {
1399 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP hints stats\n");
1400 err = PTR_ERR(hstats);
1401 goto err_hints_stats_get;
1402 }
1403
1404 /* Very basic criterion for now. */
1405 if (hstats->root_count < ostats->root_count)
1406 *p_rehash_needed = true;
1407
1408 err = 0;
1409
1410 objagg_stats_put(hstats);
1411err_hints_stats_get:
1412 objagg_stats_put(ostats);
1413 return err;
1414}
1415
1416void *
1417mlxsw_sp_acl_erp_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion)
1418{
1419 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1420 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1421 struct objagg_hints *hints;
1422 bool rehash_needed;
1423 int err;
1424
1425 mutex_lock(&erp_table->objagg_lock);
1426 hints = objagg_hints_get(erp_table->objagg,
1427 OBJAGG_OPT_ALGO_SIMPLE_GREEDY);
1428 mutex_unlock(&erp_table->objagg_lock);
1429 if (IS_ERR(hints)) {
1430 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to create ERP hints\n");
1431 return ERR_CAST(hints);
1432 }
1433 err = mlxsw_sp_acl_erp_hints_check(mlxsw_sp, aregion, hints,
1434 &rehash_needed);
1435 if (err)
1436 goto errout;
1437
1438 if (!rehash_needed) {
1439 err = -EAGAIN;
1440 goto errout;
1441 }
1442 return hints;
1443
1444errout:
1445 objagg_hints_put(hints);
1446 return ERR_PTR(err);
1447}
1448
1449void mlxsw_sp_acl_erp_rehash_hints_put(void *hints_priv)
1450{
1451 struct objagg_hints *hints = hints_priv;
1452
1453 objagg_hints_put(hints);
1454}
1455
1456int mlxsw_sp_acl_erp_region_init(struct mlxsw_sp_acl_atcam_region *aregion,
1457 void *hints_priv)
1458{
1459 struct mlxsw_sp_acl_erp_table *erp_table;
1460 struct objagg_hints *hints = hints_priv;
1461 int err;
1462
1463 erp_table = mlxsw_sp_acl_erp_table_create(aregion, hints);
1464 if (IS_ERR(erp_table))
1465 return PTR_ERR(erp_table);
1466 aregion->erp_table = erp_table;
1467
1468 /* Initialize the region's master mask to all zeroes */
1469 err = mlxsw_sp_acl_erp_master_mask_init(aregion);
1470 if (err)
1471 goto err_erp_master_mask_init;
1472
1473 /* Initialize the region to not use the eRP table */
1474 err = mlxsw_sp_acl_erp_region_param_init(aregion);
1475 if (err)
1476 goto err_erp_region_param_init;
1477
1478 return 0;
1479
1480err_erp_region_param_init:
1481err_erp_master_mask_init:
1482 mlxsw_sp_acl_erp_table_destroy(erp_table);
1483 return err;
1484}
1485
1486void mlxsw_sp_acl_erp_region_fini(struct mlxsw_sp_acl_atcam_region *aregion)
1487{
1488 mlxsw_sp_acl_erp_table_destroy(aregion->erp_table);
1489}
1490
1491static int
1492mlxsw_sp_acl_erp_tables_sizes_query(struct mlxsw_sp *mlxsw_sp,
1493 struct mlxsw_sp_acl_erp_core *erp_core)
1494{
1495 unsigned int size;
1496
1497 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB) ||
1498 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB) ||
1499 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB) ||
1500 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB))
1501 return -EIO;
1502
1503 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB);
1504 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB] = size;
1505
1506 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB);
1507 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB] = size;
1508
1509 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB);
1510 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB] = size;
1511
1512 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB);
1513 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB] = size;
1514
1515 return 0;
1516}
1517
1518static int mlxsw_sp_acl_erp_tables_init(struct mlxsw_sp *mlxsw_sp,
1519 struct mlxsw_sp_acl_erp_core *erp_core)
1520{
1521 unsigned int erpt_bank_size;
1522 int err;
1523
1524 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANK_SIZE) ||
1525 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANKS))
1526 return -EIO;
1527 erpt_bank_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1528 ACL_MAX_ERPT_BANK_SIZE);
1529 erp_core->num_erp_banks = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1530 ACL_MAX_ERPT_BANKS);
1531
1532 erp_core->erp_tables = gen_pool_create(0, -1);
1533 if (!erp_core->erp_tables)
1534 return -ENOMEM;
1535 gen_pool_set_algo(erp_core->erp_tables, gen_pool_best_fit, NULL);
1536
1537 err = gen_pool_add(erp_core->erp_tables,
1538 MLXSW_SP_ACL_ERP_GENALLOC_OFFSET, erpt_bank_size,
1539 -1);
1540 if (err)
1541 goto err_gen_pool_add;
1542
1543 erp_core->bf = mlxsw_sp_acl_bf_init(mlxsw_sp, erp_core->num_erp_banks);
1544 if (IS_ERR(erp_core->bf)) {
1545 err = PTR_ERR(erp_core->bf);
1546 goto err_bf_init;
1547 }
1548
1549 /* Different regions require masks of different sizes */
1550 err = mlxsw_sp_acl_erp_tables_sizes_query(mlxsw_sp, erp_core);
1551 if (err)
1552 goto err_erp_tables_sizes_query;
1553
1554 return 0;
1555
1556err_erp_tables_sizes_query:
1557 mlxsw_sp_acl_bf_fini(erp_core->bf);
1558err_bf_init:
1559err_gen_pool_add:
1560 gen_pool_destroy(erp_core->erp_tables);
1561 return err;
1562}
1563
1564static void mlxsw_sp_acl_erp_tables_fini(struct mlxsw_sp *mlxsw_sp,
1565 struct mlxsw_sp_acl_erp_core *erp_core)
1566{
1567 mlxsw_sp_acl_bf_fini(erp_core->bf);
1568 gen_pool_destroy(erp_core->erp_tables);
1569}
1570
1571int mlxsw_sp_acl_erps_init(struct mlxsw_sp *mlxsw_sp,
1572 struct mlxsw_sp_acl_atcam *atcam)
1573{
1574 struct mlxsw_sp_acl_erp_core *erp_core;
1575 int err;
1576
1577 erp_core = kzalloc(sizeof(*erp_core), GFP_KERNEL);
1578 if (!erp_core)
1579 return -ENOMEM;
1580 erp_core->mlxsw_sp = mlxsw_sp;
1581 atcam->erp_core = erp_core;
1582
1583 err = mlxsw_sp_acl_erp_tables_init(mlxsw_sp, erp_core);
1584 if (err)
1585 goto err_erp_tables_init;
1586
1587 return 0;
1588
1589err_erp_tables_init:
1590 kfree(erp_core);
1591 return err;
1592}
1593
1594void mlxsw_sp_acl_erps_fini(struct mlxsw_sp *mlxsw_sp,
1595 struct mlxsw_sp_acl_atcam *atcam)
1596{
1597 mlxsw_sp_acl_erp_tables_fini(mlxsw_sp, atcam->erp_core);
1598 kfree(atcam->erp_core);
1599}