Loading...
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
4 */
5
6#include "mlx5_ib.h"
7#include <linux/mlx5/eswitch.h>
8#include <linux/mlx5/vport.h>
9#include "counters.h"
10#include "ib_rep.h"
11#include "qp.h"
12
13struct mlx5_ib_counter {
14 const char *name;
15 size_t offset;
16 u32 type;
17};
18
19#define INIT_Q_COUNTER(_name) \
20 { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}
21
22#define INIT_VPORT_Q_COUNTER(_name) \
23 { .name = "vport_" #_name, .offset = \
24 MLX5_BYTE_OFF(query_q_counter_out, _name)}
25
26static const struct mlx5_ib_counter basic_q_cnts[] = {
27 INIT_Q_COUNTER(rx_write_requests),
28 INIT_Q_COUNTER(rx_read_requests),
29 INIT_Q_COUNTER(rx_atomic_requests),
30 INIT_Q_COUNTER(rx_dct_connect),
31 INIT_Q_COUNTER(out_of_buffer),
32};
33
34static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {
35 INIT_Q_COUNTER(out_of_sequence),
36};
37
38static const struct mlx5_ib_counter retrans_q_cnts[] = {
39 INIT_Q_COUNTER(duplicate_request),
40 INIT_Q_COUNTER(rnr_nak_retry_err),
41 INIT_Q_COUNTER(packet_seq_err),
42 INIT_Q_COUNTER(implied_nak_seq_err),
43 INIT_Q_COUNTER(local_ack_timeout_err),
44};
45
46static const struct mlx5_ib_counter vport_basic_q_cnts[] = {
47 INIT_VPORT_Q_COUNTER(rx_write_requests),
48 INIT_VPORT_Q_COUNTER(rx_read_requests),
49 INIT_VPORT_Q_COUNTER(rx_atomic_requests),
50 INIT_VPORT_Q_COUNTER(rx_dct_connect),
51 INIT_VPORT_Q_COUNTER(out_of_buffer),
52};
53
54static const struct mlx5_ib_counter vport_out_of_seq_q_cnts[] = {
55 INIT_VPORT_Q_COUNTER(out_of_sequence),
56};
57
58static const struct mlx5_ib_counter vport_retrans_q_cnts[] = {
59 INIT_VPORT_Q_COUNTER(duplicate_request),
60 INIT_VPORT_Q_COUNTER(rnr_nak_retry_err),
61 INIT_VPORT_Q_COUNTER(packet_seq_err),
62 INIT_VPORT_Q_COUNTER(implied_nak_seq_err),
63 INIT_VPORT_Q_COUNTER(local_ack_timeout_err),
64};
65
66#define INIT_CONG_COUNTER(_name) \
67 { .name = #_name, .offset = \
68 MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}
69
70static const struct mlx5_ib_counter cong_cnts[] = {
71 INIT_CONG_COUNTER(rp_cnp_ignored),
72 INIT_CONG_COUNTER(rp_cnp_handled),
73 INIT_CONG_COUNTER(np_ecn_marked_roce_packets),
74 INIT_CONG_COUNTER(np_cnp_sent),
75};
76
77static const struct mlx5_ib_counter extended_err_cnts[] = {
78 INIT_Q_COUNTER(resp_local_length_error),
79 INIT_Q_COUNTER(resp_cqe_error),
80 INIT_Q_COUNTER(req_cqe_error),
81 INIT_Q_COUNTER(req_remote_invalid_request),
82 INIT_Q_COUNTER(req_remote_access_errors),
83 INIT_Q_COUNTER(resp_remote_access_errors),
84 INIT_Q_COUNTER(resp_cqe_flush_error),
85 INIT_Q_COUNTER(req_cqe_flush_error),
86};
87
88static const struct mlx5_ib_counter roce_accl_cnts[] = {
89 INIT_Q_COUNTER(roce_adp_retrans),
90 INIT_Q_COUNTER(roce_adp_retrans_to),
91 INIT_Q_COUNTER(roce_slow_restart),
92 INIT_Q_COUNTER(roce_slow_restart_cnps),
93 INIT_Q_COUNTER(roce_slow_restart_trans),
94};
95
96static const struct mlx5_ib_counter vport_extended_err_cnts[] = {
97 INIT_VPORT_Q_COUNTER(resp_local_length_error),
98 INIT_VPORT_Q_COUNTER(resp_cqe_error),
99 INIT_VPORT_Q_COUNTER(req_cqe_error),
100 INIT_VPORT_Q_COUNTER(req_remote_invalid_request),
101 INIT_VPORT_Q_COUNTER(req_remote_access_errors),
102 INIT_VPORT_Q_COUNTER(resp_remote_access_errors),
103 INIT_VPORT_Q_COUNTER(resp_cqe_flush_error),
104 INIT_VPORT_Q_COUNTER(req_cqe_flush_error),
105};
106
107static const struct mlx5_ib_counter vport_roce_accl_cnts[] = {
108 INIT_VPORT_Q_COUNTER(roce_adp_retrans),
109 INIT_VPORT_Q_COUNTER(roce_adp_retrans_to),
110 INIT_VPORT_Q_COUNTER(roce_slow_restart),
111 INIT_VPORT_Q_COUNTER(roce_slow_restart_cnps),
112 INIT_VPORT_Q_COUNTER(roce_slow_restart_trans),
113};
114
115#define INIT_EXT_PPCNT_COUNTER(_name) \
116 { .name = #_name, .offset = \
117 MLX5_BYTE_OFF(ppcnt_reg, \
118 counter_set.eth_extended_cntrs_grp_data_layout._name##_high)}
119
120static const struct mlx5_ib_counter ext_ppcnt_cnts[] = {
121 INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated),
122};
123
124#define INIT_OP_COUNTER(_name, _type) \
125 { .name = #_name, .type = MLX5_IB_OPCOUNTER_##_type}
126
127static const struct mlx5_ib_counter basic_op_cnts[] = {
128 INIT_OP_COUNTER(cc_rx_ce_pkts, CC_RX_CE_PKTS),
129};
130
131static const struct mlx5_ib_counter rdmarx_cnp_op_cnts[] = {
132 INIT_OP_COUNTER(cc_rx_cnp_pkts, CC_RX_CNP_PKTS),
133};
134
135static const struct mlx5_ib_counter rdmatx_cnp_op_cnts[] = {
136 INIT_OP_COUNTER(cc_tx_cnp_pkts, CC_TX_CNP_PKTS),
137};
138
139static int mlx5_ib_read_counters(struct ib_counters *counters,
140 struct ib_counters_read_attr *read_attr,
141 struct uverbs_attr_bundle *attrs)
142{
143 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
144 struct mlx5_read_counters_attr mread_attr = {};
145 struct mlx5_ib_flow_counters_desc *desc;
146 int ret, i;
147
148 mutex_lock(&mcounters->mcntrs_mutex);
149 if (mcounters->cntrs_max_index > read_attr->ncounters) {
150 ret = -EINVAL;
151 goto err_bound;
152 }
153
154 mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64),
155 GFP_KERNEL);
156 if (!mread_attr.out) {
157 ret = -ENOMEM;
158 goto err_bound;
159 }
160
161 mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl;
162 mread_attr.flags = read_attr->flags;
163 ret = mcounters->read_counters(counters->device, &mread_attr);
164 if (ret)
165 goto err_read;
166
167 /* do the pass over the counters data array to assign according to the
168 * descriptions and indexing pairs
169 */
170 desc = mcounters->counters_data;
171 for (i = 0; i < mcounters->ncounters; i++)
172 read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description];
173
174err_read:
175 kfree(mread_attr.out);
176err_bound:
177 mutex_unlock(&mcounters->mcntrs_mutex);
178 return ret;
179}
180
181static int mlx5_ib_destroy_counters(struct ib_counters *counters)
182{
183 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
184
185 mlx5_ib_counters_clear_description(counters);
186 if (mcounters->hw_cntrs_hndl)
187 mlx5_fc_destroy(to_mdev(counters->device)->mdev,
188 mcounters->hw_cntrs_hndl);
189 return 0;
190}
191
192static int mlx5_ib_create_counters(struct ib_counters *counters,
193 struct uverbs_attr_bundle *attrs)
194{
195 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
196
197 mutex_init(&mcounters->mcntrs_mutex);
198 return 0;
199}
200
201static bool vport_qcounters_supported(struct mlx5_ib_dev *dev)
202{
203 return MLX5_CAP_GEN(dev->mdev, q_counter_other_vport) &&
204 MLX5_CAP_GEN(dev->mdev, q_counter_aggregation);
205}
206
207static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev,
208 u32 port_num)
209{
210 if ((is_mdev_switchdev_mode(dev->mdev) &&
211 !vport_qcounters_supported(dev)) || !port_num)
212 return &dev->port[0].cnts;
213
214 return is_mdev_switchdev_mode(dev->mdev) ?
215 &dev->port[1].cnts : &dev->port[port_num - 1].cnts;
216}
217
218/**
219 * mlx5_ib_get_counters_id - Returns counters id to use for device+port
220 * @dev: Pointer to mlx5 IB device
221 * @port_num: Zero based port number
222 *
223 * mlx5_ib_get_counters_id() Returns counters set id to use for given
224 * device port combination in switchdev and non switchdev mode of the
225 * parent device.
226 */
227u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u32 port_num)
228{
229 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num + 1);
230
231 return cnts->set_id;
232}
233
234static struct rdma_hw_stats *do_alloc_stats(const struct mlx5_ib_counters *cnts)
235{
236 struct rdma_hw_stats *stats;
237 u32 num_hw_counters;
238 int i;
239
240 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
241 cnts->num_ext_ppcnt_counters;
242 stats = rdma_alloc_hw_stats_struct(cnts->descs,
243 num_hw_counters +
244 cnts->num_op_counters,
245 RDMA_HW_STATS_DEFAULT_LIFESPAN);
246 if (!stats)
247 return NULL;
248
249 for (i = 0; i < cnts->num_op_counters; i++)
250 set_bit(num_hw_counters + i, stats->is_disabled);
251
252 return stats;
253}
254
255static struct rdma_hw_stats *
256mlx5_ib_alloc_hw_device_stats(struct ib_device *ibdev)
257{
258 struct mlx5_ib_dev *dev = to_mdev(ibdev);
259 const struct mlx5_ib_counters *cnts = &dev->port[0].cnts;
260
261 return do_alloc_stats(cnts);
262}
263
264static struct rdma_hw_stats *
265mlx5_ib_alloc_hw_port_stats(struct ib_device *ibdev, u32 port_num)
266{
267 struct mlx5_ib_dev *dev = to_mdev(ibdev);
268 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
269
270 return do_alloc_stats(cnts);
271}
272
273static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,
274 const struct mlx5_ib_counters *cnts,
275 struct rdma_hw_stats *stats,
276 u16 set_id)
277{
278 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
279 u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
280 __be32 val;
281 int ret, i;
282
283 MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
284 MLX5_SET(query_q_counter_in, in, counter_set_id, set_id);
285 ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);
286 if (ret)
287 return ret;
288
289 for (i = 0; i < cnts->num_q_counters; i++) {
290 val = *(__be32 *)((void *)out + cnts->offsets[i]);
291 stats->value[i] = (u64)be32_to_cpu(val);
292 }
293
294 return 0;
295}
296
297static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev,
298 const struct mlx5_ib_counters *cnts,
299 struct rdma_hw_stats *stats)
300{
301 int offset = cnts->num_q_counters + cnts->num_cong_counters;
302 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
303 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
304 int ret, i;
305 void *out;
306
307 out = kvzalloc(sz, GFP_KERNEL);
308 if (!out)
309 return -ENOMEM;
310
311 MLX5_SET(ppcnt_reg, in, local_port, 1);
312 MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
313 ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT,
314 0, 0);
315 if (ret)
316 goto free;
317
318 for (i = 0; i < cnts->num_ext_ppcnt_counters; i++)
319 stats->value[i + offset] =
320 be64_to_cpup((__be64 *)(out +
321 cnts->offsets[i + offset]));
322free:
323 kvfree(out);
324 return ret;
325}
326
327static int mlx5_ib_query_q_counters_vport(struct mlx5_ib_dev *dev,
328 u32 port_num,
329 const struct mlx5_ib_counters *cnts,
330 struct rdma_hw_stats *stats)
331
332{
333 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
334 u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
335 struct mlx5_core_dev *mdev;
336 __be32 val;
337 int ret, i;
338
339 if (!dev->port[port_num].rep ||
340 dev->port[port_num].rep->vport == MLX5_VPORT_UPLINK)
341 return 0;
342
343 mdev = mlx5_eswitch_get_core_dev(dev->port[port_num].rep->esw);
344 if (!mdev)
345 return -EOPNOTSUPP;
346
347 MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
348 MLX5_SET(query_q_counter_in, in, other_vport, 1);
349 MLX5_SET(query_q_counter_in, in, vport_number,
350 dev->port[port_num].rep->vport);
351 MLX5_SET(query_q_counter_in, in, aggregate, 1);
352 ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);
353 if (ret)
354 return ret;
355
356 for (i = 0; i < cnts->num_q_counters; i++) {
357 val = *(__be32 *)((void *)out + cnts->offsets[i]);
358 stats->value[i] = (u64)be32_to_cpu(val);
359 }
360
361 return 0;
362}
363
364static int do_get_hw_stats(struct ib_device *ibdev,
365 struct rdma_hw_stats *stats,
366 u32 port_num, int index)
367{
368 struct mlx5_ib_dev *dev = to_mdev(ibdev);
369 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
370 struct mlx5_core_dev *mdev;
371 int ret, num_counters;
372
373 if (!stats)
374 return -EINVAL;
375
376 num_counters = cnts->num_q_counters +
377 cnts->num_cong_counters +
378 cnts->num_ext_ppcnt_counters;
379
380 if (is_mdev_switchdev_mode(dev->mdev) && dev->is_rep && port_num != 0)
381 ret = mlx5_ib_query_q_counters_vport(dev, port_num - 1, cnts,
382 stats);
383 else
384 ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats,
385 cnts->set_id);
386 if (ret)
387 return ret;
388
389 /* We don't expose device counters over Vports */
390 if (is_mdev_switchdev_mode(dev->mdev) && port_num != 0)
391 goto done;
392
393 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
394 ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats);
395 if (ret)
396 return ret;
397 }
398
399 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
400 if (!port_num)
401 port_num = 1;
402 mdev = mlx5_ib_get_native_port_mdev(dev, port_num, NULL);
403 if (!mdev) {
404 /* If port is not affiliated yet, its in down state
405 * which doesn't have any counters yet, so it would be
406 * zero. So no need to read from the HCA.
407 */
408 goto done;
409 }
410 ret = mlx5_lag_query_cong_counters(dev->mdev,
411 stats->value +
412 cnts->num_q_counters,
413 cnts->num_cong_counters,
414 cnts->offsets +
415 cnts->num_q_counters);
416
417 mlx5_ib_put_native_port_mdev(dev, port_num);
418 if (ret)
419 return ret;
420 }
421
422done:
423 return num_counters;
424}
425
426static int do_get_op_stat(struct ib_device *ibdev,
427 struct rdma_hw_stats *stats,
428 u32 port_num, int index)
429{
430 struct mlx5_ib_dev *dev = to_mdev(ibdev);
431 const struct mlx5_ib_counters *cnts;
432 const struct mlx5_ib_op_fc *opfcs;
433 u64 packets = 0, bytes;
434 u32 type;
435 int ret;
436
437 cnts = get_counters(dev, port_num);
438
439 opfcs = cnts->opfcs;
440 type = *(u32 *)cnts->descs[index].priv;
441 if (type >= MLX5_IB_OPCOUNTER_MAX)
442 return -EINVAL;
443
444 if (!opfcs[type].fc)
445 goto out;
446
447 ret = mlx5_fc_query(dev->mdev, opfcs[type].fc,
448 &packets, &bytes);
449 if (ret)
450 return ret;
451
452out:
453 stats->value[index] = packets;
454 return index;
455}
456
457static int do_get_op_stats(struct ib_device *ibdev,
458 struct rdma_hw_stats *stats,
459 u32 port_num)
460{
461 struct mlx5_ib_dev *dev = to_mdev(ibdev);
462 const struct mlx5_ib_counters *cnts;
463 int index, ret, num_hw_counters;
464
465 cnts = get_counters(dev, port_num);
466 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
467 cnts->num_ext_ppcnt_counters;
468 for (index = num_hw_counters;
469 index < (num_hw_counters + cnts->num_op_counters); index++) {
470 ret = do_get_op_stat(ibdev, stats, port_num, index);
471 if (ret != index)
472 return ret;
473 }
474
475 return cnts->num_op_counters;
476}
477
478static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
479 struct rdma_hw_stats *stats,
480 u32 port_num, int index)
481{
482 int num_counters, num_hw_counters, num_op_counters;
483 struct mlx5_ib_dev *dev = to_mdev(ibdev);
484 const struct mlx5_ib_counters *cnts;
485
486 cnts = get_counters(dev, port_num);
487 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
488 cnts->num_ext_ppcnt_counters;
489 num_counters = num_hw_counters + cnts->num_op_counters;
490
491 if (index < 0 || index > num_counters)
492 return -EINVAL;
493 else if (index > 0 && index < num_hw_counters)
494 return do_get_hw_stats(ibdev, stats, port_num, index);
495 else if (index >= num_hw_counters && index < num_counters)
496 return do_get_op_stat(ibdev, stats, port_num, index);
497
498 num_hw_counters = do_get_hw_stats(ibdev, stats, port_num, index);
499 if (num_hw_counters < 0)
500 return num_hw_counters;
501
502 num_op_counters = do_get_op_stats(ibdev, stats, port_num);
503 if (num_op_counters < 0)
504 return num_op_counters;
505
506 return num_hw_counters + num_op_counters;
507}
508
509static struct rdma_hw_stats *
510mlx5_ib_counter_alloc_stats(struct rdma_counter *counter)
511{
512 struct mlx5_ib_dev *dev = to_mdev(counter->device);
513 const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port);
514
515 return do_alloc_stats(cnts);
516}
517
518static int mlx5_ib_counter_update_stats(struct rdma_counter *counter)
519{
520 struct mlx5_ib_dev *dev = to_mdev(counter->device);
521 const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port);
522
523 return mlx5_ib_query_q_counters(dev->mdev, cnts,
524 counter->stats, counter->id);
525}
526
527static int mlx5_ib_counter_dealloc(struct rdma_counter *counter)
528{
529 struct mlx5_ib_dev *dev = to_mdev(counter->device);
530 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
531
532 if (!counter->id)
533 return 0;
534
535 MLX5_SET(dealloc_q_counter_in, in, opcode,
536 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
537 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id);
538 return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
539}
540
541static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter,
542 struct ib_qp *qp)
543{
544 struct mlx5_ib_dev *dev = to_mdev(qp->device);
545 int err;
546
547 if (!counter->id) {
548 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
549 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
550
551 MLX5_SET(alloc_q_counter_in, in, opcode,
552 MLX5_CMD_OP_ALLOC_Q_COUNTER);
553 MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID);
554 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
555 if (err)
556 return err;
557 counter->id =
558 MLX5_GET(alloc_q_counter_out, out, counter_set_id);
559 }
560
561 err = mlx5_ib_qp_set_counter(qp, counter);
562 if (err)
563 goto fail_set_counter;
564
565 return 0;
566
567fail_set_counter:
568 mlx5_ib_counter_dealloc(counter);
569 counter->id = 0;
570
571 return err;
572}
573
574static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp)
575{
576 return mlx5_ib_qp_set_counter(qp, NULL);
577}
578
579static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
580 struct rdma_stat_desc *descs, size_t *offsets,
581 u32 port_num)
582{
583 bool is_vport = is_mdev_switchdev_mode(dev->mdev) &&
584 port_num != MLX5_VPORT_PF;
585 const struct mlx5_ib_counter *names;
586 int j = 0, i, size;
587
588 names = is_vport ? vport_basic_q_cnts : basic_q_cnts;
589 size = is_vport ? ARRAY_SIZE(vport_basic_q_cnts) :
590 ARRAY_SIZE(basic_q_cnts);
591 for (i = 0; i < size; i++, j++) {
592 descs[j].name = names[i].name;
593 offsets[j] = names[i].offset;
594 }
595
596 names = is_vport ? vport_out_of_seq_q_cnts : out_of_seq_q_cnts;
597 size = is_vport ? ARRAY_SIZE(vport_out_of_seq_q_cnts) :
598 ARRAY_SIZE(out_of_seq_q_cnts);
599 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {
600 for (i = 0; i < size; i++, j++) {
601 descs[j].name = names[i].name;
602 offsets[j] = names[i].offset;
603 }
604 }
605
606 names = is_vport ? vport_retrans_q_cnts : retrans_q_cnts;
607 size = is_vport ? ARRAY_SIZE(vport_retrans_q_cnts) :
608 ARRAY_SIZE(retrans_q_cnts);
609 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
610 for (i = 0; i < size; i++, j++) {
611 descs[j].name = names[i].name;
612 offsets[j] = names[i].offset;
613 }
614 }
615
616 names = is_vport ? vport_extended_err_cnts : extended_err_cnts;
617 size = is_vport ? ARRAY_SIZE(vport_extended_err_cnts) :
618 ARRAY_SIZE(extended_err_cnts);
619 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
620 for (i = 0; i < size; i++, j++) {
621 descs[j].name = names[i].name;
622 offsets[j] = names[i].offset;
623 }
624 }
625
626 names = is_vport ? vport_roce_accl_cnts : roce_accl_cnts;
627 size = is_vport ? ARRAY_SIZE(vport_roce_accl_cnts) :
628 ARRAY_SIZE(roce_accl_cnts);
629 if (MLX5_CAP_GEN(dev->mdev, roce_accl)) {
630 for (i = 0; i < size; i++, j++) {
631 descs[j].name = names[i].name;
632 offsets[j] = names[i].offset;
633 }
634 }
635
636 if (is_vport)
637 return;
638
639 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
640 for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {
641 descs[j].name = cong_cnts[i].name;
642 offsets[j] = cong_cnts[i].offset;
643 }
644 }
645
646 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
647 for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) {
648 descs[j].name = ext_ppcnt_cnts[i].name;
649 offsets[j] = ext_ppcnt_cnts[i].offset;
650 }
651 }
652
653 for (i = 0; i < ARRAY_SIZE(basic_op_cnts); i++, j++) {
654 descs[j].name = basic_op_cnts[i].name;
655 descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
656 descs[j].priv = &basic_op_cnts[i].type;
657 }
658
659 if (MLX5_CAP_FLOWTABLE(dev->mdev,
660 ft_field_support_2_nic_receive_rdma.bth_opcode)) {
661 for (i = 0; i < ARRAY_SIZE(rdmarx_cnp_op_cnts); i++, j++) {
662 descs[j].name = rdmarx_cnp_op_cnts[i].name;
663 descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
664 descs[j].priv = &rdmarx_cnp_op_cnts[i].type;
665 }
666 }
667
668 if (MLX5_CAP_FLOWTABLE(dev->mdev,
669 ft_field_support_2_nic_transmit_rdma.bth_opcode)) {
670 for (i = 0; i < ARRAY_SIZE(rdmatx_cnp_op_cnts); i++, j++) {
671 descs[j].name = rdmatx_cnp_op_cnts[i].name;
672 descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
673 descs[j].priv = &rdmatx_cnp_op_cnts[i].type;
674 }
675 }
676}
677
678
679static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
680 struct mlx5_ib_counters *cnts, u32 port_num)
681{
682 bool is_vport = is_mdev_switchdev_mode(dev->mdev) &&
683 port_num != MLX5_VPORT_PF;
684 u32 num_counters, num_op_counters = 0, size;
685
686 size = is_vport ? ARRAY_SIZE(vport_basic_q_cnts) :
687 ARRAY_SIZE(basic_q_cnts);
688 num_counters = size;
689
690 size = is_vport ? ARRAY_SIZE(vport_out_of_seq_q_cnts) :
691 ARRAY_SIZE(out_of_seq_q_cnts);
692 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt))
693 num_counters += size;
694
695 size = is_vport ? ARRAY_SIZE(vport_retrans_q_cnts) :
696 ARRAY_SIZE(retrans_q_cnts);
697 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))
698 num_counters += size;
699
700 size = is_vport ? ARRAY_SIZE(vport_extended_err_cnts) :
701 ARRAY_SIZE(extended_err_cnts);
702 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))
703 num_counters += size;
704
705 size = is_vport ? ARRAY_SIZE(vport_roce_accl_cnts) :
706 ARRAY_SIZE(roce_accl_cnts);
707 if (MLX5_CAP_GEN(dev->mdev, roce_accl))
708 num_counters += size;
709
710 cnts->num_q_counters = num_counters;
711
712 if (is_vport)
713 goto skip_non_qcounters;
714
715 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
716 cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);
717 num_counters += ARRAY_SIZE(cong_cnts);
718 }
719 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
720 cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts);
721 num_counters += ARRAY_SIZE(ext_ppcnt_cnts);
722 }
723
724 num_op_counters = ARRAY_SIZE(basic_op_cnts);
725
726 if (MLX5_CAP_FLOWTABLE(dev->mdev,
727 ft_field_support_2_nic_receive_rdma.bth_opcode))
728 num_op_counters += ARRAY_SIZE(rdmarx_cnp_op_cnts);
729
730 if (MLX5_CAP_FLOWTABLE(dev->mdev,
731 ft_field_support_2_nic_transmit_rdma.bth_opcode))
732 num_op_counters += ARRAY_SIZE(rdmatx_cnp_op_cnts);
733
734skip_non_qcounters:
735 cnts->num_op_counters = num_op_counters;
736 num_counters += num_op_counters;
737 cnts->descs = kcalloc(num_counters,
738 sizeof(struct rdma_stat_desc), GFP_KERNEL);
739 if (!cnts->descs)
740 return -ENOMEM;
741
742 cnts->offsets = kcalloc(num_counters,
743 sizeof(*cnts->offsets), GFP_KERNEL);
744 if (!cnts->offsets)
745 goto err;
746
747 return 0;
748
749err:
750 kfree(cnts->descs);
751 cnts->descs = NULL;
752 return -ENOMEM;
753}
754
755static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
756{
757 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
758 int num_cnt_ports = dev->num_ports;
759 int i, j;
760
761 if (is_mdev_switchdev_mode(dev->mdev))
762 num_cnt_ports = min(2, num_cnt_ports);
763
764 MLX5_SET(dealloc_q_counter_in, in, opcode,
765 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
766
767 for (i = 0; i < num_cnt_ports; i++) {
768 if (dev->port[i].cnts.set_id) {
769 MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
770 dev->port[i].cnts.set_id);
771 mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
772 }
773 kfree(dev->port[i].cnts.descs);
774 kfree(dev->port[i].cnts.offsets);
775
776 for (j = 0; j < MLX5_IB_OPCOUNTER_MAX; j++) {
777 if (!dev->port[i].cnts.opfcs[j].fc)
778 continue;
779
780 if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
781 mlx5_ib_fs_remove_op_fc(dev,
782 &dev->port[i].cnts.opfcs[j], j);
783 mlx5_fc_destroy(dev->mdev,
784 dev->port[i].cnts.opfcs[j].fc);
785 dev->port[i].cnts.opfcs[j].fc = NULL;
786 }
787 }
788}
789
790static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
791{
792 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
793 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
794 int num_cnt_ports = dev->num_ports;
795 int err = 0;
796 int i;
797 bool is_shared;
798
799 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
800 is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0;
801
802 /*
803 * In switchdev we need to allocate two ports, one that is used for
804 * the device Q_counters and it is essentially the real Q_counters of
805 * this device, while the other is used as a helper for PF to be able to
806 * query all other vports.
807 */
808 if (is_mdev_switchdev_mode(dev->mdev))
809 num_cnt_ports = min(2, num_cnt_ports);
810
811 for (i = 0; i < num_cnt_ports; i++) {
812 err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts, i);
813 if (err)
814 goto err_alloc;
815
816 mlx5_ib_fill_counters(dev, dev->port[i].cnts.descs,
817 dev->port[i].cnts.offsets, i);
818
819 MLX5_SET(alloc_q_counter_in, in, uid,
820 is_shared ? MLX5_SHARED_RESOURCE_UID : 0);
821
822 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
823 if (err) {
824 mlx5_ib_warn(dev,
825 "couldn't allocate queue counter for port %d, err %d\n",
826 i + 1, err);
827 goto err_alloc;
828 }
829
830 dev->port[i].cnts.set_id =
831 MLX5_GET(alloc_q_counter_out, out, counter_set_id);
832 }
833 return 0;
834
835err_alloc:
836 mlx5_ib_dealloc_counters(dev);
837 return err;
838}
839
840static int read_flow_counters(struct ib_device *ibdev,
841 struct mlx5_read_counters_attr *read_attr)
842{
843 struct mlx5_fc *fc = read_attr->hw_cntrs_hndl;
844 struct mlx5_ib_dev *dev = to_mdev(ibdev);
845
846 return mlx5_fc_query(dev->mdev, fc,
847 &read_attr->out[IB_COUNTER_PACKETS],
848 &read_attr->out[IB_COUNTER_BYTES]);
849}
850
851/* flow counters currently expose two counters packets and bytes */
852#define FLOW_COUNTERS_NUM 2
853static int counters_set_description(
854 struct ib_counters *counters, enum mlx5_ib_counters_type counters_type,
855 struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters)
856{
857 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
858 u32 cntrs_max_index = 0;
859 int i;
860
861 if (counters_type != MLX5_IB_COUNTERS_FLOW)
862 return -EINVAL;
863
864 /* init the fields for the object */
865 mcounters->type = counters_type;
866 mcounters->read_counters = read_flow_counters;
867 mcounters->counters_num = FLOW_COUNTERS_NUM;
868 mcounters->ncounters = ncounters;
869 /* each counter entry have both description and index pair */
870 for (i = 0; i < ncounters; i++) {
871 if (desc_data[i].description > IB_COUNTER_BYTES)
872 return -EINVAL;
873
874 if (cntrs_max_index <= desc_data[i].index)
875 cntrs_max_index = desc_data[i].index + 1;
876 }
877
878 mutex_lock(&mcounters->mcntrs_mutex);
879 mcounters->counters_data = desc_data;
880 mcounters->cntrs_max_index = cntrs_max_index;
881 mutex_unlock(&mcounters->mcntrs_mutex);
882
883 return 0;
884}
885
886#define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2))
887int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters,
888 struct mlx5_ib_create_flow *ucmd)
889{
890 struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters);
891 struct mlx5_ib_flow_counters_data *cntrs_data = NULL;
892 struct mlx5_ib_flow_counters_desc *desc_data = NULL;
893 bool hw_hndl = false;
894 int ret = 0;
895
896 if (ucmd && ucmd->ncounters_data != 0) {
897 cntrs_data = ucmd->data;
898 if (cntrs_data->ncounters > MAX_COUNTERS_NUM)
899 return -EINVAL;
900
901 desc_data = kcalloc(cntrs_data->ncounters,
902 sizeof(*desc_data),
903 GFP_KERNEL);
904 if (!desc_data)
905 return -ENOMEM;
906
907 if (copy_from_user(desc_data,
908 u64_to_user_ptr(cntrs_data->counters_data),
909 sizeof(*desc_data) * cntrs_data->ncounters)) {
910 ret = -EFAULT;
911 goto free;
912 }
913 }
914
915 if (!mcounters->hw_cntrs_hndl) {
916 mcounters->hw_cntrs_hndl = mlx5_fc_create(
917 to_mdev(ibcounters->device)->mdev, false);
918 if (IS_ERR(mcounters->hw_cntrs_hndl)) {
919 ret = PTR_ERR(mcounters->hw_cntrs_hndl);
920 goto free;
921 }
922 hw_hndl = true;
923 }
924
925 if (desc_data) {
926 /* counters already bound to at least one flow */
927 if (mcounters->cntrs_max_index) {
928 ret = -EINVAL;
929 goto free_hndl;
930 }
931
932 ret = counters_set_description(ibcounters,
933 MLX5_IB_COUNTERS_FLOW,
934 desc_data,
935 cntrs_data->ncounters);
936 if (ret)
937 goto free_hndl;
938
939 } else if (!mcounters->cntrs_max_index) {
940 /* counters not bound yet, must have udata passed */
941 ret = -EINVAL;
942 goto free_hndl;
943 }
944
945 return 0;
946
947free_hndl:
948 if (hw_hndl) {
949 mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev,
950 mcounters->hw_cntrs_hndl);
951 mcounters->hw_cntrs_hndl = NULL;
952 }
953free:
954 kfree(desc_data);
955 return ret;
956}
957
958void mlx5_ib_counters_clear_description(struct ib_counters *counters)
959{
960 struct mlx5_ib_mcounters *mcounters;
961
962 if (!counters || atomic_read(&counters->usecnt) != 1)
963 return;
964
965 mcounters = to_mcounters(counters);
966
967 mutex_lock(&mcounters->mcntrs_mutex);
968 kfree(mcounters->counters_data);
969 mcounters->counters_data = NULL;
970 mcounters->cntrs_max_index = 0;
971 mutex_unlock(&mcounters->mcntrs_mutex);
972}
973
974static int mlx5_ib_modify_stat(struct ib_device *device, u32 port,
975 unsigned int index, bool enable)
976{
977 struct mlx5_ib_dev *dev = to_mdev(device);
978 struct mlx5_ib_counters *cnts;
979 struct mlx5_ib_op_fc *opfc;
980 u32 num_hw_counters, type;
981 int ret;
982
983 cnts = &dev->port[port - 1].cnts;
984 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
985 cnts->num_ext_ppcnt_counters;
986 if (index < num_hw_counters ||
987 index >= (num_hw_counters + cnts->num_op_counters))
988 return -EINVAL;
989
990 if (!(cnts->descs[index].flags & IB_STAT_FLAG_OPTIONAL))
991 return -EINVAL;
992
993 type = *(u32 *)cnts->descs[index].priv;
994 if (type >= MLX5_IB_OPCOUNTER_MAX)
995 return -EINVAL;
996
997 opfc = &cnts->opfcs[type];
998
999 if (enable) {
1000 if (opfc->fc)
1001 return -EEXIST;
1002
1003 opfc->fc = mlx5_fc_create(dev->mdev, false);
1004 if (IS_ERR(opfc->fc))
1005 return PTR_ERR(opfc->fc);
1006
1007 ret = mlx5_ib_fs_add_op_fc(dev, port, opfc, type);
1008 if (ret) {
1009 mlx5_fc_destroy(dev->mdev, opfc->fc);
1010 opfc->fc = NULL;
1011 }
1012 return ret;
1013 }
1014
1015 if (!opfc->fc)
1016 return -EINVAL;
1017
1018 mlx5_ib_fs_remove_op_fc(dev, opfc, type);
1019 mlx5_fc_destroy(dev->mdev, opfc->fc);
1020 opfc->fc = NULL;
1021 return 0;
1022}
1023
1024static const struct ib_device_ops hw_stats_ops = {
1025 .alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats,
1026 .get_hw_stats = mlx5_ib_get_hw_stats,
1027 .counter_bind_qp = mlx5_ib_counter_bind_qp,
1028 .counter_unbind_qp = mlx5_ib_counter_unbind_qp,
1029 .counter_dealloc = mlx5_ib_counter_dealloc,
1030 .counter_alloc_stats = mlx5_ib_counter_alloc_stats,
1031 .counter_update_stats = mlx5_ib_counter_update_stats,
1032 .modify_hw_stat = IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) ?
1033 mlx5_ib_modify_stat : NULL,
1034};
1035
1036static const struct ib_device_ops hw_switchdev_vport_op = {
1037 .alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats,
1038};
1039
1040static const struct ib_device_ops hw_switchdev_stats_ops = {
1041 .alloc_hw_device_stats = mlx5_ib_alloc_hw_device_stats,
1042 .get_hw_stats = mlx5_ib_get_hw_stats,
1043 .counter_bind_qp = mlx5_ib_counter_bind_qp,
1044 .counter_unbind_qp = mlx5_ib_counter_unbind_qp,
1045 .counter_dealloc = mlx5_ib_counter_dealloc,
1046 .counter_alloc_stats = mlx5_ib_counter_alloc_stats,
1047 .counter_update_stats = mlx5_ib_counter_update_stats,
1048};
1049
1050static const struct ib_device_ops counters_ops = {
1051 .create_counters = mlx5_ib_create_counters,
1052 .destroy_counters = mlx5_ib_destroy_counters,
1053 .read_counters = mlx5_ib_read_counters,
1054
1055 INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
1056};
1057
1058int mlx5_ib_counters_init(struct mlx5_ib_dev *dev)
1059{
1060 ib_set_device_ops(&dev->ib_dev, &counters_ops);
1061
1062 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
1063 return 0;
1064
1065 if (is_mdev_switchdev_mode(dev->mdev)) {
1066 ib_set_device_ops(&dev->ib_dev, &hw_switchdev_stats_ops);
1067 if (vport_qcounters_supported(dev))
1068 ib_set_device_ops(&dev->ib_dev, &hw_switchdev_vport_op);
1069 } else
1070 ib_set_device_ops(&dev->ib_dev, &hw_stats_ops);
1071 return mlx5_ib_alloc_counters(dev);
1072}
1073
1074void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev)
1075{
1076 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
1077 return;
1078
1079 mlx5_ib_dealloc_counters(dev);
1080}
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
4 */
5
6#include "mlx5_ib.h"
7#include <linux/mlx5/eswitch.h>
8#include "counters.h"
9#include "ib_rep.h"
10#include "qp.h"
11
12struct mlx5_ib_counter {
13 const char *name;
14 size_t offset;
15 u32 type;
16};
17
18#define INIT_Q_COUNTER(_name) \
19 { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}
20
21static const struct mlx5_ib_counter basic_q_cnts[] = {
22 INIT_Q_COUNTER(rx_write_requests),
23 INIT_Q_COUNTER(rx_read_requests),
24 INIT_Q_COUNTER(rx_atomic_requests),
25 INIT_Q_COUNTER(out_of_buffer),
26};
27
28static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {
29 INIT_Q_COUNTER(out_of_sequence),
30};
31
32static const struct mlx5_ib_counter retrans_q_cnts[] = {
33 INIT_Q_COUNTER(duplicate_request),
34 INIT_Q_COUNTER(rnr_nak_retry_err),
35 INIT_Q_COUNTER(packet_seq_err),
36 INIT_Q_COUNTER(implied_nak_seq_err),
37 INIT_Q_COUNTER(local_ack_timeout_err),
38};
39
40#define INIT_CONG_COUNTER(_name) \
41 { .name = #_name, .offset = \
42 MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}
43
44static const struct mlx5_ib_counter cong_cnts[] = {
45 INIT_CONG_COUNTER(rp_cnp_ignored),
46 INIT_CONG_COUNTER(rp_cnp_handled),
47 INIT_CONG_COUNTER(np_ecn_marked_roce_packets),
48 INIT_CONG_COUNTER(np_cnp_sent),
49};
50
51static const struct mlx5_ib_counter extended_err_cnts[] = {
52 INIT_Q_COUNTER(resp_local_length_error),
53 INIT_Q_COUNTER(resp_cqe_error),
54 INIT_Q_COUNTER(req_cqe_error),
55 INIT_Q_COUNTER(req_remote_invalid_request),
56 INIT_Q_COUNTER(req_remote_access_errors),
57 INIT_Q_COUNTER(resp_remote_access_errors),
58 INIT_Q_COUNTER(resp_cqe_flush_error),
59 INIT_Q_COUNTER(req_cqe_flush_error),
60};
61
62static const struct mlx5_ib_counter roce_accl_cnts[] = {
63 INIT_Q_COUNTER(roce_adp_retrans),
64 INIT_Q_COUNTER(roce_adp_retrans_to),
65 INIT_Q_COUNTER(roce_slow_restart),
66 INIT_Q_COUNTER(roce_slow_restart_cnps),
67 INIT_Q_COUNTER(roce_slow_restart_trans),
68};
69
70#define INIT_EXT_PPCNT_COUNTER(_name) \
71 { .name = #_name, .offset = \
72 MLX5_BYTE_OFF(ppcnt_reg, \
73 counter_set.eth_extended_cntrs_grp_data_layout._name##_high)}
74
75static const struct mlx5_ib_counter ext_ppcnt_cnts[] = {
76 INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated),
77};
78
79#define INIT_OP_COUNTER(_name, _type) \
80 { .name = #_name, .type = MLX5_IB_OPCOUNTER_##_type}
81
82static const struct mlx5_ib_counter basic_op_cnts[] = {
83 INIT_OP_COUNTER(cc_rx_ce_pkts, CC_RX_CE_PKTS),
84};
85
86static const struct mlx5_ib_counter rdmarx_cnp_op_cnts[] = {
87 INIT_OP_COUNTER(cc_rx_cnp_pkts, CC_RX_CNP_PKTS),
88};
89
90static const struct mlx5_ib_counter rdmatx_cnp_op_cnts[] = {
91 INIT_OP_COUNTER(cc_tx_cnp_pkts, CC_TX_CNP_PKTS),
92};
93
94static int mlx5_ib_read_counters(struct ib_counters *counters,
95 struct ib_counters_read_attr *read_attr,
96 struct uverbs_attr_bundle *attrs)
97{
98 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
99 struct mlx5_read_counters_attr mread_attr = {};
100 struct mlx5_ib_flow_counters_desc *desc;
101 int ret, i;
102
103 mutex_lock(&mcounters->mcntrs_mutex);
104 if (mcounters->cntrs_max_index > read_attr->ncounters) {
105 ret = -EINVAL;
106 goto err_bound;
107 }
108
109 mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64),
110 GFP_KERNEL);
111 if (!mread_attr.out) {
112 ret = -ENOMEM;
113 goto err_bound;
114 }
115
116 mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl;
117 mread_attr.flags = read_attr->flags;
118 ret = mcounters->read_counters(counters->device, &mread_attr);
119 if (ret)
120 goto err_read;
121
122 /* do the pass over the counters data array to assign according to the
123 * descriptions and indexing pairs
124 */
125 desc = mcounters->counters_data;
126 for (i = 0; i < mcounters->ncounters; i++)
127 read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description];
128
129err_read:
130 kfree(mread_attr.out);
131err_bound:
132 mutex_unlock(&mcounters->mcntrs_mutex);
133 return ret;
134}
135
136static int mlx5_ib_destroy_counters(struct ib_counters *counters)
137{
138 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
139
140 mlx5_ib_counters_clear_description(counters);
141 if (mcounters->hw_cntrs_hndl)
142 mlx5_fc_destroy(to_mdev(counters->device)->mdev,
143 mcounters->hw_cntrs_hndl);
144 return 0;
145}
146
147static int mlx5_ib_create_counters(struct ib_counters *counters,
148 struct uverbs_attr_bundle *attrs)
149{
150 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
151
152 mutex_init(&mcounters->mcntrs_mutex);
153 return 0;
154}
155
156
157static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev,
158 u32 port_num)
159{
160 return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts :
161 &dev->port[port_num].cnts;
162}
163
164/**
165 * mlx5_ib_get_counters_id - Returns counters id to use for device+port
166 * @dev: Pointer to mlx5 IB device
167 * @port_num: Zero based port number
168 *
169 * mlx5_ib_get_counters_id() Returns counters set id to use for given
170 * device port combination in switchdev and non switchdev mode of the
171 * parent device.
172 */
173u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u32 port_num)
174{
175 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
176
177 return cnts->set_id;
178}
179
180static struct rdma_hw_stats *do_alloc_stats(const struct mlx5_ib_counters *cnts)
181{
182 struct rdma_hw_stats *stats;
183 u32 num_hw_counters;
184 int i;
185
186 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
187 cnts->num_ext_ppcnt_counters;
188 stats = rdma_alloc_hw_stats_struct(cnts->descs,
189 num_hw_counters +
190 cnts->num_op_counters,
191 RDMA_HW_STATS_DEFAULT_LIFESPAN);
192 if (!stats)
193 return NULL;
194
195 for (i = 0; i < cnts->num_op_counters; i++)
196 set_bit(num_hw_counters + i, stats->is_disabled);
197
198 return stats;
199}
200
201static struct rdma_hw_stats *
202mlx5_ib_alloc_hw_device_stats(struct ib_device *ibdev)
203{
204 struct mlx5_ib_dev *dev = to_mdev(ibdev);
205 const struct mlx5_ib_counters *cnts = &dev->port[0].cnts;
206
207 return do_alloc_stats(cnts);
208}
209
210static struct rdma_hw_stats *
211mlx5_ib_alloc_hw_port_stats(struct ib_device *ibdev, u32 port_num)
212{
213 struct mlx5_ib_dev *dev = to_mdev(ibdev);
214 const struct mlx5_ib_counters *cnts = &dev->port[port_num - 1].cnts;
215
216 return do_alloc_stats(cnts);
217}
218
219static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,
220 const struct mlx5_ib_counters *cnts,
221 struct rdma_hw_stats *stats,
222 u16 set_id)
223{
224 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
225 u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
226 __be32 val;
227 int ret, i;
228
229 MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
230 MLX5_SET(query_q_counter_in, in, counter_set_id, set_id);
231 ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);
232 if (ret)
233 return ret;
234
235 for (i = 0; i < cnts->num_q_counters; i++) {
236 val = *(__be32 *)((void *)out + cnts->offsets[i]);
237 stats->value[i] = (u64)be32_to_cpu(val);
238 }
239
240 return 0;
241}
242
243static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev,
244 const struct mlx5_ib_counters *cnts,
245 struct rdma_hw_stats *stats)
246{
247 int offset = cnts->num_q_counters + cnts->num_cong_counters;
248 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
249 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
250 int ret, i;
251 void *out;
252
253 out = kvzalloc(sz, GFP_KERNEL);
254 if (!out)
255 return -ENOMEM;
256
257 MLX5_SET(ppcnt_reg, in, local_port, 1);
258 MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
259 ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT,
260 0, 0);
261 if (ret)
262 goto free;
263
264 for (i = 0; i < cnts->num_ext_ppcnt_counters; i++)
265 stats->value[i + offset] =
266 be64_to_cpup((__be64 *)(out +
267 cnts->offsets[i + offset]));
268free:
269 kvfree(out);
270 return ret;
271}
272
273static int do_get_hw_stats(struct ib_device *ibdev,
274 struct rdma_hw_stats *stats,
275 u32 port_num, int index)
276{
277 struct mlx5_ib_dev *dev = to_mdev(ibdev);
278 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1);
279 struct mlx5_core_dev *mdev;
280 int ret, num_counters;
281
282 if (!stats)
283 return -EINVAL;
284
285 num_counters = cnts->num_q_counters +
286 cnts->num_cong_counters +
287 cnts->num_ext_ppcnt_counters;
288
289 /* q_counters are per IB device, query the master mdev */
290 ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, cnts->set_id);
291 if (ret)
292 return ret;
293
294 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
295 ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats);
296 if (ret)
297 return ret;
298 }
299
300 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
301 if (!port_num)
302 port_num = 1;
303 mdev = mlx5_ib_get_native_port_mdev(dev, port_num, NULL);
304 if (!mdev) {
305 /* If port is not affiliated yet, its in down state
306 * which doesn't have any counters yet, so it would be
307 * zero. So no need to read from the HCA.
308 */
309 goto done;
310 }
311 ret = mlx5_lag_query_cong_counters(dev->mdev,
312 stats->value +
313 cnts->num_q_counters,
314 cnts->num_cong_counters,
315 cnts->offsets +
316 cnts->num_q_counters);
317
318 mlx5_ib_put_native_port_mdev(dev, port_num);
319 if (ret)
320 return ret;
321 }
322
323done:
324 return num_counters;
325}
326
327static int do_get_op_stat(struct ib_device *ibdev,
328 struct rdma_hw_stats *stats,
329 u32 port_num, int index)
330{
331 struct mlx5_ib_dev *dev = to_mdev(ibdev);
332 const struct mlx5_ib_counters *cnts;
333 const struct mlx5_ib_op_fc *opfcs;
334 u64 packets = 0, bytes;
335 u32 type;
336 int ret;
337
338 cnts = get_counters(dev, port_num - 1);
339 opfcs = cnts->opfcs;
340 type = *(u32 *)cnts->descs[index].priv;
341 if (type >= MLX5_IB_OPCOUNTER_MAX)
342 return -EINVAL;
343
344 if (!opfcs[type].fc)
345 goto out;
346
347 ret = mlx5_fc_query(dev->mdev, opfcs[type].fc,
348 &packets, &bytes);
349 if (ret)
350 return ret;
351
352out:
353 stats->value[index] = packets;
354 return index;
355}
356
357static int do_get_op_stats(struct ib_device *ibdev,
358 struct rdma_hw_stats *stats,
359 u32 port_num)
360{
361 struct mlx5_ib_dev *dev = to_mdev(ibdev);
362 const struct mlx5_ib_counters *cnts;
363 int index, ret, num_hw_counters;
364
365 cnts = get_counters(dev, port_num - 1);
366 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
367 cnts->num_ext_ppcnt_counters;
368 for (index = num_hw_counters;
369 index < (num_hw_counters + cnts->num_op_counters); index++) {
370 ret = do_get_op_stat(ibdev, stats, port_num, index);
371 if (ret != index)
372 return ret;
373 }
374
375 return cnts->num_op_counters;
376}
377
378static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
379 struct rdma_hw_stats *stats,
380 u32 port_num, int index)
381{
382 int num_counters, num_hw_counters, num_op_counters;
383 struct mlx5_ib_dev *dev = to_mdev(ibdev);
384 const struct mlx5_ib_counters *cnts;
385
386 cnts = get_counters(dev, port_num - 1);
387 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
388 cnts->num_ext_ppcnt_counters;
389 num_counters = num_hw_counters + cnts->num_op_counters;
390
391 if (index < 0 || index > num_counters)
392 return -EINVAL;
393 else if (index > 0 && index < num_hw_counters)
394 return do_get_hw_stats(ibdev, stats, port_num, index);
395 else if (index >= num_hw_counters && index < num_counters)
396 return do_get_op_stat(ibdev, stats, port_num, index);
397
398 num_hw_counters = do_get_hw_stats(ibdev, stats, port_num, index);
399 if (num_hw_counters < 0)
400 return num_hw_counters;
401
402 num_op_counters = do_get_op_stats(ibdev, stats, port_num);
403 if (num_op_counters < 0)
404 return num_op_counters;
405
406 return num_hw_counters + num_op_counters;
407}
408
409static struct rdma_hw_stats *
410mlx5_ib_counter_alloc_stats(struct rdma_counter *counter)
411{
412 struct mlx5_ib_dev *dev = to_mdev(counter->device);
413 const struct mlx5_ib_counters *cnts =
414 get_counters(dev, counter->port - 1);
415
416 return do_alloc_stats(cnts);
417}
418
419static int mlx5_ib_counter_update_stats(struct rdma_counter *counter)
420{
421 struct mlx5_ib_dev *dev = to_mdev(counter->device);
422 const struct mlx5_ib_counters *cnts =
423 get_counters(dev, counter->port - 1);
424
425 return mlx5_ib_query_q_counters(dev->mdev, cnts,
426 counter->stats, counter->id);
427}
428
429static int mlx5_ib_counter_dealloc(struct rdma_counter *counter)
430{
431 struct mlx5_ib_dev *dev = to_mdev(counter->device);
432 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
433
434 if (!counter->id)
435 return 0;
436
437 MLX5_SET(dealloc_q_counter_in, in, opcode,
438 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
439 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id);
440 return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
441}
442
443static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter,
444 struct ib_qp *qp)
445{
446 struct mlx5_ib_dev *dev = to_mdev(qp->device);
447 int err;
448
449 if (!counter->id) {
450 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
451 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
452
453 MLX5_SET(alloc_q_counter_in, in, opcode,
454 MLX5_CMD_OP_ALLOC_Q_COUNTER);
455 MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID);
456 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
457 if (err)
458 return err;
459 counter->id =
460 MLX5_GET(alloc_q_counter_out, out, counter_set_id);
461 }
462
463 err = mlx5_ib_qp_set_counter(qp, counter);
464 if (err)
465 goto fail_set_counter;
466
467 return 0;
468
469fail_set_counter:
470 mlx5_ib_counter_dealloc(counter);
471 counter->id = 0;
472
473 return err;
474}
475
476static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp)
477{
478 return mlx5_ib_qp_set_counter(qp, NULL);
479}
480
481static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
482 struct rdma_stat_desc *descs, size_t *offsets)
483{
484 int i;
485 int j = 0;
486
487 for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) {
488 descs[j].name = basic_q_cnts[i].name;
489 offsets[j] = basic_q_cnts[i].offset;
490 }
491
492 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {
493 for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) {
494 descs[j].name = out_of_seq_q_cnts[i].name;
495 offsets[j] = out_of_seq_q_cnts[i].offset;
496 }
497 }
498
499 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
500 for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) {
501 descs[j].name = retrans_q_cnts[i].name;
502 offsets[j] = retrans_q_cnts[i].offset;
503 }
504 }
505
506 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
507 for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) {
508 descs[j].name = extended_err_cnts[i].name;
509 offsets[j] = extended_err_cnts[i].offset;
510 }
511 }
512
513 if (MLX5_CAP_GEN(dev->mdev, roce_accl)) {
514 for (i = 0; i < ARRAY_SIZE(roce_accl_cnts); i++, j++) {
515 descs[j].name = roce_accl_cnts[i].name;
516 offsets[j] = roce_accl_cnts[i].offset;
517 }
518 }
519
520 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
521 for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {
522 descs[j].name = cong_cnts[i].name;
523 offsets[j] = cong_cnts[i].offset;
524 }
525 }
526
527 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
528 for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) {
529 descs[j].name = ext_ppcnt_cnts[i].name;
530 offsets[j] = ext_ppcnt_cnts[i].offset;
531 }
532 }
533
534 for (i = 0; i < ARRAY_SIZE(basic_op_cnts); i++, j++) {
535 descs[j].name = basic_op_cnts[i].name;
536 descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
537 descs[j].priv = &basic_op_cnts[i].type;
538 }
539
540 if (MLX5_CAP_FLOWTABLE(dev->mdev,
541 ft_field_support_2_nic_receive_rdma.bth_opcode)) {
542 for (i = 0; i < ARRAY_SIZE(rdmarx_cnp_op_cnts); i++, j++) {
543 descs[j].name = rdmarx_cnp_op_cnts[i].name;
544 descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
545 descs[j].priv = &rdmarx_cnp_op_cnts[i].type;
546 }
547 }
548
549 if (MLX5_CAP_FLOWTABLE(dev->mdev,
550 ft_field_support_2_nic_transmit_rdma.bth_opcode)) {
551 for (i = 0; i < ARRAY_SIZE(rdmatx_cnp_op_cnts); i++, j++) {
552 descs[j].name = rdmatx_cnp_op_cnts[i].name;
553 descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
554 descs[j].priv = &rdmatx_cnp_op_cnts[i].type;
555 }
556 }
557}
558
559
560static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
561 struct mlx5_ib_counters *cnts)
562{
563 u32 num_counters, num_op_counters;
564
565 num_counters = ARRAY_SIZE(basic_q_cnts);
566
567 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt))
568 num_counters += ARRAY_SIZE(out_of_seq_q_cnts);
569
570 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))
571 num_counters += ARRAY_SIZE(retrans_q_cnts);
572
573 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))
574 num_counters += ARRAY_SIZE(extended_err_cnts);
575
576 if (MLX5_CAP_GEN(dev->mdev, roce_accl))
577 num_counters += ARRAY_SIZE(roce_accl_cnts);
578
579 cnts->num_q_counters = num_counters;
580
581 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
582 cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);
583 num_counters += ARRAY_SIZE(cong_cnts);
584 }
585 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
586 cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts);
587 num_counters += ARRAY_SIZE(ext_ppcnt_cnts);
588 }
589
590 num_op_counters = ARRAY_SIZE(basic_op_cnts);
591
592 if (MLX5_CAP_FLOWTABLE(dev->mdev,
593 ft_field_support_2_nic_receive_rdma.bth_opcode))
594 num_op_counters += ARRAY_SIZE(rdmarx_cnp_op_cnts);
595
596 if (MLX5_CAP_FLOWTABLE(dev->mdev,
597 ft_field_support_2_nic_transmit_rdma.bth_opcode))
598 num_op_counters += ARRAY_SIZE(rdmatx_cnp_op_cnts);
599
600 cnts->num_op_counters = num_op_counters;
601 num_counters += num_op_counters;
602 cnts->descs = kcalloc(num_counters,
603 sizeof(struct rdma_stat_desc), GFP_KERNEL);
604 if (!cnts->descs)
605 return -ENOMEM;
606
607 cnts->offsets = kcalloc(num_counters,
608 sizeof(*cnts->offsets), GFP_KERNEL);
609 if (!cnts->offsets)
610 goto err;
611
612 return 0;
613
614err:
615 kfree(cnts->descs);
616 cnts->descs = NULL;
617 return -ENOMEM;
618}
619
620static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
621{
622 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
623 int num_cnt_ports;
624 int i, j;
625
626 num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
627
628 MLX5_SET(dealloc_q_counter_in, in, opcode,
629 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
630
631 for (i = 0; i < num_cnt_ports; i++) {
632 if (dev->port[i].cnts.set_id) {
633 MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
634 dev->port[i].cnts.set_id);
635 mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
636 }
637 kfree(dev->port[i].cnts.descs);
638 kfree(dev->port[i].cnts.offsets);
639
640 for (j = 0; j < MLX5_IB_OPCOUNTER_MAX; j++) {
641 if (!dev->port[i].cnts.opfcs[j].fc)
642 continue;
643
644 if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
645 mlx5_ib_fs_remove_op_fc(dev,
646 &dev->port[i].cnts.opfcs[j], j);
647 mlx5_fc_destroy(dev->mdev,
648 dev->port[i].cnts.opfcs[j].fc);
649 dev->port[i].cnts.opfcs[j].fc = NULL;
650 }
651 }
652}
653
654static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
655{
656 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
657 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
658 int num_cnt_ports;
659 int err = 0;
660 int i;
661 bool is_shared;
662
663 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
664 is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0;
665 num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
666
667 for (i = 0; i < num_cnt_ports; i++) {
668 err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts);
669 if (err)
670 goto err_alloc;
671
672 mlx5_ib_fill_counters(dev, dev->port[i].cnts.descs,
673 dev->port[i].cnts.offsets);
674
675 MLX5_SET(alloc_q_counter_in, in, uid,
676 is_shared ? MLX5_SHARED_RESOURCE_UID : 0);
677
678 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
679 if (err) {
680 mlx5_ib_warn(dev,
681 "couldn't allocate queue counter for port %d, err %d\n",
682 i + 1, err);
683 goto err_alloc;
684 }
685
686 dev->port[i].cnts.set_id =
687 MLX5_GET(alloc_q_counter_out, out, counter_set_id);
688 }
689 return 0;
690
691err_alloc:
692 mlx5_ib_dealloc_counters(dev);
693 return err;
694}
695
696static int read_flow_counters(struct ib_device *ibdev,
697 struct mlx5_read_counters_attr *read_attr)
698{
699 struct mlx5_fc *fc = read_attr->hw_cntrs_hndl;
700 struct mlx5_ib_dev *dev = to_mdev(ibdev);
701
702 return mlx5_fc_query(dev->mdev, fc,
703 &read_attr->out[IB_COUNTER_PACKETS],
704 &read_attr->out[IB_COUNTER_BYTES]);
705}
706
707/* flow counters currently expose two counters packets and bytes */
708#define FLOW_COUNTERS_NUM 2
709static int counters_set_description(
710 struct ib_counters *counters, enum mlx5_ib_counters_type counters_type,
711 struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters)
712{
713 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
714 u32 cntrs_max_index = 0;
715 int i;
716
717 if (counters_type != MLX5_IB_COUNTERS_FLOW)
718 return -EINVAL;
719
720 /* init the fields for the object */
721 mcounters->type = counters_type;
722 mcounters->read_counters = read_flow_counters;
723 mcounters->counters_num = FLOW_COUNTERS_NUM;
724 mcounters->ncounters = ncounters;
725 /* each counter entry have both description and index pair */
726 for (i = 0; i < ncounters; i++) {
727 if (desc_data[i].description > IB_COUNTER_BYTES)
728 return -EINVAL;
729
730 if (cntrs_max_index <= desc_data[i].index)
731 cntrs_max_index = desc_data[i].index + 1;
732 }
733
734 mutex_lock(&mcounters->mcntrs_mutex);
735 mcounters->counters_data = desc_data;
736 mcounters->cntrs_max_index = cntrs_max_index;
737 mutex_unlock(&mcounters->mcntrs_mutex);
738
739 return 0;
740}
741
742#define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2))
743int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters,
744 struct mlx5_ib_create_flow *ucmd)
745{
746 struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters);
747 struct mlx5_ib_flow_counters_data *cntrs_data = NULL;
748 struct mlx5_ib_flow_counters_desc *desc_data = NULL;
749 bool hw_hndl = false;
750 int ret = 0;
751
752 if (ucmd && ucmd->ncounters_data != 0) {
753 cntrs_data = ucmd->data;
754 if (cntrs_data->ncounters > MAX_COUNTERS_NUM)
755 return -EINVAL;
756
757 desc_data = kcalloc(cntrs_data->ncounters,
758 sizeof(*desc_data),
759 GFP_KERNEL);
760 if (!desc_data)
761 return -ENOMEM;
762
763 if (copy_from_user(desc_data,
764 u64_to_user_ptr(cntrs_data->counters_data),
765 sizeof(*desc_data) * cntrs_data->ncounters)) {
766 ret = -EFAULT;
767 goto free;
768 }
769 }
770
771 if (!mcounters->hw_cntrs_hndl) {
772 mcounters->hw_cntrs_hndl = mlx5_fc_create(
773 to_mdev(ibcounters->device)->mdev, false);
774 if (IS_ERR(mcounters->hw_cntrs_hndl)) {
775 ret = PTR_ERR(mcounters->hw_cntrs_hndl);
776 goto free;
777 }
778 hw_hndl = true;
779 }
780
781 if (desc_data) {
782 /* counters already bound to at least one flow */
783 if (mcounters->cntrs_max_index) {
784 ret = -EINVAL;
785 goto free_hndl;
786 }
787
788 ret = counters_set_description(ibcounters,
789 MLX5_IB_COUNTERS_FLOW,
790 desc_data,
791 cntrs_data->ncounters);
792 if (ret)
793 goto free_hndl;
794
795 } else if (!mcounters->cntrs_max_index) {
796 /* counters not bound yet, must have udata passed */
797 ret = -EINVAL;
798 goto free_hndl;
799 }
800
801 return 0;
802
803free_hndl:
804 if (hw_hndl) {
805 mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev,
806 mcounters->hw_cntrs_hndl);
807 mcounters->hw_cntrs_hndl = NULL;
808 }
809free:
810 kfree(desc_data);
811 return ret;
812}
813
814void mlx5_ib_counters_clear_description(struct ib_counters *counters)
815{
816 struct mlx5_ib_mcounters *mcounters;
817
818 if (!counters || atomic_read(&counters->usecnt) != 1)
819 return;
820
821 mcounters = to_mcounters(counters);
822
823 mutex_lock(&mcounters->mcntrs_mutex);
824 kfree(mcounters->counters_data);
825 mcounters->counters_data = NULL;
826 mcounters->cntrs_max_index = 0;
827 mutex_unlock(&mcounters->mcntrs_mutex);
828}
829
830static int mlx5_ib_modify_stat(struct ib_device *device, u32 port,
831 unsigned int index, bool enable)
832{
833 struct mlx5_ib_dev *dev = to_mdev(device);
834 struct mlx5_ib_counters *cnts;
835 struct mlx5_ib_op_fc *opfc;
836 u32 num_hw_counters, type;
837 int ret;
838
839 cnts = &dev->port[port - 1].cnts;
840 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
841 cnts->num_ext_ppcnt_counters;
842 if (index < num_hw_counters ||
843 index >= (num_hw_counters + cnts->num_op_counters))
844 return -EINVAL;
845
846 if (!(cnts->descs[index].flags & IB_STAT_FLAG_OPTIONAL))
847 return -EINVAL;
848
849 type = *(u32 *)cnts->descs[index].priv;
850 if (type >= MLX5_IB_OPCOUNTER_MAX)
851 return -EINVAL;
852
853 opfc = &cnts->opfcs[type];
854
855 if (enable) {
856 if (opfc->fc)
857 return -EEXIST;
858
859 opfc->fc = mlx5_fc_create(dev->mdev, false);
860 if (IS_ERR(opfc->fc))
861 return PTR_ERR(opfc->fc);
862
863 ret = mlx5_ib_fs_add_op_fc(dev, port, opfc, type);
864 if (ret) {
865 mlx5_fc_destroy(dev->mdev, opfc->fc);
866 opfc->fc = NULL;
867 }
868 return ret;
869 }
870
871 if (!opfc->fc)
872 return -EINVAL;
873
874 mlx5_ib_fs_remove_op_fc(dev, opfc, type);
875 mlx5_fc_destroy(dev->mdev, opfc->fc);
876 opfc->fc = NULL;
877 return 0;
878}
879
880static const struct ib_device_ops hw_stats_ops = {
881 .alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats,
882 .get_hw_stats = mlx5_ib_get_hw_stats,
883 .counter_bind_qp = mlx5_ib_counter_bind_qp,
884 .counter_unbind_qp = mlx5_ib_counter_unbind_qp,
885 .counter_dealloc = mlx5_ib_counter_dealloc,
886 .counter_alloc_stats = mlx5_ib_counter_alloc_stats,
887 .counter_update_stats = mlx5_ib_counter_update_stats,
888 .modify_hw_stat = IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) ?
889 mlx5_ib_modify_stat : NULL,
890};
891
892static const struct ib_device_ops hw_switchdev_stats_ops = {
893 .alloc_hw_device_stats = mlx5_ib_alloc_hw_device_stats,
894 .get_hw_stats = mlx5_ib_get_hw_stats,
895 .counter_bind_qp = mlx5_ib_counter_bind_qp,
896 .counter_unbind_qp = mlx5_ib_counter_unbind_qp,
897 .counter_dealloc = mlx5_ib_counter_dealloc,
898 .counter_alloc_stats = mlx5_ib_counter_alloc_stats,
899 .counter_update_stats = mlx5_ib_counter_update_stats,
900};
901
902static const struct ib_device_ops counters_ops = {
903 .create_counters = mlx5_ib_create_counters,
904 .destroy_counters = mlx5_ib_destroy_counters,
905 .read_counters = mlx5_ib_read_counters,
906
907 INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
908};
909
910int mlx5_ib_counters_init(struct mlx5_ib_dev *dev)
911{
912 ib_set_device_ops(&dev->ib_dev, &counters_ops);
913
914 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
915 return 0;
916
917 if (is_mdev_switchdev_mode(dev->mdev))
918 ib_set_device_ops(&dev->ib_dev, &hw_switchdev_stats_ops);
919 else
920 ib_set_device_ops(&dev->ib_dev, &hw_stats_ops);
921 return mlx5_ib_alloc_counters(dev);
922}
923
924void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev)
925{
926 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
927 return;
928
929 mlx5_ib_dealloc_counters(dev);
930}