Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Huawei HiNIC PCI Express Linux driver
4 * Copyright(c) 2017 Huawei Technologies Co., Ltd
5 */
6
7#include <linux/kernel.h>
8#include <linux/types.h>
9#include <linux/errno.h>
10#include <linux/pci.h>
11#include <linux/device.h>
12#include <linux/slab.h>
13#include <linux/dma-mapping.h>
14#include <linux/bitops.h>
15#include <linux/err.h>
16#include <linux/jiffies.h>
17#include <linux/delay.h>
18#include <linux/log2.h>
19#include <linux/semaphore.h>
20#include <asm/byteorder.h>
21#include <asm/barrier.h>
22
23#include "hinic_hw_csr.h"
24#include "hinic_hw_if.h"
25#include "hinic_hw_api_cmd.h"
26
27#define API_CHAIN_NUM_CELLS 32
28
29#define API_CMD_CELL_SIZE_SHIFT 6
30#define API_CMD_CELL_SIZE_MIN (BIT(API_CMD_CELL_SIZE_SHIFT))
31
32#define API_CMD_CELL_SIZE(cell_size) \
33 (((cell_size) >= API_CMD_CELL_SIZE_MIN) ? \
34 (1 << (fls(cell_size - 1))) : API_CMD_CELL_SIZE_MIN)
35
36#define API_CMD_CELL_SIZE_VAL(size) \
37 ilog2((size) >> API_CMD_CELL_SIZE_SHIFT)
38
39#define API_CMD_BUF_SIZE 2048
40
41/* Sizes of the members in hinic_api_cmd_cell */
42#define API_CMD_CELL_DESC_SIZE 8
43#define API_CMD_CELL_DATA_ADDR_SIZE 8
44
45#define API_CMD_CELL_ALIGNMENT 8
46
47#define API_CMD_TIMEOUT 1000
48
49#define MASKED_IDX(chain, idx) ((idx) & ((chain)->num_cells - 1))
50
51#define SIZE_8BYTES(size) (ALIGN((size), 8) >> 3)
52#define SIZE_4BYTES(size) (ALIGN((size), 4) >> 2)
53
54#define RD_DMA_ATTR_DEFAULT 0
55#define WR_DMA_ATTR_DEFAULT 0
56
57enum api_cmd_data_format {
58 SGE_DATA = 1, /* cell data is passed by hw address */
59};
60
61enum api_cmd_type {
62 API_CMD_WRITE = 0,
63};
64
65enum api_cmd_bypass {
66 NO_BYPASS = 0,
67 BYPASS = 1,
68};
69
70enum api_cmd_xor_chk_level {
71 XOR_CHK_DIS = 0,
72
73 XOR_CHK_ALL = 3,
74};
75
76static u8 xor_chksum_set(void *data)
77{
78 int idx;
79 u8 *val, checksum = 0;
80
81 val = data;
82
83 for (idx = 0; idx < 7; idx++)
84 checksum ^= val[idx];
85
86 return checksum;
87}
88
89static void set_prod_idx(struct hinic_api_cmd_chain *chain)
90{
91 enum hinic_api_cmd_chain_type chain_type = chain->chain_type;
92 struct hinic_hwif *hwif = chain->hwif;
93 u32 addr, prod_idx;
94
95 addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain_type);
96 prod_idx = hinic_hwif_read_reg(hwif, addr);
97
98 prod_idx = HINIC_API_CMD_PI_CLEAR(prod_idx, IDX);
99
100 prod_idx |= HINIC_API_CMD_PI_SET(chain->prod_idx, IDX);
101
102 hinic_hwif_write_reg(hwif, addr, prod_idx);
103}
104
105static u32 get_hw_cons_idx(struct hinic_api_cmd_chain *chain)
106{
107 u32 addr, val;
108
109 addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type);
110 val = hinic_hwif_read_reg(chain->hwif, addr);
111
112 return HINIC_API_CMD_STATUS_GET(val, CONS_IDX);
113}
114
115static void dump_api_chain_reg(struct hinic_api_cmd_chain *chain)
116{
117 u32 addr, val;
118
119 addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type);
120 val = hinic_hwif_read_reg(chain->hwif, addr);
121
122 dev_err(&chain->hwif->pdev->dev, "Chain type: 0x%x, cpld error: 0x%x, check error: 0x%x, current fsm: 0x%x\n",
123 chain->chain_type, HINIC_API_CMD_STATUS_GET(val, CPLD_ERR),
124 HINIC_API_CMD_STATUS_GET(val, CHKSUM_ERR),
125 HINIC_API_CMD_STATUS_GET(val, FSM));
126
127 dev_err(&chain->hwif->pdev->dev, "Chain hw current ci: 0x%x\n",
128 HINIC_API_CMD_STATUS_GET(val, CONS_IDX));
129
130 addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain->chain_type);
131 val = hinic_hwif_read_reg(chain->hwif, addr);
132 dev_err(&chain->hwif->pdev->dev, "Chain hw current pi: 0x%x\n", val);
133}
134
135/**
136 * chain_busy - check if the chain is still processing last requests
137 * @chain: chain to check
138 *
139 * Return 0 - Success, negative - Failure
140 **/
141static int chain_busy(struct hinic_api_cmd_chain *chain)
142{
143 struct hinic_hwif *hwif = chain->hwif;
144 struct pci_dev *pdev = hwif->pdev;
145 u32 prod_idx;
146
147 switch (chain->chain_type) {
148 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
149 chain->cons_idx = get_hw_cons_idx(chain);
150 prod_idx = chain->prod_idx;
151
152 /* check for a space for a new command */
153 if (chain->cons_idx == MASKED_IDX(chain, prod_idx + 1)) {
154 dev_err(&pdev->dev, "API CMD chain %d is busy, cons_idx: %d, prod_idx: %d\n",
155 chain->chain_type, chain->cons_idx,
156 chain->prod_idx);
157 dump_api_chain_reg(chain);
158 return -EBUSY;
159 }
160 break;
161
162 default:
163 dev_err(&pdev->dev, "Unknown API CMD Chain type\n");
164 break;
165 }
166
167 return 0;
168}
169
170/**
171 * get_cell_data_size - get the data size of a specific cell type
172 * @type: chain type
173 *
174 * Return the data(Desc + Address) size in the cell
175 **/
176static u8 get_cell_data_size(enum hinic_api_cmd_chain_type type)
177{
178 u8 cell_data_size = 0;
179
180 switch (type) {
181 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
182 cell_data_size = ALIGN(API_CMD_CELL_DESC_SIZE +
183 API_CMD_CELL_DATA_ADDR_SIZE,
184 API_CMD_CELL_ALIGNMENT);
185 break;
186 default:
187 break;
188 }
189
190 return cell_data_size;
191}
192
193/**
194 * prepare_cell_ctrl - prepare the ctrl of the cell for the command
195 * @cell_ctrl: the control of the cell to set the control value into it
196 * @data_size: the size of the data in the cell
197 **/
198static void prepare_cell_ctrl(u64 *cell_ctrl, u16 data_size)
199{
200 u8 chksum;
201 u64 ctrl;
202
203 ctrl = HINIC_API_CMD_CELL_CTRL_SET(SIZE_8BYTES(data_size), DATA_SZ) |
204 HINIC_API_CMD_CELL_CTRL_SET(RD_DMA_ATTR_DEFAULT, RD_DMA_ATTR) |
205 HINIC_API_CMD_CELL_CTRL_SET(WR_DMA_ATTR_DEFAULT, WR_DMA_ATTR);
206
207 chksum = xor_chksum_set(&ctrl);
208
209 ctrl |= HINIC_API_CMD_CELL_CTRL_SET(chksum, XOR_CHKSUM);
210
211 /* The data in the HW should be in Big Endian Format */
212 *cell_ctrl = cpu_to_be64(ctrl);
213}
214
215/**
216 * prepare_api_cmd - prepare API CMD command
217 * @chain: chain for the command
218 * @dest: destination node on the card that will receive the command
219 * @cmd: command data
220 * @cmd_size: the command size
221 **/
222static void prepare_api_cmd(struct hinic_api_cmd_chain *chain,
223 enum hinic_node_id dest,
224 void *cmd, u16 cmd_size)
225{
226 struct hinic_api_cmd_cell *cell = chain->curr_node;
227 struct hinic_api_cmd_cell_ctxt *cell_ctxt;
228 struct hinic_hwif *hwif = chain->hwif;
229 struct pci_dev *pdev = hwif->pdev;
230
231 cell_ctxt = &chain->cell_ctxt[chain->prod_idx];
232
233 switch (chain->chain_type) {
234 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
235 cell->desc = HINIC_API_CMD_DESC_SET(SGE_DATA, API_TYPE) |
236 HINIC_API_CMD_DESC_SET(API_CMD_WRITE, RD_WR) |
237 HINIC_API_CMD_DESC_SET(NO_BYPASS, MGMT_BYPASS);
238 break;
239
240 default:
241 dev_err(&pdev->dev, "unknown Chain type\n");
242 return;
243 }
244
245 cell->desc |= HINIC_API_CMD_DESC_SET(dest, DEST) |
246 HINIC_API_CMD_DESC_SET(SIZE_4BYTES(cmd_size), SIZE);
247
248 cell->desc |= HINIC_API_CMD_DESC_SET(xor_chksum_set(&cell->desc),
249 XOR_CHKSUM);
250
251 /* The data in the HW should be in Big Endian Format */
252 cell->desc = cpu_to_be64(cell->desc);
253
254 memcpy(cell_ctxt->api_cmd_vaddr, cmd, cmd_size);
255}
256
257/**
258 * prepare_cell - prepare cell ctrl and cmd in the current cell
259 * @chain: chain for the command
260 * @dest: destination node on the card that will receive the command
261 * @cmd: command data
262 * @cmd_size: the command size
263 *
264 * Return 0 - Success, negative - Failure
265 **/
266static void prepare_cell(struct hinic_api_cmd_chain *chain,
267 enum hinic_node_id dest,
268 void *cmd, u16 cmd_size)
269{
270 struct hinic_api_cmd_cell *curr_node = chain->curr_node;
271 u16 data_size = get_cell_data_size(chain->chain_type);
272
273 prepare_cell_ctrl(&curr_node->ctrl, data_size);
274 prepare_api_cmd(chain, dest, cmd, cmd_size);
275}
276
277static inline void cmd_chain_prod_idx_inc(struct hinic_api_cmd_chain *chain)
278{
279 chain->prod_idx = MASKED_IDX(chain, chain->prod_idx + 1);
280}
281
282/**
283 * api_cmd_status_update - update the status in the chain struct
284 * @chain: chain to update
285 **/
286static void api_cmd_status_update(struct hinic_api_cmd_chain *chain)
287{
288 enum hinic_api_cmd_chain_type chain_type;
289 struct hinic_api_cmd_status *wb_status;
290 struct hinic_hwif *hwif = chain->hwif;
291 struct pci_dev *pdev = hwif->pdev;
292 u64 status_header;
293 u32 status;
294
295 wb_status = chain->wb_status;
296 status_header = be64_to_cpu(wb_status->header);
297
298 status = be32_to_cpu(wb_status->status);
299 if (HINIC_API_CMD_STATUS_GET(status, CHKSUM_ERR)) {
300 dev_err(&pdev->dev, "API CMD status: Xor check error\n");
301 return;
302 }
303
304 chain_type = HINIC_API_CMD_STATUS_HEADER_GET(status_header, CHAIN_ID);
305 if (chain_type >= HINIC_API_CMD_MAX) {
306 dev_err(&pdev->dev, "unknown API CMD Chain %d\n", chain_type);
307 return;
308 }
309
310 chain->cons_idx = HINIC_API_CMD_STATUS_GET(status, CONS_IDX);
311}
312
313/**
314 * wait_for_status_poll - wait for write to api cmd command to complete
315 * @chain: the chain of the command
316 *
317 * Return 0 - Success, negative - Failure
318 **/
319static int wait_for_status_poll(struct hinic_api_cmd_chain *chain)
320{
321 int err = -ETIMEDOUT;
322 unsigned long end;
323
324 end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);
325 do {
326 api_cmd_status_update(chain);
327
328 /* wait for CI to be updated - sign for completion */
329 if (chain->cons_idx == chain->prod_idx) {
330 err = 0;
331 break;
332 }
333
334 msleep(20);
335 } while (time_before(jiffies, end));
336
337 return err;
338}
339
340/**
341 * wait_for_api_cmd_completion - wait for command to complete
342 * @chain: chain for the command
343 *
344 * Return 0 - Success, negative - Failure
345 **/
346static int wait_for_api_cmd_completion(struct hinic_api_cmd_chain *chain)
347{
348 struct hinic_hwif *hwif = chain->hwif;
349 struct pci_dev *pdev = hwif->pdev;
350 int err;
351
352 switch (chain->chain_type) {
353 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
354 err = wait_for_status_poll(chain);
355 if (err) {
356 dev_err(&pdev->dev, "API CMD Poll status timeout\n");
357 dump_api_chain_reg(chain);
358 break;
359 }
360 break;
361
362 default:
363 dev_err(&pdev->dev, "unknown API CMD Chain type\n");
364 err = -EINVAL;
365 break;
366 }
367
368 return err;
369}
370
371/**
372 * api_cmd - API CMD command
373 * @chain: chain for the command
374 * @dest: destination node on the card that will receive the command
375 * @cmd: command data
376 * @cmd_size: the command size
377 *
378 * Return 0 - Success, negative - Failure
379 **/
380static int api_cmd(struct hinic_api_cmd_chain *chain,
381 enum hinic_node_id dest, u8 *cmd, u16 cmd_size)
382{
383 struct hinic_api_cmd_cell_ctxt *ctxt;
384 int err;
385
386 down(&chain->sem);
387 if (chain_busy(chain)) {
388 up(&chain->sem);
389 return -EBUSY;
390 }
391
392 prepare_cell(chain, dest, cmd, cmd_size);
393 cmd_chain_prod_idx_inc(chain);
394
395 wmb(); /* inc pi before issue the command */
396
397 set_prod_idx(chain); /* issue the command */
398
399 ctxt = &chain->cell_ctxt[chain->prod_idx];
400
401 chain->curr_node = ctxt->cell_vaddr;
402
403 err = wait_for_api_cmd_completion(chain);
404
405 up(&chain->sem);
406 return err;
407}
408
409/**
410 * hinic_api_cmd_write - Write API CMD command
411 * @chain: chain for write command
412 * @dest: destination node on the card that will receive the command
413 * @cmd: command data
414 * @size: the command size
415 *
416 * Return 0 - Success, negative - Failure
417 **/
418int hinic_api_cmd_write(struct hinic_api_cmd_chain *chain,
419 enum hinic_node_id dest, u8 *cmd, u16 size)
420{
421 /* Verify the chain type */
422 if (chain->chain_type == HINIC_API_CMD_WRITE_TO_MGMT_CPU)
423 return api_cmd(chain, dest, cmd, size);
424
425 return -EINVAL;
426}
427
428/**
429 * api_cmd_hw_restart - restart the chain in the HW
430 * @chain: the API CMD specific chain to restart
431 *
432 * Return 0 - Success, negative - Failure
433 **/
434static int api_cmd_hw_restart(struct hinic_api_cmd_chain *chain)
435{
436 struct hinic_hwif *hwif = chain->hwif;
437 int err = -ETIMEDOUT;
438 unsigned long end;
439 u32 reg_addr, val;
440
441 /* Read Modify Write */
442 reg_addr = HINIC_CSR_API_CMD_CHAIN_REQ_ADDR(chain->chain_type);
443 val = hinic_hwif_read_reg(hwif, reg_addr);
444
445 val = HINIC_API_CMD_CHAIN_REQ_CLEAR(val, RESTART);
446 val |= HINIC_API_CMD_CHAIN_REQ_SET(1, RESTART);
447
448 hinic_hwif_write_reg(hwif, reg_addr, val);
449
450 end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);
451 do {
452 val = hinic_hwif_read_reg(hwif, reg_addr);
453
454 if (!HINIC_API_CMD_CHAIN_REQ_GET(val, RESTART)) {
455 err = 0;
456 break;
457 }
458
459 msleep(20);
460 } while (time_before(jiffies, end));
461
462 return err;
463}
464
465/**
466 * api_cmd_ctrl_init - set the control register of a chain
467 * @chain: the API CMD specific chain to set control register for
468 **/
469static void api_cmd_ctrl_init(struct hinic_api_cmd_chain *chain)
470{
471 struct hinic_hwif *hwif = chain->hwif;
472 u32 addr, ctrl;
473 u16 cell_size;
474
475 /* Read Modify Write */
476 addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);
477
478 cell_size = API_CMD_CELL_SIZE_VAL(chain->cell_size);
479
480 ctrl = hinic_hwif_read_reg(hwif, addr);
481
482 ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) &
483 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR) &
484 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) &
485 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) &
486 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);
487
488 ctrl |= HINIC_API_CMD_CHAIN_CTRL_SET(1, XOR_ERR) |
489 HINIC_API_CMD_CHAIN_CTRL_SET(XOR_CHK_ALL, XOR_CHK_EN) |
490 HINIC_API_CMD_CHAIN_CTRL_SET(cell_size, CELL_SIZE);
491
492 hinic_hwif_write_reg(hwif, addr, ctrl);
493}
494
495/**
496 * api_cmd_set_status_addr - set the status address of a chain in the HW
497 * @chain: the API CMD specific chain to set in HW status address for
498 **/
499static void api_cmd_set_status_addr(struct hinic_api_cmd_chain *chain)
500{
501 struct hinic_hwif *hwif = chain->hwif;
502 u32 addr, val;
503
504 addr = HINIC_CSR_API_CMD_STATUS_HI_ADDR(chain->chain_type);
505 val = upper_32_bits(chain->wb_status_paddr);
506 hinic_hwif_write_reg(hwif, addr, val);
507
508 addr = HINIC_CSR_API_CMD_STATUS_LO_ADDR(chain->chain_type);
509 val = lower_32_bits(chain->wb_status_paddr);
510 hinic_hwif_write_reg(hwif, addr, val);
511}
512
513/**
514 * api_cmd_set_num_cells - set the number cells of a chain in the HW
515 * @chain: the API CMD specific chain to set in HW the number of cells for
516 **/
517static void api_cmd_set_num_cells(struct hinic_api_cmd_chain *chain)
518{
519 struct hinic_hwif *hwif = chain->hwif;
520 u32 addr, val;
521
522 addr = HINIC_CSR_API_CMD_CHAIN_NUM_CELLS_ADDR(chain->chain_type);
523 val = chain->num_cells;
524 hinic_hwif_write_reg(hwif, addr, val);
525}
526
527/**
528 * api_cmd_head_init - set the head of a chain in the HW
529 * @chain: the API CMD specific chain to set in HW the head for
530 **/
531static void api_cmd_head_init(struct hinic_api_cmd_chain *chain)
532{
533 struct hinic_hwif *hwif = chain->hwif;
534 u32 addr, val;
535
536 addr = HINIC_CSR_API_CMD_CHAIN_HEAD_HI_ADDR(chain->chain_type);
537 val = upper_32_bits(chain->head_cell_paddr);
538 hinic_hwif_write_reg(hwif, addr, val);
539
540 addr = HINIC_CSR_API_CMD_CHAIN_HEAD_LO_ADDR(chain->chain_type);
541 val = lower_32_bits(chain->head_cell_paddr);
542 hinic_hwif_write_reg(hwif, addr, val);
543}
544
545/**
546 * api_cmd_chain_hw_clean - clean the HW
547 * @chain: the API CMD specific chain
548 **/
549static void api_cmd_chain_hw_clean(struct hinic_api_cmd_chain *chain)
550{
551 struct hinic_hwif *hwif = chain->hwif;
552 u32 addr, ctrl;
553
554 addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);
555
556 ctrl = hinic_hwif_read_reg(hwif, addr);
557 ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) &
558 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR) &
559 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) &
560 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) &
561 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);
562
563 hinic_hwif_write_reg(hwif, addr, ctrl);
564}
565
566/**
567 * api_cmd_chain_hw_init - initialize the chain in the HW
568 * @chain: the API CMD specific chain to initialize in HW
569 *
570 * Return 0 - Success, negative - Failure
571 **/
572static int api_cmd_chain_hw_init(struct hinic_api_cmd_chain *chain)
573{
574 struct hinic_hwif *hwif = chain->hwif;
575 struct pci_dev *pdev = hwif->pdev;
576 int err;
577
578 api_cmd_chain_hw_clean(chain);
579
580 api_cmd_set_status_addr(chain);
581
582 err = api_cmd_hw_restart(chain);
583 if (err) {
584 dev_err(&pdev->dev, "Failed to restart API CMD HW\n");
585 return err;
586 }
587
588 api_cmd_ctrl_init(chain);
589 api_cmd_set_num_cells(chain);
590 api_cmd_head_init(chain);
591 return 0;
592}
593
594/**
595 * free_cmd_buf - free the dma buffer of API CMD command
596 * @chain: the API CMD specific chain of the cmd
597 * @cell_idx: the cell index of the cmd
598 **/
599static void free_cmd_buf(struct hinic_api_cmd_chain *chain, int cell_idx)
600{
601 struct hinic_api_cmd_cell_ctxt *cell_ctxt;
602 struct hinic_hwif *hwif = chain->hwif;
603 struct pci_dev *pdev = hwif->pdev;
604
605 cell_ctxt = &chain->cell_ctxt[cell_idx];
606
607 dma_free_coherent(&pdev->dev, API_CMD_BUF_SIZE,
608 cell_ctxt->api_cmd_vaddr,
609 cell_ctxt->api_cmd_paddr);
610}
611
612/**
613 * alloc_cmd_buf - allocate a dma buffer for API CMD command
614 * @chain: the API CMD specific chain for the cmd
615 * @cell: the cell in the HW for the cmd
616 * @cell_idx: the index of the cell
617 *
618 * Return 0 - Success, negative - Failure
619 **/
620static int alloc_cmd_buf(struct hinic_api_cmd_chain *chain,
621 struct hinic_api_cmd_cell *cell, int cell_idx)
622{
623 struct hinic_api_cmd_cell_ctxt *cell_ctxt;
624 struct hinic_hwif *hwif = chain->hwif;
625 struct pci_dev *pdev = hwif->pdev;
626 dma_addr_t cmd_paddr;
627 u8 *cmd_vaddr;
628 int err = 0;
629
630 cmd_vaddr = dma_alloc_coherent(&pdev->dev, API_CMD_BUF_SIZE,
631 &cmd_paddr, GFP_KERNEL);
632 if (!cmd_vaddr)
633 return -ENOMEM;
634
635 cell_ctxt = &chain->cell_ctxt[cell_idx];
636
637 cell_ctxt->api_cmd_vaddr = cmd_vaddr;
638 cell_ctxt->api_cmd_paddr = cmd_paddr;
639
640 /* set the cmd DMA address in the cell */
641 switch (chain->chain_type) {
642 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
643 /* The data in the HW should be in Big Endian Format */
644 cell->write.hw_cmd_paddr = cpu_to_be64(cmd_paddr);
645 break;
646
647 default:
648 dev_err(&pdev->dev, "Unsupported API CMD chain type\n");
649 free_cmd_buf(chain, cell_idx);
650 err = -EINVAL;
651 break;
652 }
653
654 return err;
655}
656
657/**
658 * api_cmd_create_cell - create API CMD cell for specific chain
659 * @chain: the API CMD specific chain to create its cell
660 * @cell_idx: the index of the cell to create
661 * @pre_node: previous cell
662 * @node_vaddr: the returned virt addr of the cell
663 *
664 * Return 0 - Success, negative - Failure
665 **/
666static int api_cmd_create_cell(struct hinic_api_cmd_chain *chain,
667 int cell_idx,
668 struct hinic_api_cmd_cell *pre_node,
669 struct hinic_api_cmd_cell **node_vaddr)
670{
671 struct hinic_api_cmd_cell_ctxt *cell_ctxt;
672 struct hinic_hwif *hwif = chain->hwif;
673 struct pci_dev *pdev = hwif->pdev;
674 struct hinic_api_cmd_cell *node;
675 dma_addr_t node_paddr;
676 int err;
677
678 node = dma_alloc_coherent(&pdev->dev, chain->cell_size, &node_paddr,
679 GFP_KERNEL);
680 if (!node)
681 return -ENOMEM;
682
683 node->read.hw_wb_resp_paddr = 0;
684
685 cell_ctxt = &chain->cell_ctxt[cell_idx];
686 cell_ctxt->cell_vaddr = node;
687 cell_ctxt->cell_paddr = node_paddr;
688
689 if (!pre_node) {
690 chain->head_cell_paddr = node_paddr;
691 chain->head_node = node;
692 } else {
693 /* The data in the HW should be in Big Endian Format */
694 pre_node->next_cell_paddr = cpu_to_be64(node_paddr);
695 }
696
697 switch (chain->chain_type) {
698 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
699 err = alloc_cmd_buf(chain, node, cell_idx);
700 if (err) {
701 dev_err(&pdev->dev, "Failed to allocate cmd buffer\n");
702 goto err_alloc_cmd_buf;
703 }
704 break;
705
706 default:
707 dev_err(&pdev->dev, "Unsupported API CMD chain type\n");
708 err = -EINVAL;
709 goto err_alloc_cmd_buf;
710 }
711
712 *node_vaddr = node;
713 return 0;
714
715err_alloc_cmd_buf:
716 dma_free_coherent(&pdev->dev, chain->cell_size, node, node_paddr);
717 return err;
718}
719
720/**
721 * api_cmd_destroy_cell - destroy API CMD cell of specific chain
722 * @chain: the API CMD specific chain to destroy its cell
723 * @cell_idx: the cell to destroy
724 **/
725static void api_cmd_destroy_cell(struct hinic_api_cmd_chain *chain,
726 int cell_idx)
727{
728 struct hinic_api_cmd_cell_ctxt *cell_ctxt;
729 struct hinic_hwif *hwif = chain->hwif;
730 struct pci_dev *pdev = hwif->pdev;
731 struct hinic_api_cmd_cell *node;
732 dma_addr_t node_paddr;
733 size_t node_size;
734
735 cell_ctxt = &chain->cell_ctxt[cell_idx];
736
737 node = cell_ctxt->cell_vaddr;
738 node_paddr = cell_ctxt->cell_paddr;
739 node_size = chain->cell_size;
740
741 if (cell_ctxt->api_cmd_vaddr) {
742 switch (chain->chain_type) {
743 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
744 free_cmd_buf(chain, cell_idx);
745 break;
746 default:
747 dev_err(&pdev->dev, "Unsupported API CMD chain type\n");
748 break;
749 }
750
751 dma_free_coherent(&pdev->dev, node_size, node,
752 node_paddr);
753 }
754}
755
756/**
757 * api_cmd_destroy_cells - destroy API CMD cells of specific chain
758 * @chain: the API CMD specific chain to destroy its cells
759 * @num_cells: number of cells to destroy
760 **/
761static void api_cmd_destroy_cells(struct hinic_api_cmd_chain *chain,
762 int num_cells)
763{
764 int cell_idx;
765
766 for (cell_idx = 0; cell_idx < num_cells; cell_idx++)
767 api_cmd_destroy_cell(chain, cell_idx);
768}
769
770/**
771 * api_cmd_create_cells - create API CMD cells for specific chain
772 * @chain: the API CMD specific chain
773 *
774 * Return 0 - Success, negative - Failure
775 **/
776static int api_cmd_create_cells(struct hinic_api_cmd_chain *chain)
777{
778 struct hinic_api_cmd_cell *node = NULL, *pre_node = NULL;
779 struct hinic_hwif *hwif = chain->hwif;
780 struct pci_dev *pdev = hwif->pdev;
781 int err, cell_idx;
782
783 for (cell_idx = 0; cell_idx < chain->num_cells; cell_idx++) {
784 err = api_cmd_create_cell(chain, cell_idx, pre_node, &node);
785 if (err) {
786 dev_err(&pdev->dev, "Failed to create API CMD cell\n");
787 goto err_create_cell;
788 }
789
790 pre_node = node;
791 }
792
793 /* set the Final node to point on the start */
794 node->next_cell_paddr = cpu_to_be64(chain->head_cell_paddr);
795
796 /* set the current node to be the head */
797 chain->curr_node = chain->head_node;
798 return 0;
799
800err_create_cell:
801 api_cmd_destroy_cells(chain, cell_idx);
802 return err;
803}
804
805/**
806 * api_chain_init - initialize API CMD specific chain
807 * @chain: the API CMD specific chain to initialize
808 * @attr: attributes to set in the chain
809 *
810 * Return 0 - Success, negative - Failure
811 **/
812static int api_chain_init(struct hinic_api_cmd_chain *chain,
813 struct hinic_api_cmd_chain_attr *attr)
814{
815 struct hinic_hwif *hwif = attr->hwif;
816 struct pci_dev *pdev = hwif->pdev;
817
818 chain->hwif = hwif;
819 chain->chain_type = attr->chain_type;
820 chain->num_cells = attr->num_cells;
821 chain->cell_size = attr->cell_size;
822
823 chain->prod_idx = 0;
824 chain->cons_idx = 0;
825
826 sema_init(&chain->sem, 1);
827
828 chain->cell_ctxt = devm_kcalloc(&pdev->dev, chain->num_cells,
829 sizeof(*chain->cell_ctxt), GFP_KERNEL);
830 if (!chain->cell_ctxt)
831 return -ENOMEM;
832
833 chain->wb_status = dma_alloc_coherent(&pdev->dev,
834 sizeof(*chain->wb_status),
835 &chain->wb_status_paddr,
836 GFP_KERNEL);
837 if (!chain->wb_status) {
838 dev_err(&pdev->dev, "Failed to allocate DMA wb status\n");
839 return -ENOMEM;
840 }
841
842 return 0;
843}
844
845/**
846 * api_chain_free - free API CMD specific chain
847 * @chain: the API CMD specific chain to free
848 **/
849static void api_chain_free(struct hinic_api_cmd_chain *chain)
850{
851 struct hinic_hwif *hwif = chain->hwif;
852 struct pci_dev *pdev = hwif->pdev;
853
854 dma_free_coherent(&pdev->dev, sizeof(*chain->wb_status),
855 chain->wb_status, chain->wb_status_paddr);
856}
857
858/**
859 * api_cmd_create_chain - create API CMD specific chain
860 * @attr: attributes to set the chain
861 *
862 * Return the created chain
863 **/
864static struct hinic_api_cmd_chain *
865 api_cmd_create_chain(struct hinic_api_cmd_chain_attr *attr)
866{
867 struct hinic_hwif *hwif = attr->hwif;
868 struct pci_dev *pdev = hwif->pdev;
869 struct hinic_api_cmd_chain *chain;
870 int err;
871
872 if (attr->num_cells & (attr->num_cells - 1)) {
873 dev_err(&pdev->dev, "Invalid number of cells, must be power of 2\n");
874 return ERR_PTR(-EINVAL);
875 }
876
877 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
878 if (!chain)
879 return ERR_PTR(-ENOMEM);
880
881 err = api_chain_init(chain, attr);
882 if (err) {
883 dev_err(&pdev->dev, "Failed to initialize chain\n");
884 return ERR_PTR(err);
885 }
886
887 err = api_cmd_create_cells(chain);
888 if (err) {
889 dev_err(&pdev->dev, "Failed to create cells for API CMD chain\n");
890 goto err_create_cells;
891 }
892
893 err = api_cmd_chain_hw_init(chain);
894 if (err) {
895 dev_err(&pdev->dev, "Failed to initialize chain HW\n");
896 goto err_chain_hw_init;
897 }
898
899 return chain;
900
901err_chain_hw_init:
902 api_cmd_destroy_cells(chain, chain->num_cells);
903
904err_create_cells:
905 api_chain_free(chain);
906 return ERR_PTR(err);
907}
908
909/**
910 * api_cmd_destroy_chain - destroy API CMD specific chain
911 * @chain: the API CMD specific chain to destroy
912 **/
913static void api_cmd_destroy_chain(struct hinic_api_cmd_chain *chain)
914{
915 api_cmd_chain_hw_clean(chain);
916 api_cmd_destroy_cells(chain, chain->num_cells);
917 api_chain_free(chain);
918}
919
920/**
921 * hinic_api_cmd_init - Initialize all the API CMD chains
922 * @chain: the API CMD chains that are initialized
923 * @hwif: the hardware interface of a pci function device
924 *
925 * Return 0 - Success, negative - Failure
926 **/
927int hinic_api_cmd_init(struct hinic_api_cmd_chain **chain,
928 struct hinic_hwif *hwif)
929{
930 enum hinic_api_cmd_chain_type type, chain_type;
931 struct hinic_api_cmd_chain_attr attr;
932 struct pci_dev *pdev = hwif->pdev;
933 size_t hw_cell_sz;
934 int err;
935
936 hw_cell_sz = sizeof(struct hinic_api_cmd_cell);
937
938 attr.hwif = hwif;
939 attr.num_cells = API_CHAIN_NUM_CELLS;
940 attr.cell_size = API_CMD_CELL_SIZE(hw_cell_sz);
941
942 chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;
943 for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) {
944 attr.chain_type = chain_type;
945
946 if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)
947 continue;
948
949 chain[chain_type] = api_cmd_create_chain(&attr);
950 if (IS_ERR(chain[chain_type])) {
951 dev_err(&pdev->dev, "Failed to create chain %d\n",
952 chain_type);
953 err = PTR_ERR(chain[chain_type]);
954 goto err_create_chain;
955 }
956 }
957
958 return 0;
959
960err_create_chain:
961 type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;
962 for ( ; type < chain_type; type++) {
963 if (type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)
964 continue;
965
966 api_cmd_destroy_chain(chain[type]);
967 }
968
969 return err;
970}
971
972/**
973 * hinic_api_cmd_free - free the API CMD chains
974 * @chain: the API CMD chains that are freed
975 **/
976void hinic_api_cmd_free(struct hinic_api_cmd_chain **chain)
977{
978 enum hinic_api_cmd_chain_type chain_type;
979
980 chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;
981 for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) {
982 if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)
983 continue;
984
985 api_cmd_destroy_chain(chain[chain_type]);
986 }
987}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Huawei HiNIC PCI Express Linux driver
4 * Copyright(c) 2017 Huawei Technologies Co., Ltd
5 */
6
7#include <linux/kernel.h>
8#include <linux/types.h>
9#include <linux/errno.h>
10#include <linux/pci.h>
11#include <linux/device.h>
12#include <linux/slab.h>
13#include <linux/dma-mapping.h>
14#include <linux/bitops.h>
15#include <linux/err.h>
16#include <linux/jiffies.h>
17#include <linux/delay.h>
18#include <linux/log2.h>
19#include <linux/semaphore.h>
20#include <asm/byteorder.h>
21#include <asm/barrier.h>
22
23#include "hinic_hw_csr.h"
24#include "hinic_hw_if.h"
25#include "hinic_hw_api_cmd.h"
26
27#define API_CHAIN_NUM_CELLS 32
28
29#define API_CMD_CELL_SIZE_SHIFT 6
30#define API_CMD_CELL_SIZE_MIN (BIT(API_CMD_CELL_SIZE_SHIFT))
31
32#define API_CMD_CELL_SIZE(cell_size) \
33 (((cell_size) >= API_CMD_CELL_SIZE_MIN) ? \
34 (1 << (fls(cell_size - 1))) : API_CMD_CELL_SIZE_MIN)
35
36#define API_CMD_CELL_SIZE_VAL(size) \
37 ilog2((size) >> API_CMD_CELL_SIZE_SHIFT)
38
39#define API_CMD_BUF_SIZE 2048
40
41/* Sizes of the members in hinic_api_cmd_cell */
42#define API_CMD_CELL_DESC_SIZE 8
43#define API_CMD_CELL_DATA_ADDR_SIZE 8
44
45#define API_CMD_CELL_ALIGNMENT 8
46
47#define API_CMD_TIMEOUT 1000
48
49#define MASKED_IDX(chain, idx) ((idx) & ((chain)->num_cells - 1))
50
51#define SIZE_8BYTES(size) (ALIGN((size), 8) >> 3)
52#define SIZE_4BYTES(size) (ALIGN((size), 4) >> 2)
53
54#define RD_DMA_ATTR_DEFAULT 0
55#define WR_DMA_ATTR_DEFAULT 0
56
57enum api_cmd_data_format {
58 SGE_DATA = 1, /* cell data is passed by hw address */
59};
60
61enum api_cmd_type {
62 API_CMD_WRITE = 0,
63};
64
65enum api_cmd_bypass {
66 NO_BYPASS = 0,
67 BYPASS = 1,
68};
69
70enum api_cmd_xor_chk_level {
71 XOR_CHK_DIS = 0,
72
73 XOR_CHK_ALL = 3,
74};
75
76static u8 xor_chksum_set(void *data)
77{
78 int idx;
79 u8 *val, checksum = 0;
80
81 val = data;
82
83 for (idx = 0; idx < 7; idx++)
84 checksum ^= val[idx];
85
86 return checksum;
87}
88
89static void set_prod_idx(struct hinic_api_cmd_chain *chain)
90{
91 enum hinic_api_cmd_chain_type chain_type = chain->chain_type;
92 struct hinic_hwif *hwif = chain->hwif;
93 u32 addr, prod_idx;
94
95 addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain_type);
96 prod_idx = hinic_hwif_read_reg(hwif, addr);
97
98 prod_idx = HINIC_API_CMD_PI_CLEAR(prod_idx, IDX);
99
100 prod_idx |= HINIC_API_CMD_PI_SET(chain->prod_idx, IDX);
101
102 hinic_hwif_write_reg(hwif, addr, prod_idx);
103}
104
105static u32 get_hw_cons_idx(struct hinic_api_cmd_chain *chain)
106{
107 u32 addr, val;
108
109 addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type);
110 val = hinic_hwif_read_reg(chain->hwif, addr);
111
112 return HINIC_API_CMD_STATUS_GET(val, CONS_IDX);
113}
114
115/**
116 * chain_busy - check if the chain is still processing last requests
117 * @chain: chain to check
118 *
119 * Return 0 - Success, negative - Failure
120 **/
121static int chain_busy(struct hinic_api_cmd_chain *chain)
122{
123 struct hinic_hwif *hwif = chain->hwif;
124 struct pci_dev *pdev = hwif->pdev;
125 u32 prod_idx;
126
127 switch (chain->chain_type) {
128 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
129 chain->cons_idx = get_hw_cons_idx(chain);
130 prod_idx = chain->prod_idx;
131
132 /* check for a space for a new command */
133 if (chain->cons_idx == MASKED_IDX(chain, prod_idx + 1)) {
134 dev_err(&pdev->dev, "API CMD chain %d is busy\n",
135 chain->chain_type);
136 return -EBUSY;
137 }
138 break;
139
140 default:
141 dev_err(&pdev->dev, "Unknown API CMD Chain type\n");
142 break;
143 }
144
145 return 0;
146}
147
148/**
149 * get_cell_data_size - get the data size of a specific cell type
150 * @type: chain type
151 *
152 * Return the data(Desc + Address) size in the cell
153 **/
154static u8 get_cell_data_size(enum hinic_api_cmd_chain_type type)
155{
156 u8 cell_data_size = 0;
157
158 switch (type) {
159 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
160 cell_data_size = ALIGN(API_CMD_CELL_DESC_SIZE +
161 API_CMD_CELL_DATA_ADDR_SIZE,
162 API_CMD_CELL_ALIGNMENT);
163 break;
164 default:
165 break;
166 }
167
168 return cell_data_size;
169}
170
171/**
172 * prepare_cell_ctrl - prepare the ctrl of the cell for the command
173 * @cell_ctrl: the control of the cell to set the control value into it
174 * @data_size: the size of the data in the cell
175 **/
176static void prepare_cell_ctrl(u64 *cell_ctrl, u16 data_size)
177{
178 u8 chksum;
179 u64 ctrl;
180
181 ctrl = HINIC_API_CMD_CELL_CTRL_SET(SIZE_8BYTES(data_size), DATA_SZ) |
182 HINIC_API_CMD_CELL_CTRL_SET(RD_DMA_ATTR_DEFAULT, RD_DMA_ATTR) |
183 HINIC_API_CMD_CELL_CTRL_SET(WR_DMA_ATTR_DEFAULT, WR_DMA_ATTR);
184
185 chksum = xor_chksum_set(&ctrl);
186
187 ctrl |= HINIC_API_CMD_CELL_CTRL_SET(chksum, XOR_CHKSUM);
188
189 /* The data in the HW should be in Big Endian Format */
190 *cell_ctrl = cpu_to_be64(ctrl);
191}
192
193/**
194 * prepare_api_cmd - prepare API CMD command
195 * @chain: chain for the command
196 * @dest: destination node on the card that will receive the command
197 * @cmd: command data
198 * @cmd_size: the command size
199 **/
200static void prepare_api_cmd(struct hinic_api_cmd_chain *chain,
201 enum hinic_node_id dest,
202 void *cmd, u16 cmd_size)
203{
204 struct hinic_api_cmd_cell *cell = chain->curr_node;
205 struct hinic_api_cmd_cell_ctxt *cell_ctxt;
206 struct hinic_hwif *hwif = chain->hwif;
207 struct pci_dev *pdev = hwif->pdev;
208
209 cell_ctxt = &chain->cell_ctxt[chain->prod_idx];
210
211 switch (chain->chain_type) {
212 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
213 cell->desc = HINIC_API_CMD_DESC_SET(SGE_DATA, API_TYPE) |
214 HINIC_API_CMD_DESC_SET(API_CMD_WRITE, RD_WR) |
215 HINIC_API_CMD_DESC_SET(NO_BYPASS, MGMT_BYPASS);
216 break;
217
218 default:
219 dev_err(&pdev->dev, "unknown Chain type\n");
220 return;
221 }
222
223 cell->desc |= HINIC_API_CMD_DESC_SET(dest, DEST) |
224 HINIC_API_CMD_DESC_SET(SIZE_4BYTES(cmd_size), SIZE);
225
226 cell->desc |= HINIC_API_CMD_DESC_SET(xor_chksum_set(&cell->desc),
227 XOR_CHKSUM);
228
229 /* The data in the HW should be in Big Endian Format */
230 cell->desc = cpu_to_be64(cell->desc);
231
232 memcpy(cell_ctxt->api_cmd_vaddr, cmd, cmd_size);
233}
234
235/**
236 * prepare_cell - prepare cell ctrl and cmd in the current cell
237 * @chain: chain for the command
238 * @dest: destination node on the card that will receive the command
239 * @cmd: command data
240 * @cmd_size: the command size
241 *
242 * Return 0 - Success, negative - Failure
243 **/
244static void prepare_cell(struct hinic_api_cmd_chain *chain,
245 enum hinic_node_id dest,
246 void *cmd, u16 cmd_size)
247{
248 struct hinic_api_cmd_cell *curr_node = chain->curr_node;
249 u16 data_size = get_cell_data_size(chain->chain_type);
250
251 prepare_cell_ctrl(&curr_node->ctrl, data_size);
252 prepare_api_cmd(chain, dest, cmd, cmd_size);
253}
254
255static inline void cmd_chain_prod_idx_inc(struct hinic_api_cmd_chain *chain)
256{
257 chain->prod_idx = MASKED_IDX(chain, chain->prod_idx + 1);
258}
259
260/**
261 * api_cmd_status_update - update the status in the chain struct
262 * @chain: chain to update
263 **/
264static void api_cmd_status_update(struct hinic_api_cmd_chain *chain)
265{
266 enum hinic_api_cmd_chain_type chain_type;
267 struct hinic_api_cmd_status *wb_status;
268 struct hinic_hwif *hwif = chain->hwif;
269 struct pci_dev *pdev = hwif->pdev;
270 u64 status_header;
271 u32 status;
272
273 wb_status = chain->wb_status;
274 status_header = be64_to_cpu(wb_status->header);
275
276 status = be32_to_cpu(wb_status->status);
277 if (HINIC_API_CMD_STATUS_GET(status, CHKSUM_ERR)) {
278 dev_err(&pdev->dev, "API CMD status: Xor check error\n");
279 return;
280 }
281
282 chain_type = HINIC_API_CMD_STATUS_HEADER_GET(status_header, CHAIN_ID);
283 if (chain_type >= HINIC_API_CMD_MAX) {
284 dev_err(&pdev->dev, "unknown API CMD Chain %d\n", chain_type);
285 return;
286 }
287
288 chain->cons_idx = HINIC_API_CMD_STATUS_GET(status, CONS_IDX);
289}
290
291/**
292 * wait_for_status_poll - wait for write to api cmd command to complete
293 * @chain: the chain of the command
294 *
295 * Return 0 - Success, negative - Failure
296 **/
297static int wait_for_status_poll(struct hinic_api_cmd_chain *chain)
298{
299 int err = -ETIMEDOUT;
300 unsigned long end;
301
302 end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);
303 do {
304 api_cmd_status_update(chain);
305
306 /* wait for CI to be updated - sign for completion */
307 if (chain->cons_idx == chain->prod_idx) {
308 err = 0;
309 break;
310 }
311
312 msleep(20);
313 } while (time_before(jiffies, end));
314
315 return err;
316}
317
318/**
319 * wait_for_api_cmd_completion - wait for command to complete
320 * @chain: chain for the command
321 *
322 * Return 0 - Success, negative - Failure
323 **/
324static int wait_for_api_cmd_completion(struct hinic_api_cmd_chain *chain)
325{
326 struct hinic_hwif *hwif = chain->hwif;
327 struct pci_dev *pdev = hwif->pdev;
328 int err;
329
330 switch (chain->chain_type) {
331 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
332 err = wait_for_status_poll(chain);
333 if (err) {
334 dev_err(&pdev->dev, "API CMD Poll status timeout\n");
335 break;
336 }
337 break;
338
339 default:
340 dev_err(&pdev->dev, "unknown API CMD Chain type\n");
341 err = -EINVAL;
342 break;
343 }
344
345 return err;
346}
347
348/**
349 * api_cmd - API CMD command
350 * @chain: chain for the command
351 * @dest: destination node on the card that will receive the command
352 * @cmd: command data
353 * @size: the command size
354 *
355 * Return 0 - Success, negative - Failure
356 **/
357static int api_cmd(struct hinic_api_cmd_chain *chain,
358 enum hinic_node_id dest, u8 *cmd, u16 cmd_size)
359{
360 struct hinic_api_cmd_cell_ctxt *ctxt;
361 int err;
362
363 down(&chain->sem);
364 if (chain_busy(chain)) {
365 up(&chain->sem);
366 return -EBUSY;
367 }
368
369 prepare_cell(chain, dest, cmd, cmd_size);
370 cmd_chain_prod_idx_inc(chain);
371
372 wmb(); /* inc pi before issue the command */
373
374 set_prod_idx(chain); /* issue the command */
375
376 ctxt = &chain->cell_ctxt[chain->prod_idx];
377
378 chain->curr_node = ctxt->cell_vaddr;
379
380 err = wait_for_api_cmd_completion(chain);
381
382 up(&chain->sem);
383 return err;
384}
385
386/**
387 * hinic_api_cmd_write - Write API CMD command
388 * @chain: chain for write command
389 * @dest: destination node on the card that will receive the command
390 * @cmd: command data
391 * @size: the command size
392 *
393 * Return 0 - Success, negative - Failure
394 **/
395int hinic_api_cmd_write(struct hinic_api_cmd_chain *chain,
396 enum hinic_node_id dest, u8 *cmd, u16 size)
397{
398 /* Verify the chain type */
399 if (chain->chain_type == HINIC_API_CMD_WRITE_TO_MGMT_CPU)
400 return api_cmd(chain, dest, cmd, size);
401
402 return -EINVAL;
403}
404
405/**
406 * api_cmd_hw_restart - restart the chain in the HW
407 * @chain: the API CMD specific chain to restart
408 *
409 * Return 0 - Success, negative - Failure
410 **/
411static int api_cmd_hw_restart(struct hinic_api_cmd_chain *chain)
412{
413 struct hinic_hwif *hwif = chain->hwif;
414 int err = -ETIMEDOUT;
415 unsigned long end;
416 u32 reg_addr, val;
417
418 /* Read Modify Write */
419 reg_addr = HINIC_CSR_API_CMD_CHAIN_REQ_ADDR(chain->chain_type);
420 val = hinic_hwif_read_reg(hwif, reg_addr);
421
422 val = HINIC_API_CMD_CHAIN_REQ_CLEAR(val, RESTART);
423 val |= HINIC_API_CMD_CHAIN_REQ_SET(1, RESTART);
424
425 hinic_hwif_write_reg(hwif, reg_addr, val);
426
427 end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);
428 do {
429 val = hinic_hwif_read_reg(hwif, reg_addr);
430
431 if (!HINIC_API_CMD_CHAIN_REQ_GET(val, RESTART)) {
432 err = 0;
433 break;
434 }
435
436 msleep(20);
437 } while (time_before(jiffies, end));
438
439 return err;
440}
441
442/**
443 * api_cmd_ctrl_init - set the control register of a chain
444 * @chain: the API CMD specific chain to set control register for
445 **/
446static void api_cmd_ctrl_init(struct hinic_api_cmd_chain *chain)
447{
448 struct hinic_hwif *hwif = chain->hwif;
449 u32 addr, ctrl;
450 u16 cell_size;
451
452 /* Read Modify Write */
453 addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);
454
455 cell_size = API_CMD_CELL_SIZE_VAL(chain->cell_size);
456
457 ctrl = hinic_hwif_read_reg(hwif, addr);
458
459 ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) &
460 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR) &
461 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) &
462 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) &
463 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);
464
465 ctrl |= HINIC_API_CMD_CHAIN_CTRL_SET(1, XOR_ERR) |
466 HINIC_API_CMD_CHAIN_CTRL_SET(XOR_CHK_ALL, XOR_CHK_EN) |
467 HINIC_API_CMD_CHAIN_CTRL_SET(cell_size, CELL_SIZE);
468
469 hinic_hwif_write_reg(hwif, addr, ctrl);
470}
471
472/**
473 * api_cmd_set_status_addr - set the status address of a chain in the HW
474 * @chain: the API CMD specific chain to set in HW status address for
475 **/
476static void api_cmd_set_status_addr(struct hinic_api_cmd_chain *chain)
477{
478 struct hinic_hwif *hwif = chain->hwif;
479 u32 addr, val;
480
481 addr = HINIC_CSR_API_CMD_STATUS_HI_ADDR(chain->chain_type);
482 val = upper_32_bits(chain->wb_status_paddr);
483 hinic_hwif_write_reg(hwif, addr, val);
484
485 addr = HINIC_CSR_API_CMD_STATUS_LO_ADDR(chain->chain_type);
486 val = lower_32_bits(chain->wb_status_paddr);
487 hinic_hwif_write_reg(hwif, addr, val);
488}
489
490/**
491 * api_cmd_set_num_cells - set the number cells of a chain in the HW
492 * @chain: the API CMD specific chain to set in HW the number of cells for
493 **/
494static void api_cmd_set_num_cells(struct hinic_api_cmd_chain *chain)
495{
496 struct hinic_hwif *hwif = chain->hwif;
497 u32 addr, val;
498
499 addr = HINIC_CSR_API_CMD_CHAIN_NUM_CELLS_ADDR(chain->chain_type);
500 val = chain->num_cells;
501 hinic_hwif_write_reg(hwif, addr, val);
502}
503
504/**
505 * api_cmd_head_init - set the head of a chain in the HW
506 * @chain: the API CMD specific chain to set in HW the head for
507 **/
508static void api_cmd_head_init(struct hinic_api_cmd_chain *chain)
509{
510 struct hinic_hwif *hwif = chain->hwif;
511 u32 addr, val;
512
513 addr = HINIC_CSR_API_CMD_CHAIN_HEAD_HI_ADDR(chain->chain_type);
514 val = upper_32_bits(chain->head_cell_paddr);
515 hinic_hwif_write_reg(hwif, addr, val);
516
517 addr = HINIC_CSR_API_CMD_CHAIN_HEAD_LO_ADDR(chain->chain_type);
518 val = lower_32_bits(chain->head_cell_paddr);
519 hinic_hwif_write_reg(hwif, addr, val);
520}
521
522/**
523 * api_cmd_chain_hw_clean - clean the HW
524 * @chain: the API CMD specific chain
525 **/
526static void api_cmd_chain_hw_clean(struct hinic_api_cmd_chain *chain)
527{
528 struct hinic_hwif *hwif = chain->hwif;
529 u32 addr, ctrl;
530
531 addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);
532
533 ctrl = hinic_hwif_read_reg(hwif, addr);
534 ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) &
535 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR) &
536 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) &
537 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) &
538 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);
539
540 hinic_hwif_write_reg(hwif, addr, ctrl);
541}
542
543/**
544 * api_cmd_chain_hw_init - initialize the chain in the HW
545 * @chain: the API CMD specific chain to initialize in HW
546 *
547 * Return 0 - Success, negative - Failure
548 **/
549static int api_cmd_chain_hw_init(struct hinic_api_cmd_chain *chain)
550{
551 struct hinic_hwif *hwif = chain->hwif;
552 struct pci_dev *pdev = hwif->pdev;
553 int err;
554
555 api_cmd_chain_hw_clean(chain);
556
557 api_cmd_set_status_addr(chain);
558
559 err = api_cmd_hw_restart(chain);
560 if (err) {
561 dev_err(&pdev->dev, "Failed to restart API CMD HW\n");
562 return err;
563 }
564
565 api_cmd_ctrl_init(chain);
566 api_cmd_set_num_cells(chain);
567 api_cmd_head_init(chain);
568 return 0;
569}
570
571/**
572 * free_cmd_buf - free the dma buffer of API CMD command
573 * @chain: the API CMD specific chain of the cmd
574 * @cell_idx: the cell index of the cmd
575 **/
576static void free_cmd_buf(struct hinic_api_cmd_chain *chain, int cell_idx)
577{
578 struct hinic_api_cmd_cell_ctxt *cell_ctxt;
579 struct hinic_hwif *hwif = chain->hwif;
580 struct pci_dev *pdev = hwif->pdev;
581
582 cell_ctxt = &chain->cell_ctxt[cell_idx];
583
584 dma_free_coherent(&pdev->dev, API_CMD_BUF_SIZE,
585 cell_ctxt->api_cmd_vaddr,
586 cell_ctxt->api_cmd_paddr);
587}
588
589/**
590 * alloc_cmd_buf - allocate a dma buffer for API CMD command
591 * @chain: the API CMD specific chain for the cmd
592 * @cell: the cell in the HW for the cmd
593 * @cell_idx: the index of the cell
594 *
595 * Return 0 - Success, negative - Failure
596 **/
597static int alloc_cmd_buf(struct hinic_api_cmd_chain *chain,
598 struct hinic_api_cmd_cell *cell, int cell_idx)
599{
600 struct hinic_api_cmd_cell_ctxt *cell_ctxt;
601 struct hinic_hwif *hwif = chain->hwif;
602 struct pci_dev *pdev = hwif->pdev;
603 dma_addr_t cmd_paddr;
604 u8 *cmd_vaddr;
605 int err = 0;
606
607 cmd_vaddr = dma_alloc_coherent(&pdev->dev, API_CMD_BUF_SIZE,
608 &cmd_paddr, GFP_KERNEL);
609 if (!cmd_vaddr) {
610 dev_err(&pdev->dev, "Failed to allocate API CMD DMA memory\n");
611 return -ENOMEM;
612 }
613
614 cell_ctxt = &chain->cell_ctxt[cell_idx];
615
616 cell_ctxt->api_cmd_vaddr = cmd_vaddr;
617 cell_ctxt->api_cmd_paddr = cmd_paddr;
618
619 /* set the cmd DMA address in the cell */
620 switch (chain->chain_type) {
621 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
622 /* The data in the HW should be in Big Endian Format */
623 cell->write.hw_cmd_paddr = cpu_to_be64(cmd_paddr);
624 break;
625
626 default:
627 dev_err(&pdev->dev, "Unsupported API CMD chain type\n");
628 free_cmd_buf(chain, cell_idx);
629 err = -EINVAL;
630 break;
631 }
632
633 return err;
634}
635
636/**
637 * api_cmd_create_cell - create API CMD cell for specific chain
638 * @chain: the API CMD specific chain to create its cell
639 * @cell_idx: the index of the cell to create
640 * @pre_node: previous cell
641 * @node_vaddr: the returned virt addr of the cell
642 *
643 * Return 0 - Success, negative - Failure
644 **/
645static int api_cmd_create_cell(struct hinic_api_cmd_chain *chain,
646 int cell_idx,
647 struct hinic_api_cmd_cell *pre_node,
648 struct hinic_api_cmd_cell **node_vaddr)
649{
650 struct hinic_api_cmd_cell_ctxt *cell_ctxt;
651 struct hinic_hwif *hwif = chain->hwif;
652 struct pci_dev *pdev = hwif->pdev;
653 struct hinic_api_cmd_cell *node;
654 dma_addr_t node_paddr;
655 int err;
656
657 node = dma_alloc_coherent(&pdev->dev, chain->cell_size, &node_paddr,
658 GFP_KERNEL);
659 if (!node) {
660 dev_err(&pdev->dev, "Failed to allocate dma API CMD cell\n");
661 return -ENOMEM;
662 }
663
664 node->read.hw_wb_resp_paddr = 0;
665
666 cell_ctxt = &chain->cell_ctxt[cell_idx];
667 cell_ctxt->cell_vaddr = node;
668 cell_ctxt->cell_paddr = node_paddr;
669
670 if (!pre_node) {
671 chain->head_cell_paddr = node_paddr;
672 chain->head_node = node;
673 } else {
674 /* The data in the HW should be in Big Endian Format */
675 pre_node->next_cell_paddr = cpu_to_be64(node_paddr);
676 }
677
678 switch (chain->chain_type) {
679 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
680 err = alloc_cmd_buf(chain, node, cell_idx);
681 if (err) {
682 dev_err(&pdev->dev, "Failed to allocate cmd buffer\n");
683 goto err_alloc_cmd_buf;
684 }
685 break;
686
687 default:
688 dev_err(&pdev->dev, "Unsupported API CMD chain type\n");
689 err = -EINVAL;
690 goto err_alloc_cmd_buf;
691 }
692
693 *node_vaddr = node;
694 return 0;
695
696err_alloc_cmd_buf:
697 dma_free_coherent(&pdev->dev, chain->cell_size, node, node_paddr);
698 return err;
699}
700
701/**
702 * api_cmd_destroy_cell - destroy API CMD cell of specific chain
703 * @chain: the API CMD specific chain to destroy its cell
704 * @cell_idx: the cell to destroy
705 **/
706static void api_cmd_destroy_cell(struct hinic_api_cmd_chain *chain,
707 int cell_idx)
708{
709 struct hinic_api_cmd_cell_ctxt *cell_ctxt;
710 struct hinic_hwif *hwif = chain->hwif;
711 struct pci_dev *pdev = hwif->pdev;
712 struct hinic_api_cmd_cell *node;
713 dma_addr_t node_paddr;
714 size_t node_size;
715
716 cell_ctxt = &chain->cell_ctxt[cell_idx];
717
718 node = cell_ctxt->cell_vaddr;
719 node_paddr = cell_ctxt->cell_paddr;
720 node_size = chain->cell_size;
721
722 if (cell_ctxt->api_cmd_vaddr) {
723 switch (chain->chain_type) {
724 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
725 free_cmd_buf(chain, cell_idx);
726 break;
727 default:
728 dev_err(&pdev->dev, "Unsupported API CMD chain type\n");
729 break;
730 }
731
732 dma_free_coherent(&pdev->dev, node_size, node,
733 node_paddr);
734 }
735}
736
737/**
738 * api_cmd_destroy_cells - destroy API CMD cells of specific chain
739 * @chain: the API CMD specific chain to destroy its cells
740 * @num_cells: number of cells to destroy
741 **/
742static void api_cmd_destroy_cells(struct hinic_api_cmd_chain *chain,
743 int num_cells)
744{
745 int cell_idx;
746
747 for (cell_idx = 0; cell_idx < num_cells; cell_idx++)
748 api_cmd_destroy_cell(chain, cell_idx);
749}
750
751/**
752 * api_cmd_create_cells - create API CMD cells for specific chain
753 * @chain: the API CMD specific chain
754 *
755 * Return 0 - Success, negative - Failure
756 **/
757static int api_cmd_create_cells(struct hinic_api_cmd_chain *chain)
758{
759 struct hinic_api_cmd_cell *node = NULL, *pre_node = NULL;
760 struct hinic_hwif *hwif = chain->hwif;
761 struct pci_dev *pdev = hwif->pdev;
762 int err, cell_idx;
763
764 for (cell_idx = 0; cell_idx < chain->num_cells; cell_idx++) {
765 err = api_cmd_create_cell(chain, cell_idx, pre_node, &node);
766 if (err) {
767 dev_err(&pdev->dev, "Failed to create API CMD cell\n");
768 goto err_create_cell;
769 }
770
771 pre_node = node;
772 }
773
774 /* set the Final node to point on the start */
775 node->next_cell_paddr = cpu_to_be64(chain->head_cell_paddr);
776
777 /* set the current node to be the head */
778 chain->curr_node = chain->head_node;
779 return 0;
780
781err_create_cell:
782 api_cmd_destroy_cells(chain, cell_idx);
783 return err;
784}
785
786/**
787 * api_chain_init - initialize API CMD specific chain
788 * @chain: the API CMD specific chain to initialize
789 * @attr: attributes to set in the chain
790 *
791 * Return 0 - Success, negative - Failure
792 **/
793static int api_chain_init(struct hinic_api_cmd_chain *chain,
794 struct hinic_api_cmd_chain_attr *attr)
795{
796 struct hinic_hwif *hwif = attr->hwif;
797 struct pci_dev *pdev = hwif->pdev;
798 size_t cell_ctxt_size;
799
800 chain->hwif = hwif;
801 chain->chain_type = attr->chain_type;
802 chain->num_cells = attr->num_cells;
803 chain->cell_size = attr->cell_size;
804
805 chain->prod_idx = 0;
806 chain->cons_idx = 0;
807
808 sema_init(&chain->sem, 1);
809
810 cell_ctxt_size = chain->num_cells * sizeof(*chain->cell_ctxt);
811 chain->cell_ctxt = devm_kzalloc(&pdev->dev, cell_ctxt_size, GFP_KERNEL);
812 if (!chain->cell_ctxt)
813 return -ENOMEM;
814
815 chain->wb_status = dma_alloc_coherent(&pdev->dev,
816 sizeof(*chain->wb_status),
817 &chain->wb_status_paddr,
818 GFP_KERNEL);
819 if (!chain->wb_status) {
820 dev_err(&pdev->dev, "Failed to allocate DMA wb status\n");
821 return -ENOMEM;
822 }
823
824 return 0;
825}
826
827/**
828 * api_chain_free - free API CMD specific chain
829 * @chain: the API CMD specific chain to free
830 **/
831static void api_chain_free(struct hinic_api_cmd_chain *chain)
832{
833 struct hinic_hwif *hwif = chain->hwif;
834 struct pci_dev *pdev = hwif->pdev;
835
836 dma_free_coherent(&pdev->dev, sizeof(*chain->wb_status),
837 chain->wb_status, chain->wb_status_paddr);
838}
839
840/**
841 * api_cmd_create_chain - create API CMD specific chain
842 * @attr: attributes to set the chain
843 *
844 * Return the created chain
845 **/
846static struct hinic_api_cmd_chain *
847 api_cmd_create_chain(struct hinic_api_cmd_chain_attr *attr)
848{
849 struct hinic_hwif *hwif = attr->hwif;
850 struct pci_dev *pdev = hwif->pdev;
851 struct hinic_api_cmd_chain *chain;
852 int err;
853
854 if (attr->num_cells & (attr->num_cells - 1)) {
855 dev_err(&pdev->dev, "Invalid number of cells, must be power of 2\n");
856 return ERR_PTR(-EINVAL);
857 }
858
859 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
860 if (!chain)
861 return ERR_PTR(-ENOMEM);
862
863 err = api_chain_init(chain, attr);
864 if (err) {
865 dev_err(&pdev->dev, "Failed to initialize chain\n");
866 return ERR_PTR(err);
867 }
868
869 err = api_cmd_create_cells(chain);
870 if (err) {
871 dev_err(&pdev->dev, "Failed to create cells for API CMD chain\n");
872 goto err_create_cells;
873 }
874
875 err = api_cmd_chain_hw_init(chain);
876 if (err) {
877 dev_err(&pdev->dev, "Failed to initialize chain HW\n");
878 goto err_chain_hw_init;
879 }
880
881 return chain;
882
883err_chain_hw_init:
884 api_cmd_destroy_cells(chain, chain->num_cells);
885
886err_create_cells:
887 api_chain_free(chain);
888 return ERR_PTR(err);
889}
890
891/**
892 * api_cmd_destroy_chain - destroy API CMD specific chain
893 * @chain: the API CMD specific chain to destroy
894 **/
895static void api_cmd_destroy_chain(struct hinic_api_cmd_chain *chain)
896{
897 api_cmd_chain_hw_clean(chain);
898 api_cmd_destroy_cells(chain, chain->num_cells);
899 api_chain_free(chain);
900}
901
902/**
903 * hinic_api_cmd_init - Initialize all the API CMD chains
904 * @chain: the API CMD chains that are initialized
905 * @hwif: the hardware interface of a pci function device
906 *
907 * Return 0 - Success, negative - Failure
908 **/
909int hinic_api_cmd_init(struct hinic_api_cmd_chain **chain,
910 struct hinic_hwif *hwif)
911{
912 enum hinic_api_cmd_chain_type type, chain_type;
913 struct hinic_api_cmd_chain_attr attr;
914 struct pci_dev *pdev = hwif->pdev;
915 size_t hw_cell_sz;
916 int err;
917
918 hw_cell_sz = sizeof(struct hinic_api_cmd_cell);
919
920 attr.hwif = hwif;
921 attr.num_cells = API_CHAIN_NUM_CELLS;
922 attr.cell_size = API_CMD_CELL_SIZE(hw_cell_sz);
923
924 chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;
925 for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) {
926 attr.chain_type = chain_type;
927
928 if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)
929 continue;
930
931 chain[chain_type] = api_cmd_create_chain(&attr);
932 if (IS_ERR(chain[chain_type])) {
933 dev_err(&pdev->dev, "Failed to create chain %d\n",
934 chain_type);
935 err = PTR_ERR(chain[chain_type]);
936 goto err_create_chain;
937 }
938 }
939
940 return 0;
941
942err_create_chain:
943 type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;
944 for ( ; type < chain_type; type++) {
945 if (type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)
946 continue;
947
948 api_cmd_destroy_chain(chain[type]);
949 }
950
951 return err;
952}
953
954/**
955 * hinic_api_cmd_free - free the API CMD chains
956 * @chain: the API CMD chains that are freed
957 **/
958void hinic_api_cmd_free(struct hinic_api_cmd_chain **chain)
959{
960 enum hinic_api_cmd_chain_type chain_type;
961
962 chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;
963 for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) {
964 if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)
965 continue;
966
967 api_cmd_destroy_chain(chain[chain_type]);
968 }
969}