Loading...
1/**********************************************************************
2 * Author: Cavium, Inc.
3 *
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
6 *
7 * Copyright (c) 2003-2016 Cavium, Inc.
8 *
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
12 *
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more details.
17 ***********************************************************************/
18#include <linux/pci.h>
19#include <linux/netdevice.h>
20#include "liquidio_common.h"
21#include "octeon_droq.h"
22#include "octeon_iq.h"
23#include "response_manager.h"
24#include "octeon_device.h"
25#include "octeon_main.h"
26#include "cn66xx_regs.h"
27#include "cn66xx_device.h"
28
29int lio_cn6xxx_soft_reset(struct octeon_device *oct)
30{
31 octeon_write_csr64(oct, CN6XXX_WIN_WR_MASK_REG, 0xFF);
32
33 dev_dbg(&oct->pci_dev->dev, "BIST enabled for soft reset\n");
34
35 lio_pci_writeq(oct, 1, CN6XXX_CIU_SOFT_BIST);
36 octeon_write_csr64(oct, CN6XXX_SLI_SCRATCH1, 0x1234ULL);
37
38 lio_pci_readq(oct, CN6XXX_CIU_SOFT_RST);
39 lio_pci_writeq(oct, 1, CN6XXX_CIU_SOFT_RST);
40
41 /* Wait for 10ms as Octeon resets. */
42 mdelay(100);
43
44 if (octeon_read_csr64(oct, CN6XXX_SLI_SCRATCH1)) {
45 dev_err(&oct->pci_dev->dev, "Soft reset failed\n");
46 return 1;
47 }
48
49 dev_dbg(&oct->pci_dev->dev, "Reset completed\n");
50 octeon_write_csr64(oct, CN6XXX_WIN_WR_MASK_REG, 0xFF);
51
52 return 0;
53}
54
55void lio_cn6xxx_enable_error_reporting(struct octeon_device *oct)
56{
57 u32 val;
58
59 pci_read_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, &val);
60 if (val & 0x000c0000) {
61 dev_err(&oct->pci_dev->dev, "PCI-E Link error detected: 0x%08x\n",
62 val & 0x000c0000);
63 }
64
65 val |= 0xf; /* Enable Link error reporting */
66
67 dev_dbg(&oct->pci_dev->dev, "Enabling PCI-E error reporting..\n");
68 pci_write_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, val);
69}
70
71void lio_cn6xxx_setup_pcie_mps(struct octeon_device *oct,
72 enum octeon_pcie_mps mps)
73{
74 u32 val;
75 u64 r64;
76
77 /* Read config register for MPS */
78 pci_read_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, &val);
79
80 if (mps == PCIE_MPS_DEFAULT) {
81 mps = ((val & (0x7 << 5)) >> 5);
82 } else {
83 val &= ~(0x7 << 5); /* Turn off any MPS bits */
84 val |= (mps << 5); /* Set MPS */
85 pci_write_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, val);
86 }
87
88 /* Set MPS in DPI_SLI_PRT0_CFG to the same value. */
89 r64 = lio_pci_readq(oct, CN6XXX_DPI_SLI_PRTX_CFG(oct->pcie_port));
90 r64 |= (mps << 4);
91 lio_pci_writeq(oct, r64, CN6XXX_DPI_SLI_PRTX_CFG(oct->pcie_port));
92}
93
94void lio_cn6xxx_setup_pcie_mrrs(struct octeon_device *oct,
95 enum octeon_pcie_mrrs mrrs)
96{
97 u32 val;
98 u64 r64;
99
100 /* Read config register for MRRS */
101 pci_read_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, &val);
102
103 if (mrrs == PCIE_MRRS_DEFAULT) {
104 mrrs = ((val & (0x7 << 12)) >> 12);
105 } else {
106 val &= ~(0x7 << 12); /* Turn off any MRRS bits */
107 val |= (mrrs << 12); /* Set MRRS */
108 pci_write_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, val);
109 }
110
111 /* Set MRRS in SLI_S2M_PORT0_CTL to the same value. */
112 r64 = octeon_read_csr64(oct, CN6XXX_SLI_S2M_PORTX_CTL(oct->pcie_port));
113 r64 |= mrrs;
114 octeon_write_csr64(oct, CN6XXX_SLI_S2M_PORTX_CTL(oct->pcie_port), r64);
115
116 /* Set MRRS in DPI_SLI_PRT0_CFG to the same value. */
117 r64 = lio_pci_readq(oct, CN6XXX_DPI_SLI_PRTX_CFG(oct->pcie_port));
118 r64 |= mrrs;
119 lio_pci_writeq(oct, r64, CN6XXX_DPI_SLI_PRTX_CFG(oct->pcie_port));
120}
121
122u32 lio_cn6xxx_coprocessor_clock(struct octeon_device *oct)
123{
124 /* Bits 29:24 of MIO_RST_BOOT holds the ref. clock multiplier
125 * for SLI.
126 */
127 return ((lio_pci_readq(oct, CN6XXX_MIO_RST_BOOT) >> 24) & 0x3f) * 50;
128}
129
130u32 lio_cn6xxx_get_oq_ticks(struct octeon_device *oct,
131 u32 time_intr_in_us)
132{
133 /* This gives the SLI clock per microsec */
134 u32 oqticks_per_us = lio_cn6xxx_coprocessor_clock(oct);
135
136 /* core clock per us / oq ticks will be fractional. TO avoid that
137 * we use the method below.
138 */
139
140 /* This gives the clock cycles per millisecond */
141 oqticks_per_us *= 1000;
142
143 /* This gives the oq ticks (1024 core clock cycles) per millisecond */
144 oqticks_per_us /= 1024;
145
146 /* time_intr is in microseconds. The next 2 steps gives the oq ticks
147 * corressponding to time_intr.
148 */
149 oqticks_per_us *= time_intr_in_us;
150 oqticks_per_us /= 1000;
151
152 return oqticks_per_us;
153}
154
155void lio_cn6xxx_setup_global_input_regs(struct octeon_device *oct)
156{
157 /* Select Round-Robin Arb, ES, RO, NS for Input Queues */
158 octeon_write_csr(oct, CN6XXX_SLI_PKT_INPUT_CONTROL,
159 CN6XXX_INPUT_CTL_MASK);
160
161 /* Instruction Read Size - Max 4 instructions per PCIE Read */
162 octeon_write_csr64(oct, CN6XXX_SLI_PKT_INSTR_RD_SIZE,
163 0xFFFFFFFFFFFFFFFFULL);
164
165 /* Select PCIE Port for all Input rings. */
166 octeon_write_csr64(oct, CN6XXX_SLI_IN_PCIE_PORT,
167 (oct->pcie_port * 0x5555555555555555ULL));
168}
169
170static void lio_cn66xx_setup_pkt_ctl_regs(struct octeon_device *oct)
171{
172 u64 pktctl;
173
174 struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
175
176 pktctl = octeon_read_csr64(oct, CN6XXX_SLI_PKT_CTL);
177
178 /* 66XX SPECIFIC */
179 if (CFG_GET_OQ_MAX_Q(cn6xxx->conf) <= 4)
180 /* Disable RING_EN if only upto 4 rings are used. */
181 pktctl &= ~(1 << 4);
182 else
183 pktctl |= (1 << 4);
184
185 if (CFG_GET_IS_SLI_BP_ON(cn6xxx->conf))
186 pktctl |= 0xF;
187 else
188 /* Disable per-port backpressure. */
189 pktctl &= ~0xF;
190 octeon_write_csr64(oct, CN6XXX_SLI_PKT_CTL, pktctl);
191}
192
193void lio_cn6xxx_setup_global_output_regs(struct octeon_device *oct)
194{
195 u32 time_threshold;
196 struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
197
198 /* / Select PCI-E Port for all Output queues */
199 octeon_write_csr64(oct, CN6XXX_SLI_PKT_PCIE_PORT64,
200 (oct->pcie_port * 0x5555555555555555ULL));
201
202 if (CFG_GET_IS_SLI_BP_ON(cn6xxx->conf)) {
203 octeon_write_csr64(oct, CN6XXX_SLI_OQ_WMARK, 32);
204 } else {
205 /* / Set Output queue watermark to 0 to disable backpressure */
206 octeon_write_csr64(oct, CN6XXX_SLI_OQ_WMARK, 0);
207 }
208
209 /* / Select Packet count instead of bytes for SLI_PKTi_CNTS[CNT] */
210 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_BMODE, 0);
211
212 /* Select ES, RO, NS setting from register for Output Queue Packet
213 * Address
214 */
215 octeon_write_csr(oct, CN6XXX_SLI_PKT_DPADDR, 0xFFFFFFFF);
216
217 /* No Relaxed Ordering, No Snoop, 64-bit swap for Output
218 * Queue ScatterList
219 */
220 octeon_write_csr(oct, CN6XXX_SLI_PKT_SLIST_ROR, 0);
221 octeon_write_csr(oct, CN6XXX_SLI_PKT_SLIST_NS, 0);
222
223 /* / ENDIAN_SPECIFIC CHANGES - 0 works for LE. */
224#ifdef __BIG_ENDIAN_BITFIELD
225 octeon_write_csr64(oct, CN6XXX_SLI_PKT_SLIST_ES64,
226 0x5555555555555555ULL);
227#else
228 octeon_write_csr64(oct, CN6XXX_SLI_PKT_SLIST_ES64, 0ULL);
229#endif
230
231 /* / No Relaxed Ordering, No Snoop, 64-bit swap for Output Queue Data */
232 octeon_write_csr(oct, CN6XXX_SLI_PKT_DATA_OUT_ROR, 0);
233 octeon_write_csr(oct, CN6XXX_SLI_PKT_DATA_OUT_NS, 0);
234 octeon_write_csr64(oct, CN6XXX_SLI_PKT_DATA_OUT_ES64,
235 0x5555555555555555ULL);
236
237 /* / Set up interrupt packet and time threshold */
238 octeon_write_csr(oct, CN6XXX_SLI_OQ_INT_LEVEL_PKTS,
239 (u32)CFG_GET_OQ_INTR_PKT(cn6xxx->conf));
240 time_threshold =
241 lio_cn6xxx_get_oq_ticks(oct, (u32)
242 CFG_GET_OQ_INTR_TIME(cn6xxx->conf));
243
244 octeon_write_csr(oct, CN6XXX_SLI_OQ_INT_LEVEL_TIME, time_threshold);
245}
246
247static int lio_cn6xxx_setup_device_regs(struct octeon_device *oct)
248{
249 lio_cn6xxx_setup_pcie_mps(oct, PCIE_MPS_DEFAULT);
250 lio_cn6xxx_setup_pcie_mrrs(oct, PCIE_MRRS_512B);
251 lio_cn6xxx_enable_error_reporting(oct);
252
253 lio_cn6xxx_setup_global_input_regs(oct);
254 lio_cn66xx_setup_pkt_ctl_regs(oct);
255 lio_cn6xxx_setup_global_output_regs(oct);
256
257 /* Default error timeout value should be 0x200000 to avoid host hang
258 * when reads invalid register
259 */
260 octeon_write_csr64(oct, CN6XXX_SLI_WINDOW_CTL, 0x200000ULL);
261 return 0;
262}
263
264void lio_cn6xxx_setup_iq_regs(struct octeon_device *oct, u32 iq_no)
265{
266 struct octeon_instr_queue *iq = oct->instr_queue[iq_no];
267
268 octeon_write_csr64(oct, CN6XXX_SLI_IQ_PKT_INSTR_HDR64(iq_no), 0);
269
270 /* Write the start of the input queue's ring and its size */
271 octeon_write_csr64(oct, CN6XXX_SLI_IQ_BASE_ADDR64(iq_no),
272 iq->base_addr_dma);
273 octeon_write_csr(oct, CN6XXX_SLI_IQ_SIZE(iq_no), iq->max_count);
274
275 /* Remember the doorbell & instruction count register addr for this
276 * queue
277 */
278 iq->doorbell_reg = oct->mmio[0].hw_addr + CN6XXX_SLI_IQ_DOORBELL(iq_no);
279 iq->inst_cnt_reg = oct->mmio[0].hw_addr
280 + CN6XXX_SLI_IQ_INSTR_COUNT(iq_no);
281 dev_dbg(&oct->pci_dev->dev, "InstQ[%d]:dbell reg @ 0x%p instcnt_reg @ 0x%p\n",
282 iq_no, iq->doorbell_reg, iq->inst_cnt_reg);
283
284 /* Store the current instruction counter
285 * (used in flush_iq calculation)
286 */
287 iq->reset_instr_cnt = readl(iq->inst_cnt_reg);
288}
289
290static void lio_cn66xx_setup_iq_regs(struct octeon_device *oct, u32 iq_no)
291{
292 lio_cn6xxx_setup_iq_regs(oct, iq_no);
293
294 /* Backpressure for this queue - WMARK set to all F's. This effectively
295 * disables the backpressure mechanism.
296 */
297 octeon_write_csr64(oct, CN66XX_SLI_IQ_BP64(iq_no),
298 (0xFFFFFFFFULL << 32));
299}
300
301void lio_cn6xxx_setup_oq_regs(struct octeon_device *oct, u32 oq_no)
302{
303 u32 intr;
304 struct octeon_droq *droq = oct->droq[oq_no];
305
306 octeon_write_csr64(oct, CN6XXX_SLI_OQ_BASE_ADDR64(oq_no),
307 droq->desc_ring_dma);
308 octeon_write_csr(oct, CN6XXX_SLI_OQ_SIZE(oq_no), droq->max_count);
309
310 octeon_write_csr(oct, CN6XXX_SLI_OQ_BUFF_INFO_SIZE(oq_no),
311 droq->buffer_size);
312
313 /* Get the mapped address of the pkt_sent and pkts_credit regs */
314 droq->pkts_sent_reg =
315 oct->mmio[0].hw_addr + CN6XXX_SLI_OQ_PKTS_SENT(oq_no);
316 droq->pkts_credit_reg =
317 oct->mmio[0].hw_addr + CN6XXX_SLI_OQ_PKTS_CREDIT(oq_no);
318
319 /* Enable this output queue to generate Packet Timer Interrupt */
320 intr = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB);
321 intr |= (1 << oq_no);
322 octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB, intr);
323
324 /* Enable this output queue to generate Packet Timer Interrupt */
325 intr = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB);
326 intr |= (1 << oq_no);
327 octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB, intr);
328}
329
330int lio_cn6xxx_enable_io_queues(struct octeon_device *oct)
331{
332 u32 mask;
333
334 mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_INSTR_SIZE);
335 mask |= oct->io_qmask.iq64B;
336 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_SIZE, mask);
337
338 mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB);
339 mask |= oct->io_qmask.iq;
340 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, mask);
341
342 mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_OUT_ENB);
343 mask |= oct->io_qmask.oq;
344 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, mask);
345
346 return 0;
347}
348
349void lio_cn6xxx_disable_io_queues(struct octeon_device *oct)
350{
351 int i;
352 u32 mask, loop = HZ;
353 u32 d32;
354
355 /* Reset the Enable bits for Input Queues. */
356 mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB);
357 mask ^= oct->io_qmask.iq;
358 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, mask);
359
360 /* Wait until hardware indicates that the queues are out of reset. */
361 mask = (u32)oct->io_qmask.iq;
362 d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_IQ);
363 while (((d32 & mask) != mask) && loop--) {
364 d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_IQ);
365 schedule_timeout_uninterruptible(1);
366 }
367
368 /* Reset the doorbell register for each Input queue. */
369 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
370 if (!(oct->io_qmask.iq & BIT_ULL(i)))
371 continue;
372 octeon_write_csr(oct, CN6XXX_SLI_IQ_DOORBELL(i), 0xFFFFFFFF);
373 d32 = octeon_read_csr(oct, CN6XXX_SLI_IQ_DOORBELL(i));
374 }
375
376 /* Reset the Enable bits for Output Queues. */
377 mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_OUT_ENB);
378 mask ^= oct->io_qmask.oq;
379 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, mask);
380
381 /* Wait until hardware indicates that the queues are out of reset. */
382 loop = HZ;
383 mask = (u32)oct->io_qmask.oq;
384 d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_OQ);
385 while (((d32 & mask) != mask) && loop--) {
386 d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_OQ);
387 schedule_timeout_uninterruptible(1);
388 }
389 ;
390
391 /* Reset the doorbell register for each Output queue. */
392 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
393 if (!(oct->io_qmask.oq & BIT_ULL(i)))
394 continue;
395 octeon_write_csr(oct, CN6XXX_SLI_OQ_PKTS_CREDIT(i), 0xFFFFFFFF);
396 d32 = octeon_read_csr(oct, CN6XXX_SLI_OQ_PKTS_CREDIT(i));
397
398 d32 = octeon_read_csr(oct, CN6XXX_SLI_OQ_PKTS_SENT(i));
399 octeon_write_csr(oct, CN6XXX_SLI_OQ_PKTS_SENT(i), d32);
400 }
401
402 d32 = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT);
403 if (d32)
404 octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT, d32);
405
406 d32 = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT);
407 if (d32)
408 octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT, d32);
409}
410
411void
412lio_cn6xxx_bar1_idx_setup(struct octeon_device *oct,
413 u64 core_addr,
414 u32 idx,
415 int valid)
416{
417 u64 bar1;
418
419 if (valid == 0) {
420 bar1 = lio_pci_readq(oct, CN6XXX_BAR1_REG(idx, oct->pcie_port));
421 lio_pci_writeq(oct, (bar1 & 0xFFFFFFFEULL),
422 CN6XXX_BAR1_REG(idx, oct->pcie_port));
423 bar1 = lio_pci_readq(oct, CN6XXX_BAR1_REG(idx, oct->pcie_port));
424 return;
425 }
426
427 /* Bits 17:4 of the PCI_BAR1_INDEXx stores bits 35:22 of
428 * the Core Addr
429 */
430 lio_pci_writeq(oct, (((core_addr >> 22) << 4) | PCI_BAR1_MASK),
431 CN6XXX_BAR1_REG(idx, oct->pcie_port));
432
433 bar1 = lio_pci_readq(oct, CN6XXX_BAR1_REG(idx, oct->pcie_port));
434}
435
436void lio_cn6xxx_bar1_idx_write(struct octeon_device *oct,
437 u32 idx,
438 u32 mask)
439{
440 lio_pci_writeq(oct, mask, CN6XXX_BAR1_REG(idx, oct->pcie_port));
441}
442
443u32 lio_cn6xxx_bar1_idx_read(struct octeon_device *oct, u32 idx)
444{
445 return (u32)lio_pci_readq(oct, CN6XXX_BAR1_REG(idx, oct->pcie_port));
446}
447
448u32
449lio_cn6xxx_update_read_index(struct octeon_instr_queue *iq)
450{
451 u32 new_idx = readl(iq->inst_cnt_reg);
452
453 /* The new instr cnt reg is a 32-bit counter that can roll over. We have
454 * noted the counter's initial value at init time into
455 * reset_instr_cnt
456 */
457 if (iq->reset_instr_cnt < new_idx)
458 new_idx -= iq->reset_instr_cnt;
459 else
460 new_idx += (0xffffffff - iq->reset_instr_cnt) + 1;
461
462 /* Modulo of the new index with the IQ size will give us
463 * the new index.
464 */
465 new_idx %= iq->max_count;
466
467 return new_idx;
468}
469
470void lio_cn6xxx_enable_interrupt(struct octeon_device *oct,
471 u8 unused __attribute__((unused)))
472{
473 struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
474 u64 mask = cn6xxx->intr_mask64 | CN6XXX_INTR_DMA0_FORCE;
475
476 /* Enable Interrupt */
477 writeq(mask, cn6xxx->intr_enb_reg64);
478}
479
480void lio_cn6xxx_disable_interrupt(struct octeon_device *oct,
481 u8 unused __attribute__((unused)))
482{
483 struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
484
485 /* Disable Interrupts */
486 writeq(0, cn6xxx->intr_enb_reg64);
487}
488
489static void lio_cn6xxx_get_pcie_qlmport(struct octeon_device *oct)
490{
491 /* CN63xx Pass2 and newer parts implements the SLI_MAC_NUMBER register
492 * to determine the PCIE port #
493 */
494 oct->pcie_port = octeon_read_csr(oct, CN6XXX_SLI_MAC_NUMBER) & 0xff;
495
496 dev_dbg(&oct->pci_dev->dev, "Using PCIE Port %d\n", oct->pcie_port);
497}
498
499static void
500lio_cn6xxx_process_pcie_error_intr(struct octeon_device *oct, u64 intr64)
501{
502 dev_err(&oct->pci_dev->dev, "Error Intr: 0x%016llx\n",
503 CVM_CAST64(intr64));
504}
505
506static int lio_cn6xxx_process_droq_intr_regs(struct octeon_device *oct)
507{
508 struct octeon_droq *droq;
509 int oq_no;
510 u32 pkt_count, droq_time_mask, droq_mask, droq_int_enb;
511 u32 droq_cnt_enb, droq_cnt_mask;
512
513 droq_cnt_enb = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB);
514 droq_cnt_mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT);
515 droq_mask = droq_cnt_mask & droq_cnt_enb;
516
517 droq_time_mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT);
518 droq_int_enb = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB);
519 droq_mask |= (droq_time_mask & droq_int_enb);
520
521 droq_mask &= oct->io_qmask.oq;
522
523 oct->droq_intr = 0;
524
525 for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES(oct); oq_no++) {
526 if (!(droq_mask & BIT_ULL(oq_no)))
527 continue;
528
529 droq = oct->droq[oq_no];
530 pkt_count = octeon_droq_check_hw_for_pkts(droq);
531 if (pkt_count) {
532 oct->droq_intr |= BIT_ULL(oq_no);
533 if (droq->ops.poll_mode) {
534 u32 value;
535 u32 reg;
536
537 struct octeon_cn6xxx *cn6xxx =
538 (struct octeon_cn6xxx *)oct->chip;
539
540 /* disable interrupts for this droq */
541 spin_lock
542 (&cn6xxx->lock_for_droq_int_enb_reg);
543 reg = CN6XXX_SLI_PKT_TIME_INT_ENB;
544 value = octeon_read_csr(oct, reg);
545 value &= ~(1 << oq_no);
546 octeon_write_csr(oct, reg, value);
547 reg = CN6XXX_SLI_PKT_CNT_INT_ENB;
548 value = octeon_read_csr(oct, reg);
549 value &= ~(1 << oq_no);
550 octeon_write_csr(oct, reg, value);
551
552 spin_unlock(&cn6xxx->lock_for_droq_int_enb_reg);
553 }
554 }
555 }
556
557 droq_time_mask &= oct->io_qmask.oq;
558 droq_cnt_mask &= oct->io_qmask.oq;
559
560 /* Reset the PKT_CNT/TIME_INT registers. */
561 if (droq_time_mask)
562 octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT, droq_time_mask);
563
564 if (droq_cnt_mask) /* reset PKT_CNT register:66xx */
565 octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT, droq_cnt_mask);
566
567 return 0;
568}
569
570irqreturn_t lio_cn6xxx_process_interrupt_regs(void *dev)
571{
572 struct octeon_device *oct = (struct octeon_device *)dev;
573 struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
574 u64 intr64;
575
576 intr64 = readq(cn6xxx->intr_sum_reg64);
577
578 /* If our device has interrupted, then proceed.
579 * Also check for all f's if interrupt was triggered on an error
580 * and the PCI read fails.
581 */
582 if (!intr64 || (intr64 == 0xFFFFFFFFFFFFFFFFULL))
583 return IRQ_NONE;
584
585 oct->int_status = 0;
586
587 if (intr64 & CN6XXX_INTR_ERR)
588 lio_cn6xxx_process_pcie_error_intr(oct, intr64);
589
590 if (intr64 & CN6XXX_INTR_PKT_DATA) {
591 lio_cn6xxx_process_droq_intr_regs(oct);
592 oct->int_status |= OCT_DEV_INTR_PKT_DATA;
593 }
594
595 if (intr64 & CN6XXX_INTR_DMA0_FORCE)
596 oct->int_status |= OCT_DEV_INTR_DMA0_FORCE;
597
598 if (intr64 & CN6XXX_INTR_DMA1_FORCE)
599 oct->int_status |= OCT_DEV_INTR_DMA1_FORCE;
600
601 /* Clear the current interrupts */
602 writeq(intr64, cn6xxx->intr_sum_reg64);
603
604 return IRQ_HANDLED;
605}
606
607void lio_cn6xxx_setup_reg_address(struct octeon_device *oct,
608 void *chip,
609 struct octeon_reg_list *reg_list)
610{
611 u8 __iomem *bar0_pciaddr = oct->mmio[0].hw_addr;
612 struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)chip;
613
614 reg_list->pci_win_wr_addr_hi =
615 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_ADDR_HI);
616 reg_list->pci_win_wr_addr_lo =
617 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_ADDR_LO);
618 reg_list->pci_win_wr_addr =
619 (u64 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_ADDR64);
620
621 reg_list->pci_win_rd_addr_hi =
622 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_ADDR_HI);
623 reg_list->pci_win_rd_addr_lo =
624 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_ADDR_LO);
625 reg_list->pci_win_rd_addr =
626 (u64 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_ADDR64);
627
628 reg_list->pci_win_wr_data_hi =
629 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_DATA_HI);
630 reg_list->pci_win_wr_data_lo =
631 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_DATA_LO);
632 reg_list->pci_win_wr_data =
633 (u64 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_DATA64);
634
635 reg_list->pci_win_rd_data_hi =
636 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_DATA_HI);
637 reg_list->pci_win_rd_data_lo =
638 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_DATA_LO);
639 reg_list->pci_win_rd_data =
640 (u64 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_DATA64);
641
642 lio_cn6xxx_get_pcie_qlmport(oct);
643
644 cn6xxx->intr_sum_reg64 = bar0_pciaddr + CN6XXX_SLI_INT_SUM64;
645 cn6xxx->intr_mask64 = CN6XXX_INTR_MASK;
646 cn6xxx->intr_enb_reg64 =
647 bar0_pciaddr + CN6XXX_SLI_INT_ENB64(oct->pcie_port);
648}
649
650int lio_setup_cn66xx_octeon_device(struct octeon_device *oct)
651{
652 struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
653
654 if (octeon_map_pci_barx(oct, 0, 0))
655 return 1;
656
657 if (octeon_map_pci_barx(oct, 1, MAX_BAR1_IOREMAP_SIZE)) {
658 dev_err(&oct->pci_dev->dev, "%s CN66XX BAR1 map failed\n",
659 __func__);
660 octeon_unmap_pci_barx(oct, 0);
661 return 1;
662 }
663
664 spin_lock_init(&cn6xxx->lock_for_droq_int_enb_reg);
665
666 oct->fn_list.setup_iq_regs = lio_cn66xx_setup_iq_regs;
667 oct->fn_list.setup_oq_regs = lio_cn6xxx_setup_oq_regs;
668
669 oct->fn_list.soft_reset = lio_cn6xxx_soft_reset;
670 oct->fn_list.setup_device_regs = lio_cn6xxx_setup_device_regs;
671 oct->fn_list.update_iq_read_idx = lio_cn6xxx_update_read_index;
672
673 oct->fn_list.bar1_idx_setup = lio_cn6xxx_bar1_idx_setup;
674 oct->fn_list.bar1_idx_write = lio_cn6xxx_bar1_idx_write;
675 oct->fn_list.bar1_idx_read = lio_cn6xxx_bar1_idx_read;
676
677 oct->fn_list.process_interrupt_regs = lio_cn6xxx_process_interrupt_regs;
678 oct->fn_list.enable_interrupt = lio_cn6xxx_enable_interrupt;
679 oct->fn_list.disable_interrupt = lio_cn6xxx_disable_interrupt;
680
681 oct->fn_list.enable_io_queues = lio_cn6xxx_enable_io_queues;
682 oct->fn_list.disable_io_queues = lio_cn6xxx_disable_io_queues;
683
684 lio_cn6xxx_setup_reg_address(oct, oct->chip, &oct->reg_list);
685
686 cn6xxx->conf = (struct octeon_config *)
687 oct_get_config_info(oct, LIO_210SV);
688 if (!cn6xxx->conf) {
689 dev_err(&oct->pci_dev->dev, "%s No Config found for CN66XX\n",
690 __func__);
691 octeon_unmap_pci_barx(oct, 0);
692 octeon_unmap_pci_barx(oct, 1);
693 return 1;
694 }
695
696 oct->coproc_clock_rate = 1000000ULL * lio_cn6xxx_coprocessor_clock(oct);
697
698 return 0;
699}
700EXPORT_SYMBOL_GPL(lio_setup_cn66xx_octeon_device);
701
702int lio_validate_cn6xxx_config_info(struct octeon_device *oct,
703 struct octeon_config *conf6xxx)
704{
705 if (CFG_GET_IQ_MAX_Q(conf6xxx) > CN6XXX_MAX_INPUT_QUEUES) {
706 dev_err(&oct->pci_dev->dev, "%s: Num IQ (%d) exceeds Max (%d)\n",
707 __func__, CFG_GET_IQ_MAX_Q(conf6xxx),
708 CN6XXX_MAX_INPUT_QUEUES);
709 return 1;
710 }
711
712 if (CFG_GET_OQ_MAX_Q(conf6xxx) > CN6XXX_MAX_OUTPUT_QUEUES) {
713 dev_err(&oct->pci_dev->dev, "%s: Num OQ (%d) exceeds Max (%d)\n",
714 __func__, CFG_GET_OQ_MAX_Q(conf6xxx),
715 CN6XXX_MAX_OUTPUT_QUEUES);
716 return 1;
717 }
718
719 if (CFG_GET_IQ_INSTR_TYPE(conf6xxx) != OCTEON_32BYTE_INSTR &&
720 CFG_GET_IQ_INSTR_TYPE(conf6xxx) != OCTEON_64BYTE_INSTR) {
721 dev_err(&oct->pci_dev->dev, "%s: Invalid instr type for IQ\n",
722 __func__);
723 return 1;
724 }
725 if (!CFG_GET_OQ_REFILL_THRESHOLD(conf6xxx)) {
726 dev_err(&oct->pci_dev->dev, "%s: Invalid parameter for OQ\n",
727 __func__);
728 return 1;
729 }
730
731 if (!(CFG_GET_OQ_INTR_TIME(conf6xxx))) {
732 dev_err(&oct->pci_dev->dev, "%s: No Time Interrupt for OQ\n",
733 __func__);
734 return 1;
735 }
736
737 return 0;
738}
1/**********************************************************************
2 * Author: Cavium, Inc.
3 *
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
6 *
7 * Copyright (c) 2003-2016 Cavium, Inc.
8 *
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
12 *
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more details.
17 ***********************************************************************/
18#include <linux/pci.h>
19#include <linux/netdevice.h>
20#include "liquidio_common.h"
21#include "octeon_droq.h"
22#include "octeon_iq.h"
23#include "response_manager.h"
24#include "octeon_device.h"
25#include "octeon_main.h"
26#include "cn66xx_regs.h"
27#include "cn66xx_device.h"
28
29int lio_cn6xxx_soft_reset(struct octeon_device *oct)
30{
31 octeon_write_csr64(oct, CN6XXX_WIN_WR_MASK_REG, 0xFF);
32
33 dev_dbg(&oct->pci_dev->dev, "BIST enabled for soft reset\n");
34
35 lio_pci_writeq(oct, 1, CN6XXX_CIU_SOFT_BIST);
36 octeon_write_csr64(oct, CN6XXX_SLI_SCRATCH1, 0x1234ULL);
37
38 lio_pci_readq(oct, CN6XXX_CIU_SOFT_RST);
39 lio_pci_writeq(oct, 1, CN6XXX_CIU_SOFT_RST);
40
41 /* make sure that the reset is written before starting timer */
42 mmiowb();
43
44 /* Wait for 10ms as Octeon resets. */
45 mdelay(100);
46
47 if (octeon_read_csr64(oct, CN6XXX_SLI_SCRATCH1) == 0x1234ULL) {
48 dev_err(&oct->pci_dev->dev, "Soft reset failed\n");
49 return 1;
50 }
51
52 dev_dbg(&oct->pci_dev->dev, "Reset completed\n");
53 octeon_write_csr64(oct, CN6XXX_WIN_WR_MASK_REG, 0xFF);
54
55 return 0;
56}
57
58void lio_cn6xxx_enable_error_reporting(struct octeon_device *oct)
59{
60 u32 val;
61
62 pci_read_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, &val);
63 if (val & 0x000c0000) {
64 dev_err(&oct->pci_dev->dev, "PCI-E Link error detected: 0x%08x\n",
65 val & 0x000c0000);
66 }
67
68 val |= 0xf; /* Enable Link error reporting */
69
70 dev_dbg(&oct->pci_dev->dev, "Enabling PCI-E error reporting..\n");
71 pci_write_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, val);
72}
73
74void lio_cn6xxx_setup_pcie_mps(struct octeon_device *oct,
75 enum octeon_pcie_mps mps)
76{
77 u32 val;
78 u64 r64;
79
80 /* Read config register for MPS */
81 pci_read_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, &val);
82
83 if (mps == PCIE_MPS_DEFAULT) {
84 mps = ((val & (0x7 << 5)) >> 5);
85 } else {
86 val &= ~(0x7 << 5); /* Turn off any MPS bits */
87 val |= (mps << 5); /* Set MPS */
88 pci_write_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, val);
89 }
90
91 /* Set MPS in DPI_SLI_PRT0_CFG to the same value. */
92 r64 = lio_pci_readq(oct, CN6XXX_DPI_SLI_PRTX_CFG(oct->pcie_port));
93 r64 |= (mps << 4);
94 lio_pci_writeq(oct, r64, CN6XXX_DPI_SLI_PRTX_CFG(oct->pcie_port));
95}
96
97void lio_cn6xxx_setup_pcie_mrrs(struct octeon_device *oct,
98 enum octeon_pcie_mrrs mrrs)
99{
100 u32 val;
101 u64 r64;
102
103 /* Read config register for MRRS */
104 pci_read_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, &val);
105
106 if (mrrs == PCIE_MRRS_DEFAULT) {
107 mrrs = ((val & (0x7 << 12)) >> 12);
108 } else {
109 val &= ~(0x7 << 12); /* Turn off any MRRS bits */
110 val |= (mrrs << 12); /* Set MRRS */
111 pci_write_config_dword(oct->pci_dev, CN6XXX_PCIE_DEVCTL, val);
112 }
113
114 /* Set MRRS in SLI_S2M_PORT0_CTL to the same value. */
115 r64 = octeon_read_csr64(oct, CN6XXX_SLI_S2M_PORTX_CTL(oct->pcie_port));
116 r64 |= mrrs;
117 octeon_write_csr64(oct, CN6XXX_SLI_S2M_PORTX_CTL(oct->pcie_port), r64);
118
119 /* Set MRRS in DPI_SLI_PRT0_CFG to the same value. */
120 r64 = lio_pci_readq(oct, CN6XXX_DPI_SLI_PRTX_CFG(oct->pcie_port));
121 r64 |= mrrs;
122 lio_pci_writeq(oct, r64, CN6XXX_DPI_SLI_PRTX_CFG(oct->pcie_port));
123}
124
125u32 lio_cn6xxx_coprocessor_clock(struct octeon_device *oct)
126{
127 /* Bits 29:24 of MIO_RST_BOOT holds the ref. clock multiplier
128 * for SLI.
129 */
130 return ((lio_pci_readq(oct, CN6XXX_MIO_RST_BOOT) >> 24) & 0x3f) * 50;
131}
132
133u32 lio_cn6xxx_get_oq_ticks(struct octeon_device *oct,
134 u32 time_intr_in_us)
135{
136 /* This gives the SLI clock per microsec */
137 u32 oqticks_per_us = lio_cn6xxx_coprocessor_clock(oct);
138
139 /* core clock per us / oq ticks will be fractional. TO avoid that
140 * we use the method below.
141 */
142
143 /* This gives the clock cycles per millisecond */
144 oqticks_per_us *= 1000;
145
146 /* This gives the oq ticks (1024 core clock cycles) per millisecond */
147 oqticks_per_us /= 1024;
148
149 /* time_intr is in microseconds. The next 2 steps gives the oq ticks
150 * corressponding to time_intr.
151 */
152 oqticks_per_us *= time_intr_in_us;
153 oqticks_per_us /= 1000;
154
155 return oqticks_per_us;
156}
157
158void lio_cn6xxx_setup_global_input_regs(struct octeon_device *oct)
159{
160 /* Select Round-Robin Arb, ES, RO, NS for Input Queues */
161 octeon_write_csr(oct, CN6XXX_SLI_PKT_INPUT_CONTROL,
162 CN6XXX_INPUT_CTL_MASK);
163
164 /* Instruction Read Size - Max 4 instructions per PCIE Read */
165 octeon_write_csr64(oct, CN6XXX_SLI_PKT_INSTR_RD_SIZE,
166 0xFFFFFFFFFFFFFFFFULL);
167
168 /* Select PCIE Port for all Input rings. */
169 octeon_write_csr64(oct, CN6XXX_SLI_IN_PCIE_PORT,
170 (oct->pcie_port * 0x5555555555555555ULL));
171}
172
173static void lio_cn66xx_setup_pkt_ctl_regs(struct octeon_device *oct)
174{
175 u64 pktctl;
176
177 struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
178
179 pktctl = octeon_read_csr64(oct, CN6XXX_SLI_PKT_CTL);
180
181 /* 66XX SPECIFIC */
182 if (CFG_GET_OQ_MAX_Q(cn6xxx->conf) <= 4)
183 /* Disable RING_EN if only upto 4 rings are used. */
184 pktctl &= ~(1 << 4);
185 else
186 pktctl |= (1 << 4);
187
188 if (CFG_GET_IS_SLI_BP_ON(cn6xxx->conf))
189 pktctl |= 0xF;
190 else
191 /* Disable per-port backpressure. */
192 pktctl &= ~0xF;
193 octeon_write_csr64(oct, CN6XXX_SLI_PKT_CTL, pktctl);
194}
195
196void lio_cn6xxx_setup_global_output_regs(struct octeon_device *oct)
197{
198 u32 time_threshold;
199 struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
200
201 /* / Select PCI-E Port for all Output queues */
202 octeon_write_csr64(oct, CN6XXX_SLI_PKT_PCIE_PORT64,
203 (oct->pcie_port * 0x5555555555555555ULL));
204
205 if (CFG_GET_IS_SLI_BP_ON(cn6xxx->conf)) {
206 octeon_write_csr64(oct, CN6XXX_SLI_OQ_WMARK, 32);
207 } else {
208 /* / Set Output queue watermark to 0 to disable backpressure */
209 octeon_write_csr64(oct, CN6XXX_SLI_OQ_WMARK, 0);
210 }
211
212 /* / Select Info Ptr for length & data */
213 octeon_write_csr(oct, CN6XXX_SLI_PKT_IPTR, 0xFFFFFFFF);
214
215 /* / Select Packet count instead of bytes for SLI_PKTi_CNTS[CNT] */
216 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_BMODE, 0);
217
218 /* Select ES, RO, NS setting from register for Output Queue Packet
219 * Address
220 */
221 octeon_write_csr(oct, CN6XXX_SLI_PKT_DPADDR, 0xFFFFFFFF);
222
223 /* No Relaxed Ordering, No Snoop, 64-bit swap for Output
224 * Queue ScatterList
225 */
226 octeon_write_csr(oct, CN6XXX_SLI_PKT_SLIST_ROR, 0);
227 octeon_write_csr(oct, CN6XXX_SLI_PKT_SLIST_NS, 0);
228
229 /* / ENDIAN_SPECIFIC CHANGES - 0 works for LE. */
230#ifdef __BIG_ENDIAN_BITFIELD
231 octeon_write_csr64(oct, CN6XXX_SLI_PKT_SLIST_ES64,
232 0x5555555555555555ULL);
233#else
234 octeon_write_csr64(oct, CN6XXX_SLI_PKT_SLIST_ES64, 0ULL);
235#endif
236
237 /* / No Relaxed Ordering, No Snoop, 64-bit swap for Output Queue Data */
238 octeon_write_csr(oct, CN6XXX_SLI_PKT_DATA_OUT_ROR, 0);
239 octeon_write_csr(oct, CN6XXX_SLI_PKT_DATA_OUT_NS, 0);
240 octeon_write_csr64(oct, CN6XXX_SLI_PKT_DATA_OUT_ES64,
241 0x5555555555555555ULL);
242
243 /* / Set up interrupt packet and time threshold */
244 octeon_write_csr(oct, CN6XXX_SLI_OQ_INT_LEVEL_PKTS,
245 (u32)CFG_GET_OQ_INTR_PKT(cn6xxx->conf));
246 time_threshold =
247 lio_cn6xxx_get_oq_ticks(oct, (u32)
248 CFG_GET_OQ_INTR_TIME(cn6xxx->conf));
249
250 octeon_write_csr(oct, CN6XXX_SLI_OQ_INT_LEVEL_TIME, time_threshold);
251}
252
253static int lio_cn6xxx_setup_device_regs(struct octeon_device *oct)
254{
255 lio_cn6xxx_setup_pcie_mps(oct, PCIE_MPS_DEFAULT);
256 lio_cn6xxx_setup_pcie_mrrs(oct, PCIE_MRRS_512B);
257 lio_cn6xxx_enable_error_reporting(oct);
258
259 lio_cn6xxx_setup_global_input_regs(oct);
260 lio_cn66xx_setup_pkt_ctl_regs(oct);
261 lio_cn6xxx_setup_global_output_regs(oct);
262
263 /* Default error timeout value should be 0x200000 to avoid host hang
264 * when reads invalid register
265 */
266 octeon_write_csr64(oct, CN6XXX_SLI_WINDOW_CTL, 0x200000ULL);
267 return 0;
268}
269
270void lio_cn6xxx_setup_iq_regs(struct octeon_device *oct, u32 iq_no)
271{
272 struct octeon_instr_queue *iq = oct->instr_queue[iq_no];
273
274 octeon_write_csr64(oct, CN6XXX_SLI_IQ_PKT_INSTR_HDR64(iq_no), 0);
275
276 /* Write the start of the input queue's ring and its size */
277 octeon_write_csr64(oct, CN6XXX_SLI_IQ_BASE_ADDR64(iq_no),
278 iq->base_addr_dma);
279 octeon_write_csr(oct, CN6XXX_SLI_IQ_SIZE(iq_no), iq->max_count);
280
281 /* Remember the doorbell & instruction count register addr for this
282 * queue
283 */
284 iq->doorbell_reg = oct->mmio[0].hw_addr + CN6XXX_SLI_IQ_DOORBELL(iq_no);
285 iq->inst_cnt_reg = oct->mmio[0].hw_addr
286 + CN6XXX_SLI_IQ_INSTR_COUNT(iq_no);
287 dev_dbg(&oct->pci_dev->dev, "InstQ[%d]:dbell reg @ 0x%p instcnt_reg @ 0x%p\n",
288 iq_no, iq->doorbell_reg, iq->inst_cnt_reg);
289
290 /* Store the current instruction counter
291 * (used in flush_iq calculation)
292 */
293 iq->reset_instr_cnt = readl(iq->inst_cnt_reg);
294}
295
296static void lio_cn66xx_setup_iq_regs(struct octeon_device *oct, u32 iq_no)
297{
298 lio_cn6xxx_setup_iq_regs(oct, iq_no);
299
300 /* Backpressure for this queue - WMARK set to all F's. This effectively
301 * disables the backpressure mechanism.
302 */
303 octeon_write_csr64(oct, CN66XX_SLI_IQ_BP64(iq_no),
304 (0xFFFFFFFFULL << 32));
305}
306
307void lio_cn6xxx_setup_oq_regs(struct octeon_device *oct, u32 oq_no)
308{
309 u32 intr;
310 struct octeon_droq *droq = oct->droq[oq_no];
311
312 octeon_write_csr64(oct, CN6XXX_SLI_OQ_BASE_ADDR64(oq_no),
313 droq->desc_ring_dma);
314 octeon_write_csr(oct, CN6XXX_SLI_OQ_SIZE(oq_no), droq->max_count);
315
316 octeon_write_csr(oct, CN6XXX_SLI_OQ_BUFF_INFO_SIZE(oq_no),
317 (droq->buffer_size | (OCT_RH_SIZE << 16)));
318
319 /* Get the mapped address of the pkt_sent and pkts_credit regs */
320 droq->pkts_sent_reg =
321 oct->mmio[0].hw_addr + CN6XXX_SLI_OQ_PKTS_SENT(oq_no);
322 droq->pkts_credit_reg =
323 oct->mmio[0].hw_addr + CN6XXX_SLI_OQ_PKTS_CREDIT(oq_no);
324
325 /* Enable this output queue to generate Packet Timer Interrupt */
326 intr = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB);
327 intr |= (1 << oq_no);
328 octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB, intr);
329
330 /* Enable this output queue to generate Packet Timer Interrupt */
331 intr = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB);
332 intr |= (1 << oq_no);
333 octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB, intr);
334}
335
336int lio_cn6xxx_enable_io_queues(struct octeon_device *oct)
337{
338 u32 mask;
339
340 mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_INSTR_SIZE);
341 mask |= oct->io_qmask.iq64B;
342 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_SIZE, mask);
343
344 mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB);
345 mask |= oct->io_qmask.iq;
346 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, mask);
347
348 mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_OUT_ENB);
349 mask |= oct->io_qmask.oq;
350 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, mask);
351
352 return 0;
353}
354
355void lio_cn6xxx_disable_io_queues(struct octeon_device *oct)
356{
357 int i;
358 u32 mask, loop = HZ;
359 u32 d32;
360
361 /* Reset the Enable bits for Input Queues. */
362 mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB);
363 mask ^= oct->io_qmask.iq;
364 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, mask);
365
366 /* Wait until hardware indicates that the queues are out of reset. */
367 mask = (u32)oct->io_qmask.iq;
368 d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_IQ);
369 while (((d32 & mask) != mask) && loop--) {
370 d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_IQ);
371 schedule_timeout_uninterruptible(1);
372 }
373
374 /* Reset the doorbell register for each Input queue. */
375 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
376 if (!(oct->io_qmask.iq & BIT_ULL(i)))
377 continue;
378 octeon_write_csr(oct, CN6XXX_SLI_IQ_DOORBELL(i), 0xFFFFFFFF);
379 d32 = octeon_read_csr(oct, CN6XXX_SLI_IQ_DOORBELL(i));
380 }
381
382 /* Reset the Enable bits for Output Queues. */
383 mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_OUT_ENB);
384 mask ^= oct->io_qmask.oq;
385 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, mask);
386
387 /* Wait until hardware indicates that the queues are out of reset. */
388 loop = HZ;
389 mask = (u32)oct->io_qmask.oq;
390 d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_OQ);
391 while (((d32 & mask) != mask) && loop--) {
392 d32 = octeon_read_csr(oct, CN6XXX_SLI_PORT_IN_RST_OQ);
393 schedule_timeout_uninterruptible(1);
394 }
395 ;
396
397 /* Reset the doorbell register for each Output queue. */
398 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
399 if (!(oct->io_qmask.oq & BIT_ULL(i)))
400 continue;
401 octeon_write_csr(oct, CN6XXX_SLI_OQ_PKTS_CREDIT(i), 0xFFFFFFFF);
402 d32 = octeon_read_csr(oct, CN6XXX_SLI_OQ_PKTS_CREDIT(i));
403
404 d32 = octeon_read_csr(oct, CN6XXX_SLI_OQ_PKTS_SENT(i));
405 octeon_write_csr(oct, CN6XXX_SLI_OQ_PKTS_SENT(i), d32);
406 }
407
408 d32 = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT);
409 if (d32)
410 octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT, d32);
411
412 d32 = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT);
413 if (d32)
414 octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT, d32);
415}
416
417void
418lio_cn6xxx_bar1_idx_setup(struct octeon_device *oct,
419 u64 core_addr,
420 u32 idx,
421 int valid)
422{
423 u64 bar1;
424
425 if (valid == 0) {
426 bar1 = lio_pci_readq(oct, CN6XXX_BAR1_REG(idx, oct->pcie_port));
427 lio_pci_writeq(oct, (bar1 & 0xFFFFFFFEULL),
428 CN6XXX_BAR1_REG(idx, oct->pcie_port));
429 bar1 = lio_pci_readq(oct, CN6XXX_BAR1_REG(idx, oct->pcie_port));
430 return;
431 }
432
433 /* Bits 17:4 of the PCI_BAR1_INDEXx stores bits 35:22 of
434 * the Core Addr
435 */
436 lio_pci_writeq(oct, (((core_addr >> 22) << 4) | PCI_BAR1_MASK),
437 CN6XXX_BAR1_REG(idx, oct->pcie_port));
438
439 bar1 = lio_pci_readq(oct, CN6XXX_BAR1_REG(idx, oct->pcie_port));
440}
441
442void lio_cn6xxx_bar1_idx_write(struct octeon_device *oct,
443 u32 idx,
444 u32 mask)
445{
446 lio_pci_writeq(oct, mask, CN6XXX_BAR1_REG(idx, oct->pcie_port));
447}
448
449u32 lio_cn6xxx_bar1_idx_read(struct octeon_device *oct, u32 idx)
450{
451 return (u32)lio_pci_readq(oct, CN6XXX_BAR1_REG(idx, oct->pcie_port));
452}
453
454u32
455lio_cn6xxx_update_read_index(struct octeon_instr_queue *iq)
456{
457 u32 new_idx = readl(iq->inst_cnt_reg);
458
459 /* The new instr cnt reg is a 32-bit counter that can roll over. We have
460 * noted the counter's initial value at init time into
461 * reset_instr_cnt
462 */
463 if (iq->reset_instr_cnt < new_idx)
464 new_idx -= iq->reset_instr_cnt;
465 else
466 new_idx += (0xffffffff - iq->reset_instr_cnt) + 1;
467
468 /* Modulo of the new index with the IQ size will give us
469 * the new index.
470 */
471 new_idx %= iq->max_count;
472
473 return new_idx;
474}
475
476void lio_cn6xxx_enable_interrupt(struct octeon_device *oct,
477 u8 unused __attribute__((unused)))
478{
479 struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
480 u64 mask = cn6xxx->intr_mask64 | CN6XXX_INTR_DMA0_FORCE;
481
482 /* Enable Interrupt */
483 writeq(mask, cn6xxx->intr_enb_reg64);
484}
485
486void lio_cn6xxx_disable_interrupt(struct octeon_device *oct,
487 u8 unused __attribute__((unused)))
488{
489 struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
490
491 /* Disable Interrupts */
492 writeq(0, cn6xxx->intr_enb_reg64);
493
494 /* make sure interrupts are really disabled */
495 mmiowb();
496}
497
498static void lio_cn6xxx_get_pcie_qlmport(struct octeon_device *oct)
499{
500 /* CN63xx Pass2 and newer parts implements the SLI_MAC_NUMBER register
501 * to determine the PCIE port #
502 */
503 oct->pcie_port = octeon_read_csr(oct, CN6XXX_SLI_MAC_NUMBER) & 0xff;
504
505 dev_dbg(&oct->pci_dev->dev, "Using PCIE Port %d\n", oct->pcie_port);
506}
507
508static void
509lio_cn6xxx_process_pcie_error_intr(struct octeon_device *oct, u64 intr64)
510{
511 dev_err(&oct->pci_dev->dev, "Error Intr: 0x%016llx\n",
512 CVM_CAST64(intr64));
513}
514
515static int lio_cn6xxx_process_droq_intr_regs(struct octeon_device *oct)
516{
517 struct octeon_droq *droq;
518 int oq_no;
519 u32 pkt_count, droq_time_mask, droq_mask, droq_int_enb;
520 u32 droq_cnt_enb, droq_cnt_mask;
521
522 droq_cnt_enb = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB);
523 droq_cnt_mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT);
524 droq_mask = droq_cnt_mask & droq_cnt_enb;
525
526 droq_time_mask = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT);
527 droq_int_enb = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB);
528 droq_mask |= (droq_time_mask & droq_int_enb);
529
530 droq_mask &= oct->io_qmask.oq;
531
532 oct->droq_intr = 0;
533
534 for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES(oct); oq_no++) {
535 if (!(droq_mask & BIT_ULL(oq_no)))
536 continue;
537
538 droq = oct->droq[oq_no];
539 pkt_count = octeon_droq_check_hw_for_pkts(droq);
540 if (pkt_count) {
541 oct->droq_intr |= BIT_ULL(oq_no);
542 if (droq->ops.poll_mode) {
543 u32 value;
544 u32 reg;
545
546 struct octeon_cn6xxx *cn6xxx =
547 (struct octeon_cn6xxx *)oct->chip;
548
549 /* disable interrupts for this droq */
550 spin_lock
551 (&cn6xxx->lock_for_droq_int_enb_reg);
552 reg = CN6XXX_SLI_PKT_TIME_INT_ENB;
553 value = octeon_read_csr(oct, reg);
554 value &= ~(1 << oq_no);
555 octeon_write_csr(oct, reg, value);
556 reg = CN6XXX_SLI_PKT_CNT_INT_ENB;
557 value = octeon_read_csr(oct, reg);
558 value &= ~(1 << oq_no);
559 octeon_write_csr(oct, reg, value);
560
561 /* Ensure that the enable register is written.
562 */
563 mmiowb();
564
565 spin_unlock(&cn6xxx->lock_for_droq_int_enb_reg);
566 }
567 }
568 }
569
570 droq_time_mask &= oct->io_qmask.oq;
571 droq_cnt_mask &= oct->io_qmask.oq;
572
573 /* Reset the PKT_CNT/TIME_INT registers. */
574 if (droq_time_mask)
575 octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT, droq_time_mask);
576
577 if (droq_cnt_mask) /* reset PKT_CNT register:66xx */
578 octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT, droq_cnt_mask);
579
580 return 0;
581}
582
583irqreturn_t lio_cn6xxx_process_interrupt_regs(void *dev)
584{
585 struct octeon_device *oct = (struct octeon_device *)dev;
586 struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
587 u64 intr64;
588
589 intr64 = readq(cn6xxx->intr_sum_reg64);
590
591 /* If our device has interrupted, then proceed.
592 * Also check for all f's if interrupt was triggered on an error
593 * and the PCI read fails.
594 */
595 if (!intr64 || (intr64 == 0xFFFFFFFFFFFFFFFFULL))
596 return IRQ_NONE;
597
598 oct->int_status = 0;
599
600 if (intr64 & CN6XXX_INTR_ERR)
601 lio_cn6xxx_process_pcie_error_intr(oct, intr64);
602
603 if (intr64 & CN6XXX_INTR_PKT_DATA) {
604 lio_cn6xxx_process_droq_intr_regs(oct);
605 oct->int_status |= OCT_DEV_INTR_PKT_DATA;
606 }
607
608 if (intr64 & CN6XXX_INTR_DMA0_FORCE)
609 oct->int_status |= OCT_DEV_INTR_DMA0_FORCE;
610
611 if (intr64 & CN6XXX_INTR_DMA1_FORCE)
612 oct->int_status |= OCT_DEV_INTR_DMA1_FORCE;
613
614 /* Clear the current interrupts */
615 writeq(intr64, cn6xxx->intr_sum_reg64);
616
617 return IRQ_HANDLED;
618}
619
620void lio_cn6xxx_setup_reg_address(struct octeon_device *oct,
621 void *chip,
622 struct octeon_reg_list *reg_list)
623{
624 u8 __iomem *bar0_pciaddr = oct->mmio[0].hw_addr;
625 struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)chip;
626
627 reg_list->pci_win_wr_addr_hi =
628 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_ADDR_HI);
629 reg_list->pci_win_wr_addr_lo =
630 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_ADDR_LO);
631 reg_list->pci_win_wr_addr =
632 (u64 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_ADDR64);
633
634 reg_list->pci_win_rd_addr_hi =
635 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_ADDR_HI);
636 reg_list->pci_win_rd_addr_lo =
637 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_ADDR_LO);
638 reg_list->pci_win_rd_addr =
639 (u64 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_ADDR64);
640
641 reg_list->pci_win_wr_data_hi =
642 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_DATA_HI);
643 reg_list->pci_win_wr_data_lo =
644 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_DATA_LO);
645 reg_list->pci_win_wr_data =
646 (u64 __iomem *)(bar0_pciaddr + CN6XXX_WIN_WR_DATA64);
647
648 reg_list->pci_win_rd_data_hi =
649 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_DATA_HI);
650 reg_list->pci_win_rd_data_lo =
651 (u32 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_DATA_LO);
652 reg_list->pci_win_rd_data =
653 (u64 __iomem *)(bar0_pciaddr + CN6XXX_WIN_RD_DATA64);
654
655 lio_cn6xxx_get_pcie_qlmport(oct);
656
657 cn6xxx->intr_sum_reg64 = bar0_pciaddr + CN6XXX_SLI_INT_SUM64;
658 cn6xxx->intr_mask64 = CN6XXX_INTR_MASK;
659 cn6xxx->intr_enb_reg64 =
660 bar0_pciaddr + CN6XXX_SLI_INT_ENB64(oct->pcie_port);
661}
662
663int lio_setup_cn66xx_octeon_device(struct octeon_device *oct)
664{
665 struct octeon_cn6xxx *cn6xxx = (struct octeon_cn6xxx *)oct->chip;
666
667 if (octeon_map_pci_barx(oct, 0, 0))
668 return 1;
669
670 if (octeon_map_pci_barx(oct, 1, MAX_BAR1_IOREMAP_SIZE)) {
671 dev_err(&oct->pci_dev->dev, "%s CN66XX BAR1 map failed\n",
672 __func__);
673 octeon_unmap_pci_barx(oct, 0);
674 return 1;
675 }
676
677 spin_lock_init(&cn6xxx->lock_for_droq_int_enb_reg);
678
679 oct->fn_list.setup_iq_regs = lio_cn66xx_setup_iq_regs;
680 oct->fn_list.setup_oq_regs = lio_cn6xxx_setup_oq_regs;
681
682 oct->fn_list.soft_reset = lio_cn6xxx_soft_reset;
683 oct->fn_list.setup_device_regs = lio_cn6xxx_setup_device_regs;
684 oct->fn_list.update_iq_read_idx = lio_cn6xxx_update_read_index;
685
686 oct->fn_list.bar1_idx_setup = lio_cn6xxx_bar1_idx_setup;
687 oct->fn_list.bar1_idx_write = lio_cn6xxx_bar1_idx_write;
688 oct->fn_list.bar1_idx_read = lio_cn6xxx_bar1_idx_read;
689
690 oct->fn_list.process_interrupt_regs = lio_cn6xxx_process_interrupt_regs;
691 oct->fn_list.enable_interrupt = lio_cn6xxx_enable_interrupt;
692 oct->fn_list.disable_interrupt = lio_cn6xxx_disable_interrupt;
693
694 oct->fn_list.enable_io_queues = lio_cn6xxx_enable_io_queues;
695 oct->fn_list.disable_io_queues = lio_cn6xxx_disable_io_queues;
696
697 lio_cn6xxx_setup_reg_address(oct, oct->chip, &oct->reg_list);
698
699 cn6xxx->conf = (struct octeon_config *)
700 oct_get_config_info(oct, LIO_210SV);
701 if (!cn6xxx->conf) {
702 dev_err(&oct->pci_dev->dev, "%s No Config found for CN66XX\n",
703 __func__);
704 octeon_unmap_pci_barx(oct, 0);
705 octeon_unmap_pci_barx(oct, 1);
706 return 1;
707 }
708
709 oct->coproc_clock_rate = 1000000ULL * lio_cn6xxx_coprocessor_clock(oct);
710
711 return 0;
712}
713
714int lio_validate_cn6xxx_config_info(struct octeon_device *oct,
715 struct octeon_config *conf6xxx)
716{
717 if (CFG_GET_IQ_MAX_Q(conf6xxx) > CN6XXX_MAX_INPUT_QUEUES) {
718 dev_err(&oct->pci_dev->dev, "%s: Num IQ (%d) exceeds Max (%d)\n",
719 __func__, CFG_GET_IQ_MAX_Q(conf6xxx),
720 CN6XXX_MAX_INPUT_QUEUES);
721 return 1;
722 }
723
724 if (CFG_GET_OQ_MAX_Q(conf6xxx) > CN6XXX_MAX_OUTPUT_QUEUES) {
725 dev_err(&oct->pci_dev->dev, "%s: Num OQ (%d) exceeds Max (%d)\n",
726 __func__, CFG_GET_OQ_MAX_Q(conf6xxx),
727 CN6XXX_MAX_OUTPUT_QUEUES);
728 return 1;
729 }
730
731 if (CFG_GET_IQ_INSTR_TYPE(conf6xxx) != OCTEON_32BYTE_INSTR &&
732 CFG_GET_IQ_INSTR_TYPE(conf6xxx) != OCTEON_64BYTE_INSTR) {
733 dev_err(&oct->pci_dev->dev, "%s: Invalid instr type for IQ\n",
734 __func__);
735 return 1;
736 }
737 if (!(CFG_GET_OQ_INFO_PTR(conf6xxx)) ||
738 !(CFG_GET_OQ_REFILL_THRESHOLD(conf6xxx))) {
739 dev_err(&oct->pci_dev->dev, "%s: Invalid parameter for OQ\n",
740 __func__);
741 return 1;
742 }
743
744 if (!(CFG_GET_OQ_INTR_TIME(conf6xxx))) {
745 dev_err(&oct->pci_dev->dev, "%s: No Time Interrupt for OQ\n",
746 __func__);
747 return 1;
748 }
749
750 return 0;
751}