Loading...
Note: File does not exist in v5.9.
1/*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * BSD LICENSE
14 *
15 * Copyright(c) 2012 Intel Corporation. All rights reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
19 * are met:
20 *
21 * * Redistributions of source code must retain the above copyright
22 * notice, this list of conditions and the following disclaimer.
23 * * Redistributions in binary form must reproduce the above copy
24 * notice, this list of conditions and the following disclaimer in
25 * the documentation and/or other materials provided with the
26 * distribution.
27 * * Neither the name of Intel Corporation nor the names of its
28 * contributors may be used to endorse or promote products derived
29 * from this software without specific prior written permission.
30 *
31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 *
43 * Intel PCIe NTB Linux driver
44 *
45 * Contact Information:
46 * Jon Mason <jon.mason@intel.com>
47 */
48#include <linux/debugfs.h>
49#include <linux/delay.h>
50#include <linux/init.h>
51#include <linux/interrupt.h>
52#include <linux/module.h>
53#include <linux/pci.h>
54#include <linux/random.h>
55#include <linux/slab.h>
56#include "ntb_hw.h"
57#include "ntb_regs.h"
58
59#define NTB_NAME "Intel(R) PCI-E Non-Transparent Bridge Driver"
60#define NTB_VER "1.0"
61
62MODULE_DESCRIPTION(NTB_NAME);
63MODULE_VERSION(NTB_VER);
64MODULE_LICENSE("Dual BSD/GPL");
65MODULE_AUTHOR("Intel Corporation");
66
67static bool xeon_errata_workaround = true;
68module_param(xeon_errata_workaround, bool, 0644);
69MODULE_PARM_DESC(xeon_errata_workaround, "Workaround for the Xeon Errata");
70
71enum {
72 NTB_CONN_TRANSPARENT = 0,
73 NTB_CONN_B2B,
74 NTB_CONN_RP,
75};
76
77enum {
78 NTB_DEV_USD = 0,
79 NTB_DEV_DSD,
80};
81
82enum {
83 SNB_HW = 0,
84 BWD_HW,
85};
86
87static struct dentry *debugfs_dir;
88
89#define BWD_LINK_RECOVERY_TIME 500
90
91/* Translate memory window 0,1 to BAR 2,4 */
92#define MW_TO_BAR(mw) (mw * NTB_MAX_NUM_MW + 2)
93
94static const struct pci_device_id ntb_pci_tbl[] = {
95 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
96 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
97 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
98 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
99 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
100 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
101 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
102 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
103 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
104 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
105 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
106 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
107 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
108 {0}
109};
110MODULE_DEVICE_TABLE(pci, ntb_pci_tbl);
111
112/**
113 * ntb_register_event_callback() - register event callback
114 * @ndev: pointer to ntb_device instance
115 * @func: callback function to register
116 *
117 * This function registers a callback for any HW driver events such as link
118 * up/down, power management notices and etc.
119 *
120 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
121 */
122int ntb_register_event_callback(struct ntb_device *ndev,
123 void (*func)(void *handle,
124 enum ntb_hw_event event))
125{
126 if (ndev->event_cb)
127 return -EINVAL;
128
129 ndev->event_cb = func;
130
131 return 0;
132}
133
134/**
135 * ntb_unregister_event_callback() - unregisters the event callback
136 * @ndev: pointer to ntb_device instance
137 *
138 * This function unregisters the existing callback from transport
139 */
140void ntb_unregister_event_callback(struct ntb_device *ndev)
141{
142 ndev->event_cb = NULL;
143}
144
145static void ntb_irq_work(unsigned long data)
146{
147 struct ntb_db_cb *db_cb = (struct ntb_db_cb *)data;
148 int rc;
149
150 rc = db_cb->callback(db_cb->data, db_cb->db_num);
151 if (rc)
152 tasklet_schedule(&db_cb->irq_work);
153 else {
154 struct ntb_device *ndev = db_cb->ndev;
155 unsigned long mask;
156
157 mask = readw(ndev->reg_ofs.ldb_mask);
158 clear_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
159 writew(mask, ndev->reg_ofs.ldb_mask);
160 }
161}
162
163/**
164 * ntb_register_db_callback() - register a callback for doorbell interrupt
165 * @ndev: pointer to ntb_device instance
166 * @idx: doorbell index to register callback, zero based
167 * @data: pointer to be returned to caller with every callback
168 * @func: callback function to register
169 *
170 * This function registers a callback function for the doorbell interrupt
171 * on the primary side. The function will unmask the doorbell as well to
172 * allow interrupt.
173 *
174 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
175 */
176int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,
177 void *data, int (*func)(void *data, int db_num))
178{
179 unsigned long mask;
180
181 if (idx >= ndev->max_cbs || ndev->db_cb[idx].callback) {
182 dev_warn(&ndev->pdev->dev, "Invalid Index.\n");
183 return -EINVAL;
184 }
185
186 ndev->db_cb[idx].callback = func;
187 ndev->db_cb[idx].data = data;
188 ndev->db_cb[idx].ndev = ndev;
189
190 tasklet_init(&ndev->db_cb[idx].irq_work, ntb_irq_work,
191 (unsigned long) &ndev->db_cb[idx]);
192
193 /* unmask interrupt */
194 mask = readw(ndev->reg_ofs.ldb_mask);
195 clear_bit(idx * ndev->bits_per_vector, &mask);
196 writew(mask, ndev->reg_ofs.ldb_mask);
197
198 return 0;
199}
200
201/**
202 * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt
203 * @ndev: pointer to ntb_device instance
204 * @idx: doorbell index to register callback, zero based
205 *
206 * This function unregisters a callback function for the doorbell interrupt
207 * on the primary side. The function will also mask the said doorbell.
208 */
209void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx)
210{
211 unsigned long mask;
212
213 if (idx >= ndev->max_cbs || !ndev->db_cb[idx].callback)
214 return;
215
216 mask = readw(ndev->reg_ofs.ldb_mask);
217 set_bit(idx * ndev->bits_per_vector, &mask);
218 writew(mask, ndev->reg_ofs.ldb_mask);
219
220 tasklet_disable(&ndev->db_cb[idx].irq_work);
221
222 ndev->db_cb[idx].callback = NULL;
223}
224
225/**
226 * ntb_find_transport() - find the transport pointer
227 * @transport: pointer to pci device
228 *
229 * Given the pci device pointer, return the transport pointer passed in when
230 * the transport attached when it was inited.
231 *
232 * RETURNS: pointer to transport.
233 */
234void *ntb_find_transport(struct pci_dev *pdev)
235{
236 struct ntb_device *ndev = pci_get_drvdata(pdev);
237 return ndev->ntb_transport;
238}
239
240/**
241 * ntb_register_transport() - Register NTB transport with NTB HW driver
242 * @transport: transport identifier
243 *
244 * This function allows a transport to reserve the hardware driver for
245 * NTB usage.
246 *
247 * RETURNS: pointer to ntb_device, NULL on error.
248 */
249struct ntb_device *ntb_register_transport(struct pci_dev *pdev, void *transport)
250{
251 struct ntb_device *ndev = pci_get_drvdata(pdev);
252
253 if (ndev->ntb_transport)
254 return NULL;
255
256 ndev->ntb_transport = transport;
257 return ndev;
258}
259
260/**
261 * ntb_unregister_transport() - Unregister the transport with the NTB HW driver
262 * @ndev - ntb_device of the transport to be freed
263 *
264 * This function unregisters the transport from the HW driver and performs any
265 * necessary cleanups.
266 */
267void ntb_unregister_transport(struct ntb_device *ndev)
268{
269 int i;
270
271 if (!ndev->ntb_transport)
272 return;
273
274 for (i = 0; i < ndev->max_cbs; i++)
275 ntb_unregister_db_callback(ndev, i);
276
277 ntb_unregister_event_callback(ndev);
278 ndev->ntb_transport = NULL;
279}
280
281/**
282 * ntb_write_local_spad() - write to the secondary scratchpad register
283 * @ndev: pointer to ntb_device instance
284 * @idx: index to the scratchpad register, 0 based
285 * @val: the data value to put into the register
286 *
287 * This function allows writing of a 32bit value to the indexed scratchpad
288 * register. This writes over the data mirrored to the local scratchpad register
289 * by the remote system.
290 *
291 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
292 */
293int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
294{
295 if (idx >= ndev->limits.max_spads)
296 return -EINVAL;
297
298 dev_dbg(&ndev->pdev->dev, "Writing %x to local scratch pad index %d\n",
299 val, idx);
300 writel(val, ndev->reg_ofs.spad_read + idx * 4);
301
302 return 0;
303}
304
305/**
306 * ntb_read_local_spad() - read from the primary scratchpad register
307 * @ndev: pointer to ntb_device instance
308 * @idx: index to scratchpad register, 0 based
309 * @val: pointer to 32bit integer for storing the register value
310 *
311 * This function allows reading of the 32bit scratchpad register on
312 * the primary (internal) side. This allows the local system to read data
313 * written and mirrored to the scratchpad register by the remote system.
314 *
315 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
316 */
317int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
318{
319 if (idx >= ndev->limits.max_spads)
320 return -EINVAL;
321
322 *val = readl(ndev->reg_ofs.spad_write + idx * 4);
323 dev_dbg(&ndev->pdev->dev,
324 "Reading %x from local scratch pad index %d\n", *val, idx);
325
326 return 0;
327}
328
329/**
330 * ntb_write_remote_spad() - write to the secondary scratchpad register
331 * @ndev: pointer to ntb_device instance
332 * @idx: index to the scratchpad register, 0 based
333 * @val: the data value to put into the register
334 *
335 * This function allows writing of a 32bit value to the indexed scratchpad
336 * register. The register resides on the secondary (external) side. This allows
337 * the local system to write data to be mirrored to the remote systems
338 * scratchpad register.
339 *
340 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
341 */
342int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
343{
344 if (idx >= ndev->limits.max_spads)
345 return -EINVAL;
346
347 dev_dbg(&ndev->pdev->dev, "Writing %x to remote scratch pad index %d\n",
348 val, idx);
349 writel(val, ndev->reg_ofs.spad_write + idx * 4);
350
351 return 0;
352}
353
354/**
355 * ntb_read_remote_spad() - read from the primary scratchpad register
356 * @ndev: pointer to ntb_device instance
357 * @idx: index to scratchpad register, 0 based
358 * @val: pointer to 32bit integer for storing the register value
359 *
360 * This function allows reading of the 32bit scratchpad register on
361 * the primary (internal) side. This alloows the local system to read the data
362 * it wrote to be mirrored on the remote system.
363 *
364 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
365 */
366int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
367{
368 if (idx >= ndev->limits.max_spads)
369 return -EINVAL;
370
371 *val = readl(ndev->reg_ofs.spad_read + idx * 4);
372 dev_dbg(&ndev->pdev->dev,
373 "Reading %x from remote scratch pad index %d\n", *val, idx);
374
375 return 0;
376}
377
378/**
379 * ntb_get_mw_base() - get addr for the NTB memory window
380 * @ndev: pointer to ntb_device instance
381 * @mw: memory window number
382 *
383 * This function provides the base address of the memory window specified.
384 *
385 * RETURNS: address, or NULL on error.
386 */
387resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw)
388{
389 if (mw >= ntb_max_mw(ndev))
390 return 0;
391
392 return pci_resource_start(ndev->pdev, MW_TO_BAR(mw));
393}
394
395/**
396 * ntb_get_mw_vbase() - get virtual addr for the NTB memory window
397 * @ndev: pointer to ntb_device instance
398 * @mw: memory window number
399 *
400 * This function provides the base virtual address of the memory window
401 * specified.
402 *
403 * RETURNS: pointer to virtual address, or NULL on error.
404 */
405void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw)
406{
407 if (mw >= ntb_max_mw(ndev))
408 return NULL;
409
410 return ndev->mw[mw].vbase;
411}
412
413/**
414 * ntb_get_mw_size() - return size of NTB memory window
415 * @ndev: pointer to ntb_device instance
416 * @mw: memory window number
417 *
418 * This function provides the physical size of the memory window specified
419 *
420 * RETURNS: the size of the memory window or zero on error
421 */
422u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw)
423{
424 if (mw >= ntb_max_mw(ndev))
425 return 0;
426
427 return ndev->mw[mw].bar_sz;
428}
429
430/**
431 * ntb_set_mw_addr - set the memory window address
432 * @ndev: pointer to ntb_device instance
433 * @mw: memory window number
434 * @addr: base address for data
435 *
436 * This function sets the base physical address of the memory window. This
437 * memory address is where data from the remote system will be transfered into
438 * or out of depending on how the transport is configured.
439 */
440void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr)
441{
442 if (mw >= ntb_max_mw(ndev))
443 return;
444
445 dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr,
446 MW_TO_BAR(mw));
447
448 ndev->mw[mw].phys_addr = addr;
449
450 switch (MW_TO_BAR(mw)) {
451 case NTB_BAR_23:
452 writeq(addr, ndev->reg_ofs.bar2_xlat);
453 break;
454 case NTB_BAR_45:
455 writeq(addr, ndev->reg_ofs.bar4_xlat);
456 break;
457 }
458}
459
460/**
461 * ntb_ring_doorbell() - Set the doorbell on the secondary/external side
462 * @ndev: pointer to ntb_device instance
463 * @db: doorbell to ring
464 *
465 * This function allows triggering of a doorbell on the secondary/external
466 * side that will initiate an interrupt on the remote host
467 *
468 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
469 */
470void ntb_ring_doorbell(struct ntb_device *ndev, unsigned int db)
471{
472 dev_dbg(&ndev->pdev->dev, "%s: ringing doorbell %d\n", __func__, db);
473
474 if (ndev->hw_type == BWD_HW)
475 writeq((u64) 1 << db, ndev->reg_ofs.rdb);
476 else
477 writew(((1 << ndev->bits_per_vector) - 1) <<
478 (db * ndev->bits_per_vector), ndev->reg_ofs.rdb);
479}
480
481static void bwd_recover_link(struct ntb_device *ndev)
482{
483 u32 status;
484
485 /* Driver resets the NTB ModPhy lanes - magic! */
486 writeb(0xe0, ndev->reg_base + BWD_MODPHY_PCSREG6);
487 writeb(0x40, ndev->reg_base + BWD_MODPHY_PCSREG4);
488 writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG4);
489 writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG6);
490
491 /* Driver waits 100ms to allow the NTB ModPhy to settle */
492 msleep(100);
493
494 /* Clear AER Errors, write to clear */
495 status = readl(ndev->reg_base + BWD_ERRCORSTS_OFFSET);
496 dev_dbg(&ndev->pdev->dev, "ERRCORSTS = %x\n", status);
497 status &= PCI_ERR_COR_REP_ROLL;
498 writel(status, ndev->reg_base + BWD_ERRCORSTS_OFFSET);
499
500 /* Clear unexpected electrical idle event in LTSSM, write to clear */
501 status = readl(ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
502 dev_dbg(&ndev->pdev->dev, "LTSSMERRSTS0 = %x\n", status);
503 status |= BWD_LTSSMERRSTS0_UNEXPECTEDEI;
504 writel(status, ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
505
506 /* Clear DeSkew Buffer error, write to clear */
507 status = readl(ndev->reg_base + BWD_DESKEWSTS_OFFSET);
508 dev_dbg(&ndev->pdev->dev, "DESKEWSTS = %x\n", status);
509 status |= BWD_DESKEWSTS_DBERR;
510 writel(status, ndev->reg_base + BWD_DESKEWSTS_OFFSET);
511
512 status = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
513 dev_dbg(&ndev->pdev->dev, "IBSTERRRCRVSTS0 = %x\n", status);
514 status &= BWD_IBIST_ERR_OFLOW;
515 writel(status, ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
516
517 /* Releases the NTB state machine to allow the link to retrain */
518 status = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
519 dev_dbg(&ndev->pdev->dev, "LTSSMSTATEJMP = %x\n", status);
520 status &= ~BWD_LTSSMSTATEJMP_FORCEDETECT;
521 writel(status, ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
522}
523
524static void ntb_link_event(struct ntb_device *ndev, int link_state)
525{
526 unsigned int event;
527
528 if (ndev->link_status == link_state)
529 return;
530
531 if (link_state == NTB_LINK_UP) {
532 u16 status;
533
534 dev_info(&ndev->pdev->dev, "Link Up\n");
535 ndev->link_status = NTB_LINK_UP;
536 event = NTB_EVENT_HW_LINK_UP;
537
538 if (ndev->hw_type == BWD_HW ||
539 ndev->conn_type == NTB_CONN_TRANSPARENT)
540 status = readw(ndev->reg_ofs.lnk_stat);
541 else {
542 int rc = pci_read_config_word(ndev->pdev,
543 SNB_LINK_STATUS_OFFSET,
544 &status);
545 if (rc)
546 return;
547 }
548
549 ndev->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4;
550 ndev->link_speed = (status & NTB_LINK_SPEED_MASK);
551 dev_info(&ndev->pdev->dev, "Link Width %d, Link Speed %d\n",
552 ndev->link_width, ndev->link_speed);
553 } else {
554 dev_info(&ndev->pdev->dev, "Link Down\n");
555 ndev->link_status = NTB_LINK_DOWN;
556 event = NTB_EVENT_HW_LINK_DOWN;
557 /* Don't modify link width/speed, we need it in link recovery */
558 }
559
560 /* notify the upper layer if we have an event change */
561 if (ndev->event_cb)
562 ndev->event_cb(ndev->ntb_transport, event);
563}
564
565static int ntb_link_status(struct ntb_device *ndev)
566{
567 int link_state;
568
569 if (ndev->hw_type == BWD_HW) {
570 u32 ntb_cntl;
571
572 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
573 if (ntb_cntl & BWD_CNTL_LINK_DOWN)
574 link_state = NTB_LINK_DOWN;
575 else
576 link_state = NTB_LINK_UP;
577 } else {
578 u16 status;
579 int rc;
580
581 rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET,
582 &status);
583 if (rc)
584 return rc;
585
586 if (status & NTB_LINK_STATUS_ACTIVE)
587 link_state = NTB_LINK_UP;
588 else
589 link_state = NTB_LINK_DOWN;
590 }
591
592 ntb_link_event(ndev, link_state);
593
594 return 0;
595}
596
597static void bwd_link_recovery(struct work_struct *work)
598{
599 struct ntb_device *ndev = container_of(work, struct ntb_device,
600 lr_timer.work);
601 u32 status32;
602
603 bwd_recover_link(ndev);
604 /* There is a potential race between the 2 NTB devices recovering at the
605 * same time. If the times are the same, the link will not recover and
606 * the driver will be stuck in this loop forever. Add a random interval
607 * to the recovery time to prevent this race.
608 */
609 msleep(BWD_LINK_RECOVERY_TIME + prandom_u32() % BWD_LINK_RECOVERY_TIME);
610
611 status32 = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
612 if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT)
613 goto retry;
614
615 status32 = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
616 if (status32 & BWD_IBIST_ERR_OFLOW)
617 goto retry;
618
619 status32 = readl(ndev->reg_ofs.lnk_cntl);
620 if (!(status32 & BWD_CNTL_LINK_DOWN)) {
621 unsigned char speed, width;
622 u16 status16;
623
624 status16 = readw(ndev->reg_ofs.lnk_stat);
625 width = (status16 & NTB_LINK_WIDTH_MASK) >> 4;
626 speed = (status16 & NTB_LINK_SPEED_MASK);
627 if (ndev->link_width != width || ndev->link_speed != speed)
628 goto retry;
629 }
630
631 schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
632 return;
633
634retry:
635 schedule_delayed_work(&ndev->lr_timer, NTB_HB_TIMEOUT);
636}
637
638/* BWD doesn't have link status interrupt, poll on that platform */
639static void bwd_link_poll(struct work_struct *work)
640{
641 struct ntb_device *ndev = container_of(work, struct ntb_device,
642 hb_timer.work);
643 unsigned long ts = jiffies;
644
645 /* If we haven't gotten an interrupt in a while, check the BWD link
646 * status bit
647 */
648 if (ts > ndev->last_ts + NTB_HB_TIMEOUT) {
649 int rc = ntb_link_status(ndev);
650 if (rc)
651 dev_err(&ndev->pdev->dev,
652 "Error determining link status\n");
653
654 /* Check to see if a link error is the cause of the link down */
655 if (ndev->link_status == NTB_LINK_DOWN) {
656 u32 status32 = readl(ndev->reg_base +
657 BWD_LTSSMSTATEJMP_OFFSET);
658 if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT) {
659 schedule_delayed_work(&ndev->lr_timer, 0);
660 return;
661 }
662 }
663 }
664
665 schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
666}
667
668static int ntb_xeon_setup(struct ntb_device *ndev)
669{
670 int rc;
671 u8 val;
672
673 ndev->hw_type = SNB_HW;
674
675 rc = pci_read_config_byte(ndev->pdev, NTB_PPD_OFFSET, &val);
676 if (rc)
677 return rc;
678
679 if (val & SNB_PPD_DEV_TYPE)
680 ndev->dev_type = NTB_DEV_USD;
681 else
682 ndev->dev_type = NTB_DEV_DSD;
683
684 switch (val & SNB_PPD_CONN_TYPE) {
685 case NTB_CONN_B2B:
686 dev_info(&ndev->pdev->dev, "Conn Type = B2B\n");
687 ndev->conn_type = NTB_CONN_B2B;
688 ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
689 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
690 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
691 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
692 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
693 ndev->limits.max_spads = SNB_MAX_B2B_SPADS;
694
695 /* There is a Xeon hardware errata related to writes to
696 * SDOORBELL or B2BDOORBELL in conjunction with inbound access
697 * to NTB MMIO Space, which may hang the system. To workaround
698 * this use the second memory window to access the interrupt and
699 * scratch pad registers on the remote system.
700 */
701 if (xeon_errata_workaround) {
702 if (!ndev->mw[1].bar_sz)
703 return -EINVAL;
704
705 ndev->limits.max_mw = SNB_ERRATA_MAX_MW;
706 ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
707 ndev->reg_ofs.spad_write = ndev->mw[1].vbase +
708 SNB_SPAD_OFFSET;
709 ndev->reg_ofs.rdb = ndev->mw[1].vbase +
710 SNB_PDOORBELL_OFFSET;
711
712 /* Set the Limit register to 4k, the minimum size, to
713 * prevent an illegal access
714 */
715 writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base +
716 SNB_PBAR4LMT_OFFSET);
717 /* HW errata on the Limit registers. They can only be
718 * written when the base register is 4GB aligned and
719 * < 32bit. This should already be the case based on
720 * the driver defaults, but write the Limit registers
721 * first just in case.
722 */
723 } else {
724 ndev->limits.max_mw = SNB_MAX_MW;
725
726 /* HW Errata on bit 14 of b2bdoorbell register. Writes
727 * will not be mirrored to the remote system. Shrink
728 * the number of bits by one, since bit 14 is the last
729 * bit.
730 */
731 ndev->limits.max_db_bits = SNB_MAX_DB_BITS - 1;
732 ndev->reg_ofs.spad_write = ndev->reg_base +
733 SNB_B2B_SPAD_OFFSET;
734 ndev->reg_ofs.rdb = ndev->reg_base +
735 SNB_B2B_DOORBELL_OFFSET;
736
737 /* Disable the Limit register, just incase it is set to
738 * something silly
739 */
740 writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET);
741 /* HW errata on the Limit registers. They can only be
742 * written when the base register is 4GB aligned and
743 * < 32bit. This should already be the case based on
744 * the driver defaults, but write the Limit registers
745 * first just in case.
746 */
747 }
748
749 /* The Xeon errata workaround requires setting SBAR Base
750 * addresses to known values, so that the PBAR XLAT can be
751 * pointed at SBAR0 of the remote system.
752 */
753 if (ndev->dev_type == NTB_DEV_USD) {
754 writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
755 SNB_PBAR2XLAT_OFFSET);
756 if (xeon_errata_workaround)
757 writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
758 SNB_PBAR4XLAT_OFFSET);
759 else {
760 writeq(SNB_MBAR45_DSD_ADDR, ndev->reg_base +
761 SNB_PBAR4XLAT_OFFSET);
762 /* B2B_XLAT_OFFSET is a 64bit register, but can
763 * only take 32bit writes
764 */
765 writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,
766 ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
767 writel(SNB_MBAR01_DSD_ADDR >> 32,
768 ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
769 }
770
771 writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
772 SNB_SBAR0BASE_OFFSET);
773 writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
774 SNB_SBAR2BASE_OFFSET);
775 writeq(SNB_MBAR45_USD_ADDR, ndev->reg_base +
776 SNB_SBAR4BASE_OFFSET);
777 } else {
778 writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
779 SNB_PBAR2XLAT_OFFSET);
780 if (xeon_errata_workaround)
781 writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
782 SNB_PBAR4XLAT_OFFSET);
783 else {
784 writeq(SNB_MBAR45_USD_ADDR, ndev->reg_base +
785 SNB_PBAR4XLAT_OFFSET);
786 /* B2B_XLAT_OFFSET is a 64bit register, but can
787 * only take 32bit writes
788 */
789 writel(SNB_MBAR01_USD_ADDR & 0xffffffff,
790 ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
791 writel(SNB_MBAR01_USD_ADDR >> 32,
792 ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
793 }
794 writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
795 SNB_SBAR0BASE_OFFSET);
796 writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
797 SNB_SBAR2BASE_OFFSET);
798 writeq(SNB_MBAR45_DSD_ADDR, ndev->reg_base +
799 SNB_SBAR4BASE_OFFSET);
800 }
801 break;
802 case NTB_CONN_RP:
803 dev_info(&ndev->pdev->dev, "Conn Type = RP\n");
804 ndev->conn_type = NTB_CONN_RP;
805
806 if (xeon_errata_workaround) {
807 dev_err(&ndev->pdev->dev,
808 "NTB-RP disabled due to hardware errata. To disregard this warning and potentially lock-up the system, add the parameter 'xeon_errata_workaround=0'.\n");
809 return -EINVAL;
810 }
811
812 /* Scratch pads need to have exclusive access from the primary
813 * or secondary side. Halve the num spads so that each side can
814 * have an equal amount.
815 */
816 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
817 ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
818 /* Note: The SDOORBELL is the cause of the errata. You REALLY
819 * don't want to touch it.
820 */
821 ndev->reg_ofs.rdb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
822 ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
823 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
824 /* Offset the start of the spads to correspond to whether it is
825 * primary or secondary
826 */
827 ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET +
828 ndev->limits.max_spads * 4;
829 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
830 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
831 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
832 ndev->limits.max_mw = SNB_MAX_MW;
833 break;
834 case NTB_CONN_TRANSPARENT:
835 dev_info(&ndev->pdev->dev, "Conn Type = TRANSPARENT\n");
836 ndev->conn_type = NTB_CONN_TRANSPARENT;
837 /* Scratch pads need to have exclusive access from the primary
838 * or secondary side. Halve the num spads so that each side can
839 * have an equal amount.
840 */
841 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
842 ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
843 ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
844 ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
845 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET;
846 ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET;
847 /* Offset the start of the spads to correspond to whether it is
848 * primary or secondary
849 */
850 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET +
851 ndev->limits.max_spads * 4;
852 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_PBAR2XLAT_OFFSET;
853 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_PBAR4XLAT_OFFSET;
854
855 ndev->limits.max_mw = SNB_MAX_MW;
856 break;
857 default:
858 /* Most likely caused by the remote NTB-RP device not being
859 * configured
860 */
861 dev_err(&ndev->pdev->dev, "Unknown PPD %x\n", val);
862 return -EINVAL;
863 }
864
865 ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET;
866 ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET;
867 ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET;
868
869 ndev->limits.msix_cnt = SNB_MSIX_CNT;
870 ndev->bits_per_vector = SNB_DB_BITS_PER_VEC;
871
872 return 0;
873}
874
875static int ntb_bwd_setup(struct ntb_device *ndev)
876{
877 int rc;
878 u32 val;
879
880 ndev->hw_type = BWD_HW;
881
882 rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &val);
883 if (rc)
884 return rc;
885
886 switch ((val & BWD_PPD_CONN_TYPE) >> 8) {
887 case NTB_CONN_B2B:
888 ndev->conn_type = NTB_CONN_B2B;
889 break;
890 case NTB_CONN_RP:
891 default:
892 dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n");
893 return -EINVAL;
894 }
895
896 if (val & BWD_PPD_DEV_TYPE)
897 ndev->dev_type = NTB_DEV_DSD;
898 else
899 ndev->dev_type = NTB_DEV_USD;
900
901 /* Initiate PCI-E link training */
902 rc = pci_write_config_dword(ndev->pdev, NTB_PPD_OFFSET,
903 val | BWD_PPD_INIT_LINK);
904 if (rc)
905 return rc;
906
907 ndev->reg_ofs.ldb = ndev->reg_base + BWD_PDOORBELL_OFFSET;
908 ndev->reg_ofs.ldb_mask = ndev->reg_base + BWD_PDBMSK_OFFSET;
909 ndev->reg_ofs.rdb = ndev->reg_base + BWD_B2B_DOORBELL_OFFSET;
910 ndev->reg_ofs.bar2_xlat = ndev->reg_base + BWD_SBAR2XLAT_OFFSET;
911 ndev->reg_ofs.bar4_xlat = ndev->reg_base + BWD_SBAR4XLAT_OFFSET;
912 ndev->reg_ofs.lnk_cntl = ndev->reg_base + BWD_NTBCNTL_OFFSET;
913 ndev->reg_ofs.lnk_stat = ndev->reg_base + BWD_LINK_STATUS_OFFSET;
914 ndev->reg_ofs.spad_read = ndev->reg_base + BWD_SPAD_OFFSET;
915 ndev->reg_ofs.spad_write = ndev->reg_base + BWD_B2B_SPAD_OFFSET;
916 ndev->reg_ofs.spci_cmd = ndev->reg_base + BWD_PCICMD_OFFSET;
917 ndev->limits.max_mw = BWD_MAX_MW;
918 ndev->limits.max_spads = BWD_MAX_SPADS;
919 ndev->limits.max_db_bits = BWD_MAX_DB_BITS;
920 ndev->limits.msix_cnt = BWD_MSIX_CNT;
921 ndev->bits_per_vector = BWD_DB_BITS_PER_VEC;
922
923 /* Since bwd doesn't have a link interrupt, setup a poll timer */
924 INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_poll);
925 INIT_DELAYED_WORK(&ndev->lr_timer, bwd_link_recovery);
926 schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
927
928 return 0;
929}
930
931static int ntb_device_setup(struct ntb_device *ndev)
932{
933 int rc;
934
935 switch (ndev->pdev->device) {
936 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
937 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
938 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
939 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
940 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
941 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
942 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
943 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
944 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
945 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
946 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
947 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
948 rc = ntb_xeon_setup(ndev);
949 break;
950 case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
951 rc = ntb_bwd_setup(ndev);
952 break;
953 default:
954 rc = -ENODEV;
955 }
956
957 if (rc)
958 return rc;
959
960 dev_info(&ndev->pdev->dev, "Device Type = %s\n",
961 ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP");
962
963 if (ndev->conn_type == NTB_CONN_B2B)
964 /* Enable Bus Master and Memory Space on the secondary side */
965 writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
966 ndev->reg_ofs.spci_cmd);
967
968 return 0;
969}
970
971static void ntb_device_free(struct ntb_device *ndev)
972{
973 if (ndev->hw_type == BWD_HW) {
974 cancel_delayed_work_sync(&ndev->hb_timer);
975 cancel_delayed_work_sync(&ndev->lr_timer);
976 }
977}
978
979static irqreturn_t bwd_callback_msix_irq(int irq, void *data)
980{
981 struct ntb_db_cb *db_cb = data;
982 struct ntb_device *ndev = db_cb->ndev;
983 unsigned long mask;
984
985 dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
986 db_cb->db_num);
987
988 mask = readw(ndev->reg_ofs.ldb_mask);
989 set_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
990 writew(mask, ndev->reg_ofs.ldb_mask);
991
992 tasklet_schedule(&db_cb->irq_work);
993
994 /* No need to check for the specific HB irq, any interrupt means
995 * we're connected.
996 */
997 ndev->last_ts = jiffies;
998
999 writeq((u64) 1 << db_cb->db_num, ndev->reg_ofs.ldb);
1000
1001 return IRQ_HANDLED;
1002}
1003
1004static irqreturn_t xeon_callback_msix_irq(int irq, void *data)
1005{
1006 struct ntb_db_cb *db_cb = data;
1007 struct ntb_device *ndev = db_cb->ndev;
1008 unsigned long mask;
1009
1010 dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
1011 db_cb->db_num);
1012
1013 mask = readw(ndev->reg_ofs.ldb_mask);
1014 set_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
1015 writew(mask, ndev->reg_ofs.ldb_mask);
1016
1017 tasklet_schedule(&db_cb->irq_work);
1018
1019 /* On Sandybridge, there are 16 bits in the interrupt register
1020 * but only 4 vectors. So, 5 bits are assigned to the first 3
1021 * vectors, with the 4th having a single bit for link
1022 * interrupts.
1023 */
1024 writew(((1 << ndev->bits_per_vector) - 1) <<
1025 (db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.ldb);
1026
1027 return IRQ_HANDLED;
1028}
1029
1030/* Since we do not have a HW doorbell in BWD, this is only used in JF/JT */
1031static irqreturn_t xeon_event_msix_irq(int irq, void *dev)
1032{
1033 struct ntb_device *ndev = dev;
1034 int rc;
1035
1036 dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq);
1037
1038 rc = ntb_link_status(ndev);
1039 if (rc)
1040 dev_err(&ndev->pdev->dev, "Error determining link status\n");
1041
1042 /* bit 15 is always the link bit */
1043 writew(1 << SNB_LINK_DB, ndev->reg_ofs.ldb);
1044
1045 return IRQ_HANDLED;
1046}
1047
1048static irqreturn_t ntb_interrupt(int irq, void *dev)
1049{
1050 struct ntb_device *ndev = dev;
1051 unsigned int i = 0;
1052
1053 if (ndev->hw_type == BWD_HW) {
1054 u64 ldb = readq(ndev->reg_ofs.ldb);
1055
1056 dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %Lx\n", irq, ldb);
1057
1058 while (ldb) {
1059 i = __ffs(ldb);
1060 ldb &= ldb - 1;
1061 bwd_callback_msix_irq(irq, &ndev->db_cb[i]);
1062 }
1063 } else {
1064 u16 ldb = readw(ndev->reg_ofs.ldb);
1065
1066 dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %x\n", irq, ldb);
1067
1068 if (ldb & SNB_DB_HW_LINK) {
1069 xeon_event_msix_irq(irq, dev);
1070 ldb &= ~SNB_DB_HW_LINK;
1071 }
1072
1073 while (ldb) {
1074 i = __ffs(ldb);
1075 ldb &= ldb - 1;
1076 xeon_callback_msix_irq(irq, &ndev->db_cb[i]);
1077 }
1078 }
1079
1080 return IRQ_HANDLED;
1081}
1082
1083static int ntb_setup_snb_msix(struct ntb_device *ndev, int msix_entries)
1084{
1085 struct pci_dev *pdev = ndev->pdev;
1086 struct msix_entry *msix;
1087 int rc, i;
1088
1089 if (msix_entries < ndev->limits.msix_cnt)
1090 return -ENOSPC;
1091
1092 rc = pci_enable_msix_exact(pdev, ndev->msix_entries, msix_entries);
1093 if (rc < 0)
1094 return rc;
1095
1096 for (i = 0; i < msix_entries; i++) {
1097 msix = &ndev->msix_entries[i];
1098 WARN_ON(!msix->vector);
1099
1100 if (i == msix_entries - 1) {
1101 rc = request_irq(msix->vector,
1102 xeon_event_msix_irq, 0,
1103 "ntb-event-msix", ndev);
1104 if (rc)
1105 goto err;
1106 } else {
1107 rc = request_irq(msix->vector,
1108 xeon_callback_msix_irq, 0,
1109 "ntb-callback-msix",
1110 &ndev->db_cb[i]);
1111 if (rc)
1112 goto err;
1113 }
1114 }
1115
1116 ndev->num_msix = msix_entries;
1117 ndev->max_cbs = msix_entries - 1;
1118
1119 return 0;
1120
1121err:
1122 while (--i >= 0) {
1123 /* Code never reaches here for entry nr 'ndev->num_msix - 1' */
1124 msix = &ndev->msix_entries[i];
1125 free_irq(msix->vector, &ndev->db_cb[i]);
1126 }
1127
1128 pci_disable_msix(pdev);
1129 ndev->num_msix = 0;
1130
1131 return rc;
1132}
1133
1134static int ntb_setup_bwd_msix(struct ntb_device *ndev, int msix_entries)
1135{
1136 struct pci_dev *pdev = ndev->pdev;
1137 struct msix_entry *msix;
1138 int rc, i;
1139
1140 msix_entries = pci_enable_msix_range(pdev, ndev->msix_entries,
1141 1, msix_entries);
1142 if (msix_entries < 0)
1143 return msix_entries;
1144
1145 for (i = 0; i < msix_entries; i++) {
1146 msix = &ndev->msix_entries[i];
1147 WARN_ON(!msix->vector);
1148
1149 rc = request_irq(msix->vector, bwd_callback_msix_irq, 0,
1150 "ntb-callback-msix", &ndev->db_cb[i]);
1151 if (rc)
1152 goto err;
1153 }
1154
1155 ndev->num_msix = msix_entries;
1156 ndev->max_cbs = msix_entries;
1157
1158 return 0;
1159
1160err:
1161 while (--i >= 0)
1162 free_irq(msix->vector, &ndev->db_cb[i]);
1163
1164 pci_disable_msix(pdev);
1165 ndev->num_msix = 0;
1166
1167 return rc;
1168}
1169
1170static int ntb_setup_msix(struct ntb_device *ndev)
1171{
1172 struct pci_dev *pdev = ndev->pdev;
1173 int msix_entries;
1174 int rc, i;
1175
1176 msix_entries = pci_msix_vec_count(pdev);
1177 if (msix_entries < 0) {
1178 rc = msix_entries;
1179 goto err;
1180 } else if (msix_entries > ndev->limits.msix_cnt) {
1181 rc = -EINVAL;
1182 goto err;
1183 }
1184
1185 ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries,
1186 GFP_KERNEL);
1187 if (!ndev->msix_entries) {
1188 rc = -ENOMEM;
1189 goto err;
1190 }
1191
1192 for (i = 0; i < msix_entries; i++)
1193 ndev->msix_entries[i].entry = i;
1194
1195 if (ndev->hw_type == BWD_HW)
1196 rc = ntb_setup_bwd_msix(ndev, msix_entries);
1197 else
1198 rc = ntb_setup_snb_msix(ndev, msix_entries);
1199 if (rc)
1200 goto err1;
1201
1202 return 0;
1203
1204err1:
1205 kfree(ndev->msix_entries);
1206err:
1207 dev_err(&pdev->dev, "Error allocating MSI-X interrupt\n");
1208 return rc;
1209}
1210
1211static int ntb_setup_msi(struct ntb_device *ndev)
1212{
1213 struct pci_dev *pdev = ndev->pdev;
1214 int rc;
1215
1216 rc = pci_enable_msi(pdev);
1217 if (rc)
1218 return rc;
1219
1220 rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev);
1221 if (rc) {
1222 pci_disable_msi(pdev);
1223 dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
1224 return rc;
1225 }
1226
1227 return 0;
1228}
1229
1230static int ntb_setup_intx(struct ntb_device *ndev)
1231{
1232 struct pci_dev *pdev = ndev->pdev;
1233 int rc;
1234
1235 pci_msi_off(pdev);
1236
1237 /* Verify intx is enabled */
1238 pci_intx(pdev, 1);
1239
1240 rc = request_irq(pdev->irq, ntb_interrupt, IRQF_SHARED, "ntb-intx",
1241 ndev);
1242 if (rc)
1243 return rc;
1244
1245 return 0;
1246}
1247
1248static int ntb_setup_interrupts(struct ntb_device *ndev)
1249{
1250 int rc;
1251
1252 /* On BWD, disable all interrupts. On SNB, disable all but Link
1253 * Interrupt. The rest will be unmasked as callbacks are registered.
1254 */
1255 if (ndev->hw_type == BWD_HW)
1256 writeq(~0, ndev->reg_ofs.ldb_mask);
1257 else {
1258 u16 var = 1 << SNB_LINK_DB;
1259 writew(~var, ndev->reg_ofs.ldb_mask);
1260 }
1261
1262 rc = ntb_setup_msix(ndev);
1263 if (!rc)
1264 goto done;
1265
1266 ndev->bits_per_vector = 1;
1267 ndev->max_cbs = ndev->limits.max_db_bits;
1268
1269 rc = ntb_setup_msi(ndev);
1270 if (!rc)
1271 goto done;
1272
1273 rc = ntb_setup_intx(ndev);
1274 if (rc) {
1275 dev_err(&ndev->pdev->dev, "no usable interrupts\n");
1276 return rc;
1277 }
1278
1279done:
1280 return 0;
1281}
1282
1283static void ntb_free_interrupts(struct ntb_device *ndev)
1284{
1285 struct pci_dev *pdev = ndev->pdev;
1286
1287 /* mask interrupts */
1288 if (ndev->hw_type == BWD_HW)
1289 writeq(~0, ndev->reg_ofs.ldb_mask);
1290 else
1291 writew(~0, ndev->reg_ofs.ldb_mask);
1292
1293 if (ndev->num_msix) {
1294 struct msix_entry *msix;
1295 u32 i;
1296
1297 for (i = 0; i < ndev->num_msix; i++) {
1298 msix = &ndev->msix_entries[i];
1299 if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1)
1300 free_irq(msix->vector, ndev);
1301 else
1302 free_irq(msix->vector, &ndev->db_cb[i]);
1303 }
1304 pci_disable_msix(pdev);
1305 kfree(ndev->msix_entries);
1306 } else {
1307 free_irq(pdev->irq, ndev);
1308
1309 if (pci_dev_msi_enabled(pdev))
1310 pci_disable_msi(pdev);
1311 }
1312}
1313
1314static int ntb_create_callbacks(struct ntb_device *ndev)
1315{
1316 int i;
1317
1318 /* Chicken-egg issue. We won't know how many callbacks are necessary
1319 * until we see how many MSI-X vectors we get, but these pointers need
1320 * to be passed into the MSI-X register function. So, we allocate the
1321 * max, knowing that they might not all be used, to work around this.
1322 */
1323 ndev->db_cb = kcalloc(ndev->limits.max_db_bits,
1324 sizeof(struct ntb_db_cb),
1325 GFP_KERNEL);
1326 if (!ndev->db_cb)
1327 return -ENOMEM;
1328
1329 for (i = 0; i < ndev->limits.max_db_bits; i++) {
1330 ndev->db_cb[i].db_num = i;
1331 ndev->db_cb[i].ndev = ndev;
1332 }
1333
1334 return 0;
1335}
1336
1337static void ntb_free_callbacks(struct ntb_device *ndev)
1338{
1339 int i;
1340
1341 for (i = 0; i < ndev->limits.max_db_bits; i++)
1342 ntb_unregister_db_callback(ndev, i);
1343
1344 kfree(ndev->db_cb);
1345}
1346
1347static void ntb_setup_debugfs(struct ntb_device *ndev)
1348{
1349 if (!debugfs_initialized())
1350 return;
1351
1352 if (!debugfs_dir)
1353 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1354
1355 ndev->debugfs_dir = debugfs_create_dir(pci_name(ndev->pdev),
1356 debugfs_dir);
1357}
1358
1359static void ntb_free_debugfs(struct ntb_device *ndev)
1360{
1361 debugfs_remove_recursive(ndev->debugfs_dir);
1362
1363 if (debugfs_dir && simple_empty(debugfs_dir)) {
1364 debugfs_remove_recursive(debugfs_dir);
1365 debugfs_dir = NULL;
1366 }
1367}
1368
1369static void ntb_hw_link_up(struct ntb_device *ndev)
1370{
1371 if (ndev->conn_type == NTB_CONN_TRANSPARENT)
1372 ntb_link_event(ndev, NTB_LINK_UP);
1373 else {
1374 u32 ntb_cntl;
1375
1376 /* Let's bring the NTB link up */
1377 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
1378 ntb_cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK);
1379 ntb_cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP;
1380 ntb_cntl |= NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP;
1381 writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
1382 }
1383}
1384
1385static void ntb_hw_link_down(struct ntb_device *ndev)
1386{
1387 u32 ntb_cntl;
1388
1389 if (ndev->conn_type == NTB_CONN_TRANSPARENT) {
1390 ntb_link_event(ndev, NTB_LINK_DOWN);
1391 return;
1392 }
1393
1394 /* Bring NTB link down */
1395 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
1396 ntb_cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP);
1397 ntb_cntl &= ~(NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP);
1398 ntb_cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;
1399 writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
1400}
1401
1402static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1403{
1404 struct ntb_device *ndev;
1405 int rc, i;
1406
1407 ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL);
1408 if (!ndev)
1409 return -ENOMEM;
1410
1411 ndev->pdev = pdev;
1412 ndev->link_status = NTB_LINK_DOWN;
1413 pci_set_drvdata(pdev, ndev);
1414 ntb_setup_debugfs(ndev);
1415
1416 rc = pci_enable_device(pdev);
1417 if (rc)
1418 goto err;
1419
1420 pci_set_master(ndev->pdev);
1421
1422 rc = pci_request_selected_regions(pdev, NTB_BAR_MASK, KBUILD_MODNAME);
1423 if (rc)
1424 goto err1;
1425
1426 ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO);
1427 if (!ndev->reg_base) {
1428 dev_warn(&pdev->dev, "Cannot remap BAR 0\n");
1429 rc = -EIO;
1430 goto err2;
1431 }
1432
1433 for (i = 0; i < NTB_MAX_NUM_MW; i++) {
1434 ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i));
1435 ndev->mw[i].vbase =
1436 ioremap_wc(pci_resource_start(pdev, MW_TO_BAR(i)),
1437 ndev->mw[i].bar_sz);
1438 dev_info(&pdev->dev, "MW %d size %llu\n", i,
1439 (unsigned long long) ndev->mw[i].bar_sz);
1440 if (!ndev->mw[i].vbase) {
1441 dev_warn(&pdev->dev, "Cannot remap BAR %d\n",
1442 MW_TO_BAR(i));
1443 rc = -EIO;
1444 goto err3;
1445 }
1446 }
1447
1448 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1449 if (rc) {
1450 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1451 if (rc)
1452 goto err3;
1453
1454 dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1455 }
1456
1457 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1458 if (rc) {
1459 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1460 if (rc)
1461 goto err3;
1462
1463 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
1464 }
1465
1466 rc = ntb_device_setup(ndev);
1467 if (rc)
1468 goto err3;
1469
1470 rc = ntb_create_callbacks(ndev);
1471 if (rc)
1472 goto err4;
1473
1474 rc = ntb_setup_interrupts(ndev);
1475 if (rc)
1476 goto err5;
1477
1478 /* The scratchpad registers keep the values between rmmod/insmod,
1479 * blast them now
1480 */
1481 for (i = 0; i < ndev->limits.max_spads; i++) {
1482 ntb_write_local_spad(ndev, i, 0);
1483 ntb_write_remote_spad(ndev, i, 0);
1484 }
1485
1486 rc = ntb_transport_init(pdev);
1487 if (rc)
1488 goto err6;
1489
1490 ntb_hw_link_up(ndev);
1491
1492 return 0;
1493
1494err6:
1495 ntb_free_interrupts(ndev);
1496err5:
1497 ntb_free_callbacks(ndev);
1498err4:
1499 ntb_device_free(ndev);
1500err3:
1501 for (i--; i >= 0; i--)
1502 iounmap(ndev->mw[i].vbase);
1503 iounmap(ndev->reg_base);
1504err2:
1505 pci_release_selected_regions(pdev, NTB_BAR_MASK);
1506err1:
1507 pci_disable_device(pdev);
1508err:
1509 ntb_free_debugfs(ndev);
1510 kfree(ndev);
1511
1512 dev_err(&pdev->dev, "Error loading %s module\n", KBUILD_MODNAME);
1513 return rc;
1514}
1515
1516static void ntb_pci_remove(struct pci_dev *pdev)
1517{
1518 struct ntb_device *ndev = pci_get_drvdata(pdev);
1519 int i;
1520
1521 ntb_hw_link_down(ndev);
1522
1523 ntb_transport_free(ndev->ntb_transport);
1524
1525 ntb_free_interrupts(ndev);
1526 ntb_free_callbacks(ndev);
1527 ntb_device_free(ndev);
1528
1529 for (i = 0; i < NTB_MAX_NUM_MW; i++)
1530 iounmap(ndev->mw[i].vbase);
1531
1532 iounmap(ndev->reg_base);
1533 pci_release_selected_regions(pdev, NTB_BAR_MASK);
1534 pci_disable_device(pdev);
1535 ntb_free_debugfs(ndev);
1536 kfree(ndev);
1537}
1538
1539static struct pci_driver ntb_pci_driver = {
1540 .name = KBUILD_MODNAME,
1541 .id_table = ntb_pci_tbl,
1542 .probe = ntb_pci_probe,
1543 .remove = ntb_pci_remove,
1544};
1545module_pci_driver(ntb_pci_driver);