Loading...
Note: File does not exist in v6.13.7.
1/*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
13 *
14 * BSD LICENSE
15 *
16 * Copyright(c) 2012 Intel Corporation. All rights reserved.
17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
18 *
19 * Redistribution and use in source and binary forms, with or without
20 * modification, are permitted provided that the following conditions
21 * are met:
22 *
23 * * Redistributions of source code must retain the above copyright
24 * notice, this list of conditions and the following disclaimer.
25 * * Redistributions in binary form must reproduce the above copy
26 * notice, this list of conditions and the following disclaimer in
27 * the documentation and/or other materials provided with the
28 * distribution.
29 * * Neither the name of Intel Corporation nor the names of its
30 * contributors may be used to endorse or promote products derived
31 * from this software without specific prior written permission.
32 *
33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
44 *
45 * Intel PCIe NTB Linux driver
46 *
47 * Contact Information:
48 * Jon Mason <jon.mason@intel.com>
49 */
50
51#include <linux/debugfs.h>
52#include <linux/delay.h>
53#include <linux/init.h>
54#include <linux/interrupt.h>
55#include <linux/module.h>
56#include <linux/pci.h>
57#include <linux/random.h>
58#include <linux/slab.h>
59#include <linux/ntb.h>
60
61#include "ntb_hw_intel.h"
62
63#define NTB_NAME "ntb_hw_intel"
64#define NTB_DESC "Intel(R) PCI-E Non-Transparent Bridge Driver"
65#define NTB_VER "2.0"
66
67MODULE_DESCRIPTION(NTB_DESC);
68MODULE_VERSION(NTB_VER);
69MODULE_LICENSE("Dual BSD/GPL");
70MODULE_AUTHOR("Intel Corporation");
71
72#define bar0_off(base, bar) ((base) + ((bar) << 2))
73#define bar2_off(base, bar) bar0_off(base, (bar) - 2)
74
75static const struct intel_ntb_reg atom_reg;
76static const struct intel_ntb_alt_reg atom_pri_reg;
77static const struct intel_ntb_alt_reg atom_sec_reg;
78static const struct intel_ntb_alt_reg atom_b2b_reg;
79static const struct intel_ntb_xlat_reg atom_pri_xlat;
80static const struct intel_ntb_xlat_reg atom_sec_xlat;
81static const struct intel_ntb_reg xeon_reg;
82static const struct intel_ntb_alt_reg xeon_pri_reg;
83static const struct intel_ntb_alt_reg xeon_sec_reg;
84static const struct intel_ntb_alt_reg xeon_b2b_reg;
85static const struct intel_ntb_xlat_reg xeon_pri_xlat;
86static const struct intel_ntb_xlat_reg xeon_sec_xlat;
87static struct intel_b2b_addr xeon_b2b_usd_addr;
88static struct intel_b2b_addr xeon_b2b_dsd_addr;
89static const struct ntb_dev_ops intel_ntb_ops;
90
91static const struct file_operations intel_ntb_debugfs_info;
92static struct dentry *debugfs_dir;
93
94static int b2b_mw_idx = -1;
95module_param(b2b_mw_idx, int, 0644);
96MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb. A "
97 "value of zero or positive starts from first mw idx, and a "
98 "negative value starts from last mw idx. Both sides MUST "
99 "set the same value here!");
100
101static unsigned int b2b_mw_share;
102module_param(b2b_mw_share, uint, 0644);
103MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
104 "ntb so that the peer ntb only occupies the first half of "
105 "the mw, so the second half can still be used as a mw. Both "
106 "sides MUST set the same value here!");
107
108module_param_named(xeon_b2b_usd_bar2_addr64,
109 xeon_b2b_usd_addr.bar2_addr64, ullong, 0644);
110MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
111 "XEON B2B USD BAR 2 64-bit address");
112
113module_param_named(xeon_b2b_usd_bar4_addr64,
114 xeon_b2b_usd_addr.bar4_addr64, ullong, 0644);
115MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
116 "XEON B2B USD BAR 4 64-bit address");
117
118module_param_named(xeon_b2b_usd_bar4_addr32,
119 xeon_b2b_usd_addr.bar4_addr32, ullong, 0644);
120MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
121 "XEON B2B USD split-BAR 4 32-bit address");
122
123module_param_named(xeon_b2b_usd_bar5_addr32,
124 xeon_b2b_usd_addr.bar5_addr32, ullong, 0644);
125MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
126 "XEON B2B USD split-BAR 5 32-bit address");
127
128module_param_named(xeon_b2b_dsd_bar2_addr64,
129 xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644);
130MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
131 "XEON B2B DSD BAR 2 64-bit address");
132
133module_param_named(xeon_b2b_dsd_bar4_addr64,
134 xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644);
135MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
136 "XEON B2B DSD BAR 4 64-bit address");
137
138module_param_named(xeon_b2b_dsd_bar4_addr32,
139 xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644);
140MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
141 "XEON B2B DSD split-BAR 4 32-bit address");
142
143module_param_named(xeon_b2b_dsd_bar5_addr32,
144 xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644);
145MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
146 "XEON B2B DSD split-BAR 5 32-bit address");
147
148#ifndef ioread64
149#ifdef readq
150#define ioread64 readq
151#else
152#define ioread64 _ioread64
153static inline u64 _ioread64(void __iomem *mmio)
154{
155 u64 low, high;
156
157 low = ioread32(mmio);
158 high = ioread32(mmio + sizeof(u32));
159 return low | (high << 32);
160}
161#endif
162#endif
163
164#ifndef iowrite64
165#ifdef writeq
166#define iowrite64 writeq
167#else
168#define iowrite64 _iowrite64
169static inline void _iowrite64(u64 val, void __iomem *mmio)
170{
171 iowrite32(val, mmio);
172 iowrite32(val >> 32, mmio + sizeof(u32));
173}
174#endif
175#endif
176
177static inline int pdev_is_atom(struct pci_dev *pdev)
178{
179 switch (pdev->device) {
180 case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
181 return 1;
182 }
183 return 0;
184}
185
186static inline int pdev_is_xeon(struct pci_dev *pdev)
187{
188 switch (pdev->device) {
189 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
190 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
191 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
192 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
193 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
194 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
195 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
196 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
197 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
198 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
199 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
200 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
201 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
202 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
203 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
204 return 1;
205 }
206 return 0;
207}
208
209static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
210{
211 ndev->unsafe_flags = 0;
212 ndev->unsafe_flags_ignore = 0;
213
214 /* Only B2B has a workaround to avoid SDOORBELL */
215 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
216 if (!ntb_topo_is_b2b(ndev->ntb.topo))
217 ndev->unsafe_flags |= NTB_UNSAFE_DB;
218
219 /* No low level workaround to avoid SB01BASE */
220 if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
221 ndev->unsafe_flags |= NTB_UNSAFE_DB;
222 ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
223 }
224}
225
226static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
227 unsigned long flag)
228{
229 return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
230}
231
232static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
233 unsigned long flag)
234{
235 flag &= ndev->unsafe_flags;
236 ndev->unsafe_flags_ignore |= flag;
237
238 return !!flag;
239}
240
241static int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
242{
243 if (idx < 0 || idx >= ndev->mw_count)
244 return -EINVAL;
245 return ndev->reg->mw_bar[idx];
246}
247
248static inline int ndev_db_addr(struct intel_ntb_dev *ndev,
249 phys_addr_t *db_addr, resource_size_t *db_size,
250 phys_addr_t reg_addr, unsigned long reg)
251{
252 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
253 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
254
255 if (db_addr) {
256 *db_addr = reg_addr + reg;
257 dev_dbg(ndev_dev(ndev), "Peer db addr %llx\n", *db_addr);
258 }
259
260 if (db_size) {
261 *db_size = ndev->reg->db_size;
262 dev_dbg(ndev_dev(ndev), "Peer db size %llx\n", *db_size);
263 }
264
265 return 0;
266}
267
268static inline u64 ndev_db_read(struct intel_ntb_dev *ndev,
269 void __iomem *mmio)
270{
271 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
272 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
273
274 return ndev->reg->db_ioread(mmio);
275}
276
277static inline int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
278 void __iomem *mmio)
279{
280 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
281 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
282
283 if (db_bits & ~ndev->db_valid_mask)
284 return -EINVAL;
285
286 ndev->reg->db_iowrite(db_bits, mmio);
287
288 return 0;
289}
290
291static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
292 void __iomem *mmio)
293{
294 unsigned long irqflags;
295
296 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
297 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
298
299 if (db_bits & ~ndev->db_valid_mask)
300 return -EINVAL;
301
302 spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
303 {
304 ndev->db_mask |= db_bits;
305 ndev->reg->db_iowrite(ndev->db_mask, mmio);
306 }
307 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
308
309 return 0;
310}
311
312static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
313 void __iomem *mmio)
314{
315 unsigned long irqflags;
316
317 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
318 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
319
320 if (db_bits & ~ndev->db_valid_mask)
321 return -EINVAL;
322
323 spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
324 {
325 ndev->db_mask &= ~db_bits;
326 ndev->reg->db_iowrite(ndev->db_mask, mmio);
327 }
328 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
329
330 return 0;
331}
332
333static inline int ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
334{
335 u64 shift, mask;
336
337 shift = ndev->db_vec_shift;
338 mask = BIT_ULL(shift) - 1;
339
340 return mask << (shift * db_vector);
341}
342
343static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
344 phys_addr_t *spad_addr, phys_addr_t reg_addr,
345 unsigned long reg)
346{
347 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
348 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
349
350 if (idx < 0 || idx >= ndev->spad_count)
351 return -EINVAL;
352
353 if (spad_addr) {
354 *spad_addr = reg_addr + reg + (idx << 2);
355 dev_dbg(ndev_dev(ndev), "Peer spad addr %llx\n", *spad_addr);
356 }
357
358 return 0;
359}
360
361static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
362 void __iomem *mmio)
363{
364 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
365 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
366
367 if (idx < 0 || idx >= ndev->spad_count)
368 return 0;
369
370 return ioread32(mmio + (idx << 2));
371}
372
373static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
374 void __iomem *mmio)
375{
376 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
377 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
378
379 if (idx < 0 || idx >= ndev->spad_count)
380 return -EINVAL;
381
382 iowrite32(val, mmio + (idx << 2));
383
384 return 0;
385}
386
387static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
388{
389 u64 vec_mask;
390
391 vec_mask = ndev_vec_mask(ndev, vec);
392
393 dev_dbg(ndev_dev(ndev), "vec %d vec_mask %llx\n", vec, vec_mask);
394
395 ndev->last_ts = jiffies;
396
397 if (vec_mask & ndev->db_link_mask) {
398 if (ndev->reg->poll_link(ndev))
399 ntb_link_event(&ndev->ntb);
400 }
401
402 if (vec_mask & ndev->db_valid_mask)
403 ntb_db_event(&ndev->ntb, vec);
404
405 return IRQ_HANDLED;
406}
407
408static irqreturn_t ndev_vec_isr(int irq, void *dev)
409{
410 struct intel_ntb_vec *nvec = dev;
411
412 return ndev_interrupt(nvec->ndev, nvec->num);
413}
414
415static irqreturn_t ndev_irq_isr(int irq, void *dev)
416{
417 struct intel_ntb_dev *ndev = dev;
418
419 return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq);
420}
421
422static int ndev_init_isr(struct intel_ntb_dev *ndev,
423 int msix_min, int msix_max,
424 int msix_shift, int total_shift)
425{
426 struct pci_dev *pdev;
427 int rc, i, msix_count, node;
428
429 pdev = ndev_pdev(ndev);
430
431 node = dev_to_node(&pdev->dev);
432
433 /* Mask all doorbell interrupts */
434 ndev->db_mask = ndev->db_valid_mask;
435 ndev->reg->db_iowrite(ndev->db_mask,
436 ndev->self_mmio +
437 ndev->self_reg->db_mask);
438
439 /* Try to set up msix irq */
440
441 ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec),
442 GFP_KERNEL, node);
443 if (!ndev->vec)
444 goto err_msix_vec_alloc;
445
446 ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix),
447 GFP_KERNEL, node);
448 if (!ndev->msix)
449 goto err_msix_alloc;
450
451 for (i = 0; i < msix_max; ++i)
452 ndev->msix[i].entry = i;
453
454 msix_count = pci_enable_msix_range(pdev, ndev->msix,
455 msix_min, msix_max);
456 if (msix_count < 0)
457 goto err_msix_enable;
458
459 for (i = 0; i < msix_count; ++i) {
460 ndev->vec[i].ndev = ndev;
461 ndev->vec[i].num = i;
462 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
463 "ndev_vec_isr", &ndev->vec[i]);
464 if (rc)
465 goto err_msix_request;
466 }
467
468 dev_dbg(ndev_dev(ndev), "Using msix interrupts\n");
469 ndev->db_vec_count = msix_count;
470 ndev->db_vec_shift = msix_shift;
471 return 0;
472
473err_msix_request:
474 while (i-- > 0)
475 free_irq(ndev->msix[i].vector, ndev);
476 pci_disable_msix(pdev);
477err_msix_enable:
478 kfree(ndev->msix);
479err_msix_alloc:
480 kfree(ndev->vec);
481err_msix_vec_alloc:
482 ndev->msix = NULL;
483 ndev->vec = NULL;
484
485 /* Try to set up msi irq */
486
487 rc = pci_enable_msi(pdev);
488 if (rc)
489 goto err_msi_enable;
490
491 rc = request_irq(pdev->irq, ndev_irq_isr, 0,
492 "ndev_irq_isr", ndev);
493 if (rc)
494 goto err_msi_request;
495
496 dev_dbg(ndev_dev(ndev), "Using msi interrupts\n");
497 ndev->db_vec_count = 1;
498 ndev->db_vec_shift = total_shift;
499 return 0;
500
501err_msi_request:
502 pci_disable_msi(pdev);
503err_msi_enable:
504
505 /* Try to set up intx irq */
506
507 pci_intx(pdev, 1);
508
509 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
510 "ndev_irq_isr", ndev);
511 if (rc)
512 goto err_intx_request;
513
514 dev_dbg(ndev_dev(ndev), "Using intx interrupts\n");
515 ndev->db_vec_count = 1;
516 ndev->db_vec_shift = total_shift;
517 return 0;
518
519err_intx_request:
520 return rc;
521}
522
523static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
524{
525 struct pci_dev *pdev;
526 int i;
527
528 pdev = ndev_pdev(ndev);
529
530 /* Mask all doorbell interrupts */
531 ndev->db_mask = ndev->db_valid_mask;
532 ndev->reg->db_iowrite(ndev->db_mask,
533 ndev->self_mmio +
534 ndev->self_reg->db_mask);
535
536 if (ndev->msix) {
537 i = ndev->db_vec_count;
538 while (i--)
539 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
540 pci_disable_msix(pdev);
541 kfree(ndev->msix);
542 kfree(ndev->vec);
543 } else {
544 free_irq(pdev->irq, ndev);
545 if (pci_dev_msi_enabled(pdev))
546 pci_disable_msi(pdev);
547 }
548}
549
550static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
551 size_t count, loff_t *offp)
552{
553 struct intel_ntb_dev *ndev;
554 void __iomem *mmio;
555 char *buf;
556 size_t buf_size;
557 ssize_t ret, off;
558 union { u64 v64; u32 v32; u16 v16; } u;
559
560 ndev = filp->private_data;
561 mmio = ndev->self_mmio;
562
563 buf_size = min(count, 0x800ul);
564
565 buf = kmalloc(buf_size, GFP_KERNEL);
566 if (!buf)
567 return -ENOMEM;
568
569 off = 0;
570
571 off += scnprintf(buf + off, buf_size - off,
572 "NTB Device Information:\n");
573
574 off += scnprintf(buf + off, buf_size - off,
575 "Connection Topology -\t%s\n",
576 ntb_topo_string(ndev->ntb.topo));
577
578 if (ndev->b2b_idx != UINT_MAX) {
579 off += scnprintf(buf + off, buf_size - off,
580 "B2B MW Idx -\t\t%u\n", ndev->b2b_idx);
581 off += scnprintf(buf + off, buf_size - off,
582 "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
583 }
584
585 off += scnprintf(buf + off, buf_size - off,
586 "BAR4 Split -\t\t%s\n",
587 ndev->bar4_split ? "yes" : "no");
588
589 off += scnprintf(buf + off, buf_size - off,
590 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
591 off += scnprintf(buf + off, buf_size - off,
592 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
593
594 if (!ndev->reg->link_is_up(ndev)) {
595 off += scnprintf(buf + off, buf_size - off,
596 "Link Status -\t\tDown\n");
597 } else {
598 off += scnprintf(buf + off, buf_size - off,
599 "Link Status -\t\tUp\n");
600 off += scnprintf(buf + off, buf_size - off,
601 "Link Speed -\t\tPCI-E Gen %u\n",
602 NTB_LNK_STA_SPEED(ndev->lnk_sta));
603 off += scnprintf(buf + off, buf_size - off,
604 "Link Width -\t\tx%u\n",
605 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
606 }
607
608 off += scnprintf(buf + off, buf_size - off,
609 "Memory Window Count -\t%u\n", ndev->mw_count);
610 off += scnprintf(buf + off, buf_size - off,
611 "Scratchpad Count -\t%u\n", ndev->spad_count);
612 off += scnprintf(buf + off, buf_size - off,
613 "Doorbell Count -\t%u\n", ndev->db_count);
614 off += scnprintf(buf + off, buf_size - off,
615 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
616 off += scnprintf(buf + off, buf_size - off,
617 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
618
619 off += scnprintf(buf + off, buf_size - off,
620 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
621 off += scnprintf(buf + off, buf_size - off,
622 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
623 off += scnprintf(buf + off, buf_size - off,
624 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
625
626 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
627 off += scnprintf(buf + off, buf_size - off,
628 "Doorbell Mask -\t\t%#llx\n", u.v64);
629
630 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
631 off += scnprintf(buf + off, buf_size - off,
632 "Doorbell Bell -\t\t%#llx\n", u.v64);
633
634 off += scnprintf(buf + off, buf_size - off,
635 "\nNTB Incoming XLAT:\n");
636
637 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
638 off += scnprintf(buf + off, buf_size - off,
639 "XLAT23 -\t\t%#018llx\n", u.v64);
640
641 if (ndev->bar4_split) {
642 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
643 off += scnprintf(buf + off, buf_size - off,
644 "XLAT4 -\t\t\t%#06x\n", u.v32);
645
646 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5));
647 off += scnprintf(buf + off, buf_size - off,
648 "XLAT5 -\t\t\t%#06x\n", u.v32);
649 } else {
650 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
651 off += scnprintf(buf + off, buf_size - off,
652 "XLAT45 -\t\t%#018llx\n", u.v64);
653 }
654
655 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
656 off += scnprintf(buf + off, buf_size - off,
657 "LMT23 -\t\t\t%#018llx\n", u.v64);
658
659 if (ndev->bar4_split) {
660 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
661 off += scnprintf(buf + off, buf_size - off,
662 "LMT4 -\t\t\t%#06x\n", u.v32);
663 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5));
664 off += scnprintf(buf + off, buf_size - off,
665 "LMT5 -\t\t\t%#06x\n", u.v32);
666 } else {
667 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
668 off += scnprintf(buf + off, buf_size - off,
669 "LMT45 -\t\t\t%#018llx\n", u.v64);
670 }
671
672 if (pdev_is_xeon(ndev->ntb.pdev)) {
673 if (ntb_topo_is_b2b(ndev->ntb.topo)) {
674 off += scnprintf(buf + off, buf_size - off,
675 "\nNTB Outgoing B2B XLAT:\n");
676
677 u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
678 off += scnprintf(buf + off, buf_size - off,
679 "B2B XLAT23 -\t\t%#018llx\n", u.v64);
680
681 if (ndev->bar4_split) {
682 u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
683 off += scnprintf(buf + off, buf_size - off,
684 "B2B XLAT4 -\t\t%#06x\n",
685 u.v32);
686 u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
687 off += scnprintf(buf + off, buf_size - off,
688 "B2B XLAT5 -\t\t%#06x\n",
689 u.v32);
690 } else {
691 u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
692 off += scnprintf(buf + off, buf_size - off,
693 "B2B XLAT45 -\t\t%#018llx\n",
694 u.v64);
695 }
696
697 u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET);
698 off += scnprintf(buf + off, buf_size - off,
699 "B2B LMT23 -\t\t%#018llx\n", u.v64);
700
701 if (ndev->bar4_split) {
702 u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET);
703 off += scnprintf(buf + off, buf_size - off,
704 "B2B LMT4 -\t\t%#06x\n",
705 u.v32);
706 u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET);
707 off += scnprintf(buf + off, buf_size - off,
708 "B2B LMT5 -\t\t%#06x\n",
709 u.v32);
710 } else {
711 u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET);
712 off += scnprintf(buf + off, buf_size - off,
713 "B2B LMT45 -\t\t%#018llx\n",
714 u.v64);
715 }
716
717 off += scnprintf(buf + off, buf_size - off,
718 "\nNTB Secondary BAR:\n");
719
720 u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET);
721 off += scnprintf(buf + off, buf_size - off,
722 "SBAR01 -\t\t%#018llx\n", u.v64);
723
724 u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
725 off += scnprintf(buf + off, buf_size - off,
726 "SBAR23 -\t\t%#018llx\n", u.v64);
727
728 if (ndev->bar4_split) {
729 u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
730 off += scnprintf(buf + off, buf_size - off,
731 "SBAR4 -\t\t\t%#06x\n", u.v32);
732 u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
733 off += scnprintf(buf + off, buf_size - off,
734 "SBAR5 -\t\t\t%#06x\n", u.v32);
735 } else {
736 u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
737 off += scnprintf(buf + off, buf_size - off,
738 "SBAR45 -\t\t%#018llx\n",
739 u.v64);
740 }
741 }
742
743 off += scnprintf(buf + off, buf_size - off,
744 "\nXEON NTB Statistics:\n");
745
746 u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET);
747 off += scnprintf(buf + off, buf_size - off,
748 "Upstream Memory Miss -\t%u\n", u.v16);
749
750 off += scnprintf(buf + off, buf_size - off,
751 "\nXEON NTB Hardware Errors:\n");
752
753 if (!pci_read_config_word(ndev->ntb.pdev,
754 XEON_DEVSTS_OFFSET, &u.v16))
755 off += scnprintf(buf + off, buf_size - off,
756 "DEVSTS -\t\t%#06x\n", u.v16);
757
758 if (!pci_read_config_word(ndev->ntb.pdev,
759 XEON_LINK_STATUS_OFFSET, &u.v16))
760 off += scnprintf(buf + off, buf_size - off,
761 "LNKSTS -\t\t%#06x\n", u.v16);
762
763 if (!pci_read_config_dword(ndev->ntb.pdev,
764 XEON_UNCERRSTS_OFFSET, &u.v32))
765 off += scnprintf(buf + off, buf_size - off,
766 "UNCERRSTS -\t\t%#06x\n", u.v32);
767
768 if (!pci_read_config_dword(ndev->ntb.pdev,
769 XEON_CORERRSTS_OFFSET, &u.v32))
770 off += scnprintf(buf + off, buf_size - off,
771 "CORERRSTS -\t\t%#06x\n", u.v32);
772 }
773
774 ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
775 kfree(buf);
776 return ret;
777}
778
779static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
780{
781 if (!debugfs_dir) {
782 ndev->debugfs_dir = NULL;
783 ndev->debugfs_info = NULL;
784 } else {
785 ndev->debugfs_dir =
786 debugfs_create_dir(ndev_name(ndev), debugfs_dir);
787 if (!ndev->debugfs_dir)
788 ndev->debugfs_info = NULL;
789 else
790 ndev->debugfs_info =
791 debugfs_create_file("info", S_IRUSR,
792 ndev->debugfs_dir, ndev,
793 &intel_ntb_debugfs_info);
794 }
795}
796
797static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
798{
799 debugfs_remove_recursive(ndev->debugfs_dir);
800}
801
802static int intel_ntb_mw_count(struct ntb_dev *ntb)
803{
804 return ntb_ndev(ntb)->mw_count;
805}
806
807static int intel_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
808 phys_addr_t *base,
809 resource_size_t *size,
810 resource_size_t *align,
811 resource_size_t *align_size)
812{
813 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
814 int bar;
815
816 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
817 idx += 1;
818
819 bar = ndev_mw_to_bar(ndev, idx);
820 if (bar < 0)
821 return bar;
822
823 if (base)
824 *base = pci_resource_start(ndev->ntb.pdev, bar) +
825 (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
826
827 if (size)
828 *size = pci_resource_len(ndev->ntb.pdev, bar) -
829 (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
830
831 if (align)
832 *align = pci_resource_len(ndev->ntb.pdev, bar);
833
834 if (align_size)
835 *align_size = 1;
836
837 return 0;
838}
839
840static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
841 dma_addr_t addr, resource_size_t size)
842{
843 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
844 unsigned long base_reg, xlat_reg, limit_reg;
845 resource_size_t bar_size, mw_size;
846 void __iomem *mmio;
847 u64 base, limit, reg_val;
848 int bar;
849
850 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
851 idx += 1;
852
853 bar = ndev_mw_to_bar(ndev, idx);
854 if (bar < 0)
855 return bar;
856
857 bar_size = pci_resource_len(ndev->ntb.pdev, bar);
858
859 if (idx == ndev->b2b_idx)
860 mw_size = bar_size - ndev->b2b_off;
861 else
862 mw_size = bar_size;
863
864 /* hardware requires that addr is aligned to bar size */
865 if (addr & (bar_size - 1))
866 return -EINVAL;
867
868 /* make sure the range fits in the usable mw size */
869 if (size > mw_size)
870 return -EINVAL;
871
872 mmio = ndev->self_mmio;
873 base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
874 xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
875 limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
876
877 if (bar < 4 || !ndev->bar4_split) {
878 base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64;
879
880 /* Set the limit if supported, if size is not mw_size */
881 if (limit_reg && size != mw_size)
882 limit = base + size;
883 else
884 limit = 0;
885
886 /* set and verify setting the translation address */
887 iowrite64(addr, mmio + xlat_reg);
888 reg_val = ioread64(mmio + xlat_reg);
889 if (reg_val != addr) {
890 iowrite64(0, mmio + xlat_reg);
891 return -EIO;
892 }
893
894 /* set and verify setting the limit */
895 iowrite64(limit, mmio + limit_reg);
896 reg_val = ioread64(mmio + limit_reg);
897 if (reg_val != limit) {
898 iowrite64(base, mmio + limit_reg);
899 iowrite64(0, mmio + xlat_reg);
900 return -EIO;
901 }
902 } else {
903 /* split bar addr range must all be 32 bit */
904 if (addr & (~0ull << 32))
905 return -EINVAL;
906 if ((addr + size) & (~0ull << 32))
907 return -EINVAL;
908
909 base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32;
910
911 /* Set the limit if supported, if size is not mw_size */
912 if (limit_reg && size != mw_size)
913 limit = base + size;
914 else
915 limit = 0;
916
917 /* set and verify setting the translation address */
918 iowrite32(addr, mmio + xlat_reg);
919 reg_val = ioread32(mmio + xlat_reg);
920 if (reg_val != addr) {
921 iowrite32(0, mmio + xlat_reg);
922 return -EIO;
923 }
924
925 /* set and verify setting the limit */
926 iowrite32(limit, mmio + limit_reg);
927 reg_val = ioread32(mmio + limit_reg);
928 if (reg_val != limit) {
929 iowrite32(base, mmio + limit_reg);
930 iowrite32(0, mmio + xlat_reg);
931 return -EIO;
932 }
933 }
934
935 return 0;
936}
937
938static int intel_ntb_link_is_up(struct ntb_dev *ntb,
939 enum ntb_speed *speed,
940 enum ntb_width *width)
941{
942 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
943
944 if (ndev->reg->link_is_up(ndev)) {
945 if (speed)
946 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
947 if (width)
948 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
949 return 1;
950 } else {
951 /* TODO MAYBE: is it possible to observe the link speed and
952 * width while link is training? */
953 if (speed)
954 *speed = NTB_SPEED_NONE;
955 if (width)
956 *width = NTB_WIDTH_NONE;
957 return 0;
958 }
959}
960
961static int intel_ntb_link_enable(struct ntb_dev *ntb,
962 enum ntb_speed max_speed,
963 enum ntb_width max_width)
964{
965 struct intel_ntb_dev *ndev;
966 u32 ntb_ctl;
967
968 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
969
970 if (ndev->ntb.topo == NTB_TOPO_SEC)
971 return -EINVAL;
972
973 dev_dbg(ndev_dev(ndev),
974 "Enabling link with max_speed %d max_width %d\n",
975 max_speed, max_width);
976 if (max_speed != NTB_SPEED_AUTO)
977 dev_dbg(ndev_dev(ndev), "ignoring max_speed %d\n", max_speed);
978 if (max_width != NTB_WIDTH_AUTO)
979 dev_dbg(ndev_dev(ndev), "ignoring max_width %d\n", max_width);
980
981 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
982 ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
983 ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
984 ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
985 if (ndev->bar4_split)
986 ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
987 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
988
989 return 0;
990}
991
992static int intel_ntb_link_disable(struct ntb_dev *ntb)
993{
994 struct intel_ntb_dev *ndev;
995 u32 ntb_cntl;
996
997 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
998
999 if (ndev->ntb.topo == NTB_TOPO_SEC)
1000 return -EINVAL;
1001
1002 dev_dbg(ndev_dev(ndev), "Disabling link\n");
1003
1004 /* Bring NTB link down */
1005 ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1006 ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
1007 ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
1008 if (ndev->bar4_split)
1009 ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
1010 ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
1011 iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
1012
1013 return 0;
1014}
1015
1016static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
1017{
1018 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
1019}
1020
1021static u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
1022{
1023 return ntb_ndev(ntb)->db_valid_mask;
1024}
1025
1026static int intel_ntb_db_vector_count(struct ntb_dev *ntb)
1027{
1028 struct intel_ntb_dev *ndev;
1029
1030 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1031
1032 return ndev->db_vec_count;
1033}
1034
1035static u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
1036{
1037 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1038
1039 if (db_vector < 0 || db_vector > ndev->db_vec_count)
1040 return 0;
1041
1042 return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
1043}
1044
1045static u64 intel_ntb_db_read(struct ntb_dev *ntb)
1046{
1047 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1048
1049 return ndev_db_read(ndev,
1050 ndev->self_mmio +
1051 ndev->self_reg->db_bell);
1052}
1053
1054static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
1055{
1056 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1057
1058 return ndev_db_write(ndev, db_bits,
1059 ndev->self_mmio +
1060 ndev->self_reg->db_bell);
1061}
1062
1063static int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1064{
1065 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1066
1067 return ndev_db_set_mask(ndev, db_bits,
1068 ndev->self_mmio +
1069 ndev->self_reg->db_mask);
1070}
1071
1072static int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
1073{
1074 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1075
1076 return ndev_db_clear_mask(ndev, db_bits,
1077 ndev->self_mmio +
1078 ndev->self_reg->db_mask);
1079}
1080
1081static int intel_ntb_peer_db_addr(struct ntb_dev *ntb,
1082 phys_addr_t *db_addr,
1083 resource_size_t *db_size)
1084{
1085 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1086
1087 return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
1088 ndev->peer_reg->db_bell);
1089}
1090
1091static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1092{
1093 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1094
1095 return ndev_db_write(ndev, db_bits,
1096 ndev->peer_mmio +
1097 ndev->peer_reg->db_bell);
1098}
1099
1100static int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
1101{
1102 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
1103}
1104
1105static int intel_ntb_spad_count(struct ntb_dev *ntb)
1106{
1107 struct intel_ntb_dev *ndev;
1108
1109 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1110
1111 return ndev->spad_count;
1112}
1113
1114static u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
1115{
1116 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1117
1118 return ndev_spad_read(ndev, idx,
1119 ndev->self_mmio +
1120 ndev->self_reg->spad);
1121}
1122
1123static int intel_ntb_spad_write(struct ntb_dev *ntb,
1124 int idx, u32 val)
1125{
1126 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1127
1128 return ndev_spad_write(ndev, idx, val,
1129 ndev->self_mmio +
1130 ndev->self_reg->spad);
1131}
1132
1133static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx,
1134 phys_addr_t *spad_addr)
1135{
1136 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1137
1138 return ndev_spad_addr(ndev, idx, spad_addr, ndev->peer_addr,
1139 ndev->peer_reg->spad);
1140}
1141
1142static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
1143{
1144 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1145
1146 return ndev_spad_read(ndev, idx,
1147 ndev->peer_mmio +
1148 ndev->peer_reg->spad);
1149}
1150
1151static int intel_ntb_peer_spad_write(struct ntb_dev *ntb,
1152 int idx, u32 val)
1153{
1154 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1155
1156 return ndev_spad_write(ndev, idx, val,
1157 ndev->peer_mmio +
1158 ndev->peer_reg->spad);
1159}
1160
1161/* ATOM */
1162
1163static u64 atom_db_ioread(void __iomem *mmio)
1164{
1165 return ioread64(mmio);
1166}
1167
1168static void atom_db_iowrite(u64 bits, void __iomem *mmio)
1169{
1170 iowrite64(bits, mmio);
1171}
1172
1173static int atom_poll_link(struct intel_ntb_dev *ndev)
1174{
1175 u32 ntb_ctl;
1176
1177 ntb_ctl = ioread32(ndev->self_mmio + ATOM_NTBCNTL_OFFSET);
1178
1179 if (ntb_ctl == ndev->ntb_ctl)
1180 return 0;
1181
1182 ndev->ntb_ctl = ntb_ctl;
1183
1184 ndev->lnk_sta = ioread32(ndev->self_mmio + ATOM_LINK_STATUS_OFFSET);
1185
1186 return 1;
1187}
1188
1189static int atom_link_is_up(struct intel_ntb_dev *ndev)
1190{
1191 return ATOM_NTB_CTL_ACTIVE(ndev->ntb_ctl);
1192}
1193
1194static int atom_link_is_err(struct intel_ntb_dev *ndev)
1195{
1196 if (ioread32(ndev->self_mmio + ATOM_LTSSMSTATEJMP_OFFSET)
1197 & ATOM_LTSSMSTATEJMP_FORCEDETECT)
1198 return 1;
1199
1200 if (ioread32(ndev->self_mmio + ATOM_IBSTERRRCRVSTS0_OFFSET)
1201 & ATOM_IBIST_ERR_OFLOW)
1202 return 1;
1203
1204 return 0;
1205}
1206
1207static inline enum ntb_topo atom_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd)
1208{
1209 switch (ppd & ATOM_PPD_TOPO_MASK) {
1210 case ATOM_PPD_TOPO_B2B_USD:
1211 dev_dbg(ndev_dev(ndev), "PPD %d B2B USD\n", ppd);
1212 return NTB_TOPO_B2B_USD;
1213
1214 case ATOM_PPD_TOPO_B2B_DSD:
1215 dev_dbg(ndev_dev(ndev), "PPD %d B2B DSD\n", ppd);
1216 return NTB_TOPO_B2B_DSD;
1217
1218 case ATOM_PPD_TOPO_PRI_USD:
1219 case ATOM_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1220 case ATOM_PPD_TOPO_SEC_USD:
1221 case ATOM_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1222 dev_dbg(ndev_dev(ndev), "PPD %d non B2B disabled\n", ppd);
1223 return NTB_TOPO_NONE;
1224 }
1225
1226 dev_dbg(ndev_dev(ndev), "PPD %d invalid\n", ppd);
1227 return NTB_TOPO_NONE;
1228}
1229
1230static void atom_link_hb(struct work_struct *work)
1231{
1232 struct intel_ntb_dev *ndev = hb_ndev(work);
1233 unsigned long poll_ts;
1234 void __iomem *mmio;
1235 u32 status32;
1236
1237 poll_ts = ndev->last_ts + ATOM_LINK_HB_TIMEOUT;
1238
1239 /* Delay polling the link status if an interrupt was received,
1240 * unless the cached link status says the link is down.
1241 */
1242 if (time_after(poll_ts, jiffies) && atom_link_is_up(ndev)) {
1243 schedule_delayed_work(&ndev->hb_timer, poll_ts - jiffies);
1244 return;
1245 }
1246
1247 if (atom_poll_link(ndev))
1248 ntb_link_event(&ndev->ntb);
1249
1250 if (atom_link_is_up(ndev) || !atom_link_is_err(ndev)) {
1251 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1252 return;
1253 }
1254
1255 /* Link is down with error: recover the link! */
1256
1257 mmio = ndev->self_mmio;
1258
1259 /* Driver resets the NTB ModPhy lanes - magic! */
1260 iowrite8(0xe0, mmio + ATOM_MODPHY_PCSREG6);
1261 iowrite8(0x40, mmio + ATOM_MODPHY_PCSREG4);
1262 iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG4);
1263 iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG6);
1264
1265 /* Driver waits 100ms to allow the NTB ModPhy to settle */
1266 msleep(100);
1267
1268 /* Clear AER Errors, write to clear */
1269 status32 = ioread32(mmio + ATOM_ERRCORSTS_OFFSET);
1270 dev_dbg(ndev_dev(ndev), "ERRCORSTS = %x\n", status32);
1271 status32 &= PCI_ERR_COR_REP_ROLL;
1272 iowrite32(status32, mmio + ATOM_ERRCORSTS_OFFSET);
1273
1274 /* Clear unexpected electrical idle event in LTSSM, write to clear */
1275 status32 = ioread32(mmio + ATOM_LTSSMERRSTS0_OFFSET);
1276 dev_dbg(ndev_dev(ndev), "LTSSMERRSTS0 = %x\n", status32);
1277 status32 |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI;
1278 iowrite32(status32, mmio + ATOM_LTSSMERRSTS0_OFFSET);
1279
1280 /* Clear DeSkew Buffer error, write to clear */
1281 status32 = ioread32(mmio + ATOM_DESKEWSTS_OFFSET);
1282 dev_dbg(ndev_dev(ndev), "DESKEWSTS = %x\n", status32);
1283 status32 |= ATOM_DESKEWSTS_DBERR;
1284 iowrite32(status32, mmio + ATOM_DESKEWSTS_OFFSET);
1285
1286 status32 = ioread32(mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1287 dev_dbg(ndev_dev(ndev), "IBSTERRRCRVSTS0 = %x\n", status32);
1288 status32 &= ATOM_IBIST_ERR_OFLOW;
1289 iowrite32(status32, mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1290
1291 /* Releases the NTB state machine to allow the link to retrain */
1292 status32 = ioread32(mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1293 dev_dbg(ndev_dev(ndev), "LTSSMSTATEJMP = %x\n", status32);
1294 status32 &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT;
1295 iowrite32(status32, mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1296
1297 /* There is a potential race between the 2 NTB devices recovering at the
1298 * same time. If the times are the same, the link will not recover and
1299 * the driver will be stuck in this loop forever. Add a random interval
1300 * to the recovery time to prevent this race.
1301 */
1302 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_RECOVERY_TIME
1303 + prandom_u32() % ATOM_LINK_RECOVERY_TIME);
1304}
1305
1306static int atom_init_isr(struct intel_ntb_dev *ndev)
1307{
1308 int rc;
1309
1310 rc = ndev_init_isr(ndev, 1, ATOM_DB_MSIX_VECTOR_COUNT,
1311 ATOM_DB_MSIX_VECTOR_SHIFT, ATOM_DB_TOTAL_SHIFT);
1312 if (rc)
1313 return rc;
1314
1315 /* ATOM doesn't have link status interrupt, poll on that platform */
1316 ndev->last_ts = jiffies;
1317 INIT_DELAYED_WORK(&ndev->hb_timer, atom_link_hb);
1318 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1319
1320 return 0;
1321}
1322
1323static void atom_deinit_isr(struct intel_ntb_dev *ndev)
1324{
1325 cancel_delayed_work_sync(&ndev->hb_timer);
1326 ndev_deinit_isr(ndev);
1327}
1328
1329static int atom_init_ntb(struct intel_ntb_dev *ndev)
1330{
1331 ndev->mw_count = ATOM_MW_COUNT;
1332 ndev->spad_count = ATOM_SPAD_COUNT;
1333 ndev->db_count = ATOM_DB_COUNT;
1334
1335 switch (ndev->ntb.topo) {
1336 case NTB_TOPO_B2B_USD:
1337 case NTB_TOPO_B2B_DSD:
1338 ndev->self_reg = &atom_pri_reg;
1339 ndev->peer_reg = &atom_b2b_reg;
1340 ndev->xlat_reg = &atom_sec_xlat;
1341
1342 /* Enable Bus Master and Memory Space on the secondary side */
1343 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1344 ndev->self_mmio + ATOM_SPCICMD_OFFSET);
1345
1346 break;
1347
1348 default:
1349 return -EINVAL;
1350 }
1351
1352 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1353
1354 return 0;
1355}
1356
1357static int atom_init_dev(struct intel_ntb_dev *ndev)
1358{
1359 u32 ppd;
1360 int rc;
1361
1362 rc = pci_read_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET, &ppd);
1363 if (rc)
1364 return -EIO;
1365
1366 ndev->ntb.topo = atom_ppd_topo(ndev, ppd);
1367 if (ndev->ntb.topo == NTB_TOPO_NONE)
1368 return -EINVAL;
1369
1370 rc = atom_init_ntb(ndev);
1371 if (rc)
1372 return rc;
1373
1374 rc = atom_init_isr(ndev);
1375 if (rc)
1376 return rc;
1377
1378 if (ndev->ntb.topo != NTB_TOPO_SEC) {
1379 /* Initiate PCI-E link training */
1380 rc = pci_write_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET,
1381 ppd | ATOM_PPD_INIT_LINK);
1382 if (rc)
1383 return rc;
1384 }
1385
1386 return 0;
1387}
1388
1389static void atom_deinit_dev(struct intel_ntb_dev *ndev)
1390{
1391 atom_deinit_isr(ndev);
1392}
1393
1394/* XEON */
1395
1396static u64 xeon_db_ioread(void __iomem *mmio)
1397{
1398 return (u64)ioread16(mmio);
1399}
1400
1401static void xeon_db_iowrite(u64 bits, void __iomem *mmio)
1402{
1403 iowrite16((u16)bits, mmio);
1404}
1405
1406static int xeon_poll_link(struct intel_ntb_dev *ndev)
1407{
1408 u16 reg_val;
1409 int rc;
1410
1411 ndev->reg->db_iowrite(ndev->db_link_mask,
1412 ndev->self_mmio +
1413 ndev->self_reg->db_bell);
1414
1415 rc = pci_read_config_word(ndev->ntb.pdev,
1416 XEON_LINK_STATUS_OFFSET, ®_val);
1417 if (rc)
1418 return 0;
1419
1420 if (reg_val == ndev->lnk_sta)
1421 return 0;
1422
1423 ndev->lnk_sta = reg_val;
1424
1425 return 1;
1426}
1427
1428static int xeon_link_is_up(struct intel_ntb_dev *ndev)
1429{
1430 if (ndev->ntb.topo == NTB_TOPO_SEC)
1431 return 1;
1432
1433 return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
1434}
1435
1436static inline enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
1437{
1438 switch (ppd & XEON_PPD_TOPO_MASK) {
1439 case XEON_PPD_TOPO_B2B_USD:
1440 return NTB_TOPO_B2B_USD;
1441
1442 case XEON_PPD_TOPO_B2B_DSD:
1443 return NTB_TOPO_B2B_DSD;
1444
1445 case XEON_PPD_TOPO_PRI_USD:
1446 case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1447 return NTB_TOPO_PRI;
1448
1449 case XEON_PPD_TOPO_SEC_USD:
1450 case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1451 return NTB_TOPO_SEC;
1452 }
1453
1454 return NTB_TOPO_NONE;
1455}
1456
1457static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
1458{
1459 if (ppd & XEON_PPD_SPLIT_BAR_MASK) {
1460 dev_dbg(ndev_dev(ndev), "PPD %d split bar\n", ppd);
1461 return 1;
1462 }
1463 return 0;
1464}
1465
1466static int xeon_init_isr(struct intel_ntb_dev *ndev)
1467{
1468 return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT,
1469 XEON_DB_MSIX_VECTOR_COUNT,
1470 XEON_DB_MSIX_VECTOR_SHIFT,
1471 XEON_DB_TOTAL_SHIFT);
1472}
1473
1474static void xeon_deinit_isr(struct intel_ntb_dev *ndev)
1475{
1476 ndev_deinit_isr(ndev);
1477}
1478
1479static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev,
1480 const struct intel_b2b_addr *addr,
1481 const struct intel_b2b_addr *peer_addr)
1482{
1483 struct pci_dev *pdev;
1484 void __iomem *mmio;
1485 resource_size_t bar_size;
1486 phys_addr_t bar_addr;
1487 int b2b_bar;
1488 u8 bar_sz;
1489
1490 pdev = ndev_pdev(ndev);
1491 mmio = ndev->self_mmio;
1492
1493 if (ndev->b2b_idx == UINT_MAX) {
1494 dev_dbg(ndev_dev(ndev), "not using b2b mw\n");
1495 b2b_bar = 0;
1496 ndev->b2b_off = 0;
1497 } else {
1498 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
1499 if (b2b_bar < 0)
1500 return -EIO;
1501
1502 dev_dbg(ndev_dev(ndev), "using b2b mw bar %d\n", b2b_bar);
1503
1504 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
1505
1506 dev_dbg(ndev_dev(ndev), "b2b bar size %#llx\n", bar_size);
1507
1508 if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) {
1509 dev_dbg(ndev_dev(ndev),
1510 "b2b using first half of bar\n");
1511 ndev->b2b_off = bar_size >> 1;
1512 } else if (XEON_B2B_MIN_SIZE <= bar_size) {
1513 dev_dbg(ndev_dev(ndev),
1514 "b2b using whole bar\n");
1515 ndev->b2b_off = 0;
1516 --ndev->mw_count;
1517 } else {
1518 dev_dbg(ndev_dev(ndev),
1519 "b2b bar size is too small\n");
1520 return -EIO;
1521 }
1522 }
1523
1524 /* Reset the secondary bar sizes to match the primary bar sizes,
1525 * except disable or halve the size of the b2b secondary bar.
1526 *
1527 * Note: code for each specific bar size register, because the register
1528 * offsets are not in a consistent order (bar5sz comes after ppd, odd).
1529 */
1530 pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz);
1531 dev_dbg(ndev_dev(ndev), "PBAR23SZ %#x\n", bar_sz);
1532 if (b2b_bar == 2) {
1533 if (ndev->b2b_off)
1534 bar_sz -= 1;
1535 else
1536 bar_sz = 0;
1537 }
1538 pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz);
1539 pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz);
1540 dev_dbg(ndev_dev(ndev), "SBAR23SZ %#x\n", bar_sz);
1541
1542 if (!ndev->bar4_split) {
1543 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz);
1544 dev_dbg(ndev_dev(ndev), "PBAR45SZ %#x\n", bar_sz);
1545 if (b2b_bar == 4) {
1546 if (ndev->b2b_off)
1547 bar_sz -= 1;
1548 else
1549 bar_sz = 0;
1550 }
1551 pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz);
1552 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz);
1553 dev_dbg(ndev_dev(ndev), "SBAR45SZ %#x\n", bar_sz);
1554 } else {
1555 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz);
1556 dev_dbg(ndev_dev(ndev), "PBAR4SZ %#x\n", bar_sz);
1557 if (b2b_bar == 4) {
1558 if (ndev->b2b_off)
1559 bar_sz -= 1;
1560 else
1561 bar_sz = 0;
1562 }
1563 pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz);
1564 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz);
1565 dev_dbg(ndev_dev(ndev), "SBAR4SZ %#x\n", bar_sz);
1566
1567 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz);
1568 dev_dbg(ndev_dev(ndev), "PBAR5SZ %#x\n", bar_sz);
1569 if (b2b_bar == 5) {
1570 if (ndev->b2b_off)
1571 bar_sz -= 1;
1572 else
1573 bar_sz = 0;
1574 }
1575 pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz);
1576 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz);
1577 dev_dbg(ndev_dev(ndev), "SBAR5SZ %#x\n", bar_sz);
1578 }
1579
1580 /* SBAR01 hit by first part of the b2b bar */
1581 if (b2b_bar == 0)
1582 bar_addr = addr->bar0_addr;
1583 else if (b2b_bar == 2)
1584 bar_addr = addr->bar2_addr64;
1585 else if (b2b_bar == 4 && !ndev->bar4_split)
1586 bar_addr = addr->bar4_addr64;
1587 else if (b2b_bar == 4)
1588 bar_addr = addr->bar4_addr32;
1589 else if (b2b_bar == 5)
1590 bar_addr = addr->bar5_addr32;
1591 else
1592 return -EIO;
1593
1594 dev_dbg(ndev_dev(ndev), "SBAR01 %#018llx\n", bar_addr);
1595 iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET);
1596
1597 /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
1598 * The b2b bar is either disabled above, or configured half-size, and
1599 * it starts at the PBAR xlat + offset.
1600 */
1601
1602 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1603 iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET);
1604 bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
1605 dev_dbg(ndev_dev(ndev), "SBAR23 %#018llx\n", bar_addr);
1606
1607 if (!ndev->bar4_split) {
1608 bar_addr = addr->bar4_addr64 +
1609 (b2b_bar == 4 ? ndev->b2b_off : 0);
1610 iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET);
1611 bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
1612 dev_dbg(ndev_dev(ndev), "SBAR45 %#018llx\n", bar_addr);
1613 } else {
1614 bar_addr = addr->bar4_addr32 +
1615 (b2b_bar == 4 ? ndev->b2b_off : 0);
1616 iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET);
1617 bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
1618 dev_dbg(ndev_dev(ndev), "SBAR4 %#010llx\n", bar_addr);
1619
1620 bar_addr = addr->bar5_addr32 +
1621 (b2b_bar == 5 ? ndev->b2b_off : 0);
1622 iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET);
1623 bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
1624 dev_dbg(ndev_dev(ndev), "SBAR5 %#010llx\n", bar_addr);
1625 }
1626
1627 /* setup incoming bar limits == base addrs (zero length windows) */
1628
1629 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1630 iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET);
1631 bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET);
1632 dev_dbg(ndev_dev(ndev), "SBAR23LMT %#018llx\n", bar_addr);
1633
1634 if (!ndev->bar4_split) {
1635 bar_addr = addr->bar4_addr64 +
1636 (b2b_bar == 4 ? ndev->b2b_off : 0);
1637 iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET);
1638 bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET);
1639 dev_dbg(ndev_dev(ndev), "SBAR45LMT %#018llx\n", bar_addr);
1640 } else {
1641 bar_addr = addr->bar4_addr32 +
1642 (b2b_bar == 4 ? ndev->b2b_off : 0);
1643 iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET);
1644 bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET);
1645 dev_dbg(ndev_dev(ndev), "SBAR4LMT %#010llx\n", bar_addr);
1646
1647 bar_addr = addr->bar5_addr32 +
1648 (b2b_bar == 5 ? ndev->b2b_off : 0);
1649 iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET);
1650 bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET);
1651 dev_dbg(ndev_dev(ndev), "SBAR5LMT %#05llx\n", bar_addr);
1652 }
1653
1654 /* zero incoming translation addrs */
1655 iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET);
1656
1657 if (!ndev->bar4_split) {
1658 iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET);
1659 } else {
1660 iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET);
1661 iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET);
1662 }
1663
1664 /* zero outgoing translation limits (whole bar size windows) */
1665 iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET);
1666 if (!ndev->bar4_split) {
1667 iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET);
1668 } else {
1669 iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET);
1670 iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET);
1671 }
1672
1673 /* set outgoing translation offsets */
1674 bar_addr = peer_addr->bar2_addr64;
1675 iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET);
1676 bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
1677 dev_dbg(ndev_dev(ndev), "PBAR23XLAT %#018llx\n", bar_addr);
1678
1679 if (!ndev->bar4_split) {
1680 bar_addr = peer_addr->bar4_addr64;
1681 iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET);
1682 bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
1683 dev_dbg(ndev_dev(ndev), "PBAR45XLAT %#018llx\n", bar_addr);
1684 } else {
1685 bar_addr = peer_addr->bar4_addr32;
1686 iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET);
1687 bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
1688 dev_dbg(ndev_dev(ndev), "PBAR4XLAT %#010llx\n", bar_addr);
1689
1690 bar_addr = peer_addr->bar5_addr32;
1691 iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET);
1692 bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
1693 dev_dbg(ndev_dev(ndev), "PBAR5XLAT %#010llx\n", bar_addr);
1694 }
1695
1696 /* set the translation offset for b2b registers */
1697 if (b2b_bar == 0)
1698 bar_addr = peer_addr->bar0_addr;
1699 else if (b2b_bar == 2)
1700 bar_addr = peer_addr->bar2_addr64;
1701 else if (b2b_bar == 4 && !ndev->bar4_split)
1702 bar_addr = peer_addr->bar4_addr64;
1703 else if (b2b_bar == 4)
1704 bar_addr = peer_addr->bar4_addr32;
1705 else if (b2b_bar == 5)
1706 bar_addr = peer_addr->bar5_addr32;
1707 else
1708 return -EIO;
1709
1710 /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
1711 dev_dbg(ndev_dev(ndev), "B2BXLAT %#018llx\n", bar_addr);
1712 iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL);
1713 iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU);
1714
1715 if (b2b_bar) {
1716 /* map peer ntb mmio config space registers */
1717 ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
1718 XEON_B2B_MIN_SIZE);
1719 if (!ndev->peer_mmio)
1720 return -EIO;
1721 }
1722
1723 return 0;
1724}
1725
1726static int xeon_init_ntb(struct intel_ntb_dev *ndev)
1727{
1728 int rc;
1729 u32 ntb_ctl;
1730
1731 if (ndev->bar4_split)
1732 ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
1733 else
1734 ndev->mw_count = XEON_MW_COUNT;
1735
1736 ndev->spad_count = XEON_SPAD_COUNT;
1737 ndev->db_count = XEON_DB_COUNT;
1738 ndev->db_link_mask = XEON_DB_LINK_BIT;
1739
1740 switch (ndev->ntb.topo) {
1741 case NTB_TOPO_PRI:
1742 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1743 dev_err(ndev_dev(ndev), "NTB Primary config disabled\n");
1744 return -EINVAL;
1745 }
1746
1747 /* enable link to allow secondary side device to appear */
1748 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1749 ntb_ctl &= ~NTB_CTL_DISABLE;
1750 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
1751
1752 /* use half the spads for the peer */
1753 ndev->spad_count >>= 1;
1754 ndev->self_reg = &xeon_pri_reg;
1755 ndev->peer_reg = &xeon_sec_reg;
1756 ndev->xlat_reg = &xeon_sec_xlat;
1757 break;
1758
1759 case NTB_TOPO_SEC:
1760 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1761 dev_err(ndev_dev(ndev), "NTB Secondary config disabled\n");
1762 return -EINVAL;
1763 }
1764 /* use half the spads for the peer */
1765 ndev->spad_count >>= 1;
1766 ndev->self_reg = &xeon_sec_reg;
1767 ndev->peer_reg = &xeon_pri_reg;
1768 ndev->xlat_reg = &xeon_pri_xlat;
1769 break;
1770
1771 case NTB_TOPO_B2B_USD:
1772 case NTB_TOPO_B2B_DSD:
1773 ndev->self_reg = &xeon_pri_reg;
1774 ndev->peer_reg = &xeon_b2b_reg;
1775 ndev->xlat_reg = &xeon_sec_xlat;
1776
1777 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1778 ndev->peer_reg = &xeon_pri_reg;
1779
1780 if (b2b_mw_idx < 0)
1781 ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
1782 else
1783 ndev->b2b_idx = b2b_mw_idx;
1784
1785 if (ndev->b2b_idx >= ndev->mw_count) {
1786 dev_dbg(ndev_dev(ndev),
1787 "b2b_mw_idx %d invalid for mw_count %u\n",
1788 b2b_mw_idx, ndev->mw_count);
1789 return -EINVAL;
1790 }
1791
1792 dev_dbg(ndev_dev(ndev),
1793 "setting up b2b mw idx %d means %d\n",
1794 b2b_mw_idx, ndev->b2b_idx);
1795
1796 } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
1797 dev_warn(ndev_dev(ndev), "Reduce doorbell count by 1\n");
1798 ndev->db_count -= 1;
1799 }
1800
1801 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
1802 rc = xeon_setup_b2b_mw(ndev,
1803 &xeon_b2b_dsd_addr,
1804 &xeon_b2b_usd_addr);
1805 } else {
1806 rc = xeon_setup_b2b_mw(ndev,
1807 &xeon_b2b_usd_addr,
1808 &xeon_b2b_dsd_addr);
1809 }
1810 if (rc)
1811 return rc;
1812
1813 /* Enable Bus Master and Memory Space on the secondary side */
1814 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1815 ndev->self_mmio + XEON_SPCICMD_OFFSET);
1816
1817 break;
1818
1819 default:
1820 return -EINVAL;
1821 }
1822
1823 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1824
1825 ndev->reg->db_iowrite(ndev->db_valid_mask,
1826 ndev->self_mmio +
1827 ndev->self_reg->db_mask);
1828
1829 return 0;
1830}
1831
1832static int xeon_init_dev(struct intel_ntb_dev *ndev)
1833{
1834 struct pci_dev *pdev;
1835 u8 ppd;
1836 int rc, mem;
1837
1838 pdev = ndev_pdev(ndev);
1839
1840 switch (pdev->device) {
1841 /* There is a Xeon hardware errata related to writes to SDOORBELL or
1842 * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
1843 * which may hang the system. To workaround this use the second memory
1844 * window to access the interrupt and scratch pad registers on the
1845 * remote system.
1846 */
1847 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1848 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1849 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1850 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1851 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1852 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1853 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1854 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1855 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1856 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1857 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1858 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1859 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1860 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1861 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1862 ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
1863 break;
1864 }
1865
1866 switch (pdev->device) {
1867 /* There is a hardware errata related to accessing any register in
1868 * SB01BASE in the presence of bidirectional traffic crossing the NTB.
1869 */
1870 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1871 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1872 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1873 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1874 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1875 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1876 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1877 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1878 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1879 ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
1880 break;
1881 }
1882
1883 switch (pdev->device) {
1884 /* HW Errata on bit 14 of b2bdoorbell register. Writes will not be
1885 * mirrored to the remote system. Shrink the number of bits by one,
1886 * since bit 14 is the last bit.
1887 */
1888 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1889 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1890 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1891 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1892 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1893 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1894 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1895 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1896 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1897 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1898 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1899 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1900 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1901 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1902 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1903 ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
1904 break;
1905 }
1906
1907 ndev->reg = &xeon_reg;
1908
1909 rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
1910 if (rc)
1911 return -EIO;
1912
1913 ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
1914 dev_dbg(ndev_dev(ndev), "ppd %#x topo %s\n", ppd,
1915 ntb_topo_string(ndev->ntb.topo));
1916 if (ndev->ntb.topo == NTB_TOPO_NONE)
1917 return -EINVAL;
1918
1919 if (ndev->ntb.topo != NTB_TOPO_SEC) {
1920 ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd);
1921 dev_dbg(ndev_dev(ndev), "ppd %#x bar4_split %d\n",
1922 ppd, ndev->bar4_split);
1923 } else {
1924 /* This is a way for transparent BAR to figure out if we are
1925 * doing split BAR or not. There is no way for the hw on the
1926 * transparent side to know and set the PPD.
1927 */
1928 mem = pci_select_bars(pdev, IORESOURCE_MEM);
1929 ndev->bar4_split = hweight32(mem) ==
1930 HSX_SPLIT_BAR_MW_COUNT + 1;
1931 dev_dbg(ndev_dev(ndev), "mem %#x bar4_split %d\n",
1932 mem, ndev->bar4_split);
1933 }
1934
1935 rc = xeon_init_ntb(ndev);
1936 if (rc)
1937 return rc;
1938
1939 return xeon_init_isr(ndev);
1940}
1941
1942static void xeon_deinit_dev(struct intel_ntb_dev *ndev)
1943{
1944 xeon_deinit_isr(ndev);
1945}
1946
1947static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
1948{
1949 int rc;
1950
1951 pci_set_drvdata(pdev, ndev);
1952
1953 rc = pci_enable_device(pdev);
1954 if (rc)
1955 goto err_pci_enable;
1956
1957 rc = pci_request_regions(pdev, NTB_NAME);
1958 if (rc)
1959 goto err_pci_regions;
1960
1961 pci_set_master(pdev);
1962
1963 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1964 if (rc) {
1965 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1966 if (rc)
1967 goto err_dma_mask;
1968 dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n");
1969 }
1970
1971 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1972 if (rc) {
1973 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1974 if (rc)
1975 goto err_dma_mask;
1976 dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n");
1977 }
1978
1979 ndev->self_mmio = pci_iomap(pdev, 0, 0);
1980 if (!ndev->self_mmio) {
1981 rc = -EIO;
1982 goto err_mmio;
1983 }
1984 ndev->peer_mmio = ndev->self_mmio;
1985
1986 return 0;
1987
1988err_mmio:
1989err_dma_mask:
1990 pci_clear_master(pdev);
1991 pci_release_regions(pdev);
1992err_pci_regions:
1993 pci_disable_device(pdev);
1994err_pci_enable:
1995 pci_set_drvdata(pdev, NULL);
1996 return rc;
1997}
1998
1999static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
2000{
2001 struct pci_dev *pdev = ndev_pdev(ndev);
2002
2003 if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
2004 pci_iounmap(pdev, ndev->peer_mmio);
2005 pci_iounmap(pdev, ndev->self_mmio);
2006
2007 pci_clear_master(pdev);
2008 pci_release_regions(pdev);
2009 pci_disable_device(pdev);
2010 pci_set_drvdata(pdev, NULL);
2011}
2012
2013static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
2014 struct pci_dev *pdev)
2015{
2016 ndev->ntb.pdev = pdev;
2017 ndev->ntb.topo = NTB_TOPO_NONE;
2018 ndev->ntb.ops = &intel_ntb_ops;
2019
2020 ndev->b2b_off = 0;
2021 ndev->b2b_idx = UINT_MAX;
2022
2023 ndev->bar4_split = 0;
2024
2025 ndev->mw_count = 0;
2026 ndev->spad_count = 0;
2027 ndev->db_count = 0;
2028 ndev->db_vec_count = 0;
2029 ndev->db_vec_shift = 0;
2030
2031 ndev->ntb_ctl = 0;
2032 ndev->lnk_sta = 0;
2033
2034 ndev->db_valid_mask = 0;
2035 ndev->db_link_mask = 0;
2036 ndev->db_mask = 0;
2037
2038 spin_lock_init(&ndev->db_mask_lock);
2039}
2040
2041static int intel_ntb_pci_probe(struct pci_dev *pdev,
2042 const struct pci_device_id *id)
2043{
2044 struct intel_ntb_dev *ndev;
2045 int rc, node;
2046
2047 node = dev_to_node(&pdev->dev);
2048
2049 if (pdev_is_atom(pdev)) {
2050 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
2051 if (!ndev) {
2052 rc = -ENOMEM;
2053 goto err_ndev;
2054 }
2055
2056 ndev_init_struct(ndev, pdev);
2057
2058 rc = intel_ntb_init_pci(ndev, pdev);
2059 if (rc)
2060 goto err_init_pci;
2061
2062 rc = atom_init_dev(ndev);
2063 if (rc)
2064 goto err_init_dev;
2065
2066 } else if (pdev_is_xeon(pdev)) {
2067 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
2068 if (!ndev) {
2069 rc = -ENOMEM;
2070 goto err_ndev;
2071 }
2072
2073 ndev_init_struct(ndev, pdev);
2074
2075 rc = intel_ntb_init_pci(ndev, pdev);
2076 if (rc)
2077 goto err_init_pci;
2078
2079 rc = xeon_init_dev(ndev);
2080 if (rc)
2081 goto err_init_dev;
2082
2083 } else {
2084 rc = -EINVAL;
2085 goto err_ndev;
2086 }
2087
2088 ndev_reset_unsafe_flags(ndev);
2089
2090 ndev->reg->poll_link(ndev);
2091
2092 ndev_init_debugfs(ndev);
2093
2094 rc = ntb_register_device(&ndev->ntb);
2095 if (rc)
2096 goto err_register;
2097
2098 dev_info(&pdev->dev, "NTB device registered.\n");
2099
2100 return 0;
2101
2102err_register:
2103 ndev_deinit_debugfs(ndev);
2104 if (pdev_is_atom(pdev))
2105 atom_deinit_dev(ndev);
2106 else if (pdev_is_xeon(pdev))
2107 xeon_deinit_dev(ndev);
2108err_init_dev:
2109 intel_ntb_deinit_pci(ndev);
2110err_init_pci:
2111 kfree(ndev);
2112err_ndev:
2113 return rc;
2114}
2115
2116static void intel_ntb_pci_remove(struct pci_dev *pdev)
2117{
2118 struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
2119
2120 ntb_unregister_device(&ndev->ntb);
2121 ndev_deinit_debugfs(ndev);
2122 if (pdev_is_atom(pdev))
2123 atom_deinit_dev(ndev);
2124 else if (pdev_is_xeon(pdev))
2125 xeon_deinit_dev(ndev);
2126 intel_ntb_deinit_pci(ndev);
2127 kfree(ndev);
2128}
2129
2130static const struct intel_ntb_reg atom_reg = {
2131 .poll_link = atom_poll_link,
2132 .link_is_up = atom_link_is_up,
2133 .db_ioread = atom_db_ioread,
2134 .db_iowrite = atom_db_iowrite,
2135 .db_size = sizeof(u64),
2136 .ntb_ctl = ATOM_NTBCNTL_OFFSET,
2137 .mw_bar = {2, 4},
2138};
2139
2140static const struct intel_ntb_alt_reg atom_pri_reg = {
2141 .db_bell = ATOM_PDOORBELL_OFFSET,
2142 .db_mask = ATOM_PDBMSK_OFFSET,
2143 .spad = ATOM_SPAD_OFFSET,
2144};
2145
2146static const struct intel_ntb_alt_reg atom_b2b_reg = {
2147 .db_bell = ATOM_B2B_DOORBELL_OFFSET,
2148 .spad = ATOM_B2B_SPAD_OFFSET,
2149};
2150
2151static const struct intel_ntb_xlat_reg atom_sec_xlat = {
2152 /* FIXME : .bar0_base = ATOM_SBAR0BASE_OFFSET, */
2153 /* FIXME : .bar2_limit = ATOM_SBAR2LMT_OFFSET, */
2154 .bar2_xlat = ATOM_SBAR2XLAT_OFFSET,
2155};
2156
2157static const struct intel_ntb_reg xeon_reg = {
2158 .poll_link = xeon_poll_link,
2159 .link_is_up = xeon_link_is_up,
2160 .db_ioread = xeon_db_ioread,
2161 .db_iowrite = xeon_db_iowrite,
2162 .db_size = sizeof(u32),
2163 .ntb_ctl = XEON_NTBCNTL_OFFSET,
2164 .mw_bar = {2, 4, 5},
2165};
2166
2167static const struct intel_ntb_alt_reg xeon_pri_reg = {
2168 .db_bell = XEON_PDOORBELL_OFFSET,
2169 .db_mask = XEON_PDBMSK_OFFSET,
2170 .spad = XEON_SPAD_OFFSET,
2171};
2172
2173static const struct intel_ntb_alt_reg xeon_sec_reg = {
2174 .db_bell = XEON_SDOORBELL_OFFSET,
2175 .db_mask = XEON_SDBMSK_OFFSET,
2176 /* second half of the scratchpads */
2177 .spad = XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1),
2178};
2179
2180static const struct intel_ntb_alt_reg xeon_b2b_reg = {
2181 .db_bell = XEON_B2B_DOORBELL_OFFSET,
2182 .spad = XEON_B2B_SPAD_OFFSET,
2183};
2184
2185static const struct intel_ntb_xlat_reg xeon_pri_xlat = {
2186 /* Note: no primary .bar0_base visible to the secondary side.
2187 *
2188 * The secondary side cannot get the base address stored in primary
2189 * bars. The base address is necessary to set the limit register to
2190 * any value other than zero, or unlimited.
2191 *
2192 * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
2193 * window by setting the limit equal to base, nor can it limit the size
2194 * of the memory window by setting the limit to base + size.
2195 */
2196 .bar2_limit = XEON_PBAR23LMT_OFFSET,
2197 .bar2_xlat = XEON_PBAR23XLAT_OFFSET,
2198};
2199
2200static const struct intel_ntb_xlat_reg xeon_sec_xlat = {
2201 .bar0_base = XEON_SBAR0BASE_OFFSET,
2202 .bar2_limit = XEON_SBAR23LMT_OFFSET,
2203 .bar2_xlat = XEON_SBAR23XLAT_OFFSET,
2204};
2205
2206static struct intel_b2b_addr xeon_b2b_usd_addr = {
2207 .bar2_addr64 = XEON_B2B_BAR2_ADDR64,
2208 .bar4_addr64 = XEON_B2B_BAR4_ADDR64,
2209 .bar4_addr32 = XEON_B2B_BAR4_ADDR32,
2210 .bar5_addr32 = XEON_B2B_BAR5_ADDR32,
2211};
2212
2213static struct intel_b2b_addr xeon_b2b_dsd_addr = {
2214 .bar2_addr64 = XEON_B2B_BAR2_ADDR64,
2215 .bar4_addr64 = XEON_B2B_BAR4_ADDR64,
2216 .bar4_addr32 = XEON_B2B_BAR4_ADDR32,
2217 .bar5_addr32 = XEON_B2B_BAR5_ADDR32,
2218};
2219
2220/* operations for primary side of local ntb */
2221static const struct ntb_dev_ops intel_ntb_ops = {
2222 .mw_count = intel_ntb_mw_count,
2223 .mw_get_range = intel_ntb_mw_get_range,
2224 .mw_set_trans = intel_ntb_mw_set_trans,
2225 .link_is_up = intel_ntb_link_is_up,
2226 .link_enable = intel_ntb_link_enable,
2227 .link_disable = intel_ntb_link_disable,
2228 .db_is_unsafe = intel_ntb_db_is_unsafe,
2229 .db_valid_mask = intel_ntb_db_valid_mask,
2230 .db_vector_count = intel_ntb_db_vector_count,
2231 .db_vector_mask = intel_ntb_db_vector_mask,
2232 .db_read = intel_ntb_db_read,
2233 .db_clear = intel_ntb_db_clear,
2234 .db_set_mask = intel_ntb_db_set_mask,
2235 .db_clear_mask = intel_ntb_db_clear_mask,
2236 .peer_db_addr = intel_ntb_peer_db_addr,
2237 .peer_db_set = intel_ntb_peer_db_set,
2238 .spad_is_unsafe = intel_ntb_spad_is_unsafe,
2239 .spad_count = intel_ntb_spad_count,
2240 .spad_read = intel_ntb_spad_read,
2241 .spad_write = intel_ntb_spad_write,
2242 .peer_spad_addr = intel_ntb_peer_spad_addr,
2243 .peer_spad_read = intel_ntb_peer_spad_read,
2244 .peer_spad_write = intel_ntb_peer_spad_write,
2245};
2246
2247static const struct file_operations intel_ntb_debugfs_info = {
2248 .owner = THIS_MODULE,
2249 .open = simple_open,
2250 .read = ndev_debugfs_read,
2251};
2252
2253static const struct pci_device_id intel_ntb_pci_tbl[] = {
2254 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
2255 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
2256 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
2257 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
2258 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
2259 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX)},
2260 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
2261 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
2262 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
2263 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
2264 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_BDX)},
2265 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
2266 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
2267 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
2268 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
2269 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)},
2270 {0}
2271};
2272MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
2273
2274static struct pci_driver intel_ntb_pci_driver = {
2275 .name = KBUILD_MODNAME,
2276 .id_table = intel_ntb_pci_tbl,
2277 .probe = intel_ntb_pci_probe,
2278 .remove = intel_ntb_pci_remove,
2279};
2280
2281static int __init intel_ntb_pci_driver_init(void)
2282{
2283 pr_info("%s %s\n", NTB_DESC, NTB_VER);
2284
2285 if (debugfs_initialized())
2286 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
2287
2288 return pci_register_driver(&intel_ntb_pci_driver);
2289}
2290module_init(intel_ntb_pci_driver_init);
2291
2292static void __exit intel_ntb_pci_driver_exit(void)
2293{
2294 pci_unregister_driver(&intel_ntb_pci_driver);
2295
2296 debugfs_remove_recursive(debugfs_dir);
2297}
2298module_exit(intel_ntb_pci_driver_exit);
2299