Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Intel I/OAT DMA Linux driver
4 * Copyright(c) 2004 - 2015 Intel Corporation.
5 */
6
7/*
8 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
9 * copy operations.
10 */
11
12#include <linux/init.h>
13#include <linux/module.h>
14#include <linux/slab.h>
15#include <linux/pci.h>
16#include <linux/interrupt.h>
17#include <linux/dmaengine.h>
18#include <linux/delay.h>
19#include <linux/dma-mapping.h>
20#include <linux/workqueue.h>
21#include <linux/prefetch.h>
22#include <linux/sizes.h>
23#include "dma.h"
24#include "registers.h"
25#include "hw.h"
26
27#include "../dmaengine.h"
28
29static int completion_timeout = 200;
30module_param(completion_timeout, int, 0644);
31MODULE_PARM_DESC(completion_timeout,
32 "set ioat completion timeout [msec] (default 200 [msec])");
33static int idle_timeout = 2000;
34module_param(idle_timeout, int, 0644);
35MODULE_PARM_DESC(idle_timeout,
36 "set ioat idle timeout [msec] (default 2000 [msec])");
37
38#define IDLE_TIMEOUT msecs_to_jiffies(idle_timeout)
39#define COMPLETION_TIMEOUT msecs_to_jiffies(completion_timeout)
40
41static char *chanerr_str[] = {
42 "DMA Transfer Source Address Error",
43 "DMA Transfer Destination Address Error",
44 "Next Descriptor Address Error",
45 "Descriptor Error",
46 "Chan Address Value Error",
47 "CHANCMD Error",
48 "Chipset Uncorrectable Data Integrity Error",
49 "DMA Uncorrectable Data Integrity Error",
50 "Read Data Error",
51 "Write Data Error",
52 "Descriptor Control Error",
53 "Descriptor Transfer Size Error",
54 "Completion Address Error",
55 "Interrupt Configuration Error",
56 "Super extended descriptor Address Error",
57 "Unaffiliated Error",
58 "CRC or XOR P Error",
59 "XOR Q Error",
60 "Descriptor Count Error",
61 "DIF All F detect Error",
62 "Guard Tag verification Error",
63 "Application Tag verification Error",
64 "Reference Tag verification Error",
65 "Bundle Bit Error",
66 "Result DIF All F detect Error",
67 "Result Guard Tag verification Error",
68 "Result Application Tag verification Error",
69 "Result Reference Tag verification Error",
70};
71
72static void ioat_eh(struct ioatdma_chan *ioat_chan);
73
74static void ioat_print_chanerrs(struct ioatdma_chan *ioat_chan, u32 chanerr)
75{
76 int i;
77
78 for (i = 0; i < ARRAY_SIZE(chanerr_str); i++) {
79 if ((chanerr >> i) & 1) {
80 dev_err(to_dev(ioat_chan), "Err(%d): %s\n",
81 i, chanerr_str[i]);
82 }
83 }
84}
85
86/**
87 * ioat_dma_do_interrupt - handler used for single vector interrupt mode
88 * @irq: interrupt id
89 * @data: interrupt data
90 */
91irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
92{
93 struct ioatdma_device *instance = data;
94 struct ioatdma_chan *ioat_chan;
95 unsigned long attnstatus;
96 int bit;
97 u8 intrctrl;
98
99 intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
100
101 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
102 return IRQ_NONE;
103
104 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
105 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
106 return IRQ_NONE;
107 }
108
109 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
110 for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) {
111 ioat_chan = ioat_chan_by_index(instance, bit);
112 if (test_bit(IOAT_RUN, &ioat_chan->state))
113 tasklet_schedule(&ioat_chan->cleanup_task);
114 }
115
116 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
117 return IRQ_HANDLED;
118}
119
120/**
121 * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
122 * @irq: interrupt id
123 * @data: interrupt data
124 */
125irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
126{
127 struct ioatdma_chan *ioat_chan = data;
128
129 if (test_bit(IOAT_RUN, &ioat_chan->state))
130 tasklet_schedule(&ioat_chan->cleanup_task);
131
132 return IRQ_HANDLED;
133}
134
135void ioat_stop(struct ioatdma_chan *ioat_chan)
136{
137 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
138 struct pci_dev *pdev = ioat_dma->pdev;
139 int chan_id = chan_num(ioat_chan);
140 struct msix_entry *msix;
141
142 /* 1/ stop irq from firing tasklets
143 * 2/ stop the tasklet from re-arming irqs
144 */
145 clear_bit(IOAT_RUN, &ioat_chan->state);
146
147 /* flush inflight interrupts */
148 switch (ioat_dma->irq_mode) {
149 case IOAT_MSIX:
150 msix = &ioat_dma->msix_entries[chan_id];
151 synchronize_irq(msix->vector);
152 break;
153 case IOAT_MSI:
154 case IOAT_INTX:
155 synchronize_irq(pdev->irq);
156 break;
157 default:
158 break;
159 }
160
161 /* flush inflight timers */
162 del_timer_sync(&ioat_chan->timer);
163
164 /* flush inflight tasklet runs */
165 tasklet_kill(&ioat_chan->cleanup_task);
166
167 /* final cleanup now that everything is quiesced and can't re-arm */
168 ioat_cleanup_event(&ioat_chan->cleanup_task);
169}
170
171static void __ioat_issue_pending(struct ioatdma_chan *ioat_chan)
172{
173 ioat_chan->dmacount += ioat_ring_pending(ioat_chan);
174 ioat_chan->issued = ioat_chan->head;
175 writew(ioat_chan->dmacount,
176 ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
177 dev_dbg(to_dev(ioat_chan),
178 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
179 __func__, ioat_chan->head, ioat_chan->tail,
180 ioat_chan->issued, ioat_chan->dmacount);
181}
182
183void ioat_issue_pending(struct dma_chan *c)
184{
185 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
186
187 if (ioat_ring_pending(ioat_chan)) {
188 spin_lock_bh(&ioat_chan->prep_lock);
189 __ioat_issue_pending(ioat_chan);
190 spin_unlock_bh(&ioat_chan->prep_lock);
191 }
192}
193
194/**
195 * ioat_update_pending - log pending descriptors
196 * @ioat_chan: ioat+ channel
197 *
198 * Check if the number of unsubmitted descriptors has exceeded the
199 * watermark. Called with prep_lock held
200 */
201static void ioat_update_pending(struct ioatdma_chan *ioat_chan)
202{
203 if (ioat_ring_pending(ioat_chan) > ioat_pending_level)
204 __ioat_issue_pending(ioat_chan);
205}
206
207static void __ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
208{
209 struct ioat_ring_ent *desc;
210 struct ioat_dma_descriptor *hw;
211
212 if (ioat_ring_space(ioat_chan) < 1) {
213 dev_err(to_dev(ioat_chan),
214 "Unable to start null desc - ring full\n");
215 return;
216 }
217
218 dev_dbg(to_dev(ioat_chan),
219 "%s: head: %#x tail: %#x issued: %#x\n",
220 __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
221 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head);
222
223 hw = desc->hw;
224 hw->ctl = 0;
225 hw->ctl_f.null = 1;
226 hw->ctl_f.int_en = 1;
227 hw->ctl_f.compl_write = 1;
228 /* set size to non-zero value (channel returns error when size is 0) */
229 hw->size = NULL_DESC_BUFFER_SIZE;
230 hw->src_addr = 0;
231 hw->dst_addr = 0;
232 async_tx_ack(&desc->txd);
233 ioat_set_chainaddr(ioat_chan, desc->txd.phys);
234 dump_desc_dbg(ioat_chan, desc);
235 /* make sure descriptors are written before we submit */
236 wmb();
237 ioat_chan->head += 1;
238 __ioat_issue_pending(ioat_chan);
239}
240
241void ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
242{
243 spin_lock_bh(&ioat_chan->prep_lock);
244 if (!test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))
245 __ioat_start_null_desc(ioat_chan);
246 spin_unlock_bh(&ioat_chan->prep_lock);
247}
248
249static void __ioat_restart_chan(struct ioatdma_chan *ioat_chan)
250{
251 /* set the tail to be re-issued */
252 ioat_chan->issued = ioat_chan->tail;
253 ioat_chan->dmacount = 0;
254 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
255
256 dev_dbg(to_dev(ioat_chan),
257 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
258 __func__, ioat_chan->head, ioat_chan->tail,
259 ioat_chan->issued, ioat_chan->dmacount);
260
261 if (ioat_ring_pending(ioat_chan)) {
262 struct ioat_ring_ent *desc;
263
264 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
265 ioat_set_chainaddr(ioat_chan, desc->txd.phys);
266 __ioat_issue_pending(ioat_chan);
267 } else
268 __ioat_start_null_desc(ioat_chan);
269}
270
271static int ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo)
272{
273 unsigned long end = jiffies + tmo;
274 int err = 0;
275 u32 status;
276
277 status = ioat_chansts(ioat_chan);
278 if (is_ioat_active(status) || is_ioat_idle(status))
279 ioat_suspend(ioat_chan);
280 while (is_ioat_active(status) || is_ioat_idle(status)) {
281 if (tmo && time_after(jiffies, end)) {
282 err = -ETIMEDOUT;
283 break;
284 }
285 status = ioat_chansts(ioat_chan);
286 cpu_relax();
287 }
288
289 return err;
290}
291
292static int ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo)
293{
294 unsigned long end = jiffies + tmo;
295 int err = 0;
296
297 ioat_reset(ioat_chan);
298 while (ioat_reset_pending(ioat_chan)) {
299 if (end && time_after(jiffies, end)) {
300 err = -ETIMEDOUT;
301 break;
302 }
303 cpu_relax();
304 }
305
306 return err;
307}
308
309static dma_cookie_t ioat_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
310 __releases(&ioat_chan->prep_lock)
311{
312 struct dma_chan *c = tx->chan;
313 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
314 dma_cookie_t cookie;
315
316 cookie = dma_cookie_assign(tx);
317 dev_dbg(to_dev(ioat_chan), "%s: cookie: %d\n", __func__, cookie);
318
319 if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
320 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
321
322 /* make descriptor updates visible before advancing ioat->head,
323 * this is purposefully not smp_wmb() since we are also
324 * publishing the descriptor updates to a dma device
325 */
326 wmb();
327
328 ioat_chan->head += ioat_chan->produce;
329
330 ioat_update_pending(ioat_chan);
331 spin_unlock_bh(&ioat_chan->prep_lock);
332
333 return cookie;
334}
335
336static struct ioat_ring_ent *
337ioat_alloc_ring_ent(struct dma_chan *chan, int idx, gfp_t flags)
338{
339 struct ioat_dma_descriptor *hw;
340 struct ioat_ring_ent *desc;
341 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
342 int chunk;
343 dma_addr_t phys;
344 u8 *pos;
345 off_t offs;
346
347 chunk = idx / IOAT_DESCS_PER_CHUNK;
348 idx &= (IOAT_DESCS_PER_CHUNK - 1);
349 offs = idx * IOAT_DESC_SZ;
350 pos = (u8 *)ioat_chan->descs[chunk].virt + offs;
351 phys = ioat_chan->descs[chunk].hw + offs;
352 hw = (struct ioat_dma_descriptor *)pos;
353 memset(hw, 0, sizeof(*hw));
354
355 desc = kmem_cache_zalloc(ioat_cache, flags);
356 if (!desc)
357 return NULL;
358
359 dma_async_tx_descriptor_init(&desc->txd, chan);
360 desc->txd.tx_submit = ioat_tx_submit_unlock;
361 desc->hw = hw;
362 desc->txd.phys = phys;
363 return desc;
364}
365
366void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
367{
368 kmem_cache_free(ioat_cache, desc);
369}
370
371struct ioat_ring_ent **
372ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
373{
374 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
375 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
376 struct ioat_ring_ent **ring;
377 int total_descs = 1 << order;
378 int i, chunks;
379
380 /* allocate the array to hold the software ring */
381 ring = kcalloc(total_descs, sizeof(*ring), flags);
382 if (!ring)
383 return NULL;
384
385 chunks = (total_descs * IOAT_DESC_SZ) / IOAT_CHUNK_SIZE;
386 ioat_chan->desc_chunks = chunks;
387
388 for (i = 0; i < chunks; i++) {
389 struct ioat_descs *descs = &ioat_chan->descs[i];
390
391 descs->virt = dma_alloc_coherent(to_dev(ioat_chan),
392 IOAT_CHUNK_SIZE, &descs->hw, flags);
393 if (!descs->virt) {
394 int idx;
395
396 for (idx = 0; idx < i; idx++) {
397 descs = &ioat_chan->descs[idx];
398 dma_free_coherent(to_dev(ioat_chan),
399 IOAT_CHUNK_SIZE,
400 descs->virt, descs->hw);
401 descs->virt = NULL;
402 descs->hw = 0;
403 }
404
405 ioat_chan->desc_chunks = 0;
406 kfree(ring);
407 return NULL;
408 }
409 }
410
411 for (i = 0; i < total_descs; i++) {
412 ring[i] = ioat_alloc_ring_ent(c, i, flags);
413 if (!ring[i]) {
414 int idx;
415
416 while (i--)
417 ioat_free_ring_ent(ring[i], c);
418
419 for (idx = 0; idx < ioat_chan->desc_chunks; idx++) {
420 dma_free_coherent(to_dev(ioat_chan),
421 IOAT_CHUNK_SIZE,
422 ioat_chan->descs[idx].virt,
423 ioat_chan->descs[idx].hw);
424 ioat_chan->descs[idx].virt = NULL;
425 ioat_chan->descs[idx].hw = 0;
426 }
427
428 ioat_chan->desc_chunks = 0;
429 kfree(ring);
430 return NULL;
431 }
432 set_desc_id(ring[i], i);
433 }
434
435 /* link descs */
436 for (i = 0; i < total_descs-1; i++) {
437 struct ioat_ring_ent *next = ring[i+1];
438 struct ioat_dma_descriptor *hw = ring[i]->hw;
439
440 hw->next = next->txd.phys;
441 }
442 ring[i]->hw->next = ring[0]->txd.phys;
443
444 /* setup descriptor pre-fetching for v3.4 */
445 if (ioat_dma->cap & IOAT_CAP_DPS) {
446 u16 drsctl = IOAT_CHAN_DRSZ_2MB | IOAT_CHAN_DRS_EN;
447
448 if (chunks == 1)
449 drsctl |= IOAT_CHAN_DRS_AUTOWRAP;
450
451 writew(drsctl, ioat_chan->reg_base + IOAT_CHAN_DRSCTL_OFFSET);
452
453 }
454
455 return ring;
456}
457
458/**
459 * ioat_check_space_lock - verify space and grab ring producer lock
460 * @ioat_chan: ioat,3 channel (ring) to operate on
461 * @num_descs: allocation length
462 */
463int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs)
464 __acquires(&ioat_chan->prep_lock)
465{
466 spin_lock_bh(&ioat_chan->prep_lock);
467 /* never allow the last descriptor to be consumed, we need at
468 * least one free at all times to allow for on-the-fly ring
469 * resizing.
470 */
471 if (likely(ioat_ring_space(ioat_chan) > num_descs)) {
472 dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n",
473 __func__, num_descs, ioat_chan->head,
474 ioat_chan->tail, ioat_chan->issued);
475 ioat_chan->produce = num_descs;
476 return 0; /* with ioat->prep_lock held */
477 }
478 spin_unlock_bh(&ioat_chan->prep_lock);
479
480 dev_dbg_ratelimited(to_dev(ioat_chan),
481 "%s: ring full! num_descs: %d (%x:%x:%x)\n",
482 __func__, num_descs, ioat_chan->head,
483 ioat_chan->tail, ioat_chan->issued);
484
485 /* progress reclaim in the allocation failure case we may be
486 * called under bh_disabled so we need to trigger the timer
487 * event directly
488 */
489 if (time_is_before_jiffies(ioat_chan->timer.expires)
490 && timer_pending(&ioat_chan->timer)) {
491 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
492 ioat_timer_event(&ioat_chan->timer);
493 }
494
495 return -ENOMEM;
496}
497
498static bool desc_has_ext(struct ioat_ring_ent *desc)
499{
500 struct ioat_dma_descriptor *hw = desc->hw;
501
502 if (hw->ctl_f.op == IOAT_OP_XOR ||
503 hw->ctl_f.op == IOAT_OP_XOR_VAL) {
504 struct ioat_xor_descriptor *xor = desc->xor;
505
506 if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5)
507 return true;
508 } else if (hw->ctl_f.op == IOAT_OP_PQ ||
509 hw->ctl_f.op == IOAT_OP_PQ_VAL) {
510 struct ioat_pq_descriptor *pq = desc->pq;
511
512 if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3)
513 return true;
514 }
515
516 return false;
517}
518
519static void
520ioat_free_sed(struct ioatdma_device *ioat_dma, struct ioat_sed_ent *sed)
521{
522 if (!sed)
523 return;
524
525 dma_pool_free(ioat_dma->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma);
526 kmem_cache_free(ioat_sed_cache, sed);
527}
528
529static u64 ioat_get_current_completion(struct ioatdma_chan *ioat_chan)
530{
531 u64 phys_complete;
532 u64 completion;
533
534 completion = *ioat_chan->completion;
535 phys_complete = ioat_chansts_to_addr(completion);
536
537 dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n", __func__,
538 (unsigned long long) phys_complete);
539
540 return phys_complete;
541}
542
543static bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan,
544 u64 *phys_complete)
545{
546 *phys_complete = ioat_get_current_completion(ioat_chan);
547 if (*phys_complete == ioat_chan->last_completion)
548 return false;
549
550 clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
551 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
552
553 return true;
554}
555
556static void
557desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc)
558{
559 struct ioat_dma_descriptor *hw = desc->hw;
560
561 switch (hw->ctl_f.op) {
562 case IOAT_OP_PQ_VAL:
563 case IOAT_OP_PQ_VAL_16S:
564 {
565 struct ioat_pq_descriptor *pq = desc->pq;
566
567 /* check if there's error written */
568 if (!pq->dwbes_f.wbes)
569 return;
570
571 /* need to set a chanerr var for checking to clear later */
572
573 if (pq->dwbes_f.p_val_err)
574 *desc->result |= SUM_CHECK_P_RESULT;
575
576 if (pq->dwbes_f.q_val_err)
577 *desc->result |= SUM_CHECK_Q_RESULT;
578
579 return;
580 }
581 default:
582 return;
583 }
584}
585
586/**
587 * __ioat_cleanup - reclaim used descriptors
588 * @ioat_chan: channel (ring) to clean
589 * @phys_complete: zeroed (or not) completion address (from status)
590 */
591static void __ioat_cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete)
592{
593 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
594 struct ioat_ring_ent *desc;
595 bool seen_current = false;
596 int idx = ioat_chan->tail, i;
597 u16 active;
598
599 dev_dbg(to_dev(ioat_chan), "%s: head: %#x tail: %#x issued: %#x\n",
600 __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
601
602 /*
603 * At restart of the channel, the completion address and the
604 * channel status will be 0 due to starting a new chain. Since
605 * it's new chain and the first descriptor "fails", there is
606 * nothing to clean up. We do not want to reap the entire submitted
607 * chain due to this 0 address value and then BUG.
608 */
609 if (!phys_complete)
610 return;
611
612 active = ioat_ring_active(ioat_chan);
613 for (i = 0; i < active && !seen_current; i++) {
614 struct dma_async_tx_descriptor *tx;
615
616 prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1));
617 desc = ioat_get_ring_ent(ioat_chan, idx + i);
618 dump_desc_dbg(ioat_chan, desc);
619
620 /* set err stat if we are using dwbes */
621 if (ioat_dma->cap & IOAT_CAP_DWBES)
622 desc_get_errstat(ioat_chan, desc);
623
624 tx = &desc->txd;
625 if (tx->cookie) {
626 dma_cookie_complete(tx);
627 dma_descriptor_unmap(tx);
628 dmaengine_desc_get_callback_invoke(tx, NULL);
629 tx->callback = NULL;
630 tx->callback_result = NULL;
631 }
632
633 if (tx->phys == phys_complete)
634 seen_current = true;
635
636 /* skip extended descriptors */
637 if (desc_has_ext(desc)) {
638 BUG_ON(i + 1 >= active);
639 i++;
640 }
641
642 /* cleanup super extended descriptors */
643 if (desc->sed) {
644 ioat_free_sed(ioat_dma, desc->sed);
645 desc->sed = NULL;
646 }
647 }
648
649 /* finish all descriptor reads before incrementing tail */
650 smp_mb();
651 ioat_chan->tail = idx + i;
652 /* no active descs have written a completion? */
653 BUG_ON(active && !seen_current);
654 ioat_chan->last_completion = phys_complete;
655
656 if (active - i == 0) {
657 dev_dbg(to_dev(ioat_chan), "%s: cancel completion timeout\n",
658 __func__);
659 mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
660 }
661
662 /* microsecond delay by sysfs variable per pending descriptor */
663 if (ioat_chan->intr_coalesce != ioat_chan->prev_intr_coalesce) {
664 writew(min((ioat_chan->intr_coalesce * (active - i)),
665 IOAT_INTRDELAY_MASK),
666 ioat_chan->ioat_dma->reg_base + IOAT_INTRDELAY_OFFSET);
667 ioat_chan->prev_intr_coalesce = ioat_chan->intr_coalesce;
668 }
669}
670
671static void ioat_cleanup(struct ioatdma_chan *ioat_chan)
672{
673 u64 phys_complete;
674
675 spin_lock_bh(&ioat_chan->cleanup_lock);
676
677 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
678 __ioat_cleanup(ioat_chan, phys_complete);
679
680 if (is_ioat_halted(*ioat_chan->completion)) {
681 u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
682
683 if (chanerr &
684 (IOAT_CHANERR_HANDLE_MASK | IOAT_CHANERR_RECOVER_MASK)) {
685 mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
686 ioat_eh(ioat_chan);
687 }
688 }
689
690 spin_unlock_bh(&ioat_chan->cleanup_lock);
691}
692
693void ioat_cleanup_event(struct tasklet_struct *t)
694{
695 struct ioatdma_chan *ioat_chan = from_tasklet(ioat_chan, t, cleanup_task);
696
697 ioat_cleanup(ioat_chan);
698 if (!test_bit(IOAT_RUN, &ioat_chan->state))
699 return;
700 writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
701}
702
703static void ioat_restart_channel(struct ioatdma_chan *ioat_chan)
704{
705 u64 phys_complete;
706
707 /* set the completion address register again */
708 writel(lower_32_bits(ioat_chan->completion_dma),
709 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
710 writel(upper_32_bits(ioat_chan->completion_dma),
711 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
712
713 ioat_quiesce(ioat_chan, 0);
714 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
715 __ioat_cleanup(ioat_chan, phys_complete);
716
717 __ioat_restart_chan(ioat_chan);
718}
719
720
721static void ioat_abort_descs(struct ioatdma_chan *ioat_chan)
722{
723 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
724 struct ioat_ring_ent *desc;
725 u16 active;
726 int idx = ioat_chan->tail, i;
727
728 /*
729 * We assume that the failed descriptor has been processed.
730 * Now we are just returning all the remaining submitted
731 * descriptors to abort.
732 */
733 active = ioat_ring_active(ioat_chan);
734
735 /* we skip the failed descriptor that tail points to */
736 for (i = 1; i < active; i++) {
737 struct dma_async_tx_descriptor *tx;
738
739 prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1));
740 desc = ioat_get_ring_ent(ioat_chan, idx + i);
741
742 tx = &desc->txd;
743 if (tx->cookie) {
744 struct dmaengine_result res;
745
746 dma_cookie_complete(tx);
747 dma_descriptor_unmap(tx);
748 res.result = DMA_TRANS_ABORTED;
749 dmaengine_desc_get_callback_invoke(tx, &res);
750 tx->callback = NULL;
751 tx->callback_result = NULL;
752 }
753
754 /* skip extended descriptors */
755 if (desc_has_ext(desc)) {
756 WARN_ON(i + 1 >= active);
757 i++;
758 }
759
760 /* cleanup super extended descriptors */
761 if (desc->sed) {
762 ioat_free_sed(ioat_dma, desc->sed);
763 desc->sed = NULL;
764 }
765 }
766
767 smp_mb(); /* finish all descriptor reads before incrementing tail */
768 ioat_chan->tail = idx + active;
769
770 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
771 ioat_chan->last_completion = *ioat_chan->completion = desc->txd.phys;
772}
773
774static void ioat_eh(struct ioatdma_chan *ioat_chan)
775{
776 struct pci_dev *pdev = to_pdev(ioat_chan);
777 struct ioat_dma_descriptor *hw;
778 struct dma_async_tx_descriptor *tx;
779 u64 phys_complete;
780 struct ioat_ring_ent *desc;
781 u32 err_handled = 0;
782 u32 chanerr_int;
783 u32 chanerr;
784 bool abort = false;
785 struct dmaengine_result res;
786
787 /* cleanup so tail points to descriptor that caused the error */
788 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
789 __ioat_cleanup(ioat_chan, phys_complete);
790
791 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
792 pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr_int);
793
794 dev_dbg(to_dev(ioat_chan), "%s: error = %x:%x\n",
795 __func__, chanerr, chanerr_int);
796
797 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
798 hw = desc->hw;
799 dump_desc_dbg(ioat_chan, desc);
800
801 switch (hw->ctl_f.op) {
802 case IOAT_OP_XOR_VAL:
803 if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
804 *desc->result |= SUM_CHECK_P_RESULT;
805 err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR;
806 }
807 break;
808 case IOAT_OP_PQ_VAL:
809 case IOAT_OP_PQ_VAL_16S:
810 if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
811 *desc->result |= SUM_CHECK_P_RESULT;
812 err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR;
813 }
814 if (chanerr & IOAT_CHANERR_XOR_Q_ERR) {
815 *desc->result |= SUM_CHECK_Q_RESULT;
816 err_handled |= IOAT_CHANERR_XOR_Q_ERR;
817 }
818 break;
819 }
820
821 if (chanerr & IOAT_CHANERR_RECOVER_MASK) {
822 if (chanerr & IOAT_CHANERR_READ_DATA_ERR) {
823 res.result = DMA_TRANS_READ_FAILED;
824 err_handled |= IOAT_CHANERR_READ_DATA_ERR;
825 } else if (chanerr & IOAT_CHANERR_WRITE_DATA_ERR) {
826 res.result = DMA_TRANS_WRITE_FAILED;
827 err_handled |= IOAT_CHANERR_WRITE_DATA_ERR;
828 }
829
830 abort = true;
831 } else
832 res.result = DMA_TRANS_NOERROR;
833
834 /* fault on unhandled error or spurious halt */
835 if (chanerr ^ err_handled || chanerr == 0) {
836 dev_err(to_dev(ioat_chan), "%s: fatal error (%x:%x)\n",
837 __func__, chanerr, err_handled);
838 dev_err(to_dev(ioat_chan), "Errors handled:\n");
839 ioat_print_chanerrs(ioat_chan, err_handled);
840 dev_err(to_dev(ioat_chan), "Errors not handled:\n");
841 ioat_print_chanerrs(ioat_chan, (chanerr & ~err_handled));
842
843 BUG();
844 }
845
846 /* cleanup the faulty descriptor since we are continuing */
847 tx = &desc->txd;
848 if (tx->cookie) {
849 dma_cookie_complete(tx);
850 dma_descriptor_unmap(tx);
851 dmaengine_desc_get_callback_invoke(tx, &res);
852 tx->callback = NULL;
853 tx->callback_result = NULL;
854 }
855
856 /* mark faulting descriptor as complete */
857 *ioat_chan->completion = desc->txd.phys;
858
859 spin_lock_bh(&ioat_chan->prep_lock);
860 /* we need abort all descriptors */
861 if (abort) {
862 ioat_abort_descs(ioat_chan);
863 /* clean up the channel, we could be in weird state */
864 ioat_reset_hw(ioat_chan);
865 }
866
867 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
868 pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int);
869
870 ioat_restart_channel(ioat_chan);
871 spin_unlock_bh(&ioat_chan->prep_lock);
872}
873
874static void check_active(struct ioatdma_chan *ioat_chan)
875{
876 if (ioat_ring_active(ioat_chan)) {
877 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
878 return;
879 }
880
881 if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
882 mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
883}
884
885static void ioat_reboot_chan(struct ioatdma_chan *ioat_chan)
886{
887 spin_lock_bh(&ioat_chan->prep_lock);
888 set_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
889 spin_unlock_bh(&ioat_chan->prep_lock);
890
891 ioat_abort_descs(ioat_chan);
892 dev_warn(to_dev(ioat_chan), "Reset channel...\n");
893 ioat_reset_hw(ioat_chan);
894 dev_warn(to_dev(ioat_chan), "Restart channel...\n");
895 ioat_restart_channel(ioat_chan);
896
897 spin_lock_bh(&ioat_chan->prep_lock);
898 clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
899 spin_unlock_bh(&ioat_chan->prep_lock);
900}
901
902void ioat_timer_event(struct timer_list *t)
903{
904 struct ioatdma_chan *ioat_chan = from_timer(ioat_chan, t, timer);
905 dma_addr_t phys_complete;
906 u64 status;
907
908 status = ioat_chansts(ioat_chan);
909
910 /* when halted due to errors check for channel
911 * programming errors before advancing the completion state
912 */
913 if (is_ioat_halted(status)) {
914 u32 chanerr;
915
916 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
917 dev_err(to_dev(ioat_chan), "%s: Channel halted (%x)\n",
918 __func__, chanerr);
919 dev_err(to_dev(ioat_chan), "Errors:\n");
920 ioat_print_chanerrs(ioat_chan, chanerr);
921
922 if (test_bit(IOAT_RUN, &ioat_chan->state)) {
923 spin_lock_bh(&ioat_chan->cleanup_lock);
924 ioat_reboot_chan(ioat_chan);
925 spin_unlock_bh(&ioat_chan->cleanup_lock);
926 }
927
928 return;
929 }
930
931 spin_lock_bh(&ioat_chan->cleanup_lock);
932
933 /* handle the no-actives case */
934 if (!ioat_ring_active(ioat_chan)) {
935 spin_lock_bh(&ioat_chan->prep_lock);
936 check_active(ioat_chan);
937 spin_unlock_bh(&ioat_chan->prep_lock);
938 goto unlock_out;
939 }
940
941 /* handle the missed cleanup case */
942 if (ioat_cleanup_preamble(ioat_chan, &phys_complete)) {
943 /* timer restarted in ioat_cleanup_preamble
944 * and IOAT_COMPLETION_ACK cleared
945 */
946 __ioat_cleanup(ioat_chan, phys_complete);
947 goto unlock_out;
948 }
949
950 /* if we haven't made progress and we have already
951 * acknowledged a pending completion once, then be more
952 * forceful with a restart
953 */
954 if (test_bit(IOAT_COMPLETION_ACK, &ioat_chan->state)) {
955 u32 chanerr;
956
957 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
958 dev_err(to_dev(ioat_chan), "CHANSTS: %#Lx CHANERR: %#x\n",
959 status, chanerr);
960 dev_err(to_dev(ioat_chan), "Errors:\n");
961 ioat_print_chanerrs(ioat_chan, chanerr);
962
963 dev_dbg(to_dev(ioat_chan), "Active descriptors: %d\n",
964 ioat_ring_active(ioat_chan));
965
966 ioat_reboot_chan(ioat_chan);
967
968 goto unlock_out;
969 }
970
971 /* handle missed issue pending case */
972 if (ioat_ring_pending(ioat_chan)) {
973 dev_warn(to_dev(ioat_chan),
974 "Completion timeout with pending descriptors\n");
975 spin_lock_bh(&ioat_chan->prep_lock);
976 __ioat_issue_pending(ioat_chan);
977 spin_unlock_bh(&ioat_chan->prep_lock);
978 }
979
980 set_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
981 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
982unlock_out:
983 spin_unlock_bh(&ioat_chan->cleanup_lock);
984}
985
986enum dma_status
987ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie,
988 struct dma_tx_state *txstate)
989{
990 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
991 enum dma_status ret;
992
993 ret = dma_cookie_status(c, cookie, txstate);
994 if (ret == DMA_COMPLETE)
995 return ret;
996
997 ioat_cleanup(ioat_chan);
998
999 return dma_cookie_status(c, cookie, txstate);
1000}
1001
1002int ioat_reset_hw(struct ioatdma_chan *ioat_chan)
1003{
1004 /* throw away whatever the channel was doing and get it
1005 * initialized, with ioat3 specific workarounds
1006 */
1007 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
1008 struct pci_dev *pdev = ioat_dma->pdev;
1009 u32 chanerr;
1010 u16 dev_id;
1011 int err;
1012
1013 ioat_quiesce(ioat_chan, msecs_to_jiffies(100));
1014
1015 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
1016 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
1017
1018 if (ioat_dma->version < IOAT_VER_3_3) {
1019 /* clear any pending errors */
1020 err = pci_read_config_dword(pdev,
1021 IOAT_PCI_CHANERR_INT_OFFSET, &chanerr);
1022 if (err) {
1023 dev_err(&pdev->dev,
1024 "channel error register unreachable\n");
1025 return err;
1026 }
1027 pci_write_config_dword(pdev,
1028 IOAT_PCI_CHANERR_INT_OFFSET, chanerr);
1029
1030 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
1031 * (workaround for spurious config parity error after restart)
1032 */
1033 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
1034 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) {
1035 pci_write_config_dword(pdev,
1036 IOAT_PCI_DMAUNCERRSTS_OFFSET,
1037 0x10);
1038 }
1039 }
1040
1041 if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) {
1042 ioat_dma->msixtba0 = readq(ioat_dma->reg_base + 0x1000);
1043 ioat_dma->msixdata0 = readq(ioat_dma->reg_base + 0x1008);
1044 ioat_dma->msixpba = readq(ioat_dma->reg_base + 0x1800);
1045 }
1046
1047
1048 err = ioat_reset_sync(ioat_chan, msecs_to_jiffies(200));
1049 if (!err) {
1050 if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) {
1051 writeq(ioat_dma->msixtba0, ioat_dma->reg_base + 0x1000);
1052 writeq(ioat_dma->msixdata0, ioat_dma->reg_base + 0x1008);
1053 writeq(ioat_dma->msixpba, ioat_dma->reg_base + 0x1800);
1054 }
1055 }
1056
1057 if (err)
1058 dev_err(&pdev->dev, "Failed to reset: %d\n", err);
1059
1060 return err;
1061}
1/*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2009 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
20 *
21 */
22
23/*
24 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
25 * copy operations.
26 */
27
28#include <linux/init.h>
29#include <linux/module.h>
30#include <linux/slab.h>
31#include <linux/pci.h>
32#include <linux/interrupt.h>
33#include <linux/dmaengine.h>
34#include <linux/delay.h>
35#include <linux/dma-mapping.h>
36#include <linux/workqueue.h>
37#include <linux/prefetch.h>
38#include <linux/i7300_idle.h>
39#include "dma.h"
40#include "registers.h"
41#include "hw.h"
42
43#include "../dmaengine.h"
44
45int ioat_pending_level = 4;
46module_param(ioat_pending_level, int, 0644);
47MODULE_PARM_DESC(ioat_pending_level,
48 "high-water mark for pushing ioat descriptors (default: 4)");
49
50/* internal functions */
51static void ioat1_cleanup(struct ioat_dma_chan *ioat);
52static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat);
53
54/**
55 * ioat_dma_do_interrupt - handler used for single vector interrupt mode
56 * @irq: interrupt id
57 * @data: interrupt data
58 */
59static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
60{
61 struct ioatdma_device *instance = data;
62 struct ioat_chan_common *chan;
63 unsigned long attnstatus;
64 int bit;
65 u8 intrctrl;
66
67 intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
68
69 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
70 return IRQ_NONE;
71
72 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
73 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
74 return IRQ_NONE;
75 }
76
77 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
78 for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) {
79 chan = ioat_chan_by_index(instance, bit);
80 if (test_bit(IOAT_RUN, &chan->state))
81 tasklet_schedule(&chan->cleanup_task);
82 }
83
84 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
85 return IRQ_HANDLED;
86}
87
88/**
89 * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
90 * @irq: interrupt id
91 * @data: interrupt data
92 */
93static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
94{
95 struct ioat_chan_common *chan = data;
96
97 if (test_bit(IOAT_RUN, &chan->state))
98 tasklet_schedule(&chan->cleanup_task);
99
100 return IRQ_HANDLED;
101}
102
103/* common channel initialization */
104void ioat_init_channel(struct ioatdma_device *device, struct ioat_chan_common *chan, int idx)
105{
106 struct dma_device *dma = &device->common;
107 struct dma_chan *c = &chan->common;
108 unsigned long data = (unsigned long) c;
109
110 chan->device = device;
111 chan->reg_base = device->reg_base + (0x80 * (idx + 1));
112 spin_lock_init(&chan->cleanup_lock);
113 chan->common.device = dma;
114 dma_cookie_init(&chan->common);
115 list_add_tail(&chan->common.device_node, &dma->channels);
116 device->idx[idx] = chan;
117 init_timer(&chan->timer);
118 chan->timer.function = device->timer_fn;
119 chan->timer.data = data;
120 tasklet_init(&chan->cleanup_task, device->cleanup_fn, data);
121}
122
123/**
124 * ioat1_dma_enumerate_channels - find and initialize the device's channels
125 * @device: the device to be enumerated
126 */
127static int ioat1_enumerate_channels(struct ioatdma_device *device)
128{
129 u8 xfercap_scale;
130 u32 xfercap;
131 int i;
132 struct ioat_dma_chan *ioat;
133 struct device *dev = &device->pdev->dev;
134 struct dma_device *dma = &device->common;
135
136 INIT_LIST_HEAD(&dma->channels);
137 dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
138 dma->chancnt &= 0x1f; /* bits [4:0] valid */
139 if (dma->chancnt > ARRAY_SIZE(device->idx)) {
140 dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
141 dma->chancnt, ARRAY_SIZE(device->idx));
142 dma->chancnt = ARRAY_SIZE(device->idx);
143 }
144 xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
145 xfercap_scale &= 0x1f; /* bits [4:0] valid */
146 xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
147 dev_dbg(dev, "%s: xfercap = %d\n", __func__, xfercap);
148
149#ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
150 if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
151 dma->chancnt--;
152#endif
153 for (i = 0; i < dma->chancnt; i++) {
154 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
155 if (!ioat)
156 break;
157
158 ioat_init_channel(device, &ioat->base, i);
159 ioat->xfercap = xfercap;
160 spin_lock_init(&ioat->desc_lock);
161 INIT_LIST_HEAD(&ioat->free_desc);
162 INIT_LIST_HEAD(&ioat->used_desc);
163 }
164 dma->chancnt = i;
165 return i;
166}
167
168/**
169 * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
170 * descriptors to hw
171 * @chan: DMA channel handle
172 */
173static inline void
174__ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat)
175{
176 void __iomem *reg_base = ioat->base.reg_base;
177
178 dev_dbg(to_dev(&ioat->base), "%s: pending: %d\n",
179 __func__, ioat->pending);
180 ioat->pending = 0;
181 writeb(IOAT_CHANCMD_APPEND, reg_base + IOAT1_CHANCMD_OFFSET);
182}
183
184static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
185{
186 struct ioat_dma_chan *ioat = to_ioat_chan(chan);
187
188 if (ioat->pending > 0) {
189 spin_lock_bh(&ioat->desc_lock);
190 __ioat1_dma_memcpy_issue_pending(ioat);
191 spin_unlock_bh(&ioat->desc_lock);
192 }
193}
194
195/**
196 * ioat1_reset_channel - restart a channel
197 * @ioat: IOAT DMA channel handle
198 */
199static void ioat1_reset_channel(struct ioat_dma_chan *ioat)
200{
201 struct ioat_chan_common *chan = &ioat->base;
202 void __iomem *reg_base = chan->reg_base;
203 u32 chansts, chanerr;
204
205 dev_warn(to_dev(chan), "reset\n");
206 chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
207 chansts = *chan->completion & IOAT_CHANSTS_STATUS;
208 if (chanerr) {
209 dev_err(to_dev(chan),
210 "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
211 chan_num(chan), chansts, chanerr);
212 writel(chanerr, reg_base + IOAT_CHANERR_OFFSET);
213 }
214
215 /*
216 * whack it upside the head with a reset
217 * and wait for things to settle out.
218 * force the pending count to a really big negative
219 * to make sure no one forces an issue_pending
220 * while we're waiting.
221 */
222
223 ioat->pending = INT_MIN;
224 writeb(IOAT_CHANCMD_RESET,
225 reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
226 set_bit(IOAT_RESET_PENDING, &chan->state);
227 mod_timer(&chan->timer, jiffies + RESET_DELAY);
228}
229
230static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
231{
232 struct dma_chan *c = tx->chan;
233 struct ioat_dma_chan *ioat = to_ioat_chan(c);
234 struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
235 struct ioat_chan_common *chan = &ioat->base;
236 struct ioat_desc_sw *first;
237 struct ioat_desc_sw *chain_tail;
238 dma_cookie_t cookie;
239
240 spin_lock_bh(&ioat->desc_lock);
241 /* cookie incr and addition to used_list must be atomic */
242 cookie = dma_cookie_assign(tx);
243 dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
244
245 /* write address into NextDescriptor field of last desc in chain */
246 first = to_ioat_desc(desc->tx_list.next);
247 chain_tail = to_ioat_desc(ioat->used_desc.prev);
248 /* make descriptor updates globally visible before chaining */
249 wmb();
250 chain_tail->hw->next = first->txd.phys;
251 list_splice_tail_init(&desc->tx_list, &ioat->used_desc);
252 dump_desc_dbg(ioat, chain_tail);
253 dump_desc_dbg(ioat, first);
254
255 if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
256 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
257
258 ioat->active += desc->hw->tx_cnt;
259 ioat->pending += desc->hw->tx_cnt;
260 if (ioat->pending >= ioat_pending_level)
261 __ioat1_dma_memcpy_issue_pending(ioat);
262 spin_unlock_bh(&ioat->desc_lock);
263
264 return cookie;
265}
266
267/**
268 * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
269 * @ioat: the channel supplying the memory pool for the descriptors
270 * @flags: allocation flags
271 */
272static struct ioat_desc_sw *
273ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
274{
275 struct ioat_dma_descriptor *desc;
276 struct ioat_desc_sw *desc_sw;
277 struct ioatdma_device *ioatdma_device;
278 dma_addr_t phys;
279
280 ioatdma_device = ioat->base.device;
281 desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
282 if (unlikely(!desc))
283 return NULL;
284
285 desc_sw = kzalloc(sizeof(*desc_sw), flags);
286 if (unlikely(!desc_sw)) {
287 pci_pool_free(ioatdma_device->dma_pool, desc, phys);
288 return NULL;
289 }
290
291 memset(desc, 0, sizeof(*desc));
292
293 INIT_LIST_HEAD(&desc_sw->tx_list);
294 dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
295 desc_sw->txd.tx_submit = ioat1_tx_submit;
296 desc_sw->hw = desc;
297 desc_sw->txd.phys = phys;
298 set_desc_id(desc_sw, -1);
299
300 return desc_sw;
301}
302
303static int ioat_initial_desc_count = 256;
304module_param(ioat_initial_desc_count, int, 0644);
305MODULE_PARM_DESC(ioat_initial_desc_count,
306 "ioat1: initial descriptors per channel (default: 256)");
307/**
308 * ioat1_dma_alloc_chan_resources - returns the number of allocated descriptors
309 * @chan: the channel to be filled out
310 */
311static int ioat1_dma_alloc_chan_resources(struct dma_chan *c)
312{
313 struct ioat_dma_chan *ioat = to_ioat_chan(c);
314 struct ioat_chan_common *chan = &ioat->base;
315 struct ioat_desc_sw *desc;
316 u32 chanerr;
317 int i;
318 LIST_HEAD(tmp_list);
319
320 /* have we already been set up? */
321 if (!list_empty(&ioat->free_desc))
322 return ioat->desccount;
323
324 /* Setup register to interrupt and write completion status on error */
325 writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET);
326
327 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
328 if (chanerr) {
329 dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
330 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
331 }
332
333 /* Allocate descriptors */
334 for (i = 0; i < ioat_initial_desc_count; i++) {
335 desc = ioat_dma_alloc_descriptor(ioat, GFP_KERNEL);
336 if (!desc) {
337 dev_err(to_dev(chan), "Only %d initial descriptors\n", i);
338 break;
339 }
340 set_desc_id(desc, i);
341 list_add_tail(&desc->node, &tmp_list);
342 }
343 spin_lock_bh(&ioat->desc_lock);
344 ioat->desccount = i;
345 list_splice(&tmp_list, &ioat->free_desc);
346 spin_unlock_bh(&ioat->desc_lock);
347
348 /* allocate a completion writeback area */
349 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
350 chan->completion = pci_pool_alloc(chan->device->completion_pool,
351 GFP_KERNEL, &chan->completion_dma);
352 memset(chan->completion, 0, sizeof(*chan->completion));
353 writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
354 chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
355 writel(((u64) chan->completion_dma) >> 32,
356 chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
357
358 set_bit(IOAT_RUN, &chan->state);
359 ioat1_dma_start_null_desc(ioat); /* give chain to dma device */
360 dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",
361 __func__, ioat->desccount);
362 return ioat->desccount;
363}
364
365void ioat_stop(struct ioat_chan_common *chan)
366{
367 struct ioatdma_device *device = chan->device;
368 struct pci_dev *pdev = device->pdev;
369 int chan_id = chan_num(chan);
370 struct msix_entry *msix;
371
372 /* 1/ stop irq from firing tasklets
373 * 2/ stop the tasklet from re-arming irqs
374 */
375 clear_bit(IOAT_RUN, &chan->state);
376
377 /* flush inflight interrupts */
378 switch (device->irq_mode) {
379 case IOAT_MSIX:
380 msix = &device->msix_entries[chan_id];
381 synchronize_irq(msix->vector);
382 break;
383 case IOAT_MSI:
384 case IOAT_INTX:
385 synchronize_irq(pdev->irq);
386 break;
387 default:
388 break;
389 }
390
391 /* flush inflight timers */
392 del_timer_sync(&chan->timer);
393
394 /* flush inflight tasklet runs */
395 tasklet_kill(&chan->cleanup_task);
396
397 /* final cleanup now that everything is quiesced and can't re-arm */
398 device->cleanup_fn((unsigned long) &chan->common);
399}
400
401/**
402 * ioat1_dma_free_chan_resources - release all the descriptors
403 * @chan: the channel to be cleaned
404 */
405static void ioat1_dma_free_chan_resources(struct dma_chan *c)
406{
407 struct ioat_dma_chan *ioat = to_ioat_chan(c);
408 struct ioat_chan_common *chan = &ioat->base;
409 struct ioatdma_device *ioatdma_device = chan->device;
410 struct ioat_desc_sw *desc, *_desc;
411 int in_use_descs = 0;
412
413 /* Before freeing channel resources first check
414 * if they have been previously allocated for this channel.
415 */
416 if (ioat->desccount == 0)
417 return;
418
419 ioat_stop(chan);
420
421 /* Delay 100ms after reset to allow internal DMA logic to quiesce
422 * before removing DMA descriptor resources.
423 */
424 writeb(IOAT_CHANCMD_RESET,
425 chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
426 mdelay(100);
427
428 spin_lock_bh(&ioat->desc_lock);
429 list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
430 dev_dbg(to_dev(chan), "%s: freeing %d from used list\n",
431 __func__, desc_id(desc));
432 dump_desc_dbg(ioat, desc);
433 in_use_descs++;
434 list_del(&desc->node);
435 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
436 desc->txd.phys);
437 kfree(desc);
438 }
439 list_for_each_entry_safe(desc, _desc,
440 &ioat->free_desc, node) {
441 list_del(&desc->node);
442 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
443 desc->txd.phys);
444 kfree(desc);
445 }
446 spin_unlock_bh(&ioat->desc_lock);
447
448 pci_pool_free(ioatdma_device->completion_pool,
449 chan->completion,
450 chan->completion_dma);
451
452 /* one is ok since we left it on there on purpose */
453 if (in_use_descs > 1)
454 dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
455 in_use_descs - 1);
456
457 chan->last_completion = 0;
458 chan->completion_dma = 0;
459 ioat->pending = 0;
460 ioat->desccount = 0;
461}
462
463/**
464 * ioat1_dma_get_next_descriptor - return the next available descriptor
465 * @ioat: IOAT DMA channel handle
466 *
467 * Gets the next descriptor from the chain, and must be called with the
468 * channel's desc_lock held. Allocates more descriptors if the channel
469 * has run out.
470 */
471static struct ioat_desc_sw *
472ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
473{
474 struct ioat_desc_sw *new;
475
476 if (!list_empty(&ioat->free_desc)) {
477 new = to_ioat_desc(ioat->free_desc.next);
478 list_del(&new->node);
479 } else {
480 /* try to get another desc */
481 new = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC);
482 if (!new) {
483 dev_err(to_dev(&ioat->base), "alloc failed\n");
484 return NULL;
485 }
486 }
487 dev_dbg(to_dev(&ioat->base), "%s: allocated: %d\n",
488 __func__, desc_id(new));
489 prefetch(new->hw);
490 return new;
491}
492
493static struct dma_async_tx_descriptor *
494ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
495 dma_addr_t dma_src, size_t len, unsigned long flags)
496{
497 struct ioat_dma_chan *ioat = to_ioat_chan(c);
498 struct ioat_desc_sw *desc;
499 size_t copy;
500 LIST_HEAD(chain);
501 dma_addr_t src = dma_src;
502 dma_addr_t dest = dma_dest;
503 size_t total_len = len;
504 struct ioat_dma_descriptor *hw = NULL;
505 int tx_cnt = 0;
506
507 spin_lock_bh(&ioat->desc_lock);
508 desc = ioat1_dma_get_next_descriptor(ioat);
509 do {
510 if (!desc)
511 break;
512
513 tx_cnt++;
514 copy = min_t(size_t, len, ioat->xfercap);
515
516 hw = desc->hw;
517 hw->size = copy;
518 hw->ctl = 0;
519 hw->src_addr = src;
520 hw->dst_addr = dest;
521
522 list_add_tail(&desc->node, &chain);
523
524 len -= copy;
525 dest += copy;
526 src += copy;
527 if (len) {
528 struct ioat_desc_sw *next;
529
530 async_tx_ack(&desc->txd);
531 next = ioat1_dma_get_next_descriptor(ioat);
532 hw->next = next ? next->txd.phys : 0;
533 dump_desc_dbg(ioat, desc);
534 desc = next;
535 } else
536 hw->next = 0;
537 } while (len);
538
539 if (!desc) {
540 struct ioat_chan_common *chan = &ioat->base;
541
542 dev_err(to_dev(chan),
543 "chan%d - get_next_desc failed\n", chan_num(chan));
544 list_splice(&chain, &ioat->free_desc);
545 spin_unlock_bh(&ioat->desc_lock);
546 return NULL;
547 }
548 spin_unlock_bh(&ioat->desc_lock);
549
550 desc->txd.flags = flags;
551 desc->len = total_len;
552 list_splice(&chain, &desc->tx_list);
553 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
554 hw->ctl_f.compl_write = 1;
555 hw->tx_cnt = tx_cnt;
556 dump_desc_dbg(ioat, desc);
557
558 return &desc->txd;
559}
560
561static void ioat1_cleanup_event(unsigned long data)
562{
563 struct ioat_dma_chan *ioat = to_ioat_chan((void *) data);
564 struct ioat_chan_common *chan = &ioat->base;
565
566 ioat1_cleanup(ioat);
567 if (!test_bit(IOAT_RUN, &chan->state))
568 return;
569 writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
570}
571
572dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan)
573{
574 dma_addr_t phys_complete;
575 u64 completion;
576
577 completion = *chan->completion;
578 phys_complete = ioat_chansts_to_addr(completion);
579
580 dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__,
581 (unsigned long long) phys_complete);
582
583 if (is_ioat_halted(completion)) {
584 u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
585 dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
586 chanerr);
587
588 /* TODO do something to salvage the situation */
589 }
590
591 return phys_complete;
592}
593
594bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
595 dma_addr_t *phys_complete)
596{
597 *phys_complete = ioat_get_current_completion(chan);
598 if (*phys_complete == chan->last_completion)
599 return false;
600 clear_bit(IOAT_COMPLETION_ACK, &chan->state);
601 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
602
603 return true;
604}
605
606static void __cleanup(struct ioat_dma_chan *ioat, dma_addr_t phys_complete)
607{
608 struct ioat_chan_common *chan = &ioat->base;
609 struct list_head *_desc, *n;
610 struct dma_async_tx_descriptor *tx;
611
612 dev_dbg(to_dev(chan), "%s: phys_complete: %llx\n",
613 __func__, (unsigned long long) phys_complete);
614 list_for_each_safe(_desc, n, &ioat->used_desc) {
615 struct ioat_desc_sw *desc;
616
617 prefetch(n);
618 desc = list_entry(_desc, typeof(*desc), node);
619 tx = &desc->txd;
620 /*
621 * Incoming DMA requests may use multiple descriptors,
622 * due to exceeding xfercap, perhaps. If so, only the
623 * last one will have a cookie, and require unmapping.
624 */
625 dump_desc_dbg(ioat, desc);
626 if (tx->cookie) {
627 dma_cookie_complete(tx);
628 dma_descriptor_unmap(tx);
629 ioat->active -= desc->hw->tx_cnt;
630 if (tx->callback) {
631 tx->callback(tx->callback_param);
632 tx->callback = NULL;
633 }
634 }
635
636 if (tx->phys != phys_complete) {
637 /*
638 * a completed entry, but not the last, so clean
639 * up if the client is done with the descriptor
640 */
641 if (async_tx_test_ack(tx))
642 list_move_tail(&desc->node, &ioat->free_desc);
643 } else {
644 /*
645 * last used desc. Do not remove, so we can
646 * append from it.
647 */
648
649 /* if nothing else is pending, cancel the
650 * completion timeout
651 */
652 if (n == &ioat->used_desc) {
653 dev_dbg(to_dev(chan),
654 "%s cancel completion timeout\n",
655 __func__);
656 clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
657 }
658
659 /* TODO check status bits? */
660 break;
661 }
662 }
663
664 chan->last_completion = phys_complete;
665}
666
667/**
668 * ioat1_cleanup - cleanup up finished descriptors
669 * @chan: ioat channel to be cleaned up
670 *
671 * To prevent lock contention we defer cleanup when the locks are
672 * contended with a terminal timeout that forces cleanup and catches
673 * completion notification errors.
674 */
675static void ioat1_cleanup(struct ioat_dma_chan *ioat)
676{
677 struct ioat_chan_common *chan = &ioat->base;
678 dma_addr_t phys_complete;
679
680 prefetch(chan->completion);
681
682 if (!spin_trylock_bh(&chan->cleanup_lock))
683 return;
684
685 if (!ioat_cleanup_preamble(chan, &phys_complete)) {
686 spin_unlock_bh(&chan->cleanup_lock);
687 return;
688 }
689
690 if (!spin_trylock_bh(&ioat->desc_lock)) {
691 spin_unlock_bh(&chan->cleanup_lock);
692 return;
693 }
694
695 __cleanup(ioat, phys_complete);
696
697 spin_unlock_bh(&ioat->desc_lock);
698 spin_unlock_bh(&chan->cleanup_lock);
699}
700
701static void ioat1_timer_event(unsigned long data)
702{
703 struct ioat_dma_chan *ioat = to_ioat_chan((void *) data);
704 struct ioat_chan_common *chan = &ioat->base;
705
706 dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state);
707
708 spin_lock_bh(&chan->cleanup_lock);
709 if (test_and_clear_bit(IOAT_RESET_PENDING, &chan->state)) {
710 struct ioat_desc_sw *desc;
711
712 spin_lock_bh(&ioat->desc_lock);
713
714 /* restart active descriptors */
715 desc = to_ioat_desc(ioat->used_desc.prev);
716 ioat_set_chainaddr(ioat, desc->txd.phys);
717 ioat_start(chan);
718
719 ioat->pending = 0;
720 set_bit(IOAT_COMPLETION_PENDING, &chan->state);
721 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
722 spin_unlock_bh(&ioat->desc_lock);
723 } else if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
724 dma_addr_t phys_complete;
725
726 spin_lock_bh(&ioat->desc_lock);
727 /* if we haven't made progress and we have already
728 * acknowledged a pending completion once, then be more
729 * forceful with a restart
730 */
731 if (ioat_cleanup_preamble(chan, &phys_complete))
732 __cleanup(ioat, phys_complete);
733 else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
734 ioat1_reset_channel(ioat);
735 else {
736 u64 status = ioat_chansts(chan);
737
738 /* manually update the last completion address */
739 if (ioat_chansts_to_addr(status) != 0)
740 *chan->completion = status;
741
742 set_bit(IOAT_COMPLETION_ACK, &chan->state);
743 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
744 }
745 spin_unlock_bh(&ioat->desc_lock);
746 }
747 spin_unlock_bh(&chan->cleanup_lock);
748}
749
750enum dma_status
751ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
752 struct dma_tx_state *txstate)
753{
754 struct ioat_chan_common *chan = to_chan_common(c);
755 struct ioatdma_device *device = chan->device;
756 enum dma_status ret;
757
758 ret = dma_cookie_status(c, cookie, txstate);
759 if (ret == DMA_COMPLETE)
760 return ret;
761
762 device->cleanup_fn((unsigned long) c);
763
764 return dma_cookie_status(c, cookie, txstate);
765}
766
767static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat)
768{
769 struct ioat_chan_common *chan = &ioat->base;
770 struct ioat_desc_sw *desc;
771 struct ioat_dma_descriptor *hw;
772
773 spin_lock_bh(&ioat->desc_lock);
774
775 desc = ioat1_dma_get_next_descriptor(ioat);
776
777 if (!desc) {
778 dev_err(to_dev(chan),
779 "Unable to start null desc - get next desc failed\n");
780 spin_unlock_bh(&ioat->desc_lock);
781 return;
782 }
783
784 hw = desc->hw;
785 hw->ctl = 0;
786 hw->ctl_f.null = 1;
787 hw->ctl_f.int_en = 1;
788 hw->ctl_f.compl_write = 1;
789 /* set size to non-zero value (channel returns error when size is 0) */
790 hw->size = NULL_DESC_BUFFER_SIZE;
791 hw->src_addr = 0;
792 hw->dst_addr = 0;
793 async_tx_ack(&desc->txd);
794 hw->next = 0;
795 list_add_tail(&desc->node, &ioat->used_desc);
796 dump_desc_dbg(ioat, desc);
797
798 ioat_set_chainaddr(ioat, desc->txd.phys);
799 ioat_start(chan);
800 spin_unlock_bh(&ioat->desc_lock);
801}
802
803/*
804 * Perform a IOAT transaction to verify the HW works.
805 */
806#define IOAT_TEST_SIZE 2000
807
808static void ioat_dma_test_callback(void *dma_async_param)
809{
810 struct completion *cmp = dma_async_param;
811
812 complete(cmp);
813}
814
815/**
816 * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
817 * @device: device to be tested
818 */
819int ioat_dma_self_test(struct ioatdma_device *device)
820{
821 int i;
822 u8 *src;
823 u8 *dest;
824 struct dma_device *dma = &device->common;
825 struct device *dev = &device->pdev->dev;
826 struct dma_chan *dma_chan;
827 struct dma_async_tx_descriptor *tx;
828 dma_addr_t dma_dest, dma_src;
829 dma_cookie_t cookie;
830 int err = 0;
831 struct completion cmp;
832 unsigned long tmo;
833 unsigned long flags;
834
835 src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
836 if (!src)
837 return -ENOMEM;
838 dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
839 if (!dest) {
840 kfree(src);
841 return -ENOMEM;
842 }
843
844 /* Fill in src buffer */
845 for (i = 0; i < IOAT_TEST_SIZE; i++)
846 src[i] = (u8)i;
847
848 /* Start copy, using first DMA channel */
849 dma_chan = container_of(dma->channels.next, struct dma_chan,
850 device_node);
851 if (dma->device_alloc_chan_resources(dma_chan) < 1) {
852 dev_err(dev, "selftest cannot allocate chan resource\n");
853 err = -ENODEV;
854 goto out;
855 }
856
857 dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
858 if (dma_mapping_error(dev, dma_src)) {
859 dev_err(dev, "mapping src buffer failed\n");
860 goto free_resources;
861 }
862 dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
863 if (dma_mapping_error(dev, dma_dest)) {
864 dev_err(dev, "mapping dest buffer failed\n");
865 goto unmap_src;
866 }
867 flags = DMA_PREP_INTERRUPT;
868 tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
869 IOAT_TEST_SIZE, flags);
870 if (!tx) {
871 dev_err(dev, "Self-test prep failed, disabling\n");
872 err = -ENODEV;
873 goto unmap_dma;
874 }
875
876 async_tx_ack(tx);
877 init_completion(&cmp);
878 tx->callback = ioat_dma_test_callback;
879 tx->callback_param = &cmp;
880 cookie = tx->tx_submit(tx);
881 if (cookie < 0) {
882 dev_err(dev, "Self-test setup failed, disabling\n");
883 err = -ENODEV;
884 goto unmap_dma;
885 }
886 dma->device_issue_pending(dma_chan);
887
888 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
889
890 if (tmo == 0 ||
891 dma->device_tx_status(dma_chan, cookie, NULL)
892 != DMA_COMPLETE) {
893 dev_err(dev, "Self-test copy timed out, disabling\n");
894 err = -ENODEV;
895 goto unmap_dma;
896 }
897 if (memcmp(src, dest, IOAT_TEST_SIZE)) {
898 dev_err(dev, "Self-test copy failed compare, disabling\n");
899 err = -ENODEV;
900 goto free_resources;
901 }
902
903unmap_dma:
904 dma_unmap_single(dev, dma_dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
905unmap_src:
906 dma_unmap_single(dev, dma_src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
907free_resources:
908 dma->device_free_chan_resources(dma_chan);
909out:
910 kfree(src);
911 kfree(dest);
912 return err;
913}
914
915static char ioat_interrupt_style[32] = "msix";
916module_param_string(ioat_interrupt_style, ioat_interrupt_style,
917 sizeof(ioat_interrupt_style), 0644);
918MODULE_PARM_DESC(ioat_interrupt_style,
919 "set ioat interrupt style: msix (default), msi, intx");
920
921/**
922 * ioat_dma_setup_interrupts - setup interrupt handler
923 * @device: ioat device
924 */
925int ioat_dma_setup_interrupts(struct ioatdma_device *device)
926{
927 struct ioat_chan_common *chan;
928 struct pci_dev *pdev = device->pdev;
929 struct device *dev = &pdev->dev;
930 struct msix_entry *msix;
931 int i, j, msixcnt;
932 int err = -EINVAL;
933 u8 intrctrl = 0;
934
935 if (!strcmp(ioat_interrupt_style, "msix"))
936 goto msix;
937 if (!strcmp(ioat_interrupt_style, "msi"))
938 goto msi;
939 if (!strcmp(ioat_interrupt_style, "intx"))
940 goto intx;
941 dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
942 goto err_no_irq;
943
944msix:
945 /* The number of MSI-X vectors should equal the number of channels */
946 msixcnt = device->common.chancnt;
947 for (i = 0; i < msixcnt; i++)
948 device->msix_entries[i].entry = i;
949
950 err = pci_enable_msix(pdev, device->msix_entries, msixcnt);
951 if (err)
952 goto msi;
953
954 for (i = 0; i < msixcnt; i++) {
955 msix = &device->msix_entries[i];
956 chan = ioat_chan_by_index(device, i);
957 err = devm_request_irq(dev, msix->vector,
958 ioat_dma_do_interrupt_msix, 0,
959 "ioat-msix", chan);
960 if (err) {
961 for (j = 0; j < i; j++) {
962 msix = &device->msix_entries[j];
963 chan = ioat_chan_by_index(device, j);
964 devm_free_irq(dev, msix->vector, chan);
965 }
966 goto msi;
967 }
968 }
969 intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
970 device->irq_mode = IOAT_MSIX;
971 goto done;
972
973msi:
974 err = pci_enable_msi(pdev);
975 if (err)
976 goto intx;
977
978 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
979 "ioat-msi", device);
980 if (err) {
981 pci_disable_msi(pdev);
982 goto intx;
983 }
984 device->irq_mode = IOAT_MSI;
985 goto done;
986
987intx:
988 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
989 IRQF_SHARED, "ioat-intx", device);
990 if (err)
991 goto err_no_irq;
992
993 device->irq_mode = IOAT_INTX;
994done:
995 if (device->intr_quirk)
996 device->intr_quirk(device);
997 intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
998 writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
999 return 0;
1000
1001err_no_irq:
1002 /* Disable all interrupt generation */
1003 writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1004 device->irq_mode = IOAT_NOIRQ;
1005 dev_err(dev, "no usable interrupts\n");
1006 return err;
1007}
1008EXPORT_SYMBOL(ioat_dma_setup_interrupts);
1009
1010static void ioat_disable_interrupts(struct ioatdma_device *device)
1011{
1012 /* Disable all interrupt generation */
1013 writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1014}
1015
1016int ioat_probe(struct ioatdma_device *device)
1017{
1018 int err = -ENODEV;
1019 struct dma_device *dma = &device->common;
1020 struct pci_dev *pdev = device->pdev;
1021 struct device *dev = &pdev->dev;
1022
1023 /* DMA coherent memory pool for DMA descriptor allocations */
1024 device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
1025 sizeof(struct ioat_dma_descriptor),
1026 64, 0);
1027 if (!device->dma_pool) {
1028 err = -ENOMEM;
1029 goto err_dma_pool;
1030 }
1031
1032 device->completion_pool = pci_pool_create("completion_pool", pdev,
1033 sizeof(u64), SMP_CACHE_BYTES,
1034 SMP_CACHE_BYTES);
1035
1036 if (!device->completion_pool) {
1037 err = -ENOMEM;
1038 goto err_completion_pool;
1039 }
1040
1041 device->enumerate_channels(device);
1042
1043 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
1044 dma->dev = &pdev->dev;
1045
1046 if (!dma->chancnt) {
1047 dev_err(dev, "channel enumeration error\n");
1048 goto err_setup_interrupts;
1049 }
1050
1051 err = ioat_dma_setup_interrupts(device);
1052 if (err)
1053 goto err_setup_interrupts;
1054
1055 err = device->self_test(device);
1056 if (err)
1057 goto err_self_test;
1058
1059 return 0;
1060
1061err_self_test:
1062 ioat_disable_interrupts(device);
1063err_setup_interrupts:
1064 pci_pool_destroy(device->completion_pool);
1065err_completion_pool:
1066 pci_pool_destroy(device->dma_pool);
1067err_dma_pool:
1068 return err;
1069}
1070
1071int ioat_register(struct ioatdma_device *device)
1072{
1073 int err = dma_async_device_register(&device->common);
1074
1075 if (err) {
1076 ioat_disable_interrupts(device);
1077 pci_pool_destroy(device->completion_pool);
1078 pci_pool_destroy(device->dma_pool);
1079 }
1080
1081 return err;
1082}
1083
1084/* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */
1085static void ioat1_intr_quirk(struct ioatdma_device *device)
1086{
1087 struct pci_dev *pdev = device->pdev;
1088 u32 dmactrl;
1089
1090 pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
1091 if (pdev->msi_enabled)
1092 dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
1093 else
1094 dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN;
1095 pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
1096}
1097
1098static ssize_t ring_size_show(struct dma_chan *c, char *page)
1099{
1100 struct ioat_dma_chan *ioat = to_ioat_chan(c);
1101
1102 return sprintf(page, "%d\n", ioat->desccount);
1103}
1104static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
1105
1106static ssize_t ring_active_show(struct dma_chan *c, char *page)
1107{
1108 struct ioat_dma_chan *ioat = to_ioat_chan(c);
1109
1110 return sprintf(page, "%d\n", ioat->active);
1111}
1112static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
1113
1114static ssize_t cap_show(struct dma_chan *c, char *page)
1115{
1116 struct dma_device *dma = c->device;
1117
1118 return sprintf(page, "copy%s%s%s%s%s\n",
1119 dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "",
1120 dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "",
1121 dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "",
1122 dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "",
1123 dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : "");
1124
1125}
1126struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap);
1127
1128static ssize_t version_show(struct dma_chan *c, char *page)
1129{
1130 struct dma_device *dma = c->device;
1131 struct ioatdma_device *device = to_ioatdma_device(dma);
1132
1133 return sprintf(page, "%d.%d\n",
1134 device->version >> 4, device->version & 0xf);
1135}
1136struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version);
1137
1138static struct attribute *ioat1_attrs[] = {
1139 &ring_size_attr.attr,
1140 &ring_active_attr.attr,
1141 &ioat_cap_attr.attr,
1142 &ioat_version_attr.attr,
1143 NULL,
1144};
1145
1146static ssize_t
1147ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
1148{
1149 struct ioat_sysfs_entry *entry;
1150 struct ioat_chan_common *chan;
1151
1152 entry = container_of(attr, struct ioat_sysfs_entry, attr);
1153 chan = container_of(kobj, struct ioat_chan_common, kobj);
1154
1155 if (!entry->show)
1156 return -EIO;
1157 return entry->show(&chan->common, page);
1158}
1159
1160const struct sysfs_ops ioat_sysfs_ops = {
1161 .show = ioat_attr_show,
1162};
1163
1164static struct kobj_type ioat1_ktype = {
1165 .sysfs_ops = &ioat_sysfs_ops,
1166 .default_attrs = ioat1_attrs,
1167};
1168
1169void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type)
1170{
1171 struct dma_device *dma = &device->common;
1172 struct dma_chan *c;
1173
1174 list_for_each_entry(c, &dma->channels, device_node) {
1175 struct ioat_chan_common *chan = to_chan_common(c);
1176 struct kobject *parent = &c->dev->device.kobj;
1177 int err;
1178
1179 err = kobject_init_and_add(&chan->kobj, type, parent, "quickdata");
1180 if (err) {
1181 dev_warn(to_dev(chan),
1182 "sysfs init error (%d), continuing...\n", err);
1183 kobject_put(&chan->kobj);
1184 set_bit(IOAT_KOBJ_INIT_FAIL, &chan->state);
1185 }
1186 }
1187}
1188
1189void ioat_kobject_del(struct ioatdma_device *device)
1190{
1191 struct dma_device *dma = &device->common;
1192 struct dma_chan *c;
1193
1194 list_for_each_entry(c, &dma->channels, device_node) {
1195 struct ioat_chan_common *chan = to_chan_common(c);
1196
1197 if (!test_bit(IOAT_KOBJ_INIT_FAIL, &chan->state)) {
1198 kobject_del(&chan->kobj);
1199 kobject_put(&chan->kobj);
1200 }
1201 }
1202}
1203
1204int ioat1_dma_probe(struct ioatdma_device *device, int dca)
1205{
1206 struct pci_dev *pdev = device->pdev;
1207 struct dma_device *dma;
1208 int err;
1209
1210 device->intr_quirk = ioat1_intr_quirk;
1211 device->enumerate_channels = ioat1_enumerate_channels;
1212 device->self_test = ioat_dma_self_test;
1213 device->timer_fn = ioat1_timer_event;
1214 device->cleanup_fn = ioat1_cleanup_event;
1215 dma = &device->common;
1216 dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
1217 dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
1218 dma->device_alloc_chan_resources = ioat1_dma_alloc_chan_resources;
1219 dma->device_free_chan_resources = ioat1_dma_free_chan_resources;
1220 dma->device_tx_status = ioat_dma_tx_status;
1221
1222 err = ioat_probe(device);
1223 if (err)
1224 return err;
1225 ioat_set_tcp_copy_break(4096);
1226 err = ioat_register(device);
1227 if (err)
1228 return err;
1229 ioat_kobject_add(device, &ioat1_ktype);
1230
1231 if (dca)
1232 device->dca = ioat_dca_init(pdev, device->reg_base);
1233
1234 return err;
1235}
1236
1237void ioat_dma_remove(struct ioatdma_device *device)
1238{
1239 struct dma_device *dma = &device->common;
1240
1241 ioat_disable_interrupts(device);
1242
1243 ioat_kobject_del(device);
1244
1245 dma_async_device_unregister(dma);
1246
1247 pci_pool_destroy(device->dma_pool);
1248 pci_pool_destroy(device->completion_pool);
1249
1250 INIT_LIST_HEAD(&dma->channels);
1251}