Loading...
Note: File does not exist in v3.1.
1/*
2 * Intel MIC Platform Software Stack (MPSS)
3 *
4 * Copyright(c) 2015 Intel Corporation.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
14 *
15 * Intel SCIF driver.
16 *
17 */
18#include "scif_main.h"
19#include "scif_map.h"
20
21/*
22 * struct scif_dma_comp_cb - SCIF DMA completion callback
23 *
24 * @dma_completion_func: DMA completion callback
25 * @cb_cookie: DMA completion callback cookie
26 * @temp_buf: Temporary buffer
27 * @temp_buf_to_free: Temporary buffer to be freed
28 * @is_cache: Is a kmem_cache allocated buffer
29 * @dst_offset: Destination registration offset
30 * @dst_window: Destination registration window
31 * @len: Length of the temp buffer
32 * @temp_phys: DMA address of the temp buffer
33 * @sdev: The SCIF device
34 * @header_padding: padding for cache line alignment
35 */
36struct scif_dma_comp_cb {
37 void (*dma_completion_func)(void *cookie);
38 void *cb_cookie;
39 u8 *temp_buf;
40 u8 *temp_buf_to_free;
41 bool is_cache;
42 s64 dst_offset;
43 struct scif_window *dst_window;
44 size_t len;
45 dma_addr_t temp_phys;
46 struct scif_dev *sdev;
47 int header_padding;
48};
49
50/**
51 * struct scif_copy_work - Work for DMA copy
52 *
53 * @src_offset: Starting source offset
54 * @dst_offset: Starting destination offset
55 * @src_window: Starting src registered window
56 * @dst_window: Starting dst registered window
57 * @loopback: true if this is a loopback DMA transfer
58 * @len: Length of the transfer
59 * @comp_cb: DMA copy completion callback
60 * @remote_dev: The remote SCIF peer device
61 * @fence_type: polling or interrupt based
62 * @ordered: is this a tail byte ordered DMA transfer
63 */
64struct scif_copy_work {
65 s64 src_offset;
66 s64 dst_offset;
67 struct scif_window *src_window;
68 struct scif_window *dst_window;
69 int loopback;
70 size_t len;
71 struct scif_dma_comp_cb *comp_cb;
72 struct scif_dev *remote_dev;
73 int fence_type;
74 bool ordered;
75};
76
77/**
78 * scif_reserve_dma_chan:
79 * @ep: Endpoint Descriptor.
80 *
81 * This routine reserves a DMA channel for a particular
82 * endpoint. All DMA transfers for an endpoint are always
83 * programmed on the same DMA channel.
84 */
85int scif_reserve_dma_chan(struct scif_endpt *ep)
86{
87 int err = 0;
88 struct scif_dev *scifdev;
89 struct scif_hw_dev *sdev;
90 struct dma_chan *chan;
91
92 /* Loopback DMAs are not supported on the management node */
93 if (!scif_info.nodeid && scifdev_self(ep->remote_dev))
94 return 0;
95 if (scif_info.nodeid)
96 scifdev = &scif_dev[0];
97 else
98 scifdev = ep->remote_dev;
99 sdev = scifdev->sdev;
100 if (!sdev->num_dma_ch)
101 return -ENODEV;
102 chan = sdev->dma_ch[scifdev->dma_ch_idx];
103 scifdev->dma_ch_idx = (scifdev->dma_ch_idx + 1) % sdev->num_dma_ch;
104 mutex_lock(&ep->rma_info.rma_lock);
105 ep->rma_info.dma_chan = chan;
106 mutex_unlock(&ep->rma_info.rma_lock);
107 return err;
108}
109
110#ifdef CONFIG_MMU_NOTIFIER
111/**
112 * scif_rma_destroy_tcw:
113 *
114 * This routine destroys temporary cached windows
115 */
116static
117void __scif_rma_destroy_tcw(struct scif_mmu_notif *mmn,
118 u64 start, u64 len)
119{
120 struct list_head *item, *tmp;
121 struct scif_window *window;
122 u64 start_va, end_va;
123 u64 end = start + len;
124
125 if (end <= start)
126 return;
127
128 list_for_each_safe(item, tmp, &mmn->tc_reg_list) {
129 window = list_entry(item, struct scif_window, list);
130 if (!len)
131 break;
132 start_va = window->va_for_temp;
133 end_va = start_va + (window->nr_pages << PAGE_SHIFT);
134 if (start < start_va && end <= start_va)
135 break;
136 if (start >= end_va)
137 continue;
138 __scif_rma_destroy_tcw_helper(window);
139 }
140}
141
142static void scif_rma_destroy_tcw(struct scif_mmu_notif *mmn, u64 start, u64 len)
143{
144 struct scif_endpt *ep = mmn->ep;
145
146 spin_lock(&ep->rma_info.tc_lock);
147 __scif_rma_destroy_tcw(mmn, start, len);
148 spin_unlock(&ep->rma_info.tc_lock);
149}
150
151static void scif_rma_destroy_tcw_ep(struct scif_endpt *ep)
152{
153 struct list_head *item, *tmp;
154 struct scif_mmu_notif *mmn;
155
156 list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {
157 mmn = list_entry(item, struct scif_mmu_notif, list);
158 scif_rma_destroy_tcw(mmn, 0, ULONG_MAX);
159 }
160}
161
162static void __scif_rma_destroy_tcw_ep(struct scif_endpt *ep)
163{
164 struct list_head *item, *tmp;
165 struct scif_mmu_notif *mmn;
166
167 spin_lock(&ep->rma_info.tc_lock);
168 list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {
169 mmn = list_entry(item, struct scif_mmu_notif, list);
170 __scif_rma_destroy_tcw(mmn, 0, ULONG_MAX);
171 }
172 spin_unlock(&ep->rma_info.tc_lock);
173}
174
175static bool scif_rma_tc_can_cache(struct scif_endpt *ep, size_t cur_bytes)
176{
177 if ((cur_bytes >> PAGE_SHIFT) > scif_info.rma_tc_limit)
178 return false;
179 if ((atomic_read(&ep->rma_info.tcw_total_pages)
180 + (cur_bytes >> PAGE_SHIFT)) >
181 scif_info.rma_tc_limit) {
182 dev_info(scif_info.mdev.this_device,
183 "%s %d total=%d, current=%zu reached max\n",
184 __func__, __LINE__,
185 atomic_read(&ep->rma_info.tcw_total_pages),
186 (1 + (cur_bytes >> PAGE_SHIFT)));
187 scif_rma_destroy_tcw_invalid();
188 __scif_rma_destroy_tcw_ep(ep);
189 }
190 return true;
191}
192
193static void scif_mmu_notifier_release(struct mmu_notifier *mn,
194 struct mm_struct *mm)
195{
196 struct scif_mmu_notif *mmn;
197
198 mmn = container_of(mn, struct scif_mmu_notif, ep_mmu_notifier);
199 scif_rma_destroy_tcw(mmn, 0, ULONG_MAX);
200 schedule_work(&scif_info.misc_work);
201}
202
203static void scif_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
204 struct mm_struct *mm,
205 unsigned long start,
206 unsigned long end)
207{
208 struct scif_mmu_notif *mmn;
209
210 mmn = container_of(mn, struct scif_mmu_notif, ep_mmu_notifier);
211 scif_rma_destroy_tcw(mmn, start, end - start);
212}
213
214static void scif_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
215 struct mm_struct *mm,
216 unsigned long start,
217 unsigned long end)
218{
219 /*
220 * Nothing to do here, everything needed was done in
221 * invalidate_range_start.
222 */
223}
224
225static const struct mmu_notifier_ops scif_mmu_notifier_ops = {
226 .release = scif_mmu_notifier_release,
227 .clear_flush_young = NULL,
228 .invalidate_range_start = scif_mmu_notifier_invalidate_range_start,
229 .invalidate_range_end = scif_mmu_notifier_invalidate_range_end};
230
231static void scif_ep_unregister_mmu_notifier(struct scif_endpt *ep)
232{
233 struct scif_endpt_rma_info *rma = &ep->rma_info;
234 struct scif_mmu_notif *mmn = NULL;
235 struct list_head *item, *tmp;
236
237 mutex_lock(&ep->rma_info.mmn_lock);
238 list_for_each_safe(item, tmp, &rma->mmn_list) {
239 mmn = list_entry(item, struct scif_mmu_notif, list);
240 mmu_notifier_unregister(&mmn->ep_mmu_notifier, mmn->mm);
241 list_del(item);
242 kfree(mmn);
243 }
244 mutex_unlock(&ep->rma_info.mmn_lock);
245}
246
247static void scif_init_mmu_notifier(struct scif_mmu_notif *mmn,
248 struct mm_struct *mm, struct scif_endpt *ep)
249{
250 mmn->ep = ep;
251 mmn->mm = mm;
252 mmn->ep_mmu_notifier.ops = &scif_mmu_notifier_ops;
253 INIT_LIST_HEAD(&mmn->list);
254 INIT_LIST_HEAD(&mmn->tc_reg_list);
255}
256
257static struct scif_mmu_notif *
258scif_find_mmu_notifier(struct mm_struct *mm, struct scif_endpt_rma_info *rma)
259{
260 struct scif_mmu_notif *mmn;
261
262 list_for_each_entry(mmn, &rma->mmn_list, list)
263 if (mmn->mm == mm)
264 return mmn;
265 return NULL;
266}
267
268static struct scif_mmu_notif *
269scif_add_mmu_notifier(struct mm_struct *mm, struct scif_endpt *ep)
270{
271 struct scif_mmu_notif *mmn
272 = kzalloc(sizeof(*mmn), GFP_KERNEL);
273
274 if (!mmn)
275 return ERR_PTR(-ENOMEM);
276
277 scif_init_mmu_notifier(mmn, current->mm, ep);
278 if (mmu_notifier_register(&mmn->ep_mmu_notifier, current->mm)) {
279 kfree(mmn);
280 return ERR_PTR(-EBUSY);
281 }
282 list_add(&mmn->list, &ep->rma_info.mmn_list);
283 return mmn;
284}
285
286/*
287 * Called from the misc thread to destroy temporary cached windows and
288 * unregister the MMU notifier for the SCIF endpoint.
289 */
290void scif_mmu_notif_handler(struct work_struct *work)
291{
292 struct list_head *pos, *tmpq;
293 struct scif_endpt *ep;
294restart:
295 scif_rma_destroy_tcw_invalid();
296 spin_lock(&scif_info.rmalock);
297 list_for_each_safe(pos, tmpq, &scif_info.mmu_notif_cleanup) {
298 ep = list_entry(pos, struct scif_endpt, mmu_list);
299 list_del(&ep->mmu_list);
300 spin_unlock(&scif_info.rmalock);
301 scif_rma_destroy_tcw_ep(ep);
302 scif_ep_unregister_mmu_notifier(ep);
303 goto restart;
304 }
305 spin_unlock(&scif_info.rmalock);
306}
307
308static bool scif_is_set_reg_cache(int flags)
309{
310 return !!(flags & SCIF_RMA_USECACHE);
311}
312#else
313static struct scif_mmu_notif *
314scif_find_mmu_notifier(struct mm_struct *mm,
315 struct scif_endpt_rma_info *rma)
316{
317 return NULL;
318}
319
320static struct scif_mmu_notif *
321scif_add_mmu_notifier(struct mm_struct *mm, struct scif_endpt *ep)
322{
323 return NULL;
324}
325
326void scif_mmu_notif_handler(struct work_struct *work)
327{
328}
329
330static bool scif_is_set_reg_cache(int flags)
331{
332 return false;
333}
334
335static bool scif_rma_tc_can_cache(struct scif_endpt *ep, size_t cur_bytes)
336{
337 return false;
338}
339#endif
340
341/**
342 * scif_register_temp:
343 * @epd: End Point Descriptor.
344 * @addr: virtual address to/from which to copy
345 * @len: length of range to copy
346 * @out_offset: computed offset returned by reference.
347 * @out_window: allocated registered window returned by reference.
348 *
349 * Create a temporary registered window. The peer will not know about this
350 * window. This API is used for scif_vreadfrom()/scif_vwriteto() API's.
351 */
352static int
353scif_register_temp(scif_epd_t epd, unsigned long addr, size_t len, int prot,
354 off_t *out_offset, struct scif_window **out_window)
355{
356 struct scif_endpt *ep = (struct scif_endpt *)epd;
357 int err;
358 scif_pinned_pages_t pinned_pages;
359 size_t aligned_len;
360
361 aligned_len = ALIGN(len, PAGE_SIZE);
362
363 err = __scif_pin_pages((void *)(addr & PAGE_MASK),
364 aligned_len, &prot, 0, &pinned_pages);
365 if (err)
366 return err;
367
368 pinned_pages->prot = prot;
369
370 /* Compute the offset for this registration */
371 err = scif_get_window_offset(ep, 0, 0,
372 aligned_len >> PAGE_SHIFT,
373 (s64 *)out_offset);
374 if (err)
375 goto error_unpin;
376
377 /* Allocate and prepare self registration window */
378 *out_window = scif_create_window(ep, aligned_len >> PAGE_SHIFT,
379 *out_offset, true);
380 if (!*out_window) {
381 scif_free_window_offset(ep, NULL, *out_offset);
382 err = -ENOMEM;
383 goto error_unpin;
384 }
385
386 (*out_window)->pinned_pages = pinned_pages;
387 (*out_window)->nr_pages = pinned_pages->nr_pages;
388 (*out_window)->prot = pinned_pages->prot;
389
390 (*out_window)->va_for_temp = addr & PAGE_MASK;
391 err = scif_map_window(ep->remote_dev, *out_window);
392 if (err) {
393 /* Something went wrong! Rollback */
394 scif_destroy_window(ep, *out_window);
395 *out_window = NULL;
396 } else {
397 *out_offset |= (addr - (*out_window)->va_for_temp);
398 }
399 return err;
400error_unpin:
401 if (err)
402 dev_err(&ep->remote_dev->sdev->dev,
403 "%s %d err %d\n", __func__, __LINE__, err);
404 scif_unpin_pages(pinned_pages);
405 return err;
406}
407
408#define SCIF_DMA_TO (3 * HZ)
409
410/*
411 * scif_sync_dma - Program a DMA without an interrupt descriptor
412 *
413 * @dev - The address of the pointer to the device instance used
414 * for DMA registration.
415 * @chan - DMA channel to be used.
416 * @sync_wait: Wait for DMA to complete?
417 *
418 * Return 0 on success and -errno on error.
419 */
420static int scif_sync_dma(struct scif_hw_dev *sdev, struct dma_chan *chan,
421 bool sync_wait)
422{
423 int err = 0;
424 struct dma_async_tx_descriptor *tx = NULL;
425 enum dma_ctrl_flags flags = DMA_PREP_FENCE;
426 dma_cookie_t cookie;
427 struct dma_device *ddev;
428
429 if (!chan) {
430 err = -EIO;
431 dev_err(&sdev->dev, "%s %d err %d\n",
432 __func__, __LINE__, err);
433 return err;
434 }
435 ddev = chan->device;
436
437 tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, flags);
438 if (!tx) {
439 err = -ENOMEM;
440 dev_err(&sdev->dev, "%s %d err %d\n",
441 __func__, __LINE__, err);
442 goto release;
443 }
444 cookie = tx->tx_submit(tx);
445
446 if (dma_submit_error(cookie)) {
447 err = -ENOMEM;
448 dev_err(&sdev->dev, "%s %d err %d\n",
449 __func__, __LINE__, err);
450 goto release;
451 }
452 if (!sync_wait) {
453 dma_async_issue_pending(chan);
454 } else {
455 if (dma_sync_wait(chan, cookie) == DMA_COMPLETE) {
456 err = 0;
457 } else {
458 err = -EIO;
459 dev_err(&sdev->dev, "%s %d err %d\n",
460 __func__, __LINE__, err);
461 }
462 }
463release:
464 return err;
465}
466
467static void scif_dma_callback(void *arg)
468{
469 struct completion *done = (struct completion *)arg;
470
471 complete(done);
472}
473
474#define SCIF_DMA_SYNC_WAIT true
475#define SCIF_DMA_POLL BIT(0)
476#define SCIF_DMA_INTR BIT(1)
477
478/*
479 * scif_async_dma - Program a DMA with an interrupt descriptor
480 *
481 * @dev - The address of the pointer to the device instance used
482 * for DMA registration.
483 * @chan - DMA channel to be used.
484 * Return 0 on success and -errno on error.
485 */
486static int scif_async_dma(struct scif_hw_dev *sdev, struct dma_chan *chan)
487{
488 int err = 0;
489 struct dma_device *ddev;
490 struct dma_async_tx_descriptor *tx = NULL;
491 enum dma_ctrl_flags flags = DMA_PREP_INTERRUPT | DMA_PREP_FENCE;
492 DECLARE_COMPLETION_ONSTACK(done_wait);
493 dma_cookie_t cookie;
494 enum dma_status status;
495
496 if (!chan) {
497 err = -EIO;
498 dev_err(&sdev->dev, "%s %d err %d\n",
499 __func__, __LINE__, err);
500 return err;
501 }
502 ddev = chan->device;
503
504 tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, flags);
505 if (!tx) {
506 err = -ENOMEM;
507 dev_err(&sdev->dev, "%s %d err %d\n",
508 __func__, __LINE__, err);
509 goto release;
510 }
511 reinit_completion(&done_wait);
512 tx->callback = scif_dma_callback;
513 tx->callback_param = &done_wait;
514 cookie = tx->tx_submit(tx);
515
516 if (dma_submit_error(cookie)) {
517 err = -ENOMEM;
518 dev_err(&sdev->dev, "%s %d err %d\n",
519 __func__, __LINE__, err);
520 goto release;
521 }
522 dma_async_issue_pending(chan);
523
524 err = wait_for_completion_timeout(&done_wait, SCIF_DMA_TO);
525 if (!err) {
526 err = -EIO;
527 dev_err(&sdev->dev, "%s %d err %d\n",
528 __func__, __LINE__, err);
529 goto release;
530 }
531 err = 0;
532 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
533 if (status != DMA_COMPLETE) {
534 err = -EIO;
535 dev_err(&sdev->dev, "%s %d err %d\n",
536 __func__, __LINE__, err);
537 goto release;
538 }
539release:
540 return err;
541}
542
543/*
544 * scif_drain_dma_poll - Drain all outstanding DMA operations for a particular
545 * DMA channel via polling.
546 *
547 * @sdev - The SCIF device
548 * @chan - DMA channel
549 * Return 0 on success and -errno on error.
550 */
551static int scif_drain_dma_poll(struct scif_hw_dev *sdev, struct dma_chan *chan)
552{
553 if (!chan)
554 return -EINVAL;
555 return scif_sync_dma(sdev, chan, SCIF_DMA_SYNC_WAIT);
556}
557
558/*
559 * scif_drain_dma_intr - Drain all outstanding DMA operations for a particular
560 * DMA channel via interrupt based blocking wait.
561 *
562 * @sdev - The SCIF device
563 * @chan - DMA channel
564 * Return 0 on success and -errno on error.
565 */
566int scif_drain_dma_intr(struct scif_hw_dev *sdev, struct dma_chan *chan)
567{
568 if (!chan)
569 return -EINVAL;
570 return scif_async_dma(sdev, chan);
571}
572
573/**
574 * scif_rma_destroy_windows:
575 *
576 * This routine destroys all windows queued for cleanup
577 */
578void scif_rma_destroy_windows(void)
579{
580 struct list_head *item, *tmp;
581 struct scif_window *window;
582 struct scif_endpt *ep;
583 struct dma_chan *chan;
584
585 might_sleep();
586restart:
587 spin_lock(&scif_info.rmalock);
588 list_for_each_safe(item, tmp, &scif_info.rma) {
589 window = list_entry(item, struct scif_window,
590 list);
591 ep = (struct scif_endpt *)window->ep;
592 chan = ep->rma_info.dma_chan;
593
594 list_del_init(&window->list);
595 spin_unlock(&scif_info.rmalock);
596 if (!chan || !scifdev_alive(ep) ||
597 !scif_drain_dma_intr(ep->remote_dev->sdev,
598 ep->rma_info.dma_chan))
599 /* Remove window from global list */
600 window->unreg_state = OP_COMPLETED;
601 else
602 dev_warn(&ep->remote_dev->sdev->dev,
603 "DMA engine hung?\n");
604 if (window->unreg_state == OP_COMPLETED) {
605 if (window->type == SCIF_WINDOW_SELF)
606 scif_destroy_window(ep, window);
607 else
608 scif_destroy_remote_window(window);
609 atomic_dec(&ep->rma_info.tw_refcount);
610 }
611 goto restart;
612 }
613 spin_unlock(&scif_info.rmalock);
614}
615
616/**
617 * scif_rma_destroy_tcw:
618 *
619 * This routine destroys temporary cached registered windows
620 * which have been queued for cleanup.
621 */
622void scif_rma_destroy_tcw_invalid(void)
623{
624 struct list_head *item, *tmp;
625 struct scif_window *window;
626 struct scif_endpt *ep;
627 struct dma_chan *chan;
628
629 might_sleep();
630restart:
631 spin_lock(&scif_info.rmalock);
632 list_for_each_safe(item, tmp, &scif_info.rma_tc) {
633 window = list_entry(item, struct scif_window, list);
634 ep = (struct scif_endpt *)window->ep;
635 chan = ep->rma_info.dma_chan;
636 list_del_init(&window->list);
637 spin_unlock(&scif_info.rmalock);
638 mutex_lock(&ep->rma_info.rma_lock);
639 if (!chan || !scifdev_alive(ep) ||
640 !scif_drain_dma_intr(ep->remote_dev->sdev,
641 ep->rma_info.dma_chan)) {
642 atomic_sub(window->nr_pages,
643 &ep->rma_info.tcw_total_pages);
644 scif_destroy_window(ep, window);
645 atomic_dec(&ep->rma_info.tcw_refcount);
646 } else {
647 dev_warn(&ep->remote_dev->sdev->dev,
648 "DMA engine hung?\n");
649 }
650 mutex_unlock(&ep->rma_info.rma_lock);
651 goto restart;
652 }
653 spin_unlock(&scif_info.rmalock);
654}
655
656static inline
657void *_get_local_va(off_t off, struct scif_window *window, size_t len)
658{
659 int page_nr = (off - window->offset) >> PAGE_SHIFT;
660 off_t page_off = off & ~PAGE_MASK;
661 void *va = NULL;
662
663 if (window->type == SCIF_WINDOW_SELF) {
664 struct page **pages = window->pinned_pages->pages;
665
666 va = page_address(pages[page_nr]) + page_off;
667 }
668 return va;
669}
670
671static inline
672void *ioremap_remote(off_t off, struct scif_window *window,
673 size_t len, struct scif_dev *dev,
674 struct scif_window_iter *iter)
675{
676 dma_addr_t phys = scif_off_to_dma_addr(window, off, NULL, iter);
677
678 /*
679 * If the DMA address is not card relative then we need the DMA
680 * addresses to be an offset into the bar. The aperture base was already
681 * added so subtract it here since scif_ioremap is going to add it again
682 */
683 if (!scifdev_self(dev) && window->type == SCIF_WINDOW_PEER &&
684 dev->sdev->aper && !dev->sdev->card_rel_da)
685 phys = phys - dev->sdev->aper->pa;
686 return scif_ioremap(phys, len, dev);
687}
688
689static inline void
690iounmap_remote(void *virt, size_t size, struct scif_copy_work *work)
691{
692 scif_iounmap(virt, size, work->remote_dev);
693}
694
695/*
696 * Takes care of ordering issue caused by
697 * 1. Hardware: Only in the case of cpu copy from mgmt node to card
698 * because of WC memory.
699 * 2. Software: If memcpy reorders copy instructions for optimization.
700 * This could happen at both mgmt node and card.
701 */
702static inline void
703scif_ordered_memcpy_toio(char *dst, const char *src, size_t count)
704{
705 if (!count)
706 return;
707
708 memcpy_toio((void __iomem __force *)dst, src, --count);
709 /* Order the last byte with the previous stores */
710 wmb();
711 *(dst + count) = *(src + count);
712}
713
714static inline void scif_unaligned_cpy_toio(char *dst, const char *src,
715 size_t count, bool ordered)
716{
717 if (ordered)
718 scif_ordered_memcpy_toio(dst, src, count);
719 else
720 memcpy_toio((void __iomem __force *)dst, src, count);
721}
722
723static inline
724void scif_ordered_memcpy_fromio(char *dst, const char *src, size_t count)
725{
726 if (!count)
727 return;
728
729 memcpy_fromio(dst, (void __iomem __force *)src, --count);
730 /* Order the last byte with the previous loads */
731 rmb();
732 *(dst + count) = *(src + count);
733}
734
735static inline void scif_unaligned_cpy_fromio(char *dst, const char *src,
736 size_t count, bool ordered)
737{
738 if (ordered)
739 scif_ordered_memcpy_fromio(dst, src, count);
740 else
741 memcpy_fromio(dst, (void __iomem __force *)src, count);
742}
743
744#define SCIF_RMA_ERROR_CODE (~(dma_addr_t)0x0)
745
746/*
747 * scif_off_to_dma_addr:
748 * Obtain the dma_addr given the window and the offset.
749 * @window: Registered window.
750 * @off: Window offset.
751 * @nr_bytes: Return the number of contiguous bytes till next DMA addr index.
752 * @index: Return the index of the dma_addr array found.
753 * @start_off: start offset of index of the dma addr array found.
754 * The nr_bytes provides the callee an estimate of the maximum possible
755 * DMA xfer possible while the index/start_off provide faster lookups
756 * for the next iteration.
757 */
758dma_addr_t scif_off_to_dma_addr(struct scif_window *window, s64 off,
759 size_t *nr_bytes, struct scif_window_iter *iter)
760{
761 int i, page_nr;
762 s64 start, end;
763 off_t page_off;
764
765 if (window->nr_pages == window->nr_contig_chunks) {
766 page_nr = (off - window->offset) >> PAGE_SHIFT;
767 page_off = off & ~PAGE_MASK;
768
769 if (nr_bytes)
770 *nr_bytes = PAGE_SIZE - page_off;
771 return window->dma_addr[page_nr] | page_off;
772 }
773 if (iter) {
774 i = iter->index;
775 start = iter->offset;
776 } else {
777 i = 0;
778 start = window->offset;
779 }
780 for (; i < window->nr_contig_chunks; i++) {
781 end = start + (window->num_pages[i] << PAGE_SHIFT);
782 if (off >= start && off < end) {
783 if (iter) {
784 iter->index = i;
785 iter->offset = start;
786 }
787 if (nr_bytes)
788 *nr_bytes = end - off;
789 return (window->dma_addr[i] + (off - start));
790 }
791 start += (window->num_pages[i] << PAGE_SHIFT);
792 }
793 dev_err(scif_info.mdev.this_device,
794 "%s %d BUG. Addr not found? window %p off 0x%llx\n",
795 __func__, __LINE__, window, off);
796 return SCIF_RMA_ERROR_CODE;
797}
798
799/*
800 * Copy between rma window and temporary buffer
801 */
802static void scif_rma_local_cpu_copy(s64 offset, struct scif_window *window,
803 u8 *temp, size_t rem_len, bool to_temp)
804{
805 void *window_virt;
806 size_t loop_len;
807 int offset_in_page;
808 s64 end_offset;
809
810 offset_in_page = offset & ~PAGE_MASK;
811 loop_len = PAGE_SIZE - offset_in_page;
812
813 if (rem_len < loop_len)
814 loop_len = rem_len;
815
816 window_virt = _get_local_va(offset, window, loop_len);
817 if (!window_virt)
818 return;
819 if (to_temp)
820 memcpy(temp, window_virt, loop_len);
821 else
822 memcpy(window_virt, temp, loop_len);
823
824 offset += loop_len;
825 temp += loop_len;
826 rem_len -= loop_len;
827
828 end_offset = window->offset +
829 (window->nr_pages << PAGE_SHIFT);
830 while (rem_len) {
831 if (offset == end_offset) {
832 window = list_next_entry(window, list);
833 end_offset = window->offset +
834 (window->nr_pages << PAGE_SHIFT);
835 }
836 loop_len = min(PAGE_SIZE, rem_len);
837 window_virt = _get_local_va(offset, window, loop_len);
838 if (!window_virt)
839 return;
840 if (to_temp)
841 memcpy(temp, window_virt, loop_len);
842 else
843 memcpy(window_virt, temp, loop_len);
844 offset += loop_len;
845 temp += loop_len;
846 rem_len -= loop_len;
847 }
848}
849
850/**
851 * scif_rma_completion_cb:
852 * @data: RMA cookie
853 *
854 * RMA interrupt completion callback.
855 */
856static void scif_rma_completion_cb(void *data)
857{
858 struct scif_dma_comp_cb *comp_cb = data;
859
860 /* Free DMA Completion CB. */
861 if (comp_cb->dst_window)
862 scif_rma_local_cpu_copy(comp_cb->dst_offset,
863 comp_cb->dst_window,
864 comp_cb->temp_buf +
865 comp_cb->header_padding,
866 comp_cb->len, false);
867 scif_unmap_single(comp_cb->temp_phys, comp_cb->sdev,
868 SCIF_KMEM_UNALIGNED_BUF_SIZE);
869 if (comp_cb->is_cache)
870 kmem_cache_free(unaligned_cache,
871 comp_cb->temp_buf_to_free);
872 else
873 kfree(comp_cb->temp_buf_to_free);
874}
875
876/* Copies between temporary buffer and offsets provided in work */
877static int
878scif_rma_list_dma_copy_unaligned(struct scif_copy_work *work,
879 u8 *temp, struct dma_chan *chan,
880 bool src_local)
881{
882 struct scif_dma_comp_cb *comp_cb = work->comp_cb;
883 dma_addr_t window_dma_addr, temp_dma_addr;
884 dma_addr_t temp_phys = comp_cb->temp_phys;
885 size_t loop_len, nr_contig_bytes = 0, remaining_len = work->len;
886 int offset_in_ca, ret = 0;
887 s64 end_offset, offset;
888 struct scif_window *window;
889 void *window_virt_addr;
890 size_t tail_len;
891 struct dma_async_tx_descriptor *tx;
892 struct dma_device *dev = chan->device;
893 dma_cookie_t cookie;
894
895 if (src_local) {
896 offset = work->dst_offset;
897 window = work->dst_window;
898 } else {
899 offset = work->src_offset;
900 window = work->src_window;
901 }
902
903 offset_in_ca = offset & (L1_CACHE_BYTES - 1);
904 if (offset_in_ca) {
905 loop_len = L1_CACHE_BYTES - offset_in_ca;
906 loop_len = min(loop_len, remaining_len);
907 window_virt_addr = ioremap_remote(offset, window,
908 loop_len,
909 work->remote_dev,
910 NULL);
911 if (!window_virt_addr)
912 return -ENOMEM;
913 if (src_local)
914 scif_unaligned_cpy_toio(window_virt_addr, temp,
915 loop_len,
916 work->ordered &&
917 !(remaining_len - loop_len));
918 else
919 scif_unaligned_cpy_fromio(temp, window_virt_addr,
920 loop_len, work->ordered &&
921 !(remaining_len - loop_len));
922 iounmap_remote(window_virt_addr, loop_len, work);
923
924 offset += loop_len;
925 temp += loop_len;
926 temp_phys += loop_len;
927 remaining_len -= loop_len;
928 }
929
930 offset_in_ca = offset & ~PAGE_MASK;
931 end_offset = window->offset +
932 (window->nr_pages << PAGE_SHIFT);
933
934 tail_len = remaining_len & (L1_CACHE_BYTES - 1);
935 remaining_len -= tail_len;
936 while (remaining_len) {
937 if (offset == end_offset) {
938 window = list_next_entry(window, list);
939 end_offset = window->offset +
940 (window->nr_pages << PAGE_SHIFT);
941 }
942 if (scif_is_mgmt_node())
943 temp_dma_addr = temp_phys;
944 else
945 /* Fix if we ever enable IOMMU on the card */
946 temp_dma_addr = (dma_addr_t)virt_to_phys(temp);
947 window_dma_addr = scif_off_to_dma_addr(window, offset,
948 &nr_contig_bytes,
949 NULL);
950 loop_len = min(nr_contig_bytes, remaining_len);
951 if (src_local) {
952 if (work->ordered && !tail_len &&
953 !(remaining_len - loop_len) &&
954 loop_len != L1_CACHE_BYTES) {
955 /*
956 * Break up the last chunk of the transfer into
957 * two steps. if there is no tail to guarantee
958 * DMA ordering. SCIF_DMA_POLLING inserts
959 * a status update descriptor in step 1 which
960 * acts as a double sided synchronization fence
961 * for the DMA engine to ensure that the last
962 * cache line in step 2 is updated last.
963 */
964 /* Step 1) DMA: Body Length - L1_CACHE_BYTES. */
965 tx =
966 dev->device_prep_dma_memcpy(chan,
967 window_dma_addr,
968 temp_dma_addr,
969 loop_len -
970 L1_CACHE_BYTES,
971 DMA_PREP_FENCE);
972 if (!tx) {
973 ret = -ENOMEM;
974 goto err;
975 }
976 cookie = tx->tx_submit(tx);
977 if (dma_submit_error(cookie)) {
978 ret = -ENOMEM;
979 goto err;
980 }
981 dma_async_issue_pending(chan);
982 offset += (loop_len - L1_CACHE_BYTES);
983 temp_dma_addr += (loop_len - L1_CACHE_BYTES);
984 window_dma_addr += (loop_len - L1_CACHE_BYTES);
985 remaining_len -= (loop_len - L1_CACHE_BYTES);
986 loop_len = remaining_len;
987
988 /* Step 2) DMA: L1_CACHE_BYTES */
989 tx =
990 dev->device_prep_dma_memcpy(chan,
991 window_dma_addr,
992 temp_dma_addr,
993 loop_len, 0);
994 if (!tx) {
995 ret = -ENOMEM;
996 goto err;
997 }
998 cookie = tx->tx_submit(tx);
999 if (dma_submit_error(cookie)) {
1000 ret = -ENOMEM;
1001 goto err;
1002 }
1003 dma_async_issue_pending(chan);
1004 } else {
1005 tx =
1006 dev->device_prep_dma_memcpy(chan,
1007 window_dma_addr,
1008 temp_dma_addr,
1009 loop_len, 0);
1010 if (!tx) {
1011 ret = -ENOMEM;
1012 goto err;
1013 }
1014 cookie = tx->tx_submit(tx);
1015 if (dma_submit_error(cookie)) {
1016 ret = -ENOMEM;
1017 goto err;
1018 }
1019 dma_async_issue_pending(chan);
1020 }
1021 } else {
1022 tx = dev->device_prep_dma_memcpy(chan, temp_dma_addr,
1023 window_dma_addr, loop_len, 0);
1024 if (!tx) {
1025 ret = -ENOMEM;
1026 goto err;
1027 }
1028 cookie = tx->tx_submit(tx);
1029 if (dma_submit_error(cookie)) {
1030 ret = -ENOMEM;
1031 goto err;
1032 }
1033 dma_async_issue_pending(chan);
1034 }
1035 if (ret < 0)
1036 goto err;
1037 offset += loop_len;
1038 temp += loop_len;
1039 temp_phys += loop_len;
1040 remaining_len -= loop_len;
1041 offset_in_ca = 0;
1042 }
1043 if (tail_len) {
1044 if (offset == end_offset) {
1045 window = list_next_entry(window, list);
1046 end_offset = window->offset +
1047 (window->nr_pages << PAGE_SHIFT);
1048 }
1049 window_virt_addr = ioremap_remote(offset, window, tail_len,
1050 work->remote_dev,
1051 NULL);
1052 if (!window_virt_addr)
1053 return -ENOMEM;
1054 /*
1055 * The CPU copy for the tail bytes must be initiated only once
1056 * previous DMA transfers for this endpoint have completed
1057 * to guarantee ordering.
1058 */
1059 if (work->ordered) {
1060 struct scif_dev *rdev = work->remote_dev;
1061
1062 ret = scif_drain_dma_intr(rdev->sdev, chan);
1063 if (ret)
1064 return ret;
1065 }
1066 if (src_local)
1067 scif_unaligned_cpy_toio(window_virt_addr, temp,
1068 tail_len, work->ordered);
1069 else
1070 scif_unaligned_cpy_fromio(temp, window_virt_addr,
1071 tail_len, work->ordered);
1072 iounmap_remote(window_virt_addr, tail_len, work);
1073 }
1074 tx = dev->device_prep_dma_memcpy(chan, 0, 0, 0, DMA_PREP_INTERRUPT);
1075 if (!tx) {
1076 ret = -ENOMEM;
1077 return ret;
1078 }
1079 tx->callback = &scif_rma_completion_cb;
1080 tx->callback_param = comp_cb;
1081 cookie = tx->tx_submit(tx);
1082
1083 if (dma_submit_error(cookie)) {
1084 ret = -ENOMEM;
1085 return ret;
1086 }
1087 dma_async_issue_pending(chan);
1088 return 0;
1089err:
1090 dev_err(scif_info.mdev.this_device,
1091 "%s %d Desc Prog Failed ret %d\n",
1092 __func__, __LINE__, ret);
1093 return ret;
1094}
1095
1096/*
1097 * _scif_rma_list_dma_copy_aligned:
1098 *
1099 * Traverse all the windows and perform DMA copy.
1100 */
1101static int _scif_rma_list_dma_copy_aligned(struct scif_copy_work *work,
1102 struct dma_chan *chan)
1103{
1104 dma_addr_t src_dma_addr, dst_dma_addr;
1105 size_t loop_len, remaining_len, src_contig_bytes = 0;
1106 size_t dst_contig_bytes = 0;
1107 struct scif_window_iter src_win_iter;
1108 struct scif_window_iter dst_win_iter;
1109 s64 end_src_offset, end_dst_offset;
1110 struct scif_window *src_window = work->src_window;
1111 struct scif_window *dst_window = work->dst_window;
1112 s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1113 int ret = 0;
1114 struct dma_async_tx_descriptor *tx;
1115 struct dma_device *dev = chan->device;
1116 dma_cookie_t cookie;
1117
1118 remaining_len = work->len;
1119
1120 scif_init_window_iter(src_window, &src_win_iter);
1121 scif_init_window_iter(dst_window, &dst_win_iter);
1122 end_src_offset = src_window->offset +
1123 (src_window->nr_pages << PAGE_SHIFT);
1124 end_dst_offset = dst_window->offset +
1125 (dst_window->nr_pages << PAGE_SHIFT);
1126 while (remaining_len) {
1127 if (src_offset == end_src_offset) {
1128 src_window = list_next_entry(src_window, list);
1129 end_src_offset = src_window->offset +
1130 (src_window->nr_pages << PAGE_SHIFT);
1131 scif_init_window_iter(src_window, &src_win_iter);
1132 }
1133 if (dst_offset == end_dst_offset) {
1134 dst_window = list_next_entry(dst_window, list);
1135 end_dst_offset = dst_window->offset +
1136 (dst_window->nr_pages << PAGE_SHIFT);
1137 scif_init_window_iter(dst_window, &dst_win_iter);
1138 }
1139
1140 /* compute dma addresses for transfer */
1141 src_dma_addr = scif_off_to_dma_addr(src_window, src_offset,
1142 &src_contig_bytes,
1143 &src_win_iter);
1144 dst_dma_addr = scif_off_to_dma_addr(dst_window, dst_offset,
1145 &dst_contig_bytes,
1146 &dst_win_iter);
1147 loop_len = min(src_contig_bytes, dst_contig_bytes);
1148 loop_len = min(loop_len, remaining_len);
1149 if (work->ordered && !(remaining_len - loop_len)) {
1150 /*
1151 * Break up the last chunk of the transfer into two
1152 * steps to ensure that the last byte in step 2 is
1153 * updated last.
1154 */
1155 /* Step 1) DMA: Body Length - 1 */
1156 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1157 src_dma_addr,
1158 loop_len - 1,
1159 DMA_PREP_FENCE);
1160 if (!tx) {
1161 ret = -ENOMEM;
1162 goto err;
1163 }
1164 cookie = tx->tx_submit(tx);
1165 if (dma_submit_error(cookie)) {
1166 ret = -ENOMEM;
1167 goto err;
1168 }
1169 src_offset += (loop_len - 1);
1170 dst_offset += (loop_len - 1);
1171 src_dma_addr += (loop_len - 1);
1172 dst_dma_addr += (loop_len - 1);
1173 remaining_len -= (loop_len - 1);
1174 loop_len = remaining_len;
1175
1176 /* Step 2) DMA: 1 BYTES */
1177 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1178 src_dma_addr, loop_len, 0);
1179 if (!tx) {
1180 ret = -ENOMEM;
1181 goto err;
1182 }
1183 cookie = tx->tx_submit(tx);
1184 if (dma_submit_error(cookie)) {
1185 ret = -ENOMEM;
1186 goto err;
1187 }
1188 dma_async_issue_pending(chan);
1189 } else {
1190 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1191 src_dma_addr, loop_len, 0);
1192 if (!tx) {
1193 ret = -ENOMEM;
1194 goto err;
1195 }
1196 cookie = tx->tx_submit(tx);
1197 if (dma_submit_error(cookie)) {
1198 ret = -ENOMEM;
1199 goto err;
1200 }
1201 }
1202 src_offset += loop_len;
1203 dst_offset += loop_len;
1204 remaining_len -= loop_len;
1205 }
1206 return ret;
1207err:
1208 dev_err(scif_info.mdev.this_device,
1209 "%s %d Desc Prog Failed ret %d\n",
1210 __func__, __LINE__, ret);
1211 return ret;
1212}
1213
1214/*
1215 * scif_rma_list_dma_copy_aligned:
1216 *
1217 * Traverse all the windows and perform DMA copy.
1218 */
1219static int scif_rma_list_dma_copy_aligned(struct scif_copy_work *work,
1220 struct dma_chan *chan)
1221{
1222 dma_addr_t src_dma_addr, dst_dma_addr;
1223 size_t loop_len, remaining_len, tail_len, src_contig_bytes = 0;
1224 size_t dst_contig_bytes = 0;
1225 int src_cache_off;
1226 s64 end_src_offset, end_dst_offset;
1227 struct scif_window_iter src_win_iter;
1228 struct scif_window_iter dst_win_iter;
1229 void *src_virt, *dst_virt;
1230 struct scif_window *src_window = work->src_window;
1231 struct scif_window *dst_window = work->dst_window;
1232 s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1233 int ret = 0;
1234 struct dma_async_tx_descriptor *tx;
1235 struct dma_device *dev = chan->device;
1236 dma_cookie_t cookie;
1237
1238 remaining_len = work->len;
1239 scif_init_window_iter(src_window, &src_win_iter);
1240 scif_init_window_iter(dst_window, &dst_win_iter);
1241
1242 src_cache_off = src_offset & (L1_CACHE_BYTES - 1);
1243 if (src_cache_off != 0) {
1244 /* Head */
1245 loop_len = L1_CACHE_BYTES - src_cache_off;
1246 loop_len = min(loop_len, remaining_len);
1247 src_dma_addr = __scif_off_to_dma_addr(src_window, src_offset);
1248 dst_dma_addr = __scif_off_to_dma_addr(dst_window, dst_offset);
1249 if (src_window->type == SCIF_WINDOW_SELF)
1250 src_virt = _get_local_va(src_offset, src_window,
1251 loop_len);
1252 else
1253 src_virt = ioremap_remote(src_offset, src_window,
1254 loop_len,
1255 work->remote_dev, NULL);
1256 if (!src_virt)
1257 return -ENOMEM;
1258 if (dst_window->type == SCIF_WINDOW_SELF)
1259 dst_virt = _get_local_va(dst_offset, dst_window,
1260 loop_len);
1261 else
1262 dst_virt = ioremap_remote(dst_offset, dst_window,
1263 loop_len,
1264 work->remote_dev, NULL);
1265 if (!dst_virt) {
1266 if (src_window->type != SCIF_WINDOW_SELF)
1267 iounmap_remote(src_virt, loop_len, work);
1268 return -ENOMEM;
1269 }
1270 if (src_window->type == SCIF_WINDOW_SELF)
1271 scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,
1272 remaining_len == loop_len ?
1273 work->ordered : false);
1274 else
1275 scif_unaligned_cpy_fromio(dst_virt, src_virt, loop_len,
1276 remaining_len == loop_len ?
1277 work->ordered : false);
1278 if (src_window->type != SCIF_WINDOW_SELF)
1279 iounmap_remote(src_virt, loop_len, work);
1280 if (dst_window->type != SCIF_WINDOW_SELF)
1281 iounmap_remote(dst_virt, loop_len, work);
1282 src_offset += loop_len;
1283 dst_offset += loop_len;
1284 remaining_len -= loop_len;
1285 }
1286
1287 end_src_offset = src_window->offset +
1288 (src_window->nr_pages << PAGE_SHIFT);
1289 end_dst_offset = dst_window->offset +
1290 (dst_window->nr_pages << PAGE_SHIFT);
1291 tail_len = remaining_len & (L1_CACHE_BYTES - 1);
1292 remaining_len -= tail_len;
1293 while (remaining_len) {
1294 if (src_offset == end_src_offset) {
1295 src_window = list_next_entry(src_window, list);
1296 end_src_offset = src_window->offset +
1297 (src_window->nr_pages << PAGE_SHIFT);
1298 scif_init_window_iter(src_window, &src_win_iter);
1299 }
1300 if (dst_offset == end_dst_offset) {
1301 dst_window = list_next_entry(dst_window, list);
1302 end_dst_offset = dst_window->offset +
1303 (dst_window->nr_pages << PAGE_SHIFT);
1304 scif_init_window_iter(dst_window, &dst_win_iter);
1305 }
1306
1307 /* compute dma addresses for transfer */
1308 src_dma_addr = scif_off_to_dma_addr(src_window, src_offset,
1309 &src_contig_bytes,
1310 &src_win_iter);
1311 dst_dma_addr = scif_off_to_dma_addr(dst_window, dst_offset,
1312 &dst_contig_bytes,
1313 &dst_win_iter);
1314 loop_len = min(src_contig_bytes, dst_contig_bytes);
1315 loop_len = min(loop_len, remaining_len);
1316 if (work->ordered && !tail_len &&
1317 !(remaining_len - loop_len)) {
1318 /*
1319 * Break up the last chunk of the transfer into two
1320 * steps. if there is no tail to gurantee DMA ordering.
1321 * Passing SCIF_DMA_POLLING inserts a status update
1322 * descriptor in step 1 which acts as a double sided
1323 * synchronization fence for the DMA engine to ensure
1324 * that the last cache line in step 2 is updated last.
1325 */
1326 /* Step 1) DMA: Body Length - L1_CACHE_BYTES. */
1327 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1328 src_dma_addr,
1329 loop_len -
1330 L1_CACHE_BYTES,
1331 DMA_PREP_FENCE);
1332 if (!tx) {
1333 ret = -ENOMEM;
1334 goto err;
1335 }
1336 cookie = tx->tx_submit(tx);
1337 if (dma_submit_error(cookie)) {
1338 ret = -ENOMEM;
1339 goto err;
1340 }
1341 dma_async_issue_pending(chan);
1342 src_offset += (loop_len - L1_CACHE_BYTES);
1343 dst_offset += (loop_len - L1_CACHE_BYTES);
1344 src_dma_addr += (loop_len - L1_CACHE_BYTES);
1345 dst_dma_addr += (loop_len - L1_CACHE_BYTES);
1346 remaining_len -= (loop_len - L1_CACHE_BYTES);
1347 loop_len = remaining_len;
1348
1349 /* Step 2) DMA: L1_CACHE_BYTES */
1350 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1351 src_dma_addr,
1352 loop_len, 0);
1353 if (!tx) {
1354 ret = -ENOMEM;
1355 goto err;
1356 }
1357 cookie = tx->tx_submit(tx);
1358 if (dma_submit_error(cookie)) {
1359 ret = -ENOMEM;
1360 goto err;
1361 }
1362 dma_async_issue_pending(chan);
1363 } else {
1364 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1365 src_dma_addr,
1366 loop_len, 0);
1367 if (!tx) {
1368 ret = -ENOMEM;
1369 goto err;
1370 }
1371 cookie = tx->tx_submit(tx);
1372 if (dma_submit_error(cookie)) {
1373 ret = -ENOMEM;
1374 goto err;
1375 }
1376 dma_async_issue_pending(chan);
1377 }
1378 src_offset += loop_len;
1379 dst_offset += loop_len;
1380 remaining_len -= loop_len;
1381 }
1382 remaining_len = tail_len;
1383 if (remaining_len) {
1384 loop_len = remaining_len;
1385 if (src_offset == end_src_offset)
1386 src_window = list_next_entry(src_window, list);
1387 if (dst_offset == end_dst_offset)
1388 dst_window = list_next_entry(dst_window, list);
1389
1390 src_dma_addr = __scif_off_to_dma_addr(src_window, src_offset);
1391 dst_dma_addr = __scif_off_to_dma_addr(dst_window, dst_offset);
1392 /*
1393 * The CPU copy for the tail bytes must be initiated only once
1394 * previous DMA transfers for this endpoint have completed to
1395 * guarantee ordering.
1396 */
1397 if (work->ordered) {
1398 struct scif_dev *rdev = work->remote_dev;
1399
1400 ret = scif_drain_dma_poll(rdev->sdev, chan);
1401 if (ret)
1402 return ret;
1403 }
1404 if (src_window->type == SCIF_WINDOW_SELF)
1405 src_virt = _get_local_va(src_offset, src_window,
1406 loop_len);
1407 else
1408 src_virt = ioremap_remote(src_offset, src_window,
1409 loop_len,
1410 work->remote_dev, NULL);
1411 if (!src_virt)
1412 return -ENOMEM;
1413
1414 if (dst_window->type == SCIF_WINDOW_SELF)
1415 dst_virt = _get_local_va(dst_offset, dst_window,
1416 loop_len);
1417 else
1418 dst_virt = ioremap_remote(dst_offset, dst_window,
1419 loop_len,
1420 work->remote_dev, NULL);
1421 if (!dst_virt) {
1422 if (src_window->type != SCIF_WINDOW_SELF)
1423 iounmap_remote(src_virt, loop_len, work);
1424 return -ENOMEM;
1425 }
1426
1427 if (src_window->type == SCIF_WINDOW_SELF)
1428 scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,
1429 work->ordered);
1430 else
1431 scif_unaligned_cpy_fromio(dst_virt, src_virt,
1432 loop_len, work->ordered);
1433 if (src_window->type != SCIF_WINDOW_SELF)
1434 iounmap_remote(src_virt, loop_len, work);
1435
1436 if (dst_window->type != SCIF_WINDOW_SELF)
1437 iounmap_remote(dst_virt, loop_len, work);
1438 remaining_len -= loop_len;
1439 }
1440 return ret;
1441err:
1442 dev_err(scif_info.mdev.this_device,
1443 "%s %d Desc Prog Failed ret %d\n",
1444 __func__, __LINE__, ret);
1445 return ret;
1446}
1447
1448/*
1449 * scif_rma_list_cpu_copy:
1450 *
1451 * Traverse all the windows and perform CPU copy.
1452 */
1453static int scif_rma_list_cpu_copy(struct scif_copy_work *work)
1454{
1455 void *src_virt, *dst_virt;
1456 size_t loop_len, remaining_len;
1457 int src_page_off, dst_page_off;
1458 s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1459 struct scif_window *src_window = work->src_window;
1460 struct scif_window *dst_window = work->dst_window;
1461 s64 end_src_offset, end_dst_offset;
1462 int ret = 0;
1463 struct scif_window_iter src_win_iter;
1464 struct scif_window_iter dst_win_iter;
1465
1466 remaining_len = work->len;
1467
1468 scif_init_window_iter(src_window, &src_win_iter);
1469 scif_init_window_iter(dst_window, &dst_win_iter);
1470 while (remaining_len) {
1471 src_page_off = src_offset & ~PAGE_MASK;
1472 dst_page_off = dst_offset & ~PAGE_MASK;
1473 loop_len = min(PAGE_SIZE -
1474 max(src_page_off, dst_page_off),
1475 remaining_len);
1476
1477 if (src_window->type == SCIF_WINDOW_SELF)
1478 src_virt = _get_local_va(src_offset, src_window,
1479 loop_len);
1480 else
1481 src_virt = ioremap_remote(src_offset, src_window,
1482 loop_len,
1483 work->remote_dev,
1484 &src_win_iter);
1485 if (!src_virt) {
1486 ret = -ENOMEM;
1487 goto error;
1488 }
1489
1490 if (dst_window->type == SCIF_WINDOW_SELF)
1491 dst_virt = _get_local_va(dst_offset, dst_window,
1492 loop_len);
1493 else
1494 dst_virt = ioremap_remote(dst_offset, dst_window,
1495 loop_len,
1496 work->remote_dev,
1497 &dst_win_iter);
1498 if (!dst_virt) {
1499 if (src_window->type == SCIF_WINDOW_PEER)
1500 iounmap_remote(src_virt, loop_len, work);
1501 ret = -ENOMEM;
1502 goto error;
1503 }
1504
1505 if (work->loopback) {
1506 memcpy(dst_virt, src_virt, loop_len);
1507 } else {
1508 if (src_window->type == SCIF_WINDOW_SELF)
1509 memcpy_toio((void __iomem __force *)dst_virt,
1510 src_virt, loop_len);
1511 else
1512 memcpy_fromio(dst_virt,
1513 (void __iomem __force *)src_virt,
1514 loop_len);
1515 }
1516 if (src_window->type == SCIF_WINDOW_PEER)
1517 iounmap_remote(src_virt, loop_len, work);
1518
1519 if (dst_window->type == SCIF_WINDOW_PEER)
1520 iounmap_remote(dst_virt, loop_len, work);
1521
1522 src_offset += loop_len;
1523 dst_offset += loop_len;
1524 remaining_len -= loop_len;
1525 if (remaining_len) {
1526 end_src_offset = src_window->offset +
1527 (src_window->nr_pages << PAGE_SHIFT);
1528 end_dst_offset = dst_window->offset +
1529 (dst_window->nr_pages << PAGE_SHIFT);
1530 if (src_offset == end_src_offset) {
1531 src_window = list_next_entry(src_window, list);
1532 scif_init_window_iter(src_window,
1533 &src_win_iter);
1534 }
1535 if (dst_offset == end_dst_offset) {
1536 dst_window = list_next_entry(dst_window, list);
1537 scif_init_window_iter(dst_window,
1538 &dst_win_iter);
1539 }
1540 }
1541 }
1542error:
1543 return ret;
1544}
1545
1546static int scif_rma_list_dma_copy_wrapper(struct scif_endpt *epd,
1547 struct scif_copy_work *work,
1548 struct dma_chan *chan, off_t loffset)
1549{
1550 int src_cache_off, dst_cache_off;
1551 s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1552 u8 *temp = NULL;
1553 bool src_local = true, dst_local = false;
1554 struct scif_dma_comp_cb *comp_cb;
1555 dma_addr_t src_dma_addr, dst_dma_addr;
1556 int err;
1557
1558 if (is_dma_copy_aligned(chan->device, 1, 1, 1))
1559 return _scif_rma_list_dma_copy_aligned(work, chan);
1560
1561 src_cache_off = src_offset & (L1_CACHE_BYTES - 1);
1562 dst_cache_off = dst_offset & (L1_CACHE_BYTES - 1);
1563
1564 if (dst_cache_off == src_cache_off)
1565 return scif_rma_list_dma_copy_aligned(work, chan);
1566
1567 if (work->loopback)
1568 return scif_rma_list_cpu_copy(work);
1569 src_dma_addr = __scif_off_to_dma_addr(work->src_window, src_offset);
1570 dst_dma_addr = __scif_off_to_dma_addr(work->dst_window, dst_offset);
1571 src_local = work->src_window->type == SCIF_WINDOW_SELF;
1572 dst_local = work->dst_window->type == SCIF_WINDOW_SELF;
1573
1574 dst_local = dst_local;
1575 /* Allocate dma_completion cb */
1576 comp_cb = kzalloc(sizeof(*comp_cb), GFP_KERNEL);
1577 if (!comp_cb)
1578 goto error;
1579
1580 work->comp_cb = comp_cb;
1581 comp_cb->cb_cookie = comp_cb;
1582 comp_cb->dma_completion_func = &scif_rma_completion_cb;
1583
1584 if (work->len + (L1_CACHE_BYTES << 1) < SCIF_KMEM_UNALIGNED_BUF_SIZE) {
1585 comp_cb->is_cache = false;
1586 /* Allocate padding bytes to align to a cache line */
1587 temp = kmalloc(work->len + (L1_CACHE_BYTES << 1),
1588 GFP_KERNEL);
1589 if (!temp)
1590 goto free_comp_cb;
1591 comp_cb->temp_buf_to_free = temp;
1592 /* kmalloc(..) does not guarantee cache line alignment */
1593 if (!IS_ALIGNED((u64)temp, L1_CACHE_BYTES))
1594 temp = PTR_ALIGN(temp, L1_CACHE_BYTES);
1595 } else {
1596 comp_cb->is_cache = true;
1597 temp = kmem_cache_alloc(unaligned_cache, GFP_KERNEL);
1598 if (!temp)
1599 goto free_comp_cb;
1600 comp_cb->temp_buf_to_free = temp;
1601 }
1602
1603 if (src_local) {
1604 temp += dst_cache_off;
1605 scif_rma_local_cpu_copy(work->src_offset, work->src_window,
1606 temp, work->len, true);
1607 } else {
1608 comp_cb->dst_window = work->dst_window;
1609 comp_cb->dst_offset = work->dst_offset;
1610 work->src_offset = work->src_offset - src_cache_off;
1611 comp_cb->len = work->len;
1612 work->len = ALIGN(work->len + src_cache_off, L1_CACHE_BYTES);
1613 comp_cb->header_padding = src_cache_off;
1614 }
1615 comp_cb->temp_buf = temp;
1616
1617 err = scif_map_single(&comp_cb->temp_phys, temp,
1618 work->remote_dev, SCIF_KMEM_UNALIGNED_BUF_SIZE);
1619 if (err)
1620 goto free_temp_buf;
1621 comp_cb->sdev = work->remote_dev;
1622 if (scif_rma_list_dma_copy_unaligned(work, temp, chan, src_local) < 0)
1623 goto free_temp_buf;
1624 if (!src_local)
1625 work->fence_type = SCIF_DMA_INTR;
1626 return 0;
1627free_temp_buf:
1628 if (comp_cb->is_cache)
1629 kmem_cache_free(unaligned_cache, comp_cb->temp_buf_to_free);
1630 else
1631 kfree(comp_cb->temp_buf_to_free);
1632free_comp_cb:
1633 kfree(comp_cb);
1634error:
1635 return -ENOMEM;
1636}
1637
1638/**
1639 * scif_rma_copy:
1640 * @epd: end point descriptor.
1641 * @loffset: offset in local registered address space to/from which to copy
1642 * @addr: user virtual address to/from which to copy
1643 * @len: length of range to copy
1644 * @roffset: offset in remote registered address space to/from which to copy
1645 * @flags: flags
1646 * @dir: LOCAL->REMOTE or vice versa.
1647 * @last_chunk: true if this is the last chunk of a larger transfer
1648 *
1649 * Validate parameters, check if src/dst registered ranges requested for copy
1650 * are valid and initiate either CPU or DMA copy.
1651 */
1652static int scif_rma_copy(scif_epd_t epd, off_t loffset, unsigned long addr,
1653 size_t len, off_t roffset, int flags,
1654 enum scif_rma_dir dir, bool last_chunk)
1655{
1656 struct scif_endpt *ep = (struct scif_endpt *)epd;
1657 struct scif_rma_req remote_req;
1658 struct scif_rma_req req;
1659 struct scif_window *local_window = NULL;
1660 struct scif_window *remote_window = NULL;
1661 struct scif_copy_work copy_work;
1662 bool loopback;
1663 int err = 0;
1664 struct dma_chan *chan;
1665 struct scif_mmu_notif *mmn = NULL;
1666 bool cache = false;
1667 struct device *spdev;
1668
1669 err = scif_verify_epd(ep);
1670 if (err)
1671 return err;
1672
1673 if (flags && !(flags & (SCIF_RMA_USECPU | SCIF_RMA_USECACHE |
1674 SCIF_RMA_SYNC | SCIF_RMA_ORDERED)))
1675 return -EINVAL;
1676
1677 loopback = scifdev_self(ep->remote_dev) ? true : false;
1678 copy_work.fence_type = ((flags & SCIF_RMA_SYNC) && last_chunk) ?
1679 SCIF_DMA_POLL : 0;
1680 copy_work.ordered = !!((flags & SCIF_RMA_ORDERED) && last_chunk);
1681
1682 /* Use CPU for Mgmt node <-> Mgmt node copies */
1683 if (loopback && scif_is_mgmt_node()) {
1684 flags |= SCIF_RMA_USECPU;
1685 copy_work.fence_type = 0x0;
1686 }
1687
1688 cache = scif_is_set_reg_cache(flags);
1689
1690 remote_req.out_window = &remote_window;
1691 remote_req.offset = roffset;
1692 remote_req.nr_bytes = len;
1693 /*
1694 * If transfer is from local to remote then the remote window
1695 * must be writeable and vice versa.
1696 */
1697 remote_req.prot = dir == SCIF_LOCAL_TO_REMOTE ? VM_WRITE : VM_READ;
1698 remote_req.type = SCIF_WINDOW_PARTIAL;
1699 remote_req.head = &ep->rma_info.remote_reg_list;
1700
1701 spdev = scif_get_peer_dev(ep->remote_dev);
1702 if (IS_ERR(spdev)) {
1703 err = PTR_ERR(spdev);
1704 return err;
1705 }
1706
1707 if (addr && cache) {
1708 mutex_lock(&ep->rma_info.mmn_lock);
1709 mmn = scif_find_mmu_notifier(current->mm, &ep->rma_info);
1710 if (!mmn)
1711 mmn = scif_add_mmu_notifier(current->mm, ep);
1712 mutex_unlock(&ep->rma_info.mmn_lock);
1713 if (IS_ERR(mmn)) {
1714 scif_put_peer_dev(spdev);
1715 return PTR_ERR(mmn);
1716 }
1717 cache = cache && !scif_rma_tc_can_cache(ep, len);
1718 }
1719 mutex_lock(&ep->rma_info.rma_lock);
1720 if (addr) {
1721 req.out_window = &local_window;
1722 req.nr_bytes = ALIGN(len + (addr & ~PAGE_MASK),
1723 PAGE_SIZE);
1724 req.va_for_temp = addr & PAGE_MASK;
1725 req.prot = (dir == SCIF_LOCAL_TO_REMOTE ?
1726 VM_READ : VM_WRITE | VM_READ);
1727 /* Does a valid local window exist? */
1728 if (mmn) {
1729 spin_lock(&ep->rma_info.tc_lock);
1730 req.head = &mmn->tc_reg_list;
1731 err = scif_query_tcw(ep, &req);
1732 spin_unlock(&ep->rma_info.tc_lock);
1733 }
1734 if (!mmn || err) {
1735 err = scif_register_temp(epd, req.va_for_temp,
1736 req.nr_bytes, req.prot,
1737 &loffset, &local_window);
1738 if (err) {
1739 mutex_unlock(&ep->rma_info.rma_lock);
1740 goto error;
1741 }
1742 if (!cache)
1743 goto skip_cache;
1744 atomic_inc(&ep->rma_info.tcw_refcount);
1745 atomic_add_return(local_window->nr_pages,
1746 &ep->rma_info.tcw_total_pages);
1747 if (mmn) {
1748 spin_lock(&ep->rma_info.tc_lock);
1749 scif_insert_tcw(local_window,
1750 &mmn->tc_reg_list);
1751 spin_unlock(&ep->rma_info.tc_lock);
1752 }
1753 }
1754skip_cache:
1755 loffset = local_window->offset +
1756 (addr - local_window->va_for_temp);
1757 } else {
1758 req.out_window = &local_window;
1759 req.offset = loffset;
1760 /*
1761 * If transfer is from local to remote then the self window
1762 * must be readable and vice versa.
1763 */
1764 req.prot = dir == SCIF_LOCAL_TO_REMOTE ? VM_READ : VM_WRITE;
1765 req.nr_bytes = len;
1766 req.type = SCIF_WINDOW_PARTIAL;
1767 req.head = &ep->rma_info.reg_list;
1768 /* Does a valid local window exist? */
1769 err = scif_query_window(&req);
1770 if (err) {
1771 mutex_unlock(&ep->rma_info.rma_lock);
1772 goto error;
1773 }
1774 }
1775
1776 /* Does a valid remote window exist? */
1777 err = scif_query_window(&remote_req);
1778 if (err) {
1779 mutex_unlock(&ep->rma_info.rma_lock);
1780 goto error;
1781 }
1782
1783 /*
1784 * Prepare copy_work for submitting work to the DMA kernel thread
1785 * or CPU copy routine.
1786 */
1787 copy_work.len = len;
1788 copy_work.loopback = loopback;
1789 copy_work.remote_dev = ep->remote_dev;
1790 if (dir == SCIF_LOCAL_TO_REMOTE) {
1791 copy_work.src_offset = loffset;
1792 copy_work.src_window = local_window;
1793 copy_work.dst_offset = roffset;
1794 copy_work.dst_window = remote_window;
1795 } else {
1796 copy_work.src_offset = roffset;
1797 copy_work.src_window = remote_window;
1798 copy_work.dst_offset = loffset;
1799 copy_work.dst_window = local_window;
1800 }
1801
1802 if (flags & SCIF_RMA_USECPU) {
1803 scif_rma_list_cpu_copy(©_work);
1804 } else {
1805 chan = ep->rma_info.dma_chan;
1806 err = scif_rma_list_dma_copy_wrapper(epd, ©_work,
1807 chan, loffset);
1808 }
1809 if (addr && !cache)
1810 atomic_inc(&ep->rma_info.tw_refcount);
1811
1812 mutex_unlock(&ep->rma_info.rma_lock);
1813
1814 if (last_chunk) {
1815 struct scif_dev *rdev = ep->remote_dev;
1816
1817 if (copy_work.fence_type == SCIF_DMA_POLL)
1818 err = scif_drain_dma_poll(rdev->sdev,
1819 ep->rma_info.dma_chan);
1820 else if (copy_work.fence_type == SCIF_DMA_INTR)
1821 err = scif_drain_dma_intr(rdev->sdev,
1822 ep->rma_info.dma_chan);
1823 }
1824
1825 if (addr && !cache)
1826 scif_queue_for_cleanup(local_window, &scif_info.rma);
1827 scif_put_peer_dev(spdev);
1828 return err;
1829error:
1830 if (err) {
1831 if (addr && local_window && !cache)
1832 scif_destroy_window(ep, local_window);
1833 dev_err(scif_info.mdev.this_device,
1834 "%s %d err %d len 0x%lx\n",
1835 __func__, __LINE__, err, len);
1836 }
1837 scif_put_peer_dev(spdev);
1838 return err;
1839}
1840
1841int scif_readfrom(scif_epd_t epd, off_t loffset, size_t len,
1842 off_t roffset, int flags)
1843{
1844 int err;
1845
1846 dev_dbg(scif_info.mdev.this_device,
1847 "SCIFAPI readfrom: ep %p loffset 0x%lx len 0x%lx offset 0x%lx flags 0x%x\n",
1848 epd, loffset, len, roffset, flags);
1849 if (scif_unaligned(loffset, roffset)) {
1850 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1851 err = scif_rma_copy(epd, loffset, 0x0,
1852 SCIF_MAX_UNALIGNED_BUF_SIZE,
1853 roffset, flags,
1854 SCIF_REMOTE_TO_LOCAL, false);
1855 if (err)
1856 goto readfrom_err;
1857 loffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1858 roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1859 len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1860 }
1861 }
1862 err = scif_rma_copy(epd, loffset, 0x0, len,
1863 roffset, flags, SCIF_REMOTE_TO_LOCAL, true);
1864readfrom_err:
1865 return err;
1866}
1867EXPORT_SYMBOL_GPL(scif_readfrom);
1868
1869int scif_writeto(scif_epd_t epd, off_t loffset, size_t len,
1870 off_t roffset, int flags)
1871{
1872 int err;
1873
1874 dev_dbg(scif_info.mdev.this_device,
1875 "SCIFAPI writeto: ep %p loffset 0x%lx len 0x%lx roffset 0x%lx flags 0x%x\n",
1876 epd, loffset, len, roffset, flags);
1877 if (scif_unaligned(loffset, roffset)) {
1878 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1879 err = scif_rma_copy(epd, loffset, 0x0,
1880 SCIF_MAX_UNALIGNED_BUF_SIZE,
1881 roffset, flags,
1882 SCIF_LOCAL_TO_REMOTE, false);
1883 if (err)
1884 goto writeto_err;
1885 loffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1886 roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1887 len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1888 }
1889 }
1890 err = scif_rma_copy(epd, loffset, 0x0, len,
1891 roffset, flags, SCIF_LOCAL_TO_REMOTE, true);
1892writeto_err:
1893 return err;
1894}
1895EXPORT_SYMBOL_GPL(scif_writeto);
1896
1897int scif_vreadfrom(scif_epd_t epd, void *addr, size_t len,
1898 off_t roffset, int flags)
1899{
1900 int err;
1901
1902 dev_dbg(scif_info.mdev.this_device,
1903 "SCIFAPI vreadfrom: ep %p addr %p len 0x%lx roffset 0x%lx flags 0x%x\n",
1904 epd, addr, len, roffset, flags);
1905 if (scif_unaligned((off_t __force)addr, roffset)) {
1906 if (len > SCIF_MAX_UNALIGNED_BUF_SIZE)
1907 flags &= ~SCIF_RMA_USECACHE;
1908
1909 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1910 err = scif_rma_copy(epd, 0, (u64)addr,
1911 SCIF_MAX_UNALIGNED_BUF_SIZE,
1912 roffset, flags,
1913 SCIF_REMOTE_TO_LOCAL, false);
1914 if (err)
1915 goto vreadfrom_err;
1916 addr += SCIF_MAX_UNALIGNED_BUF_SIZE;
1917 roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1918 len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1919 }
1920 }
1921 err = scif_rma_copy(epd, 0, (u64)addr, len,
1922 roffset, flags, SCIF_REMOTE_TO_LOCAL, true);
1923vreadfrom_err:
1924 return err;
1925}
1926EXPORT_SYMBOL_GPL(scif_vreadfrom);
1927
1928int scif_vwriteto(scif_epd_t epd, void *addr, size_t len,
1929 off_t roffset, int flags)
1930{
1931 int err;
1932
1933 dev_dbg(scif_info.mdev.this_device,
1934 "SCIFAPI vwriteto: ep %p addr %p len 0x%lx roffset 0x%lx flags 0x%x\n",
1935 epd, addr, len, roffset, flags);
1936 if (scif_unaligned((off_t __force)addr, roffset)) {
1937 if (len > SCIF_MAX_UNALIGNED_BUF_SIZE)
1938 flags &= ~SCIF_RMA_USECACHE;
1939
1940 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1941 err = scif_rma_copy(epd, 0, (u64)addr,
1942 SCIF_MAX_UNALIGNED_BUF_SIZE,
1943 roffset, flags,
1944 SCIF_LOCAL_TO_REMOTE, false);
1945 if (err)
1946 goto vwriteto_err;
1947 addr += SCIF_MAX_UNALIGNED_BUF_SIZE;
1948 roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1949 len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1950 }
1951 }
1952 err = scif_rma_copy(epd, 0, (u64)addr, len,
1953 roffset, flags, SCIF_LOCAL_TO_REMOTE, true);
1954vwriteto_err:
1955 return err;
1956}
1957EXPORT_SYMBOL_GPL(scif_vwriteto);