Loading...
Note: File does not exist in v3.1.
1/*
2 * Intel MIC Platform Software Stack (MPSS)
3 *
4 * Copyright(c) 2015 Intel Corporation.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
14 *
15 * Intel SCIF driver.
16 *
17 */
18#include "scif_main.h"
19#include "scif_map.h"
20
21/*
22 * struct scif_dma_comp_cb - SCIF DMA completion callback
23 *
24 * @dma_completion_func: DMA completion callback
25 * @cb_cookie: DMA completion callback cookie
26 * @temp_buf: Temporary buffer
27 * @temp_buf_to_free: Temporary buffer to be freed
28 * @is_cache: Is a kmem_cache allocated buffer
29 * @dst_offset: Destination registration offset
30 * @dst_window: Destination registration window
31 * @len: Length of the temp buffer
32 * @temp_phys: DMA address of the temp buffer
33 * @sdev: The SCIF device
34 * @header_padding: padding for cache line alignment
35 */
36struct scif_dma_comp_cb {
37 void (*dma_completion_func)(void *cookie);
38 void *cb_cookie;
39 u8 *temp_buf;
40 u8 *temp_buf_to_free;
41 bool is_cache;
42 s64 dst_offset;
43 struct scif_window *dst_window;
44 size_t len;
45 dma_addr_t temp_phys;
46 struct scif_dev *sdev;
47 int header_padding;
48};
49
50/**
51 * struct scif_copy_work - Work for DMA copy
52 *
53 * @src_offset: Starting source offset
54 * @dst_offset: Starting destination offset
55 * @src_window: Starting src registered window
56 * @dst_window: Starting dst registered window
57 * @loopback: true if this is a loopback DMA transfer
58 * @len: Length of the transfer
59 * @comp_cb: DMA copy completion callback
60 * @remote_dev: The remote SCIF peer device
61 * @fence_type: polling or interrupt based
62 * @ordered: is this a tail byte ordered DMA transfer
63 */
64struct scif_copy_work {
65 s64 src_offset;
66 s64 dst_offset;
67 struct scif_window *src_window;
68 struct scif_window *dst_window;
69 int loopback;
70 size_t len;
71 struct scif_dma_comp_cb *comp_cb;
72 struct scif_dev *remote_dev;
73 int fence_type;
74 bool ordered;
75};
76
77/**
78 * scif_reserve_dma_chan:
79 * @ep: Endpoint Descriptor.
80 *
81 * This routine reserves a DMA channel for a particular
82 * endpoint. All DMA transfers for an endpoint are always
83 * programmed on the same DMA channel.
84 */
85int scif_reserve_dma_chan(struct scif_endpt *ep)
86{
87 int err = 0;
88 struct scif_dev *scifdev;
89 struct scif_hw_dev *sdev;
90 struct dma_chan *chan;
91
92 /* Loopback DMAs are not supported on the management node */
93 if (!scif_info.nodeid && scifdev_self(ep->remote_dev))
94 return 0;
95 if (scif_info.nodeid)
96 scifdev = &scif_dev[0];
97 else
98 scifdev = ep->remote_dev;
99 sdev = scifdev->sdev;
100 if (!sdev->num_dma_ch)
101 return -ENODEV;
102 chan = sdev->dma_ch[scifdev->dma_ch_idx];
103 scifdev->dma_ch_idx = (scifdev->dma_ch_idx + 1) % sdev->num_dma_ch;
104 mutex_lock(&ep->rma_info.rma_lock);
105 ep->rma_info.dma_chan = chan;
106 mutex_unlock(&ep->rma_info.rma_lock);
107 return err;
108}
109
110#ifdef CONFIG_MMU_NOTIFIER
111/**
112 * scif_rma_destroy_tcw:
113 *
114 * This routine destroys temporary cached windows
115 */
116static
117void __scif_rma_destroy_tcw(struct scif_mmu_notif *mmn,
118 u64 start, u64 len)
119{
120 struct list_head *item, *tmp;
121 struct scif_window *window;
122 u64 start_va, end_va;
123 u64 end = start + len;
124
125 if (end <= start)
126 return;
127
128 list_for_each_safe(item, tmp, &mmn->tc_reg_list) {
129 window = list_entry(item, struct scif_window, list);
130 if (!len)
131 break;
132 start_va = window->va_for_temp;
133 end_va = start_va + (window->nr_pages << PAGE_SHIFT);
134 if (start < start_va && end <= start_va)
135 break;
136 if (start >= end_va)
137 continue;
138 __scif_rma_destroy_tcw_helper(window);
139 }
140}
141
142static void scif_rma_destroy_tcw(struct scif_mmu_notif *mmn, u64 start, u64 len)
143{
144 struct scif_endpt *ep = mmn->ep;
145
146 spin_lock(&ep->rma_info.tc_lock);
147 __scif_rma_destroy_tcw(mmn, start, len);
148 spin_unlock(&ep->rma_info.tc_lock);
149}
150
151static void scif_rma_destroy_tcw_ep(struct scif_endpt *ep)
152{
153 struct list_head *item, *tmp;
154 struct scif_mmu_notif *mmn;
155
156 list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {
157 mmn = list_entry(item, struct scif_mmu_notif, list);
158 scif_rma_destroy_tcw(mmn, 0, ULONG_MAX);
159 }
160}
161
162static void __scif_rma_destroy_tcw_ep(struct scif_endpt *ep)
163{
164 struct list_head *item, *tmp;
165 struct scif_mmu_notif *mmn;
166
167 spin_lock(&ep->rma_info.tc_lock);
168 list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {
169 mmn = list_entry(item, struct scif_mmu_notif, list);
170 __scif_rma_destroy_tcw(mmn, 0, ULONG_MAX);
171 }
172 spin_unlock(&ep->rma_info.tc_lock);
173}
174
175static bool scif_rma_tc_can_cache(struct scif_endpt *ep, size_t cur_bytes)
176{
177 if ((cur_bytes >> PAGE_SHIFT) > scif_info.rma_tc_limit)
178 return false;
179 if ((atomic_read(&ep->rma_info.tcw_total_pages)
180 + (cur_bytes >> PAGE_SHIFT)) >
181 scif_info.rma_tc_limit) {
182 dev_info(scif_info.mdev.this_device,
183 "%s %d total=%d, current=%zu reached max\n",
184 __func__, __LINE__,
185 atomic_read(&ep->rma_info.tcw_total_pages),
186 (1 + (cur_bytes >> PAGE_SHIFT)));
187 scif_rma_destroy_tcw_invalid();
188 __scif_rma_destroy_tcw_ep(ep);
189 }
190 return true;
191}
192
193static void scif_mmu_notifier_release(struct mmu_notifier *mn,
194 struct mm_struct *mm)
195{
196 struct scif_mmu_notif *mmn;
197
198 mmn = container_of(mn, struct scif_mmu_notif, ep_mmu_notifier);
199 scif_rma_destroy_tcw(mmn, 0, ULONG_MAX);
200 schedule_work(&scif_info.misc_work);
201}
202
203static void scif_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
204 struct mm_struct *mm,
205 unsigned long address)
206{
207 struct scif_mmu_notif *mmn;
208
209 mmn = container_of(mn, struct scif_mmu_notif, ep_mmu_notifier);
210 scif_rma_destroy_tcw(mmn, address, PAGE_SIZE);
211}
212
213static void scif_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
214 struct mm_struct *mm,
215 unsigned long start,
216 unsigned long end)
217{
218 struct scif_mmu_notif *mmn;
219
220 mmn = container_of(mn, struct scif_mmu_notif, ep_mmu_notifier);
221 scif_rma_destroy_tcw(mmn, start, end - start);
222}
223
224static void scif_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
225 struct mm_struct *mm,
226 unsigned long start,
227 unsigned long end)
228{
229 /*
230 * Nothing to do here, everything needed was done in
231 * invalidate_range_start.
232 */
233}
234
235static const struct mmu_notifier_ops scif_mmu_notifier_ops = {
236 .release = scif_mmu_notifier_release,
237 .clear_flush_young = NULL,
238 .invalidate_page = scif_mmu_notifier_invalidate_page,
239 .invalidate_range_start = scif_mmu_notifier_invalidate_range_start,
240 .invalidate_range_end = scif_mmu_notifier_invalidate_range_end};
241
242static void scif_ep_unregister_mmu_notifier(struct scif_endpt *ep)
243{
244 struct scif_endpt_rma_info *rma = &ep->rma_info;
245 struct scif_mmu_notif *mmn = NULL;
246 struct list_head *item, *tmp;
247
248 mutex_lock(&ep->rma_info.mmn_lock);
249 list_for_each_safe(item, tmp, &rma->mmn_list) {
250 mmn = list_entry(item, struct scif_mmu_notif, list);
251 mmu_notifier_unregister(&mmn->ep_mmu_notifier, mmn->mm);
252 list_del(item);
253 kfree(mmn);
254 }
255 mutex_unlock(&ep->rma_info.mmn_lock);
256}
257
258static void scif_init_mmu_notifier(struct scif_mmu_notif *mmn,
259 struct mm_struct *mm, struct scif_endpt *ep)
260{
261 mmn->ep = ep;
262 mmn->mm = mm;
263 mmn->ep_mmu_notifier.ops = &scif_mmu_notifier_ops;
264 INIT_LIST_HEAD(&mmn->list);
265 INIT_LIST_HEAD(&mmn->tc_reg_list);
266}
267
268static struct scif_mmu_notif *
269scif_find_mmu_notifier(struct mm_struct *mm, struct scif_endpt_rma_info *rma)
270{
271 struct scif_mmu_notif *mmn;
272
273 list_for_each_entry(mmn, &rma->mmn_list, list)
274 if (mmn->mm == mm)
275 return mmn;
276 return NULL;
277}
278
279static struct scif_mmu_notif *
280scif_add_mmu_notifier(struct mm_struct *mm, struct scif_endpt *ep)
281{
282 struct scif_mmu_notif *mmn
283 = kzalloc(sizeof(*mmn), GFP_KERNEL);
284
285 if (!mmn)
286 return ERR_PTR(-ENOMEM);
287
288 scif_init_mmu_notifier(mmn, current->mm, ep);
289 if (mmu_notifier_register(&mmn->ep_mmu_notifier, current->mm)) {
290 kfree(mmn);
291 return ERR_PTR(-EBUSY);
292 }
293 list_add(&mmn->list, &ep->rma_info.mmn_list);
294 return mmn;
295}
296
297/*
298 * Called from the misc thread to destroy temporary cached windows and
299 * unregister the MMU notifier for the SCIF endpoint.
300 */
301void scif_mmu_notif_handler(struct work_struct *work)
302{
303 struct list_head *pos, *tmpq;
304 struct scif_endpt *ep;
305restart:
306 scif_rma_destroy_tcw_invalid();
307 spin_lock(&scif_info.rmalock);
308 list_for_each_safe(pos, tmpq, &scif_info.mmu_notif_cleanup) {
309 ep = list_entry(pos, struct scif_endpt, mmu_list);
310 list_del(&ep->mmu_list);
311 spin_unlock(&scif_info.rmalock);
312 scif_rma_destroy_tcw_ep(ep);
313 scif_ep_unregister_mmu_notifier(ep);
314 goto restart;
315 }
316 spin_unlock(&scif_info.rmalock);
317}
318
319static bool scif_is_set_reg_cache(int flags)
320{
321 return !!(flags & SCIF_RMA_USECACHE);
322}
323#else
324static struct scif_mmu_notif *
325scif_find_mmu_notifier(struct mm_struct *mm,
326 struct scif_endpt_rma_info *rma)
327{
328 return NULL;
329}
330
331static struct scif_mmu_notif *
332scif_add_mmu_notifier(struct mm_struct *mm, struct scif_endpt *ep)
333{
334 return NULL;
335}
336
337void scif_mmu_notif_handler(struct work_struct *work)
338{
339}
340
341static bool scif_is_set_reg_cache(int flags)
342{
343 return false;
344}
345
346static bool scif_rma_tc_can_cache(struct scif_endpt *ep, size_t cur_bytes)
347{
348 return false;
349}
350#endif
351
352/**
353 * scif_register_temp:
354 * @epd: End Point Descriptor.
355 * @addr: virtual address to/from which to copy
356 * @len: length of range to copy
357 * @out_offset: computed offset returned by reference.
358 * @out_window: allocated registered window returned by reference.
359 *
360 * Create a temporary registered window. The peer will not know about this
361 * window. This API is used for scif_vreadfrom()/scif_vwriteto() API's.
362 */
363static int
364scif_register_temp(scif_epd_t epd, unsigned long addr, size_t len, int prot,
365 off_t *out_offset, struct scif_window **out_window)
366{
367 struct scif_endpt *ep = (struct scif_endpt *)epd;
368 int err;
369 scif_pinned_pages_t pinned_pages;
370 size_t aligned_len;
371
372 aligned_len = ALIGN(len, PAGE_SIZE);
373
374 err = __scif_pin_pages((void *)(addr & PAGE_MASK),
375 aligned_len, &prot, 0, &pinned_pages);
376 if (err)
377 return err;
378
379 pinned_pages->prot = prot;
380
381 /* Compute the offset for this registration */
382 err = scif_get_window_offset(ep, 0, 0,
383 aligned_len >> PAGE_SHIFT,
384 (s64 *)out_offset);
385 if (err)
386 goto error_unpin;
387
388 /* Allocate and prepare self registration window */
389 *out_window = scif_create_window(ep, aligned_len >> PAGE_SHIFT,
390 *out_offset, true);
391 if (!*out_window) {
392 scif_free_window_offset(ep, NULL, *out_offset);
393 err = -ENOMEM;
394 goto error_unpin;
395 }
396
397 (*out_window)->pinned_pages = pinned_pages;
398 (*out_window)->nr_pages = pinned_pages->nr_pages;
399 (*out_window)->prot = pinned_pages->prot;
400
401 (*out_window)->va_for_temp = addr & PAGE_MASK;
402 err = scif_map_window(ep->remote_dev, *out_window);
403 if (err) {
404 /* Something went wrong! Rollback */
405 scif_destroy_window(ep, *out_window);
406 *out_window = NULL;
407 } else {
408 *out_offset |= (addr - (*out_window)->va_for_temp);
409 }
410 return err;
411error_unpin:
412 if (err)
413 dev_err(&ep->remote_dev->sdev->dev,
414 "%s %d err %d\n", __func__, __LINE__, err);
415 scif_unpin_pages(pinned_pages);
416 return err;
417}
418
419#define SCIF_DMA_TO (3 * HZ)
420
421/*
422 * scif_sync_dma - Program a DMA without an interrupt descriptor
423 *
424 * @dev - The address of the pointer to the device instance used
425 * for DMA registration.
426 * @chan - DMA channel to be used.
427 * @sync_wait: Wait for DMA to complete?
428 *
429 * Return 0 on success and -errno on error.
430 */
431static int scif_sync_dma(struct scif_hw_dev *sdev, struct dma_chan *chan,
432 bool sync_wait)
433{
434 int err = 0;
435 struct dma_async_tx_descriptor *tx = NULL;
436 enum dma_ctrl_flags flags = DMA_PREP_FENCE;
437 dma_cookie_t cookie;
438 struct dma_device *ddev;
439
440 if (!chan) {
441 err = -EIO;
442 dev_err(&sdev->dev, "%s %d err %d\n",
443 __func__, __LINE__, err);
444 return err;
445 }
446 ddev = chan->device;
447
448 tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, flags);
449 if (!tx) {
450 err = -ENOMEM;
451 dev_err(&sdev->dev, "%s %d err %d\n",
452 __func__, __LINE__, err);
453 goto release;
454 }
455 cookie = tx->tx_submit(tx);
456
457 if (dma_submit_error(cookie)) {
458 err = -ENOMEM;
459 dev_err(&sdev->dev, "%s %d err %d\n",
460 __func__, __LINE__, err);
461 goto release;
462 }
463 if (!sync_wait) {
464 dma_async_issue_pending(chan);
465 } else {
466 if (dma_sync_wait(chan, cookie) == DMA_COMPLETE) {
467 err = 0;
468 } else {
469 err = -EIO;
470 dev_err(&sdev->dev, "%s %d err %d\n",
471 __func__, __LINE__, err);
472 }
473 }
474release:
475 return err;
476}
477
478static void scif_dma_callback(void *arg)
479{
480 struct completion *done = (struct completion *)arg;
481
482 complete(done);
483}
484
485#define SCIF_DMA_SYNC_WAIT true
486#define SCIF_DMA_POLL BIT(0)
487#define SCIF_DMA_INTR BIT(1)
488
489/*
490 * scif_async_dma - Program a DMA with an interrupt descriptor
491 *
492 * @dev - The address of the pointer to the device instance used
493 * for DMA registration.
494 * @chan - DMA channel to be used.
495 * Return 0 on success and -errno on error.
496 */
497static int scif_async_dma(struct scif_hw_dev *sdev, struct dma_chan *chan)
498{
499 int err = 0;
500 struct dma_device *ddev;
501 struct dma_async_tx_descriptor *tx = NULL;
502 enum dma_ctrl_flags flags = DMA_PREP_INTERRUPT | DMA_PREP_FENCE;
503 DECLARE_COMPLETION_ONSTACK(done_wait);
504 dma_cookie_t cookie;
505 enum dma_status status;
506
507 if (!chan) {
508 err = -EIO;
509 dev_err(&sdev->dev, "%s %d err %d\n",
510 __func__, __LINE__, err);
511 return err;
512 }
513 ddev = chan->device;
514
515 tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, flags);
516 if (!tx) {
517 err = -ENOMEM;
518 dev_err(&sdev->dev, "%s %d err %d\n",
519 __func__, __LINE__, err);
520 goto release;
521 }
522 reinit_completion(&done_wait);
523 tx->callback = scif_dma_callback;
524 tx->callback_param = &done_wait;
525 cookie = tx->tx_submit(tx);
526
527 if (dma_submit_error(cookie)) {
528 err = -ENOMEM;
529 dev_err(&sdev->dev, "%s %d err %d\n",
530 __func__, __LINE__, err);
531 goto release;
532 }
533 dma_async_issue_pending(chan);
534
535 err = wait_for_completion_timeout(&done_wait, SCIF_DMA_TO);
536 if (!err) {
537 err = -EIO;
538 dev_err(&sdev->dev, "%s %d err %d\n",
539 __func__, __LINE__, err);
540 goto release;
541 }
542 err = 0;
543 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
544 if (status != DMA_COMPLETE) {
545 err = -EIO;
546 dev_err(&sdev->dev, "%s %d err %d\n",
547 __func__, __LINE__, err);
548 goto release;
549 }
550release:
551 return err;
552}
553
554/*
555 * scif_drain_dma_poll - Drain all outstanding DMA operations for a particular
556 * DMA channel via polling.
557 *
558 * @sdev - The SCIF device
559 * @chan - DMA channel
560 * Return 0 on success and -errno on error.
561 */
562static int scif_drain_dma_poll(struct scif_hw_dev *sdev, struct dma_chan *chan)
563{
564 if (!chan)
565 return -EINVAL;
566 return scif_sync_dma(sdev, chan, SCIF_DMA_SYNC_WAIT);
567}
568
569/*
570 * scif_drain_dma_intr - Drain all outstanding DMA operations for a particular
571 * DMA channel via interrupt based blocking wait.
572 *
573 * @sdev - The SCIF device
574 * @chan - DMA channel
575 * Return 0 on success and -errno on error.
576 */
577int scif_drain_dma_intr(struct scif_hw_dev *sdev, struct dma_chan *chan)
578{
579 if (!chan)
580 return -EINVAL;
581 return scif_async_dma(sdev, chan);
582}
583
584/**
585 * scif_rma_destroy_windows:
586 *
587 * This routine destroys all windows queued for cleanup
588 */
589void scif_rma_destroy_windows(void)
590{
591 struct list_head *item, *tmp;
592 struct scif_window *window;
593 struct scif_endpt *ep;
594 struct dma_chan *chan;
595
596 might_sleep();
597restart:
598 spin_lock(&scif_info.rmalock);
599 list_for_each_safe(item, tmp, &scif_info.rma) {
600 window = list_entry(item, struct scif_window,
601 list);
602 ep = (struct scif_endpt *)window->ep;
603 chan = ep->rma_info.dma_chan;
604
605 list_del_init(&window->list);
606 spin_unlock(&scif_info.rmalock);
607 if (!chan || !scifdev_alive(ep) ||
608 !scif_drain_dma_intr(ep->remote_dev->sdev,
609 ep->rma_info.dma_chan))
610 /* Remove window from global list */
611 window->unreg_state = OP_COMPLETED;
612 else
613 dev_warn(&ep->remote_dev->sdev->dev,
614 "DMA engine hung?\n");
615 if (window->unreg_state == OP_COMPLETED) {
616 if (window->type == SCIF_WINDOW_SELF)
617 scif_destroy_window(ep, window);
618 else
619 scif_destroy_remote_window(window);
620 atomic_dec(&ep->rma_info.tw_refcount);
621 }
622 goto restart;
623 }
624 spin_unlock(&scif_info.rmalock);
625}
626
627/**
628 * scif_rma_destroy_tcw:
629 *
630 * This routine destroys temporary cached registered windows
631 * which have been queued for cleanup.
632 */
633void scif_rma_destroy_tcw_invalid(void)
634{
635 struct list_head *item, *tmp;
636 struct scif_window *window;
637 struct scif_endpt *ep;
638 struct dma_chan *chan;
639
640 might_sleep();
641restart:
642 spin_lock(&scif_info.rmalock);
643 list_for_each_safe(item, tmp, &scif_info.rma_tc) {
644 window = list_entry(item, struct scif_window, list);
645 ep = (struct scif_endpt *)window->ep;
646 chan = ep->rma_info.dma_chan;
647 list_del_init(&window->list);
648 spin_unlock(&scif_info.rmalock);
649 mutex_lock(&ep->rma_info.rma_lock);
650 if (!chan || !scifdev_alive(ep) ||
651 !scif_drain_dma_intr(ep->remote_dev->sdev,
652 ep->rma_info.dma_chan)) {
653 atomic_sub(window->nr_pages,
654 &ep->rma_info.tcw_total_pages);
655 scif_destroy_window(ep, window);
656 atomic_dec(&ep->rma_info.tcw_refcount);
657 } else {
658 dev_warn(&ep->remote_dev->sdev->dev,
659 "DMA engine hung?\n");
660 }
661 mutex_unlock(&ep->rma_info.rma_lock);
662 goto restart;
663 }
664 spin_unlock(&scif_info.rmalock);
665}
666
667static inline
668void *_get_local_va(off_t off, struct scif_window *window, size_t len)
669{
670 int page_nr = (off - window->offset) >> PAGE_SHIFT;
671 off_t page_off = off & ~PAGE_MASK;
672 void *va = NULL;
673
674 if (window->type == SCIF_WINDOW_SELF) {
675 struct page **pages = window->pinned_pages->pages;
676
677 va = page_address(pages[page_nr]) + page_off;
678 }
679 return va;
680}
681
682static inline
683void *ioremap_remote(off_t off, struct scif_window *window,
684 size_t len, struct scif_dev *dev,
685 struct scif_window_iter *iter)
686{
687 dma_addr_t phys = scif_off_to_dma_addr(window, off, NULL, iter);
688
689 /*
690 * If the DMA address is not card relative then we need the DMA
691 * addresses to be an offset into the bar. The aperture base was already
692 * added so subtract it here since scif_ioremap is going to add it again
693 */
694 if (!scifdev_self(dev) && window->type == SCIF_WINDOW_PEER &&
695 dev->sdev->aper && !dev->sdev->card_rel_da)
696 phys = phys - dev->sdev->aper->pa;
697 return scif_ioremap(phys, len, dev);
698}
699
700static inline void
701iounmap_remote(void *virt, size_t size, struct scif_copy_work *work)
702{
703 scif_iounmap(virt, size, work->remote_dev);
704}
705
706/*
707 * Takes care of ordering issue caused by
708 * 1. Hardware: Only in the case of cpu copy from mgmt node to card
709 * because of WC memory.
710 * 2. Software: If memcpy reorders copy instructions for optimization.
711 * This could happen at both mgmt node and card.
712 */
713static inline void
714scif_ordered_memcpy_toio(char *dst, const char *src, size_t count)
715{
716 if (!count)
717 return;
718
719 memcpy_toio((void __iomem __force *)dst, src, --count);
720 /* Order the last byte with the previous stores */
721 wmb();
722 *(dst + count) = *(src + count);
723}
724
725static inline void scif_unaligned_cpy_toio(char *dst, const char *src,
726 size_t count, bool ordered)
727{
728 if (ordered)
729 scif_ordered_memcpy_toio(dst, src, count);
730 else
731 memcpy_toio((void __iomem __force *)dst, src, count);
732}
733
734static inline
735void scif_ordered_memcpy_fromio(char *dst, const char *src, size_t count)
736{
737 if (!count)
738 return;
739
740 memcpy_fromio(dst, (void __iomem __force *)src, --count);
741 /* Order the last byte with the previous loads */
742 rmb();
743 *(dst + count) = *(src + count);
744}
745
746static inline void scif_unaligned_cpy_fromio(char *dst, const char *src,
747 size_t count, bool ordered)
748{
749 if (ordered)
750 scif_ordered_memcpy_fromio(dst, src, count);
751 else
752 memcpy_fromio(dst, (void __iomem __force *)src, count);
753}
754
755#define SCIF_RMA_ERROR_CODE (~(dma_addr_t)0x0)
756
757/*
758 * scif_off_to_dma_addr:
759 * Obtain the dma_addr given the window and the offset.
760 * @window: Registered window.
761 * @off: Window offset.
762 * @nr_bytes: Return the number of contiguous bytes till next DMA addr index.
763 * @index: Return the index of the dma_addr array found.
764 * @start_off: start offset of index of the dma addr array found.
765 * The nr_bytes provides the callee an estimate of the maximum possible
766 * DMA xfer possible while the index/start_off provide faster lookups
767 * for the next iteration.
768 */
769dma_addr_t scif_off_to_dma_addr(struct scif_window *window, s64 off,
770 size_t *nr_bytes, struct scif_window_iter *iter)
771{
772 int i, page_nr;
773 s64 start, end;
774 off_t page_off;
775
776 if (window->nr_pages == window->nr_contig_chunks) {
777 page_nr = (off - window->offset) >> PAGE_SHIFT;
778 page_off = off & ~PAGE_MASK;
779
780 if (nr_bytes)
781 *nr_bytes = PAGE_SIZE - page_off;
782 return window->dma_addr[page_nr] | page_off;
783 }
784 if (iter) {
785 i = iter->index;
786 start = iter->offset;
787 } else {
788 i = 0;
789 start = window->offset;
790 }
791 for (; i < window->nr_contig_chunks; i++) {
792 end = start + (window->num_pages[i] << PAGE_SHIFT);
793 if (off >= start && off < end) {
794 if (iter) {
795 iter->index = i;
796 iter->offset = start;
797 }
798 if (nr_bytes)
799 *nr_bytes = end - off;
800 return (window->dma_addr[i] + (off - start));
801 }
802 start += (window->num_pages[i] << PAGE_SHIFT);
803 }
804 dev_err(scif_info.mdev.this_device,
805 "%s %d BUG. Addr not found? window %p off 0x%llx\n",
806 __func__, __LINE__, window, off);
807 return SCIF_RMA_ERROR_CODE;
808}
809
810/*
811 * Copy between rma window and temporary buffer
812 */
813static void scif_rma_local_cpu_copy(s64 offset, struct scif_window *window,
814 u8 *temp, size_t rem_len, bool to_temp)
815{
816 void *window_virt;
817 size_t loop_len;
818 int offset_in_page;
819 s64 end_offset;
820
821 offset_in_page = offset & ~PAGE_MASK;
822 loop_len = PAGE_SIZE - offset_in_page;
823
824 if (rem_len < loop_len)
825 loop_len = rem_len;
826
827 window_virt = _get_local_va(offset, window, loop_len);
828 if (!window_virt)
829 return;
830 if (to_temp)
831 memcpy(temp, window_virt, loop_len);
832 else
833 memcpy(window_virt, temp, loop_len);
834
835 offset += loop_len;
836 temp += loop_len;
837 rem_len -= loop_len;
838
839 end_offset = window->offset +
840 (window->nr_pages << PAGE_SHIFT);
841 while (rem_len) {
842 if (offset == end_offset) {
843 window = list_next_entry(window, list);
844 end_offset = window->offset +
845 (window->nr_pages << PAGE_SHIFT);
846 }
847 loop_len = min(PAGE_SIZE, rem_len);
848 window_virt = _get_local_va(offset, window, loop_len);
849 if (!window_virt)
850 return;
851 if (to_temp)
852 memcpy(temp, window_virt, loop_len);
853 else
854 memcpy(window_virt, temp, loop_len);
855 offset += loop_len;
856 temp += loop_len;
857 rem_len -= loop_len;
858 }
859}
860
861/**
862 * scif_rma_completion_cb:
863 * @data: RMA cookie
864 *
865 * RMA interrupt completion callback.
866 */
867static void scif_rma_completion_cb(void *data)
868{
869 struct scif_dma_comp_cb *comp_cb = data;
870
871 /* Free DMA Completion CB. */
872 if (comp_cb->dst_window)
873 scif_rma_local_cpu_copy(comp_cb->dst_offset,
874 comp_cb->dst_window,
875 comp_cb->temp_buf +
876 comp_cb->header_padding,
877 comp_cb->len, false);
878 scif_unmap_single(comp_cb->temp_phys, comp_cb->sdev,
879 SCIF_KMEM_UNALIGNED_BUF_SIZE);
880 if (comp_cb->is_cache)
881 kmem_cache_free(unaligned_cache,
882 comp_cb->temp_buf_to_free);
883 else
884 kfree(comp_cb->temp_buf_to_free);
885}
886
887/* Copies between temporary buffer and offsets provided in work */
888static int
889scif_rma_list_dma_copy_unaligned(struct scif_copy_work *work,
890 u8 *temp, struct dma_chan *chan,
891 bool src_local)
892{
893 struct scif_dma_comp_cb *comp_cb = work->comp_cb;
894 dma_addr_t window_dma_addr, temp_dma_addr;
895 dma_addr_t temp_phys = comp_cb->temp_phys;
896 size_t loop_len, nr_contig_bytes = 0, remaining_len = work->len;
897 int offset_in_ca, ret = 0;
898 s64 end_offset, offset;
899 struct scif_window *window;
900 void *window_virt_addr;
901 size_t tail_len;
902 struct dma_async_tx_descriptor *tx;
903 struct dma_device *dev = chan->device;
904 dma_cookie_t cookie;
905
906 if (src_local) {
907 offset = work->dst_offset;
908 window = work->dst_window;
909 } else {
910 offset = work->src_offset;
911 window = work->src_window;
912 }
913
914 offset_in_ca = offset & (L1_CACHE_BYTES - 1);
915 if (offset_in_ca) {
916 loop_len = L1_CACHE_BYTES - offset_in_ca;
917 loop_len = min(loop_len, remaining_len);
918 window_virt_addr = ioremap_remote(offset, window,
919 loop_len,
920 work->remote_dev,
921 NULL);
922 if (!window_virt_addr)
923 return -ENOMEM;
924 if (src_local)
925 scif_unaligned_cpy_toio(window_virt_addr, temp,
926 loop_len,
927 work->ordered &&
928 !(remaining_len - loop_len));
929 else
930 scif_unaligned_cpy_fromio(temp, window_virt_addr,
931 loop_len, work->ordered &&
932 !(remaining_len - loop_len));
933 iounmap_remote(window_virt_addr, loop_len, work);
934
935 offset += loop_len;
936 temp += loop_len;
937 temp_phys += loop_len;
938 remaining_len -= loop_len;
939 }
940
941 offset_in_ca = offset & ~PAGE_MASK;
942 end_offset = window->offset +
943 (window->nr_pages << PAGE_SHIFT);
944
945 tail_len = remaining_len & (L1_CACHE_BYTES - 1);
946 remaining_len -= tail_len;
947 while (remaining_len) {
948 if (offset == end_offset) {
949 window = list_next_entry(window, list);
950 end_offset = window->offset +
951 (window->nr_pages << PAGE_SHIFT);
952 }
953 if (scif_is_mgmt_node())
954 temp_dma_addr = temp_phys;
955 else
956 /* Fix if we ever enable IOMMU on the card */
957 temp_dma_addr = (dma_addr_t)virt_to_phys(temp);
958 window_dma_addr = scif_off_to_dma_addr(window, offset,
959 &nr_contig_bytes,
960 NULL);
961 loop_len = min(nr_contig_bytes, remaining_len);
962 if (src_local) {
963 if (work->ordered && !tail_len &&
964 !(remaining_len - loop_len) &&
965 loop_len != L1_CACHE_BYTES) {
966 /*
967 * Break up the last chunk of the transfer into
968 * two steps. if there is no tail to guarantee
969 * DMA ordering. SCIF_DMA_POLLING inserts
970 * a status update descriptor in step 1 which
971 * acts as a double sided synchronization fence
972 * for the DMA engine to ensure that the last
973 * cache line in step 2 is updated last.
974 */
975 /* Step 1) DMA: Body Length - L1_CACHE_BYTES. */
976 tx =
977 dev->device_prep_dma_memcpy(chan,
978 window_dma_addr,
979 temp_dma_addr,
980 loop_len -
981 L1_CACHE_BYTES,
982 DMA_PREP_FENCE);
983 if (!tx) {
984 ret = -ENOMEM;
985 goto err;
986 }
987 cookie = tx->tx_submit(tx);
988 if (dma_submit_error(cookie)) {
989 ret = -ENOMEM;
990 goto err;
991 }
992 dma_async_issue_pending(chan);
993 offset += (loop_len - L1_CACHE_BYTES);
994 temp_dma_addr += (loop_len - L1_CACHE_BYTES);
995 window_dma_addr += (loop_len - L1_CACHE_BYTES);
996 remaining_len -= (loop_len - L1_CACHE_BYTES);
997 loop_len = remaining_len;
998
999 /* Step 2) DMA: L1_CACHE_BYTES */
1000 tx =
1001 dev->device_prep_dma_memcpy(chan,
1002 window_dma_addr,
1003 temp_dma_addr,
1004 loop_len, 0);
1005 if (!tx) {
1006 ret = -ENOMEM;
1007 goto err;
1008 }
1009 cookie = tx->tx_submit(tx);
1010 if (dma_submit_error(cookie)) {
1011 ret = -ENOMEM;
1012 goto err;
1013 }
1014 dma_async_issue_pending(chan);
1015 } else {
1016 tx =
1017 dev->device_prep_dma_memcpy(chan,
1018 window_dma_addr,
1019 temp_dma_addr,
1020 loop_len, 0);
1021 if (!tx) {
1022 ret = -ENOMEM;
1023 goto err;
1024 }
1025 cookie = tx->tx_submit(tx);
1026 if (dma_submit_error(cookie)) {
1027 ret = -ENOMEM;
1028 goto err;
1029 }
1030 dma_async_issue_pending(chan);
1031 }
1032 } else {
1033 tx = dev->device_prep_dma_memcpy(chan, temp_dma_addr,
1034 window_dma_addr, loop_len, 0);
1035 if (!tx) {
1036 ret = -ENOMEM;
1037 goto err;
1038 }
1039 cookie = tx->tx_submit(tx);
1040 if (dma_submit_error(cookie)) {
1041 ret = -ENOMEM;
1042 goto err;
1043 }
1044 dma_async_issue_pending(chan);
1045 }
1046 if (ret < 0)
1047 goto err;
1048 offset += loop_len;
1049 temp += loop_len;
1050 temp_phys += loop_len;
1051 remaining_len -= loop_len;
1052 offset_in_ca = 0;
1053 }
1054 if (tail_len) {
1055 if (offset == end_offset) {
1056 window = list_next_entry(window, list);
1057 end_offset = window->offset +
1058 (window->nr_pages << PAGE_SHIFT);
1059 }
1060 window_virt_addr = ioremap_remote(offset, window, tail_len,
1061 work->remote_dev,
1062 NULL);
1063 if (!window_virt_addr)
1064 return -ENOMEM;
1065 /*
1066 * The CPU copy for the tail bytes must be initiated only once
1067 * previous DMA transfers for this endpoint have completed
1068 * to guarantee ordering.
1069 */
1070 if (work->ordered) {
1071 struct scif_dev *rdev = work->remote_dev;
1072
1073 ret = scif_drain_dma_intr(rdev->sdev, chan);
1074 if (ret)
1075 return ret;
1076 }
1077 if (src_local)
1078 scif_unaligned_cpy_toio(window_virt_addr, temp,
1079 tail_len, work->ordered);
1080 else
1081 scif_unaligned_cpy_fromio(temp, window_virt_addr,
1082 tail_len, work->ordered);
1083 iounmap_remote(window_virt_addr, tail_len, work);
1084 }
1085 tx = dev->device_prep_dma_memcpy(chan, 0, 0, 0, DMA_PREP_INTERRUPT);
1086 if (!tx) {
1087 ret = -ENOMEM;
1088 return ret;
1089 }
1090 tx->callback = &scif_rma_completion_cb;
1091 tx->callback_param = comp_cb;
1092 cookie = tx->tx_submit(tx);
1093
1094 if (dma_submit_error(cookie)) {
1095 ret = -ENOMEM;
1096 return ret;
1097 }
1098 dma_async_issue_pending(chan);
1099 return 0;
1100err:
1101 dev_err(scif_info.mdev.this_device,
1102 "%s %d Desc Prog Failed ret %d\n",
1103 __func__, __LINE__, ret);
1104 return ret;
1105}
1106
1107/*
1108 * _scif_rma_list_dma_copy_aligned:
1109 *
1110 * Traverse all the windows and perform DMA copy.
1111 */
1112static int _scif_rma_list_dma_copy_aligned(struct scif_copy_work *work,
1113 struct dma_chan *chan)
1114{
1115 dma_addr_t src_dma_addr, dst_dma_addr;
1116 size_t loop_len, remaining_len, src_contig_bytes = 0;
1117 size_t dst_contig_bytes = 0;
1118 struct scif_window_iter src_win_iter;
1119 struct scif_window_iter dst_win_iter;
1120 s64 end_src_offset, end_dst_offset;
1121 struct scif_window *src_window = work->src_window;
1122 struct scif_window *dst_window = work->dst_window;
1123 s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1124 int ret = 0;
1125 struct dma_async_tx_descriptor *tx;
1126 struct dma_device *dev = chan->device;
1127 dma_cookie_t cookie;
1128
1129 remaining_len = work->len;
1130
1131 scif_init_window_iter(src_window, &src_win_iter);
1132 scif_init_window_iter(dst_window, &dst_win_iter);
1133 end_src_offset = src_window->offset +
1134 (src_window->nr_pages << PAGE_SHIFT);
1135 end_dst_offset = dst_window->offset +
1136 (dst_window->nr_pages << PAGE_SHIFT);
1137 while (remaining_len) {
1138 if (src_offset == end_src_offset) {
1139 src_window = list_next_entry(src_window, list);
1140 end_src_offset = src_window->offset +
1141 (src_window->nr_pages << PAGE_SHIFT);
1142 scif_init_window_iter(src_window, &src_win_iter);
1143 }
1144 if (dst_offset == end_dst_offset) {
1145 dst_window = list_next_entry(dst_window, list);
1146 end_dst_offset = dst_window->offset +
1147 (dst_window->nr_pages << PAGE_SHIFT);
1148 scif_init_window_iter(dst_window, &dst_win_iter);
1149 }
1150
1151 /* compute dma addresses for transfer */
1152 src_dma_addr = scif_off_to_dma_addr(src_window, src_offset,
1153 &src_contig_bytes,
1154 &src_win_iter);
1155 dst_dma_addr = scif_off_to_dma_addr(dst_window, dst_offset,
1156 &dst_contig_bytes,
1157 &dst_win_iter);
1158 loop_len = min(src_contig_bytes, dst_contig_bytes);
1159 loop_len = min(loop_len, remaining_len);
1160 if (work->ordered && !(remaining_len - loop_len)) {
1161 /*
1162 * Break up the last chunk of the transfer into two
1163 * steps to ensure that the last byte in step 2 is
1164 * updated last.
1165 */
1166 /* Step 1) DMA: Body Length - 1 */
1167 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1168 src_dma_addr,
1169 loop_len - 1,
1170 DMA_PREP_FENCE);
1171 if (!tx) {
1172 ret = -ENOMEM;
1173 goto err;
1174 }
1175 cookie = tx->tx_submit(tx);
1176 if (dma_submit_error(cookie)) {
1177 ret = -ENOMEM;
1178 goto err;
1179 }
1180 src_offset += (loop_len - 1);
1181 dst_offset += (loop_len - 1);
1182 src_dma_addr += (loop_len - 1);
1183 dst_dma_addr += (loop_len - 1);
1184 remaining_len -= (loop_len - 1);
1185 loop_len = remaining_len;
1186
1187 /* Step 2) DMA: 1 BYTES */
1188 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1189 src_dma_addr, loop_len, 0);
1190 if (!tx) {
1191 ret = -ENOMEM;
1192 goto err;
1193 }
1194 cookie = tx->tx_submit(tx);
1195 if (dma_submit_error(cookie)) {
1196 ret = -ENOMEM;
1197 goto err;
1198 }
1199 dma_async_issue_pending(chan);
1200 } else {
1201 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1202 src_dma_addr, loop_len, 0);
1203 if (!tx) {
1204 ret = -ENOMEM;
1205 goto err;
1206 }
1207 cookie = tx->tx_submit(tx);
1208 if (dma_submit_error(cookie)) {
1209 ret = -ENOMEM;
1210 goto err;
1211 }
1212 }
1213 src_offset += loop_len;
1214 dst_offset += loop_len;
1215 remaining_len -= loop_len;
1216 }
1217 return ret;
1218err:
1219 dev_err(scif_info.mdev.this_device,
1220 "%s %d Desc Prog Failed ret %d\n",
1221 __func__, __LINE__, ret);
1222 return ret;
1223}
1224
1225/*
1226 * scif_rma_list_dma_copy_aligned:
1227 *
1228 * Traverse all the windows and perform DMA copy.
1229 */
1230static int scif_rma_list_dma_copy_aligned(struct scif_copy_work *work,
1231 struct dma_chan *chan)
1232{
1233 dma_addr_t src_dma_addr, dst_dma_addr;
1234 size_t loop_len, remaining_len, tail_len, src_contig_bytes = 0;
1235 size_t dst_contig_bytes = 0;
1236 int src_cache_off;
1237 s64 end_src_offset, end_dst_offset;
1238 struct scif_window_iter src_win_iter;
1239 struct scif_window_iter dst_win_iter;
1240 void *src_virt, *dst_virt;
1241 struct scif_window *src_window = work->src_window;
1242 struct scif_window *dst_window = work->dst_window;
1243 s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1244 int ret = 0;
1245 struct dma_async_tx_descriptor *tx;
1246 struct dma_device *dev = chan->device;
1247 dma_cookie_t cookie;
1248
1249 remaining_len = work->len;
1250 scif_init_window_iter(src_window, &src_win_iter);
1251 scif_init_window_iter(dst_window, &dst_win_iter);
1252
1253 src_cache_off = src_offset & (L1_CACHE_BYTES - 1);
1254 if (src_cache_off != 0) {
1255 /* Head */
1256 loop_len = L1_CACHE_BYTES - src_cache_off;
1257 loop_len = min(loop_len, remaining_len);
1258 src_dma_addr = __scif_off_to_dma_addr(src_window, src_offset);
1259 dst_dma_addr = __scif_off_to_dma_addr(dst_window, dst_offset);
1260 if (src_window->type == SCIF_WINDOW_SELF)
1261 src_virt = _get_local_va(src_offset, src_window,
1262 loop_len);
1263 else
1264 src_virt = ioremap_remote(src_offset, src_window,
1265 loop_len,
1266 work->remote_dev, NULL);
1267 if (!src_virt)
1268 return -ENOMEM;
1269 if (dst_window->type == SCIF_WINDOW_SELF)
1270 dst_virt = _get_local_va(dst_offset, dst_window,
1271 loop_len);
1272 else
1273 dst_virt = ioremap_remote(dst_offset, dst_window,
1274 loop_len,
1275 work->remote_dev, NULL);
1276 if (!dst_virt) {
1277 if (src_window->type != SCIF_WINDOW_SELF)
1278 iounmap_remote(src_virt, loop_len, work);
1279 return -ENOMEM;
1280 }
1281 if (src_window->type == SCIF_WINDOW_SELF)
1282 scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,
1283 remaining_len == loop_len ?
1284 work->ordered : false);
1285 else
1286 scif_unaligned_cpy_fromio(dst_virt, src_virt, loop_len,
1287 remaining_len == loop_len ?
1288 work->ordered : false);
1289 if (src_window->type != SCIF_WINDOW_SELF)
1290 iounmap_remote(src_virt, loop_len, work);
1291 if (dst_window->type != SCIF_WINDOW_SELF)
1292 iounmap_remote(dst_virt, loop_len, work);
1293 src_offset += loop_len;
1294 dst_offset += loop_len;
1295 remaining_len -= loop_len;
1296 }
1297
1298 end_src_offset = src_window->offset +
1299 (src_window->nr_pages << PAGE_SHIFT);
1300 end_dst_offset = dst_window->offset +
1301 (dst_window->nr_pages << PAGE_SHIFT);
1302 tail_len = remaining_len & (L1_CACHE_BYTES - 1);
1303 remaining_len -= tail_len;
1304 while (remaining_len) {
1305 if (src_offset == end_src_offset) {
1306 src_window = list_next_entry(src_window, list);
1307 end_src_offset = src_window->offset +
1308 (src_window->nr_pages << PAGE_SHIFT);
1309 scif_init_window_iter(src_window, &src_win_iter);
1310 }
1311 if (dst_offset == end_dst_offset) {
1312 dst_window = list_next_entry(dst_window, list);
1313 end_dst_offset = dst_window->offset +
1314 (dst_window->nr_pages << PAGE_SHIFT);
1315 scif_init_window_iter(dst_window, &dst_win_iter);
1316 }
1317
1318 /* compute dma addresses for transfer */
1319 src_dma_addr = scif_off_to_dma_addr(src_window, src_offset,
1320 &src_contig_bytes,
1321 &src_win_iter);
1322 dst_dma_addr = scif_off_to_dma_addr(dst_window, dst_offset,
1323 &dst_contig_bytes,
1324 &dst_win_iter);
1325 loop_len = min(src_contig_bytes, dst_contig_bytes);
1326 loop_len = min(loop_len, remaining_len);
1327 if (work->ordered && !tail_len &&
1328 !(remaining_len - loop_len)) {
1329 /*
1330 * Break up the last chunk of the transfer into two
1331 * steps. if there is no tail to gurantee DMA ordering.
1332 * Passing SCIF_DMA_POLLING inserts a status update
1333 * descriptor in step 1 which acts as a double sided
1334 * synchronization fence for the DMA engine to ensure
1335 * that the last cache line in step 2 is updated last.
1336 */
1337 /* Step 1) DMA: Body Length - L1_CACHE_BYTES. */
1338 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1339 src_dma_addr,
1340 loop_len -
1341 L1_CACHE_BYTES,
1342 DMA_PREP_FENCE);
1343 if (!tx) {
1344 ret = -ENOMEM;
1345 goto err;
1346 }
1347 cookie = tx->tx_submit(tx);
1348 if (dma_submit_error(cookie)) {
1349 ret = -ENOMEM;
1350 goto err;
1351 }
1352 dma_async_issue_pending(chan);
1353 src_offset += (loop_len - L1_CACHE_BYTES);
1354 dst_offset += (loop_len - L1_CACHE_BYTES);
1355 src_dma_addr += (loop_len - L1_CACHE_BYTES);
1356 dst_dma_addr += (loop_len - L1_CACHE_BYTES);
1357 remaining_len -= (loop_len - L1_CACHE_BYTES);
1358 loop_len = remaining_len;
1359
1360 /* Step 2) DMA: L1_CACHE_BYTES */
1361 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1362 src_dma_addr,
1363 loop_len, 0);
1364 if (!tx) {
1365 ret = -ENOMEM;
1366 goto err;
1367 }
1368 cookie = tx->tx_submit(tx);
1369 if (dma_submit_error(cookie)) {
1370 ret = -ENOMEM;
1371 goto err;
1372 }
1373 dma_async_issue_pending(chan);
1374 } else {
1375 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1376 src_dma_addr,
1377 loop_len, 0);
1378 if (!tx) {
1379 ret = -ENOMEM;
1380 goto err;
1381 }
1382 cookie = tx->tx_submit(tx);
1383 if (dma_submit_error(cookie)) {
1384 ret = -ENOMEM;
1385 goto err;
1386 }
1387 dma_async_issue_pending(chan);
1388 }
1389 src_offset += loop_len;
1390 dst_offset += loop_len;
1391 remaining_len -= loop_len;
1392 }
1393 remaining_len = tail_len;
1394 if (remaining_len) {
1395 loop_len = remaining_len;
1396 if (src_offset == end_src_offset)
1397 src_window = list_next_entry(src_window, list);
1398 if (dst_offset == end_dst_offset)
1399 dst_window = list_next_entry(dst_window, list);
1400
1401 src_dma_addr = __scif_off_to_dma_addr(src_window, src_offset);
1402 dst_dma_addr = __scif_off_to_dma_addr(dst_window, dst_offset);
1403 /*
1404 * The CPU copy for the tail bytes must be initiated only once
1405 * previous DMA transfers for this endpoint have completed to
1406 * guarantee ordering.
1407 */
1408 if (work->ordered) {
1409 struct scif_dev *rdev = work->remote_dev;
1410
1411 ret = scif_drain_dma_poll(rdev->sdev, chan);
1412 if (ret)
1413 return ret;
1414 }
1415 if (src_window->type == SCIF_WINDOW_SELF)
1416 src_virt = _get_local_va(src_offset, src_window,
1417 loop_len);
1418 else
1419 src_virt = ioremap_remote(src_offset, src_window,
1420 loop_len,
1421 work->remote_dev, NULL);
1422 if (!src_virt)
1423 return -ENOMEM;
1424
1425 if (dst_window->type == SCIF_WINDOW_SELF)
1426 dst_virt = _get_local_va(dst_offset, dst_window,
1427 loop_len);
1428 else
1429 dst_virt = ioremap_remote(dst_offset, dst_window,
1430 loop_len,
1431 work->remote_dev, NULL);
1432 if (!dst_virt) {
1433 if (src_window->type != SCIF_WINDOW_SELF)
1434 iounmap_remote(src_virt, loop_len, work);
1435 return -ENOMEM;
1436 }
1437
1438 if (src_window->type == SCIF_WINDOW_SELF)
1439 scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,
1440 work->ordered);
1441 else
1442 scif_unaligned_cpy_fromio(dst_virt, src_virt,
1443 loop_len, work->ordered);
1444 if (src_window->type != SCIF_WINDOW_SELF)
1445 iounmap_remote(src_virt, loop_len, work);
1446
1447 if (dst_window->type != SCIF_WINDOW_SELF)
1448 iounmap_remote(dst_virt, loop_len, work);
1449 remaining_len -= loop_len;
1450 }
1451 return ret;
1452err:
1453 dev_err(scif_info.mdev.this_device,
1454 "%s %d Desc Prog Failed ret %d\n",
1455 __func__, __LINE__, ret);
1456 return ret;
1457}
1458
1459/*
1460 * scif_rma_list_cpu_copy:
1461 *
1462 * Traverse all the windows and perform CPU copy.
1463 */
1464static int scif_rma_list_cpu_copy(struct scif_copy_work *work)
1465{
1466 void *src_virt, *dst_virt;
1467 size_t loop_len, remaining_len;
1468 int src_page_off, dst_page_off;
1469 s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1470 struct scif_window *src_window = work->src_window;
1471 struct scif_window *dst_window = work->dst_window;
1472 s64 end_src_offset, end_dst_offset;
1473 int ret = 0;
1474 struct scif_window_iter src_win_iter;
1475 struct scif_window_iter dst_win_iter;
1476
1477 remaining_len = work->len;
1478
1479 scif_init_window_iter(src_window, &src_win_iter);
1480 scif_init_window_iter(dst_window, &dst_win_iter);
1481 while (remaining_len) {
1482 src_page_off = src_offset & ~PAGE_MASK;
1483 dst_page_off = dst_offset & ~PAGE_MASK;
1484 loop_len = min(PAGE_SIZE -
1485 max(src_page_off, dst_page_off),
1486 remaining_len);
1487
1488 if (src_window->type == SCIF_WINDOW_SELF)
1489 src_virt = _get_local_va(src_offset, src_window,
1490 loop_len);
1491 else
1492 src_virt = ioremap_remote(src_offset, src_window,
1493 loop_len,
1494 work->remote_dev,
1495 &src_win_iter);
1496 if (!src_virt) {
1497 ret = -ENOMEM;
1498 goto error;
1499 }
1500
1501 if (dst_window->type == SCIF_WINDOW_SELF)
1502 dst_virt = _get_local_va(dst_offset, dst_window,
1503 loop_len);
1504 else
1505 dst_virt = ioremap_remote(dst_offset, dst_window,
1506 loop_len,
1507 work->remote_dev,
1508 &dst_win_iter);
1509 if (!dst_virt) {
1510 if (src_window->type == SCIF_WINDOW_PEER)
1511 iounmap_remote(src_virt, loop_len, work);
1512 ret = -ENOMEM;
1513 goto error;
1514 }
1515
1516 if (work->loopback) {
1517 memcpy(dst_virt, src_virt, loop_len);
1518 } else {
1519 if (src_window->type == SCIF_WINDOW_SELF)
1520 memcpy_toio((void __iomem __force *)dst_virt,
1521 src_virt, loop_len);
1522 else
1523 memcpy_fromio(dst_virt,
1524 (void __iomem __force *)src_virt,
1525 loop_len);
1526 }
1527 if (src_window->type == SCIF_WINDOW_PEER)
1528 iounmap_remote(src_virt, loop_len, work);
1529
1530 if (dst_window->type == SCIF_WINDOW_PEER)
1531 iounmap_remote(dst_virt, loop_len, work);
1532
1533 src_offset += loop_len;
1534 dst_offset += loop_len;
1535 remaining_len -= loop_len;
1536 if (remaining_len) {
1537 end_src_offset = src_window->offset +
1538 (src_window->nr_pages << PAGE_SHIFT);
1539 end_dst_offset = dst_window->offset +
1540 (dst_window->nr_pages << PAGE_SHIFT);
1541 if (src_offset == end_src_offset) {
1542 src_window = list_next_entry(src_window, list);
1543 scif_init_window_iter(src_window,
1544 &src_win_iter);
1545 }
1546 if (dst_offset == end_dst_offset) {
1547 dst_window = list_next_entry(dst_window, list);
1548 scif_init_window_iter(dst_window,
1549 &dst_win_iter);
1550 }
1551 }
1552 }
1553error:
1554 return ret;
1555}
1556
1557static int scif_rma_list_dma_copy_wrapper(struct scif_endpt *epd,
1558 struct scif_copy_work *work,
1559 struct dma_chan *chan, off_t loffset)
1560{
1561 int src_cache_off, dst_cache_off;
1562 s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1563 u8 *temp = NULL;
1564 bool src_local = true, dst_local = false;
1565 struct scif_dma_comp_cb *comp_cb;
1566 dma_addr_t src_dma_addr, dst_dma_addr;
1567 int err;
1568
1569 if (is_dma_copy_aligned(chan->device, 1, 1, 1))
1570 return _scif_rma_list_dma_copy_aligned(work, chan);
1571
1572 src_cache_off = src_offset & (L1_CACHE_BYTES - 1);
1573 dst_cache_off = dst_offset & (L1_CACHE_BYTES - 1);
1574
1575 if (dst_cache_off == src_cache_off)
1576 return scif_rma_list_dma_copy_aligned(work, chan);
1577
1578 if (work->loopback)
1579 return scif_rma_list_cpu_copy(work);
1580 src_dma_addr = __scif_off_to_dma_addr(work->src_window, src_offset);
1581 dst_dma_addr = __scif_off_to_dma_addr(work->dst_window, dst_offset);
1582 src_local = work->src_window->type == SCIF_WINDOW_SELF;
1583 dst_local = work->dst_window->type == SCIF_WINDOW_SELF;
1584
1585 dst_local = dst_local;
1586 /* Allocate dma_completion cb */
1587 comp_cb = kzalloc(sizeof(*comp_cb), GFP_KERNEL);
1588 if (!comp_cb)
1589 goto error;
1590
1591 work->comp_cb = comp_cb;
1592 comp_cb->cb_cookie = comp_cb;
1593 comp_cb->dma_completion_func = &scif_rma_completion_cb;
1594
1595 if (work->len + (L1_CACHE_BYTES << 1) < SCIF_KMEM_UNALIGNED_BUF_SIZE) {
1596 comp_cb->is_cache = false;
1597 /* Allocate padding bytes to align to a cache line */
1598 temp = kmalloc(work->len + (L1_CACHE_BYTES << 1),
1599 GFP_KERNEL);
1600 if (!temp)
1601 goto free_comp_cb;
1602 comp_cb->temp_buf_to_free = temp;
1603 /* kmalloc(..) does not guarantee cache line alignment */
1604 if (!IS_ALIGNED((u64)temp, L1_CACHE_BYTES))
1605 temp = PTR_ALIGN(temp, L1_CACHE_BYTES);
1606 } else {
1607 comp_cb->is_cache = true;
1608 temp = kmem_cache_alloc(unaligned_cache, GFP_KERNEL);
1609 if (!temp)
1610 goto free_comp_cb;
1611 comp_cb->temp_buf_to_free = temp;
1612 }
1613
1614 if (src_local) {
1615 temp += dst_cache_off;
1616 scif_rma_local_cpu_copy(work->src_offset, work->src_window,
1617 temp, work->len, true);
1618 } else {
1619 comp_cb->dst_window = work->dst_window;
1620 comp_cb->dst_offset = work->dst_offset;
1621 work->src_offset = work->src_offset - src_cache_off;
1622 comp_cb->len = work->len;
1623 work->len = ALIGN(work->len + src_cache_off, L1_CACHE_BYTES);
1624 comp_cb->header_padding = src_cache_off;
1625 }
1626 comp_cb->temp_buf = temp;
1627
1628 err = scif_map_single(&comp_cb->temp_phys, temp,
1629 work->remote_dev, SCIF_KMEM_UNALIGNED_BUF_SIZE);
1630 if (err)
1631 goto free_temp_buf;
1632 comp_cb->sdev = work->remote_dev;
1633 if (scif_rma_list_dma_copy_unaligned(work, temp, chan, src_local) < 0)
1634 goto free_temp_buf;
1635 if (!src_local)
1636 work->fence_type = SCIF_DMA_INTR;
1637 return 0;
1638free_temp_buf:
1639 if (comp_cb->is_cache)
1640 kmem_cache_free(unaligned_cache, comp_cb->temp_buf_to_free);
1641 else
1642 kfree(comp_cb->temp_buf_to_free);
1643free_comp_cb:
1644 kfree(comp_cb);
1645error:
1646 return -ENOMEM;
1647}
1648
1649/**
1650 * scif_rma_copy:
1651 * @epd: end point descriptor.
1652 * @loffset: offset in local registered address space to/from which to copy
1653 * @addr: user virtual address to/from which to copy
1654 * @len: length of range to copy
1655 * @roffset: offset in remote registered address space to/from which to copy
1656 * @flags: flags
1657 * @dir: LOCAL->REMOTE or vice versa.
1658 * @last_chunk: true if this is the last chunk of a larger transfer
1659 *
1660 * Validate parameters, check if src/dst registered ranges requested for copy
1661 * are valid and initiate either CPU or DMA copy.
1662 */
1663static int scif_rma_copy(scif_epd_t epd, off_t loffset, unsigned long addr,
1664 size_t len, off_t roffset, int flags,
1665 enum scif_rma_dir dir, bool last_chunk)
1666{
1667 struct scif_endpt *ep = (struct scif_endpt *)epd;
1668 struct scif_rma_req remote_req;
1669 struct scif_rma_req req;
1670 struct scif_window *local_window = NULL;
1671 struct scif_window *remote_window = NULL;
1672 struct scif_copy_work copy_work;
1673 bool loopback;
1674 int err = 0;
1675 struct dma_chan *chan;
1676 struct scif_mmu_notif *mmn = NULL;
1677 bool cache = false;
1678 struct device *spdev;
1679
1680 err = scif_verify_epd(ep);
1681 if (err)
1682 return err;
1683
1684 if (flags && !(flags & (SCIF_RMA_USECPU | SCIF_RMA_USECACHE |
1685 SCIF_RMA_SYNC | SCIF_RMA_ORDERED)))
1686 return -EINVAL;
1687
1688 loopback = scifdev_self(ep->remote_dev) ? true : false;
1689 copy_work.fence_type = ((flags & SCIF_RMA_SYNC) && last_chunk) ?
1690 SCIF_DMA_POLL : 0;
1691 copy_work.ordered = !!((flags & SCIF_RMA_ORDERED) && last_chunk);
1692
1693 /* Use CPU for Mgmt node <-> Mgmt node copies */
1694 if (loopback && scif_is_mgmt_node()) {
1695 flags |= SCIF_RMA_USECPU;
1696 copy_work.fence_type = 0x0;
1697 }
1698
1699 cache = scif_is_set_reg_cache(flags);
1700
1701 remote_req.out_window = &remote_window;
1702 remote_req.offset = roffset;
1703 remote_req.nr_bytes = len;
1704 /*
1705 * If transfer is from local to remote then the remote window
1706 * must be writeable and vice versa.
1707 */
1708 remote_req.prot = dir == SCIF_LOCAL_TO_REMOTE ? VM_WRITE : VM_READ;
1709 remote_req.type = SCIF_WINDOW_PARTIAL;
1710 remote_req.head = &ep->rma_info.remote_reg_list;
1711
1712 spdev = scif_get_peer_dev(ep->remote_dev);
1713 if (IS_ERR(spdev)) {
1714 err = PTR_ERR(spdev);
1715 return err;
1716 }
1717
1718 if (addr && cache) {
1719 mutex_lock(&ep->rma_info.mmn_lock);
1720 mmn = scif_find_mmu_notifier(current->mm, &ep->rma_info);
1721 if (!mmn)
1722 mmn = scif_add_mmu_notifier(current->mm, ep);
1723 mutex_unlock(&ep->rma_info.mmn_lock);
1724 if (IS_ERR(mmn)) {
1725 scif_put_peer_dev(spdev);
1726 return PTR_ERR(mmn);
1727 }
1728 cache = cache && !scif_rma_tc_can_cache(ep, len);
1729 }
1730 mutex_lock(&ep->rma_info.rma_lock);
1731 if (addr) {
1732 req.out_window = &local_window;
1733 req.nr_bytes = ALIGN(len + (addr & ~PAGE_MASK),
1734 PAGE_SIZE);
1735 req.va_for_temp = addr & PAGE_MASK;
1736 req.prot = (dir == SCIF_LOCAL_TO_REMOTE ?
1737 VM_READ : VM_WRITE | VM_READ);
1738 /* Does a valid local window exist? */
1739 if (mmn) {
1740 spin_lock(&ep->rma_info.tc_lock);
1741 req.head = &mmn->tc_reg_list;
1742 err = scif_query_tcw(ep, &req);
1743 spin_unlock(&ep->rma_info.tc_lock);
1744 }
1745 if (!mmn || err) {
1746 err = scif_register_temp(epd, req.va_for_temp,
1747 req.nr_bytes, req.prot,
1748 &loffset, &local_window);
1749 if (err) {
1750 mutex_unlock(&ep->rma_info.rma_lock);
1751 goto error;
1752 }
1753 if (!cache)
1754 goto skip_cache;
1755 atomic_inc(&ep->rma_info.tcw_refcount);
1756 atomic_add_return(local_window->nr_pages,
1757 &ep->rma_info.tcw_total_pages);
1758 if (mmn) {
1759 spin_lock(&ep->rma_info.tc_lock);
1760 scif_insert_tcw(local_window,
1761 &mmn->tc_reg_list);
1762 spin_unlock(&ep->rma_info.tc_lock);
1763 }
1764 }
1765skip_cache:
1766 loffset = local_window->offset +
1767 (addr - local_window->va_for_temp);
1768 } else {
1769 req.out_window = &local_window;
1770 req.offset = loffset;
1771 /*
1772 * If transfer is from local to remote then the self window
1773 * must be readable and vice versa.
1774 */
1775 req.prot = dir == SCIF_LOCAL_TO_REMOTE ? VM_READ : VM_WRITE;
1776 req.nr_bytes = len;
1777 req.type = SCIF_WINDOW_PARTIAL;
1778 req.head = &ep->rma_info.reg_list;
1779 /* Does a valid local window exist? */
1780 err = scif_query_window(&req);
1781 if (err) {
1782 mutex_unlock(&ep->rma_info.rma_lock);
1783 goto error;
1784 }
1785 }
1786
1787 /* Does a valid remote window exist? */
1788 err = scif_query_window(&remote_req);
1789 if (err) {
1790 mutex_unlock(&ep->rma_info.rma_lock);
1791 goto error;
1792 }
1793
1794 /*
1795 * Prepare copy_work for submitting work to the DMA kernel thread
1796 * or CPU copy routine.
1797 */
1798 copy_work.len = len;
1799 copy_work.loopback = loopback;
1800 copy_work.remote_dev = ep->remote_dev;
1801 if (dir == SCIF_LOCAL_TO_REMOTE) {
1802 copy_work.src_offset = loffset;
1803 copy_work.src_window = local_window;
1804 copy_work.dst_offset = roffset;
1805 copy_work.dst_window = remote_window;
1806 } else {
1807 copy_work.src_offset = roffset;
1808 copy_work.src_window = remote_window;
1809 copy_work.dst_offset = loffset;
1810 copy_work.dst_window = local_window;
1811 }
1812
1813 if (flags & SCIF_RMA_USECPU) {
1814 scif_rma_list_cpu_copy(©_work);
1815 } else {
1816 chan = ep->rma_info.dma_chan;
1817 err = scif_rma_list_dma_copy_wrapper(epd, ©_work,
1818 chan, loffset);
1819 }
1820 if (addr && !cache)
1821 atomic_inc(&ep->rma_info.tw_refcount);
1822
1823 mutex_unlock(&ep->rma_info.rma_lock);
1824
1825 if (last_chunk) {
1826 struct scif_dev *rdev = ep->remote_dev;
1827
1828 if (copy_work.fence_type == SCIF_DMA_POLL)
1829 err = scif_drain_dma_poll(rdev->sdev,
1830 ep->rma_info.dma_chan);
1831 else if (copy_work.fence_type == SCIF_DMA_INTR)
1832 err = scif_drain_dma_intr(rdev->sdev,
1833 ep->rma_info.dma_chan);
1834 }
1835
1836 if (addr && !cache)
1837 scif_queue_for_cleanup(local_window, &scif_info.rma);
1838 scif_put_peer_dev(spdev);
1839 return err;
1840error:
1841 if (err) {
1842 if (addr && local_window && !cache)
1843 scif_destroy_window(ep, local_window);
1844 dev_err(scif_info.mdev.this_device,
1845 "%s %d err %d len 0x%lx\n",
1846 __func__, __LINE__, err, len);
1847 }
1848 scif_put_peer_dev(spdev);
1849 return err;
1850}
1851
1852int scif_readfrom(scif_epd_t epd, off_t loffset, size_t len,
1853 off_t roffset, int flags)
1854{
1855 int err;
1856
1857 dev_dbg(scif_info.mdev.this_device,
1858 "SCIFAPI readfrom: ep %p loffset 0x%lx len 0x%lx offset 0x%lx flags 0x%x\n",
1859 epd, loffset, len, roffset, flags);
1860 if (scif_unaligned(loffset, roffset)) {
1861 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1862 err = scif_rma_copy(epd, loffset, 0x0,
1863 SCIF_MAX_UNALIGNED_BUF_SIZE,
1864 roffset, flags,
1865 SCIF_REMOTE_TO_LOCAL, false);
1866 if (err)
1867 goto readfrom_err;
1868 loffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1869 roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1870 len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1871 }
1872 }
1873 err = scif_rma_copy(epd, loffset, 0x0, len,
1874 roffset, flags, SCIF_REMOTE_TO_LOCAL, true);
1875readfrom_err:
1876 return err;
1877}
1878EXPORT_SYMBOL_GPL(scif_readfrom);
1879
1880int scif_writeto(scif_epd_t epd, off_t loffset, size_t len,
1881 off_t roffset, int flags)
1882{
1883 int err;
1884
1885 dev_dbg(scif_info.mdev.this_device,
1886 "SCIFAPI writeto: ep %p loffset 0x%lx len 0x%lx roffset 0x%lx flags 0x%x\n",
1887 epd, loffset, len, roffset, flags);
1888 if (scif_unaligned(loffset, roffset)) {
1889 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1890 err = scif_rma_copy(epd, loffset, 0x0,
1891 SCIF_MAX_UNALIGNED_BUF_SIZE,
1892 roffset, flags,
1893 SCIF_LOCAL_TO_REMOTE, false);
1894 if (err)
1895 goto writeto_err;
1896 loffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1897 roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1898 len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1899 }
1900 }
1901 err = scif_rma_copy(epd, loffset, 0x0, len,
1902 roffset, flags, SCIF_LOCAL_TO_REMOTE, true);
1903writeto_err:
1904 return err;
1905}
1906EXPORT_SYMBOL_GPL(scif_writeto);
1907
1908int scif_vreadfrom(scif_epd_t epd, void *addr, size_t len,
1909 off_t roffset, int flags)
1910{
1911 int err;
1912
1913 dev_dbg(scif_info.mdev.this_device,
1914 "SCIFAPI vreadfrom: ep %p addr %p len 0x%lx roffset 0x%lx flags 0x%x\n",
1915 epd, addr, len, roffset, flags);
1916 if (scif_unaligned((off_t __force)addr, roffset)) {
1917 if (len > SCIF_MAX_UNALIGNED_BUF_SIZE)
1918 flags &= ~SCIF_RMA_USECACHE;
1919
1920 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1921 err = scif_rma_copy(epd, 0, (u64)addr,
1922 SCIF_MAX_UNALIGNED_BUF_SIZE,
1923 roffset, flags,
1924 SCIF_REMOTE_TO_LOCAL, false);
1925 if (err)
1926 goto vreadfrom_err;
1927 addr += SCIF_MAX_UNALIGNED_BUF_SIZE;
1928 roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1929 len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1930 }
1931 }
1932 err = scif_rma_copy(epd, 0, (u64)addr, len,
1933 roffset, flags, SCIF_REMOTE_TO_LOCAL, true);
1934vreadfrom_err:
1935 return err;
1936}
1937EXPORT_SYMBOL_GPL(scif_vreadfrom);
1938
1939int scif_vwriteto(scif_epd_t epd, void *addr, size_t len,
1940 off_t roffset, int flags)
1941{
1942 int err;
1943
1944 dev_dbg(scif_info.mdev.this_device,
1945 "SCIFAPI vwriteto: ep %p addr %p len 0x%lx roffset 0x%lx flags 0x%x\n",
1946 epd, addr, len, roffset, flags);
1947 if (scif_unaligned((off_t __force)addr, roffset)) {
1948 if (len > SCIF_MAX_UNALIGNED_BUF_SIZE)
1949 flags &= ~SCIF_RMA_USECACHE;
1950
1951 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1952 err = scif_rma_copy(epd, 0, (u64)addr,
1953 SCIF_MAX_UNALIGNED_BUF_SIZE,
1954 roffset, flags,
1955 SCIF_LOCAL_TO_REMOTE, false);
1956 if (err)
1957 goto vwriteto_err;
1958 addr += SCIF_MAX_UNALIGNED_BUF_SIZE;
1959 roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1960 len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1961 }
1962 }
1963 err = scif_rma_copy(epd, 0, (u64)addr, len,
1964 roffset, flags, SCIF_LOCAL_TO_REMOTE, true);
1965vwriteto_err:
1966 return err;
1967}
1968EXPORT_SYMBOL_GPL(scif_vwriteto);