Loading...
Note: File does not exist in v3.1.
1/*
2 * Intel MIC Platform Software Stack (MPSS)
3 *
4 * Copyright(c) 2015 Intel Corporation.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
14 *
15 * Intel SCIF driver.
16 *
17 */
18#include "scif_main.h"
19#include "scif_map.h"
20
21/*
22 * struct scif_dma_comp_cb - SCIF DMA completion callback
23 *
24 * @dma_completion_func: DMA completion callback
25 * @cb_cookie: DMA completion callback cookie
26 * @temp_buf: Temporary buffer
27 * @temp_buf_to_free: Temporary buffer to be freed
28 * @is_cache: Is a kmem_cache allocated buffer
29 * @dst_offset: Destination registration offset
30 * @dst_window: Destination registration window
31 * @len: Length of the temp buffer
32 * @temp_phys: DMA address of the temp buffer
33 * @sdev: The SCIF device
34 * @header_padding: padding for cache line alignment
35 */
36struct scif_dma_comp_cb {
37 void (*dma_completion_func)(void *cookie);
38 void *cb_cookie;
39 u8 *temp_buf;
40 u8 *temp_buf_to_free;
41 bool is_cache;
42 s64 dst_offset;
43 struct scif_window *dst_window;
44 size_t len;
45 dma_addr_t temp_phys;
46 struct scif_dev *sdev;
47 int header_padding;
48};
49
50/**
51 * struct scif_copy_work - Work for DMA copy
52 *
53 * @src_offset: Starting source offset
54 * @dst_offset: Starting destination offset
55 * @src_window: Starting src registered window
56 * @dst_window: Starting dst registered window
57 * @loopback: true if this is a loopback DMA transfer
58 * @len: Length of the transfer
59 * @comp_cb: DMA copy completion callback
60 * @remote_dev: The remote SCIF peer device
61 * @fence_type: polling or interrupt based
62 * @ordered: is this a tail byte ordered DMA transfer
63 */
64struct scif_copy_work {
65 s64 src_offset;
66 s64 dst_offset;
67 struct scif_window *src_window;
68 struct scif_window *dst_window;
69 int loopback;
70 size_t len;
71 struct scif_dma_comp_cb *comp_cb;
72 struct scif_dev *remote_dev;
73 int fence_type;
74 bool ordered;
75};
76
77/**
78 * scif_reserve_dma_chan:
79 * @ep: Endpoint Descriptor.
80 *
81 * This routine reserves a DMA channel for a particular
82 * endpoint. All DMA transfers for an endpoint are always
83 * programmed on the same DMA channel.
84 */
85int scif_reserve_dma_chan(struct scif_endpt *ep)
86{
87 int err = 0;
88 struct scif_dev *scifdev;
89 struct scif_hw_dev *sdev;
90 struct dma_chan *chan;
91
92 /* Loopback DMAs are not supported on the management node */
93 if (!scif_info.nodeid && scifdev_self(ep->remote_dev))
94 return 0;
95 if (scif_info.nodeid)
96 scifdev = &scif_dev[0];
97 else
98 scifdev = ep->remote_dev;
99 sdev = scifdev->sdev;
100 if (!sdev->num_dma_ch)
101 return -ENODEV;
102 chan = sdev->dma_ch[scifdev->dma_ch_idx];
103 scifdev->dma_ch_idx = (scifdev->dma_ch_idx + 1) % sdev->num_dma_ch;
104 mutex_lock(&ep->rma_info.rma_lock);
105 ep->rma_info.dma_chan = chan;
106 mutex_unlock(&ep->rma_info.rma_lock);
107 return err;
108}
109
110#ifdef CONFIG_MMU_NOTIFIER
111/**
112 * scif_rma_destroy_tcw:
113 *
114 * This routine destroys temporary cached windows
115 */
116static
117void __scif_rma_destroy_tcw(struct scif_mmu_notif *mmn,
118 struct scif_endpt *ep,
119 u64 start, u64 len)
120{
121 struct list_head *item, *tmp;
122 struct scif_window *window;
123 u64 start_va, end_va;
124 u64 end = start + len;
125
126 if (end <= start)
127 return;
128
129 list_for_each_safe(item, tmp, &mmn->tc_reg_list) {
130 window = list_entry(item, struct scif_window, list);
131 ep = (struct scif_endpt *)window->ep;
132 if (!len)
133 break;
134 start_va = window->va_for_temp;
135 end_va = start_va + (window->nr_pages << PAGE_SHIFT);
136 if (start < start_va && end <= start_va)
137 break;
138 if (start >= end_va)
139 continue;
140 __scif_rma_destroy_tcw_helper(window);
141 }
142}
143
144static void scif_rma_destroy_tcw(struct scif_mmu_notif *mmn, u64 start, u64 len)
145{
146 struct scif_endpt *ep = mmn->ep;
147
148 spin_lock(&ep->rma_info.tc_lock);
149 __scif_rma_destroy_tcw(mmn, ep, start, len);
150 spin_unlock(&ep->rma_info.tc_lock);
151}
152
153static void scif_rma_destroy_tcw_ep(struct scif_endpt *ep)
154{
155 struct list_head *item, *tmp;
156 struct scif_mmu_notif *mmn;
157
158 list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {
159 mmn = list_entry(item, struct scif_mmu_notif, list);
160 scif_rma_destroy_tcw(mmn, 0, ULONG_MAX);
161 }
162}
163
164static void __scif_rma_destroy_tcw_ep(struct scif_endpt *ep)
165{
166 struct list_head *item, *tmp;
167 struct scif_mmu_notif *mmn;
168
169 spin_lock(&ep->rma_info.tc_lock);
170 list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {
171 mmn = list_entry(item, struct scif_mmu_notif, list);
172 __scif_rma_destroy_tcw(mmn, ep, 0, ULONG_MAX);
173 }
174 spin_unlock(&ep->rma_info.tc_lock);
175}
176
177static bool scif_rma_tc_can_cache(struct scif_endpt *ep, size_t cur_bytes)
178{
179 if ((cur_bytes >> PAGE_SHIFT) > scif_info.rma_tc_limit)
180 return false;
181 if ((atomic_read(&ep->rma_info.tcw_total_pages)
182 + (cur_bytes >> PAGE_SHIFT)) >
183 scif_info.rma_tc_limit) {
184 dev_info(scif_info.mdev.this_device,
185 "%s %d total=%d, current=%zu reached max\n",
186 __func__, __LINE__,
187 atomic_read(&ep->rma_info.tcw_total_pages),
188 (1 + (cur_bytes >> PAGE_SHIFT)));
189 scif_rma_destroy_tcw_invalid();
190 __scif_rma_destroy_tcw_ep(ep);
191 }
192 return true;
193}
194
195static void scif_mmu_notifier_release(struct mmu_notifier *mn,
196 struct mm_struct *mm)
197{
198 struct scif_mmu_notif *mmn;
199
200 mmn = container_of(mn, struct scif_mmu_notif, ep_mmu_notifier);
201 scif_rma_destroy_tcw(mmn, 0, ULONG_MAX);
202 schedule_work(&scif_info.misc_work);
203}
204
205static void scif_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
206 struct mm_struct *mm,
207 unsigned long address)
208{
209 struct scif_mmu_notif *mmn;
210
211 mmn = container_of(mn, struct scif_mmu_notif, ep_mmu_notifier);
212 scif_rma_destroy_tcw(mmn, address, PAGE_SIZE);
213}
214
215static void scif_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
216 struct mm_struct *mm,
217 unsigned long start,
218 unsigned long end)
219{
220 struct scif_mmu_notif *mmn;
221
222 mmn = container_of(mn, struct scif_mmu_notif, ep_mmu_notifier);
223 scif_rma_destroy_tcw(mmn, start, end - start);
224}
225
226static void scif_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
227 struct mm_struct *mm,
228 unsigned long start,
229 unsigned long end)
230{
231 /*
232 * Nothing to do here, everything needed was done in
233 * invalidate_range_start.
234 */
235}
236
237static const struct mmu_notifier_ops scif_mmu_notifier_ops = {
238 .release = scif_mmu_notifier_release,
239 .clear_flush_young = NULL,
240 .invalidate_page = scif_mmu_notifier_invalidate_page,
241 .invalidate_range_start = scif_mmu_notifier_invalidate_range_start,
242 .invalidate_range_end = scif_mmu_notifier_invalidate_range_end};
243
244static void scif_ep_unregister_mmu_notifier(struct scif_endpt *ep)
245{
246 struct scif_endpt_rma_info *rma = &ep->rma_info;
247 struct scif_mmu_notif *mmn = NULL;
248 struct list_head *item, *tmp;
249
250 mutex_lock(&ep->rma_info.mmn_lock);
251 list_for_each_safe(item, tmp, &rma->mmn_list) {
252 mmn = list_entry(item, struct scif_mmu_notif, list);
253 mmu_notifier_unregister(&mmn->ep_mmu_notifier, mmn->mm);
254 list_del(item);
255 kfree(mmn);
256 }
257 mutex_unlock(&ep->rma_info.mmn_lock);
258}
259
260static void scif_init_mmu_notifier(struct scif_mmu_notif *mmn,
261 struct mm_struct *mm, struct scif_endpt *ep)
262{
263 mmn->ep = ep;
264 mmn->mm = mm;
265 mmn->ep_mmu_notifier.ops = &scif_mmu_notifier_ops;
266 INIT_LIST_HEAD(&mmn->list);
267 INIT_LIST_HEAD(&mmn->tc_reg_list);
268}
269
270static struct scif_mmu_notif *
271scif_find_mmu_notifier(struct mm_struct *mm, struct scif_endpt_rma_info *rma)
272{
273 struct scif_mmu_notif *mmn;
274
275 list_for_each_entry(mmn, &rma->mmn_list, list)
276 if (mmn->mm == mm)
277 return mmn;
278 return NULL;
279}
280
281static struct scif_mmu_notif *
282scif_add_mmu_notifier(struct mm_struct *mm, struct scif_endpt *ep)
283{
284 struct scif_mmu_notif *mmn
285 = kzalloc(sizeof(*mmn), GFP_KERNEL);
286
287 if (!mmn)
288 return ERR_PTR(-ENOMEM);
289
290 scif_init_mmu_notifier(mmn, current->mm, ep);
291 if (mmu_notifier_register(&mmn->ep_mmu_notifier, current->mm)) {
292 kfree(mmn);
293 return ERR_PTR(-EBUSY);
294 }
295 list_add(&mmn->list, &ep->rma_info.mmn_list);
296 return mmn;
297}
298
299/*
300 * Called from the misc thread to destroy temporary cached windows and
301 * unregister the MMU notifier for the SCIF endpoint.
302 */
303void scif_mmu_notif_handler(struct work_struct *work)
304{
305 struct list_head *pos, *tmpq;
306 struct scif_endpt *ep;
307restart:
308 scif_rma_destroy_tcw_invalid();
309 spin_lock(&scif_info.rmalock);
310 list_for_each_safe(pos, tmpq, &scif_info.mmu_notif_cleanup) {
311 ep = list_entry(pos, struct scif_endpt, mmu_list);
312 list_del(&ep->mmu_list);
313 spin_unlock(&scif_info.rmalock);
314 scif_rma_destroy_tcw_ep(ep);
315 scif_ep_unregister_mmu_notifier(ep);
316 goto restart;
317 }
318 spin_unlock(&scif_info.rmalock);
319}
320
321static bool scif_is_set_reg_cache(int flags)
322{
323 return !!(flags & SCIF_RMA_USECACHE);
324}
325#else
326static struct scif_mmu_notif *
327scif_find_mmu_notifier(struct mm_struct *mm,
328 struct scif_endpt_rma_info *rma)
329{
330 return NULL;
331}
332
333static struct scif_mmu_notif *
334scif_add_mmu_notifier(struct mm_struct *mm, struct scif_endpt *ep)
335{
336 return NULL;
337}
338
339void scif_mmu_notif_handler(struct work_struct *work)
340{
341}
342
343static bool scif_is_set_reg_cache(int flags)
344{
345 return false;
346}
347
348static bool scif_rma_tc_can_cache(struct scif_endpt *ep, size_t cur_bytes)
349{
350 return false;
351}
352#endif
353
354/**
355 * scif_register_temp:
356 * @epd: End Point Descriptor.
357 * @addr: virtual address to/from which to copy
358 * @len: length of range to copy
359 * @out_offset: computed offset returned by reference.
360 * @out_window: allocated registered window returned by reference.
361 *
362 * Create a temporary registered window. The peer will not know about this
363 * window. This API is used for scif_vreadfrom()/scif_vwriteto() API's.
364 */
365static int
366scif_register_temp(scif_epd_t epd, unsigned long addr, size_t len, int prot,
367 off_t *out_offset, struct scif_window **out_window)
368{
369 struct scif_endpt *ep = (struct scif_endpt *)epd;
370 int err;
371 scif_pinned_pages_t pinned_pages;
372 size_t aligned_len;
373
374 aligned_len = ALIGN(len, PAGE_SIZE);
375
376 err = __scif_pin_pages((void *)(addr & PAGE_MASK),
377 aligned_len, &prot, 0, &pinned_pages);
378 if (err)
379 return err;
380
381 pinned_pages->prot = prot;
382
383 /* Compute the offset for this registration */
384 err = scif_get_window_offset(ep, 0, 0,
385 aligned_len >> PAGE_SHIFT,
386 (s64 *)out_offset);
387 if (err)
388 goto error_unpin;
389
390 /* Allocate and prepare self registration window */
391 *out_window = scif_create_window(ep, aligned_len >> PAGE_SHIFT,
392 *out_offset, true);
393 if (!*out_window) {
394 scif_free_window_offset(ep, NULL, *out_offset);
395 err = -ENOMEM;
396 goto error_unpin;
397 }
398
399 (*out_window)->pinned_pages = pinned_pages;
400 (*out_window)->nr_pages = pinned_pages->nr_pages;
401 (*out_window)->prot = pinned_pages->prot;
402
403 (*out_window)->va_for_temp = addr & PAGE_MASK;
404 err = scif_map_window(ep->remote_dev, *out_window);
405 if (err) {
406 /* Something went wrong! Rollback */
407 scif_destroy_window(ep, *out_window);
408 *out_window = NULL;
409 } else {
410 *out_offset |= (addr - (*out_window)->va_for_temp);
411 }
412 return err;
413error_unpin:
414 if (err)
415 dev_err(&ep->remote_dev->sdev->dev,
416 "%s %d err %d\n", __func__, __LINE__, err);
417 scif_unpin_pages(pinned_pages);
418 return err;
419}
420
421#define SCIF_DMA_TO (3 * HZ)
422
423/*
424 * scif_sync_dma - Program a DMA without an interrupt descriptor
425 *
426 * @dev - The address of the pointer to the device instance used
427 * for DMA registration.
428 * @chan - DMA channel to be used.
429 * @sync_wait: Wait for DMA to complete?
430 *
431 * Return 0 on success and -errno on error.
432 */
433static int scif_sync_dma(struct scif_hw_dev *sdev, struct dma_chan *chan,
434 bool sync_wait)
435{
436 int err = 0;
437 struct dma_async_tx_descriptor *tx = NULL;
438 enum dma_ctrl_flags flags = DMA_PREP_FENCE;
439 dma_cookie_t cookie;
440 struct dma_device *ddev;
441
442 if (!chan) {
443 err = -EIO;
444 dev_err(&sdev->dev, "%s %d err %d\n",
445 __func__, __LINE__, err);
446 return err;
447 }
448 ddev = chan->device;
449
450 tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, flags);
451 if (!tx) {
452 err = -ENOMEM;
453 dev_err(&sdev->dev, "%s %d err %d\n",
454 __func__, __LINE__, err);
455 goto release;
456 }
457 cookie = tx->tx_submit(tx);
458
459 if (dma_submit_error(cookie)) {
460 err = -ENOMEM;
461 dev_err(&sdev->dev, "%s %d err %d\n",
462 __func__, __LINE__, err);
463 goto release;
464 }
465 if (!sync_wait) {
466 dma_async_issue_pending(chan);
467 } else {
468 if (dma_sync_wait(chan, cookie) == DMA_COMPLETE) {
469 err = 0;
470 } else {
471 err = -EIO;
472 dev_err(&sdev->dev, "%s %d err %d\n",
473 __func__, __LINE__, err);
474 }
475 }
476release:
477 return err;
478}
479
480static void scif_dma_callback(void *arg)
481{
482 struct completion *done = (struct completion *)arg;
483
484 complete(done);
485}
486
487#define SCIF_DMA_SYNC_WAIT true
488#define SCIF_DMA_POLL BIT(0)
489#define SCIF_DMA_INTR BIT(1)
490
491/*
492 * scif_async_dma - Program a DMA with an interrupt descriptor
493 *
494 * @dev - The address of the pointer to the device instance used
495 * for DMA registration.
496 * @chan - DMA channel to be used.
497 * Return 0 on success and -errno on error.
498 */
499static int scif_async_dma(struct scif_hw_dev *sdev, struct dma_chan *chan)
500{
501 int err = 0;
502 struct dma_device *ddev;
503 struct dma_async_tx_descriptor *tx = NULL;
504 enum dma_ctrl_flags flags = DMA_PREP_INTERRUPT | DMA_PREP_FENCE;
505 DECLARE_COMPLETION_ONSTACK(done_wait);
506 dma_cookie_t cookie;
507 enum dma_status status;
508
509 if (!chan) {
510 err = -EIO;
511 dev_err(&sdev->dev, "%s %d err %d\n",
512 __func__, __LINE__, err);
513 return err;
514 }
515 ddev = chan->device;
516
517 tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, flags);
518 if (!tx) {
519 err = -ENOMEM;
520 dev_err(&sdev->dev, "%s %d err %d\n",
521 __func__, __LINE__, err);
522 goto release;
523 }
524 reinit_completion(&done_wait);
525 tx->callback = scif_dma_callback;
526 tx->callback_param = &done_wait;
527 cookie = tx->tx_submit(tx);
528
529 if (dma_submit_error(cookie)) {
530 err = -ENOMEM;
531 dev_err(&sdev->dev, "%s %d err %d\n",
532 __func__, __LINE__, err);
533 goto release;
534 }
535 dma_async_issue_pending(chan);
536
537 err = wait_for_completion_timeout(&done_wait, SCIF_DMA_TO);
538 if (!err) {
539 err = -EIO;
540 dev_err(&sdev->dev, "%s %d err %d\n",
541 __func__, __LINE__, err);
542 goto release;
543 }
544 err = 0;
545 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
546 if (status != DMA_COMPLETE) {
547 err = -EIO;
548 dev_err(&sdev->dev, "%s %d err %d\n",
549 __func__, __LINE__, err);
550 goto release;
551 }
552release:
553 return err;
554}
555
556/*
557 * scif_drain_dma_poll - Drain all outstanding DMA operations for a particular
558 * DMA channel via polling.
559 *
560 * @sdev - The SCIF device
561 * @chan - DMA channel
562 * Return 0 on success and -errno on error.
563 */
564static int scif_drain_dma_poll(struct scif_hw_dev *sdev, struct dma_chan *chan)
565{
566 if (!chan)
567 return -EINVAL;
568 return scif_sync_dma(sdev, chan, SCIF_DMA_SYNC_WAIT);
569}
570
571/*
572 * scif_drain_dma_intr - Drain all outstanding DMA operations for a particular
573 * DMA channel via interrupt based blocking wait.
574 *
575 * @sdev - The SCIF device
576 * @chan - DMA channel
577 * Return 0 on success and -errno on error.
578 */
579int scif_drain_dma_intr(struct scif_hw_dev *sdev, struct dma_chan *chan)
580{
581 if (!chan)
582 return -EINVAL;
583 return scif_async_dma(sdev, chan);
584}
585
586/**
587 * scif_rma_destroy_windows:
588 *
589 * This routine destroys all windows queued for cleanup
590 */
591void scif_rma_destroy_windows(void)
592{
593 struct list_head *item, *tmp;
594 struct scif_window *window;
595 struct scif_endpt *ep;
596 struct dma_chan *chan;
597
598 might_sleep();
599restart:
600 spin_lock(&scif_info.rmalock);
601 list_for_each_safe(item, tmp, &scif_info.rma) {
602 window = list_entry(item, struct scif_window,
603 list);
604 ep = (struct scif_endpt *)window->ep;
605 chan = ep->rma_info.dma_chan;
606
607 list_del_init(&window->list);
608 spin_unlock(&scif_info.rmalock);
609 if (!chan || !scifdev_alive(ep) ||
610 !scif_drain_dma_intr(ep->remote_dev->sdev,
611 ep->rma_info.dma_chan))
612 /* Remove window from global list */
613 window->unreg_state = OP_COMPLETED;
614 else
615 dev_warn(&ep->remote_dev->sdev->dev,
616 "DMA engine hung?\n");
617 if (window->unreg_state == OP_COMPLETED) {
618 if (window->type == SCIF_WINDOW_SELF)
619 scif_destroy_window(ep, window);
620 else
621 scif_destroy_remote_window(window);
622 atomic_dec(&ep->rma_info.tw_refcount);
623 }
624 goto restart;
625 }
626 spin_unlock(&scif_info.rmalock);
627}
628
629/**
630 * scif_rma_destroy_tcw:
631 *
632 * This routine destroys temporary cached registered windows
633 * which have been queued for cleanup.
634 */
635void scif_rma_destroy_tcw_invalid(void)
636{
637 struct list_head *item, *tmp;
638 struct scif_window *window;
639 struct scif_endpt *ep;
640 struct dma_chan *chan;
641
642 might_sleep();
643restart:
644 spin_lock(&scif_info.rmalock);
645 list_for_each_safe(item, tmp, &scif_info.rma_tc) {
646 window = list_entry(item, struct scif_window, list);
647 ep = (struct scif_endpt *)window->ep;
648 chan = ep->rma_info.dma_chan;
649 list_del_init(&window->list);
650 spin_unlock(&scif_info.rmalock);
651 mutex_lock(&ep->rma_info.rma_lock);
652 if (!chan || !scifdev_alive(ep) ||
653 !scif_drain_dma_intr(ep->remote_dev->sdev,
654 ep->rma_info.dma_chan)) {
655 atomic_sub(window->nr_pages,
656 &ep->rma_info.tcw_total_pages);
657 scif_destroy_window(ep, window);
658 atomic_dec(&ep->rma_info.tcw_refcount);
659 } else {
660 dev_warn(&ep->remote_dev->sdev->dev,
661 "DMA engine hung?\n");
662 }
663 mutex_unlock(&ep->rma_info.rma_lock);
664 goto restart;
665 }
666 spin_unlock(&scif_info.rmalock);
667}
668
669static inline
670void *_get_local_va(off_t off, struct scif_window *window, size_t len)
671{
672 int page_nr = (off - window->offset) >> PAGE_SHIFT;
673 off_t page_off = off & ~PAGE_MASK;
674 void *va = NULL;
675
676 if (window->type == SCIF_WINDOW_SELF) {
677 struct page **pages = window->pinned_pages->pages;
678
679 va = page_address(pages[page_nr]) + page_off;
680 }
681 return va;
682}
683
684static inline
685void *ioremap_remote(off_t off, struct scif_window *window,
686 size_t len, struct scif_dev *dev,
687 struct scif_window_iter *iter)
688{
689 dma_addr_t phys = scif_off_to_dma_addr(window, off, NULL, iter);
690
691 /*
692 * If the DMA address is not card relative then we need the DMA
693 * addresses to be an offset into the bar. The aperture base was already
694 * added so subtract it here since scif_ioremap is going to add it again
695 */
696 if (!scifdev_self(dev) && window->type == SCIF_WINDOW_PEER &&
697 dev->sdev->aper && !dev->sdev->card_rel_da)
698 phys = phys - dev->sdev->aper->pa;
699 return scif_ioremap(phys, len, dev);
700}
701
702static inline void
703iounmap_remote(void *virt, size_t size, struct scif_copy_work *work)
704{
705 scif_iounmap(virt, size, work->remote_dev);
706}
707
708/*
709 * Takes care of ordering issue caused by
710 * 1. Hardware: Only in the case of cpu copy from mgmt node to card
711 * because of WC memory.
712 * 2. Software: If memcpy reorders copy instructions for optimization.
713 * This could happen at both mgmt node and card.
714 */
715static inline void
716scif_ordered_memcpy_toio(char *dst, const char *src, size_t count)
717{
718 if (!count)
719 return;
720
721 memcpy_toio((void __iomem __force *)dst, src, --count);
722 /* Order the last byte with the previous stores */
723 wmb();
724 *(dst + count) = *(src + count);
725}
726
727static inline void scif_unaligned_cpy_toio(char *dst, const char *src,
728 size_t count, bool ordered)
729{
730 if (ordered)
731 scif_ordered_memcpy_toio(dst, src, count);
732 else
733 memcpy_toio((void __iomem __force *)dst, src, count);
734}
735
736static inline
737void scif_ordered_memcpy_fromio(char *dst, const char *src, size_t count)
738{
739 if (!count)
740 return;
741
742 memcpy_fromio(dst, (void __iomem __force *)src, --count);
743 /* Order the last byte with the previous loads */
744 rmb();
745 *(dst + count) = *(src + count);
746}
747
748static inline void scif_unaligned_cpy_fromio(char *dst, const char *src,
749 size_t count, bool ordered)
750{
751 if (ordered)
752 scif_ordered_memcpy_fromio(dst, src, count);
753 else
754 memcpy_fromio(dst, (void __iomem __force *)src, count);
755}
756
757#define SCIF_RMA_ERROR_CODE (~(dma_addr_t)0x0)
758
759/*
760 * scif_off_to_dma_addr:
761 * Obtain the dma_addr given the window and the offset.
762 * @window: Registered window.
763 * @off: Window offset.
764 * @nr_bytes: Return the number of contiguous bytes till next DMA addr index.
765 * @index: Return the index of the dma_addr array found.
766 * @start_off: start offset of index of the dma addr array found.
767 * The nr_bytes provides the callee an estimate of the maximum possible
768 * DMA xfer possible while the index/start_off provide faster lookups
769 * for the next iteration.
770 */
771dma_addr_t scif_off_to_dma_addr(struct scif_window *window, s64 off,
772 size_t *nr_bytes, struct scif_window_iter *iter)
773{
774 int i, page_nr;
775 s64 start, end;
776 off_t page_off;
777
778 if (window->nr_pages == window->nr_contig_chunks) {
779 page_nr = (off - window->offset) >> PAGE_SHIFT;
780 page_off = off & ~PAGE_MASK;
781
782 if (nr_bytes)
783 *nr_bytes = PAGE_SIZE - page_off;
784 return window->dma_addr[page_nr] | page_off;
785 }
786 if (iter) {
787 i = iter->index;
788 start = iter->offset;
789 } else {
790 i = 0;
791 start = window->offset;
792 }
793 for (; i < window->nr_contig_chunks; i++) {
794 end = start + (window->num_pages[i] << PAGE_SHIFT);
795 if (off >= start && off < end) {
796 if (iter) {
797 iter->index = i;
798 iter->offset = start;
799 }
800 if (nr_bytes)
801 *nr_bytes = end - off;
802 return (window->dma_addr[i] + (off - start));
803 }
804 start += (window->num_pages[i] << PAGE_SHIFT);
805 }
806 dev_err(scif_info.mdev.this_device,
807 "%s %d BUG. Addr not found? window %p off 0x%llx\n",
808 __func__, __LINE__, window, off);
809 return SCIF_RMA_ERROR_CODE;
810}
811
812/*
813 * Copy between rma window and temporary buffer
814 */
815static void scif_rma_local_cpu_copy(s64 offset, struct scif_window *window,
816 u8 *temp, size_t rem_len, bool to_temp)
817{
818 void *window_virt;
819 size_t loop_len;
820 int offset_in_page;
821 s64 end_offset;
822
823 offset_in_page = offset & ~PAGE_MASK;
824 loop_len = PAGE_SIZE - offset_in_page;
825
826 if (rem_len < loop_len)
827 loop_len = rem_len;
828
829 window_virt = _get_local_va(offset, window, loop_len);
830 if (!window_virt)
831 return;
832 if (to_temp)
833 memcpy(temp, window_virt, loop_len);
834 else
835 memcpy(window_virt, temp, loop_len);
836
837 offset += loop_len;
838 temp += loop_len;
839 rem_len -= loop_len;
840
841 end_offset = window->offset +
842 (window->nr_pages << PAGE_SHIFT);
843 while (rem_len) {
844 if (offset == end_offset) {
845 window = list_next_entry(window, list);
846 end_offset = window->offset +
847 (window->nr_pages << PAGE_SHIFT);
848 }
849 loop_len = min(PAGE_SIZE, rem_len);
850 window_virt = _get_local_va(offset, window, loop_len);
851 if (!window_virt)
852 return;
853 if (to_temp)
854 memcpy(temp, window_virt, loop_len);
855 else
856 memcpy(window_virt, temp, loop_len);
857 offset += loop_len;
858 temp += loop_len;
859 rem_len -= loop_len;
860 }
861}
862
863/**
864 * scif_rma_completion_cb:
865 * @data: RMA cookie
866 *
867 * RMA interrupt completion callback.
868 */
869static void scif_rma_completion_cb(void *data)
870{
871 struct scif_dma_comp_cb *comp_cb = data;
872
873 /* Free DMA Completion CB. */
874 if (comp_cb->dst_window)
875 scif_rma_local_cpu_copy(comp_cb->dst_offset,
876 comp_cb->dst_window,
877 comp_cb->temp_buf +
878 comp_cb->header_padding,
879 comp_cb->len, false);
880 scif_unmap_single(comp_cb->temp_phys, comp_cb->sdev,
881 SCIF_KMEM_UNALIGNED_BUF_SIZE);
882 if (comp_cb->is_cache)
883 kmem_cache_free(unaligned_cache,
884 comp_cb->temp_buf_to_free);
885 else
886 kfree(comp_cb->temp_buf_to_free);
887}
888
889/* Copies between temporary buffer and offsets provided in work */
890static int
891scif_rma_list_dma_copy_unaligned(struct scif_copy_work *work,
892 u8 *temp, struct dma_chan *chan,
893 bool src_local)
894{
895 struct scif_dma_comp_cb *comp_cb = work->comp_cb;
896 dma_addr_t window_dma_addr, temp_dma_addr;
897 dma_addr_t temp_phys = comp_cb->temp_phys;
898 size_t loop_len, nr_contig_bytes = 0, remaining_len = work->len;
899 int offset_in_ca, ret = 0;
900 s64 end_offset, offset;
901 struct scif_window *window;
902 void *window_virt_addr;
903 size_t tail_len;
904 struct dma_async_tx_descriptor *tx;
905 struct dma_device *dev = chan->device;
906 dma_cookie_t cookie;
907
908 if (src_local) {
909 offset = work->dst_offset;
910 window = work->dst_window;
911 } else {
912 offset = work->src_offset;
913 window = work->src_window;
914 }
915
916 offset_in_ca = offset & (L1_CACHE_BYTES - 1);
917 if (offset_in_ca) {
918 loop_len = L1_CACHE_BYTES - offset_in_ca;
919 loop_len = min(loop_len, remaining_len);
920 window_virt_addr = ioremap_remote(offset, window,
921 loop_len,
922 work->remote_dev,
923 NULL);
924 if (!window_virt_addr)
925 return -ENOMEM;
926 if (src_local)
927 scif_unaligned_cpy_toio(window_virt_addr, temp,
928 loop_len,
929 work->ordered &&
930 !(remaining_len - loop_len));
931 else
932 scif_unaligned_cpy_fromio(temp, window_virt_addr,
933 loop_len, work->ordered &&
934 !(remaining_len - loop_len));
935 iounmap_remote(window_virt_addr, loop_len, work);
936
937 offset += loop_len;
938 temp += loop_len;
939 temp_phys += loop_len;
940 remaining_len -= loop_len;
941 }
942
943 offset_in_ca = offset & ~PAGE_MASK;
944 end_offset = window->offset +
945 (window->nr_pages << PAGE_SHIFT);
946
947 tail_len = remaining_len & (L1_CACHE_BYTES - 1);
948 remaining_len -= tail_len;
949 while (remaining_len) {
950 if (offset == end_offset) {
951 window = list_next_entry(window, list);
952 end_offset = window->offset +
953 (window->nr_pages << PAGE_SHIFT);
954 }
955 if (scif_is_mgmt_node())
956 temp_dma_addr = temp_phys;
957 else
958 /* Fix if we ever enable IOMMU on the card */
959 temp_dma_addr = (dma_addr_t)virt_to_phys(temp);
960 window_dma_addr = scif_off_to_dma_addr(window, offset,
961 &nr_contig_bytes,
962 NULL);
963 loop_len = min(nr_contig_bytes, remaining_len);
964 if (src_local) {
965 if (work->ordered && !tail_len &&
966 !(remaining_len - loop_len) &&
967 loop_len != L1_CACHE_BYTES) {
968 /*
969 * Break up the last chunk of the transfer into
970 * two steps. if there is no tail to guarantee
971 * DMA ordering. SCIF_DMA_POLLING inserts
972 * a status update descriptor in step 1 which
973 * acts as a double sided synchronization fence
974 * for the DMA engine to ensure that the last
975 * cache line in step 2 is updated last.
976 */
977 /* Step 1) DMA: Body Length - L1_CACHE_BYTES. */
978 tx =
979 dev->device_prep_dma_memcpy(chan,
980 window_dma_addr,
981 temp_dma_addr,
982 loop_len -
983 L1_CACHE_BYTES,
984 DMA_PREP_FENCE);
985 if (!tx) {
986 ret = -ENOMEM;
987 goto err;
988 }
989 cookie = tx->tx_submit(tx);
990 if (dma_submit_error(cookie)) {
991 ret = -ENOMEM;
992 goto err;
993 }
994 dma_async_issue_pending(chan);
995 offset += (loop_len - L1_CACHE_BYTES);
996 temp_dma_addr += (loop_len - L1_CACHE_BYTES);
997 window_dma_addr += (loop_len - L1_CACHE_BYTES);
998 remaining_len -= (loop_len - L1_CACHE_BYTES);
999 loop_len = remaining_len;
1000
1001 /* Step 2) DMA: L1_CACHE_BYTES */
1002 tx =
1003 dev->device_prep_dma_memcpy(chan,
1004 window_dma_addr,
1005 temp_dma_addr,
1006 loop_len, 0);
1007 if (!tx) {
1008 ret = -ENOMEM;
1009 goto err;
1010 }
1011 cookie = tx->tx_submit(tx);
1012 if (dma_submit_error(cookie)) {
1013 ret = -ENOMEM;
1014 goto err;
1015 }
1016 dma_async_issue_pending(chan);
1017 } else {
1018 tx =
1019 dev->device_prep_dma_memcpy(chan,
1020 window_dma_addr,
1021 temp_dma_addr,
1022 loop_len, 0);
1023 if (!tx) {
1024 ret = -ENOMEM;
1025 goto err;
1026 }
1027 cookie = tx->tx_submit(tx);
1028 if (dma_submit_error(cookie)) {
1029 ret = -ENOMEM;
1030 goto err;
1031 }
1032 dma_async_issue_pending(chan);
1033 }
1034 } else {
1035 tx = dev->device_prep_dma_memcpy(chan, temp_dma_addr,
1036 window_dma_addr, loop_len, 0);
1037 if (!tx) {
1038 ret = -ENOMEM;
1039 goto err;
1040 }
1041 cookie = tx->tx_submit(tx);
1042 if (dma_submit_error(cookie)) {
1043 ret = -ENOMEM;
1044 goto err;
1045 }
1046 dma_async_issue_pending(chan);
1047 }
1048 if (ret < 0)
1049 goto err;
1050 offset += loop_len;
1051 temp += loop_len;
1052 temp_phys += loop_len;
1053 remaining_len -= loop_len;
1054 offset_in_ca = 0;
1055 }
1056 if (tail_len) {
1057 if (offset == end_offset) {
1058 window = list_next_entry(window, list);
1059 end_offset = window->offset +
1060 (window->nr_pages << PAGE_SHIFT);
1061 }
1062 window_virt_addr = ioremap_remote(offset, window, tail_len,
1063 work->remote_dev,
1064 NULL);
1065 if (!window_virt_addr)
1066 return -ENOMEM;
1067 /*
1068 * The CPU copy for the tail bytes must be initiated only once
1069 * previous DMA transfers for this endpoint have completed
1070 * to guarantee ordering.
1071 */
1072 if (work->ordered) {
1073 struct scif_dev *rdev = work->remote_dev;
1074
1075 ret = scif_drain_dma_intr(rdev->sdev, chan);
1076 if (ret)
1077 return ret;
1078 }
1079 if (src_local)
1080 scif_unaligned_cpy_toio(window_virt_addr, temp,
1081 tail_len, work->ordered);
1082 else
1083 scif_unaligned_cpy_fromio(temp, window_virt_addr,
1084 tail_len, work->ordered);
1085 iounmap_remote(window_virt_addr, tail_len, work);
1086 }
1087 tx = dev->device_prep_dma_memcpy(chan, 0, 0, 0, DMA_PREP_INTERRUPT);
1088 if (!tx) {
1089 ret = -ENOMEM;
1090 return ret;
1091 }
1092 tx->callback = &scif_rma_completion_cb;
1093 tx->callback_param = comp_cb;
1094 cookie = tx->tx_submit(tx);
1095
1096 if (dma_submit_error(cookie)) {
1097 ret = -ENOMEM;
1098 return ret;
1099 }
1100 dma_async_issue_pending(chan);
1101 return 0;
1102err:
1103 dev_err(scif_info.mdev.this_device,
1104 "%s %d Desc Prog Failed ret %d\n",
1105 __func__, __LINE__, ret);
1106 return ret;
1107}
1108
1109/*
1110 * _scif_rma_list_dma_copy_aligned:
1111 *
1112 * Traverse all the windows and perform DMA copy.
1113 */
1114static int _scif_rma_list_dma_copy_aligned(struct scif_copy_work *work,
1115 struct dma_chan *chan)
1116{
1117 dma_addr_t src_dma_addr, dst_dma_addr;
1118 size_t loop_len, remaining_len, src_contig_bytes = 0;
1119 size_t dst_contig_bytes = 0;
1120 struct scif_window_iter src_win_iter;
1121 struct scif_window_iter dst_win_iter;
1122 s64 end_src_offset, end_dst_offset;
1123 struct scif_window *src_window = work->src_window;
1124 struct scif_window *dst_window = work->dst_window;
1125 s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1126 int ret = 0;
1127 struct dma_async_tx_descriptor *tx;
1128 struct dma_device *dev = chan->device;
1129 dma_cookie_t cookie;
1130
1131 remaining_len = work->len;
1132
1133 scif_init_window_iter(src_window, &src_win_iter);
1134 scif_init_window_iter(dst_window, &dst_win_iter);
1135 end_src_offset = src_window->offset +
1136 (src_window->nr_pages << PAGE_SHIFT);
1137 end_dst_offset = dst_window->offset +
1138 (dst_window->nr_pages << PAGE_SHIFT);
1139 while (remaining_len) {
1140 if (src_offset == end_src_offset) {
1141 src_window = list_next_entry(src_window, list);
1142 end_src_offset = src_window->offset +
1143 (src_window->nr_pages << PAGE_SHIFT);
1144 scif_init_window_iter(src_window, &src_win_iter);
1145 }
1146 if (dst_offset == end_dst_offset) {
1147 dst_window = list_next_entry(dst_window, list);
1148 end_dst_offset = dst_window->offset +
1149 (dst_window->nr_pages << PAGE_SHIFT);
1150 scif_init_window_iter(dst_window, &dst_win_iter);
1151 }
1152
1153 /* compute dma addresses for transfer */
1154 src_dma_addr = scif_off_to_dma_addr(src_window, src_offset,
1155 &src_contig_bytes,
1156 &src_win_iter);
1157 dst_dma_addr = scif_off_to_dma_addr(dst_window, dst_offset,
1158 &dst_contig_bytes,
1159 &dst_win_iter);
1160 loop_len = min(src_contig_bytes, dst_contig_bytes);
1161 loop_len = min(loop_len, remaining_len);
1162 if (work->ordered && !(remaining_len - loop_len)) {
1163 /*
1164 * Break up the last chunk of the transfer into two
1165 * steps to ensure that the last byte in step 2 is
1166 * updated last.
1167 */
1168 /* Step 1) DMA: Body Length - 1 */
1169 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1170 src_dma_addr,
1171 loop_len - 1,
1172 DMA_PREP_FENCE);
1173 if (!tx) {
1174 ret = -ENOMEM;
1175 goto err;
1176 }
1177 cookie = tx->tx_submit(tx);
1178 if (dma_submit_error(cookie)) {
1179 ret = -ENOMEM;
1180 goto err;
1181 }
1182 src_offset += (loop_len - 1);
1183 dst_offset += (loop_len - 1);
1184 src_dma_addr += (loop_len - 1);
1185 dst_dma_addr += (loop_len - 1);
1186 remaining_len -= (loop_len - 1);
1187 loop_len = remaining_len;
1188
1189 /* Step 2) DMA: 1 BYTES */
1190 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1191 src_dma_addr, loop_len, 0);
1192 if (!tx) {
1193 ret = -ENOMEM;
1194 goto err;
1195 }
1196 cookie = tx->tx_submit(tx);
1197 if (dma_submit_error(cookie)) {
1198 ret = -ENOMEM;
1199 goto err;
1200 }
1201 dma_async_issue_pending(chan);
1202 } else {
1203 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1204 src_dma_addr, loop_len, 0);
1205 if (!tx) {
1206 ret = -ENOMEM;
1207 goto err;
1208 }
1209 cookie = tx->tx_submit(tx);
1210 if (dma_submit_error(cookie)) {
1211 ret = -ENOMEM;
1212 goto err;
1213 }
1214 }
1215 src_offset += loop_len;
1216 dst_offset += loop_len;
1217 remaining_len -= loop_len;
1218 }
1219 return ret;
1220err:
1221 dev_err(scif_info.mdev.this_device,
1222 "%s %d Desc Prog Failed ret %d\n",
1223 __func__, __LINE__, ret);
1224 return ret;
1225}
1226
1227/*
1228 * scif_rma_list_dma_copy_aligned:
1229 *
1230 * Traverse all the windows and perform DMA copy.
1231 */
1232static int scif_rma_list_dma_copy_aligned(struct scif_copy_work *work,
1233 struct dma_chan *chan)
1234{
1235 dma_addr_t src_dma_addr, dst_dma_addr;
1236 size_t loop_len, remaining_len, tail_len, src_contig_bytes = 0;
1237 size_t dst_contig_bytes = 0;
1238 int src_cache_off;
1239 s64 end_src_offset, end_dst_offset;
1240 struct scif_window_iter src_win_iter;
1241 struct scif_window_iter dst_win_iter;
1242 void *src_virt, *dst_virt;
1243 struct scif_window *src_window = work->src_window;
1244 struct scif_window *dst_window = work->dst_window;
1245 s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1246 int ret = 0;
1247 struct dma_async_tx_descriptor *tx;
1248 struct dma_device *dev = chan->device;
1249 dma_cookie_t cookie;
1250
1251 remaining_len = work->len;
1252 scif_init_window_iter(src_window, &src_win_iter);
1253 scif_init_window_iter(dst_window, &dst_win_iter);
1254
1255 src_cache_off = src_offset & (L1_CACHE_BYTES - 1);
1256 if (src_cache_off != 0) {
1257 /* Head */
1258 loop_len = L1_CACHE_BYTES - src_cache_off;
1259 loop_len = min(loop_len, remaining_len);
1260 src_dma_addr = __scif_off_to_dma_addr(src_window, src_offset);
1261 dst_dma_addr = __scif_off_to_dma_addr(dst_window, dst_offset);
1262 if (src_window->type == SCIF_WINDOW_SELF)
1263 src_virt = _get_local_va(src_offset, src_window,
1264 loop_len);
1265 else
1266 src_virt = ioremap_remote(src_offset, src_window,
1267 loop_len,
1268 work->remote_dev, NULL);
1269 if (!src_virt)
1270 return -ENOMEM;
1271 if (dst_window->type == SCIF_WINDOW_SELF)
1272 dst_virt = _get_local_va(dst_offset, dst_window,
1273 loop_len);
1274 else
1275 dst_virt = ioremap_remote(dst_offset, dst_window,
1276 loop_len,
1277 work->remote_dev, NULL);
1278 if (!dst_virt) {
1279 if (src_window->type != SCIF_WINDOW_SELF)
1280 iounmap_remote(src_virt, loop_len, work);
1281 return -ENOMEM;
1282 }
1283 if (src_window->type == SCIF_WINDOW_SELF)
1284 scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,
1285 remaining_len == loop_len ?
1286 work->ordered : false);
1287 else
1288 scif_unaligned_cpy_fromio(dst_virt, src_virt, loop_len,
1289 remaining_len == loop_len ?
1290 work->ordered : false);
1291 if (src_window->type != SCIF_WINDOW_SELF)
1292 iounmap_remote(src_virt, loop_len, work);
1293 if (dst_window->type != SCIF_WINDOW_SELF)
1294 iounmap_remote(dst_virt, loop_len, work);
1295 src_offset += loop_len;
1296 dst_offset += loop_len;
1297 remaining_len -= loop_len;
1298 }
1299
1300 end_src_offset = src_window->offset +
1301 (src_window->nr_pages << PAGE_SHIFT);
1302 end_dst_offset = dst_window->offset +
1303 (dst_window->nr_pages << PAGE_SHIFT);
1304 tail_len = remaining_len & (L1_CACHE_BYTES - 1);
1305 remaining_len -= tail_len;
1306 while (remaining_len) {
1307 if (src_offset == end_src_offset) {
1308 src_window = list_next_entry(src_window, list);
1309 end_src_offset = src_window->offset +
1310 (src_window->nr_pages << PAGE_SHIFT);
1311 scif_init_window_iter(src_window, &src_win_iter);
1312 }
1313 if (dst_offset == end_dst_offset) {
1314 dst_window = list_next_entry(dst_window, list);
1315 end_dst_offset = dst_window->offset +
1316 (dst_window->nr_pages << PAGE_SHIFT);
1317 scif_init_window_iter(dst_window, &dst_win_iter);
1318 }
1319
1320 /* compute dma addresses for transfer */
1321 src_dma_addr = scif_off_to_dma_addr(src_window, src_offset,
1322 &src_contig_bytes,
1323 &src_win_iter);
1324 dst_dma_addr = scif_off_to_dma_addr(dst_window, dst_offset,
1325 &dst_contig_bytes,
1326 &dst_win_iter);
1327 loop_len = min(src_contig_bytes, dst_contig_bytes);
1328 loop_len = min(loop_len, remaining_len);
1329 if (work->ordered && !tail_len &&
1330 !(remaining_len - loop_len)) {
1331 /*
1332 * Break up the last chunk of the transfer into two
1333 * steps. if there is no tail to gurantee DMA ordering.
1334 * Passing SCIF_DMA_POLLING inserts a status update
1335 * descriptor in step 1 which acts as a double sided
1336 * synchronization fence for the DMA engine to ensure
1337 * that the last cache line in step 2 is updated last.
1338 */
1339 /* Step 1) DMA: Body Length - L1_CACHE_BYTES. */
1340 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1341 src_dma_addr,
1342 loop_len -
1343 L1_CACHE_BYTES,
1344 DMA_PREP_FENCE);
1345 if (!tx) {
1346 ret = -ENOMEM;
1347 goto err;
1348 }
1349 cookie = tx->tx_submit(tx);
1350 if (dma_submit_error(cookie)) {
1351 ret = -ENOMEM;
1352 goto err;
1353 }
1354 dma_async_issue_pending(chan);
1355 src_offset += (loop_len - L1_CACHE_BYTES);
1356 dst_offset += (loop_len - L1_CACHE_BYTES);
1357 src_dma_addr += (loop_len - L1_CACHE_BYTES);
1358 dst_dma_addr += (loop_len - L1_CACHE_BYTES);
1359 remaining_len -= (loop_len - L1_CACHE_BYTES);
1360 loop_len = remaining_len;
1361
1362 /* Step 2) DMA: L1_CACHE_BYTES */
1363 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1364 src_dma_addr,
1365 loop_len, 0);
1366 if (!tx) {
1367 ret = -ENOMEM;
1368 goto err;
1369 }
1370 cookie = tx->tx_submit(tx);
1371 if (dma_submit_error(cookie)) {
1372 ret = -ENOMEM;
1373 goto err;
1374 }
1375 dma_async_issue_pending(chan);
1376 } else {
1377 tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1378 src_dma_addr,
1379 loop_len, 0);
1380 if (!tx) {
1381 ret = -ENOMEM;
1382 goto err;
1383 }
1384 cookie = tx->tx_submit(tx);
1385 if (dma_submit_error(cookie)) {
1386 ret = -ENOMEM;
1387 goto err;
1388 }
1389 dma_async_issue_pending(chan);
1390 }
1391 src_offset += loop_len;
1392 dst_offset += loop_len;
1393 remaining_len -= loop_len;
1394 }
1395 remaining_len = tail_len;
1396 if (remaining_len) {
1397 loop_len = remaining_len;
1398 if (src_offset == end_src_offset)
1399 src_window = list_next_entry(src_window, list);
1400 if (dst_offset == end_dst_offset)
1401 dst_window = list_next_entry(dst_window, list);
1402
1403 src_dma_addr = __scif_off_to_dma_addr(src_window, src_offset);
1404 dst_dma_addr = __scif_off_to_dma_addr(dst_window, dst_offset);
1405 /*
1406 * The CPU copy for the tail bytes must be initiated only once
1407 * previous DMA transfers for this endpoint have completed to
1408 * guarantee ordering.
1409 */
1410 if (work->ordered) {
1411 struct scif_dev *rdev = work->remote_dev;
1412
1413 ret = scif_drain_dma_poll(rdev->sdev, chan);
1414 if (ret)
1415 return ret;
1416 }
1417 if (src_window->type == SCIF_WINDOW_SELF)
1418 src_virt = _get_local_va(src_offset, src_window,
1419 loop_len);
1420 else
1421 src_virt = ioremap_remote(src_offset, src_window,
1422 loop_len,
1423 work->remote_dev, NULL);
1424 if (!src_virt)
1425 return -ENOMEM;
1426
1427 if (dst_window->type == SCIF_WINDOW_SELF)
1428 dst_virt = _get_local_va(dst_offset, dst_window,
1429 loop_len);
1430 else
1431 dst_virt = ioremap_remote(dst_offset, dst_window,
1432 loop_len,
1433 work->remote_dev, NULL);
1434 if (!dst_virt) {
1435 if (src_window->type != SCIF_WINDOW_SELF)
1436 iounmap_remote(src_virt, loop_len, work);
1437 return -ENOMEM;
1438 }
1439
1440 if (src_window->type == SCIF_WINDOW_SELF)
1441 scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,
1442 work->ordered);
1443 else
1444 scif_unaligned_cpy_fromio(dst_virt, src_virt,
1445 loop_len, work->ordered);
1446 if (src_window->type != SCIF_WINDOW_SELF)
1447 iounmap_remote(src_virt, loop_len, work);
1448
1449 if (dst_window->type != SCIF_WINDOW_SELF)
1450 iounmap_remote(dst_virt, loop_len, work);
1451 remaining_len -= loop_len;
1452 }
1453 return ret;
1454err:
1455 dev_err(scif_info.mdev.this_device,
1456 "%s %d Desc Prog Failed ret %d\n",
1457 __func__, __LINE__, ret);
1458 return ret;
1459}
1460
1461/*
1462 * scif_rma_list_cpu_copy:
1463 *
1464 * Traverse all the windows and perform CPU copy.
1465 */
1466static int scif_rma_list_cpu_copy(struct scif_copy_work *work)
1467{
1468 void *src_virt, *dst_virt;
1469 size_t loop_len, remaining_len;
1470 int src_page_off, dst_page_off;
1471 s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1472 struct scif_window *src_window = work->src_window;
1473 struct scif_window *dst_window = work->dst_window;
1474 s64 end_src_offset, end_dst_offset;
1475 int ret = 0;
1476 struct scif_window_iter src_win_iter;
1477 struct scif_window_iter dst_win_iter;
1478
1479 remaining_len = work->len;
1480
1481 scif_init_window_iter(src_window, &src_win_iter);
1482 scif_init_window_iter(dst_window, &dst_win_iter);
1483 while (remaining_len) {
1484 src_page_off = src_offset & ~PAGE_MASK;
1485 dst_page_off = dst_offset & ~PAGE_MASK;
1486 loop_len = min(PAGE_SIZE -
1487 max(src_page_off, dst_page_off),
1488 remaining_len);
1489
1490 if (src_window->type == SCIF_WINDOW_SELF)
1491 src_virt = _get_local_va(src_offset, src_window,
1492 loop_len);
1493 else
1494 src_virt = ioremap_remote(src_offset, src_window,
1495 loop_len,
1496 work->remote_dev,
1497 &src_win_iter);
1498 if (!src_virt) {
1499 ret = -ENOMEM;
1500 goto error;
1501 }
1502
1503 if (dst_window->type == SCIF_WINDOW_SELF)
1504 dst_virt = _get_local_va(dst_offset, dst_window,
1505 loop_len);
1506 else
1507 dst_virt = ioremap_remote(dst_offset, dst_window,
1508 loop_len,
1509 work->remote_dev,
1510 &dst_win_iter);
1511 if (!dst_virt) {
1512 if (src_window->type == SCIF_WINDOW_PEER)
1513 iounmap_remote(src_virt, loop_len, work);
1514 ret = -ENOMEM;
1515 goto error;
1516 }
1517
1518 if (work->loopback) {
1519 memcpy(dst_virt, src_virt, loop_len);
1520 } else {
1521 if (src_window->type == SCIF_WINDOW_SELF)
1522 memcpy_toio((void __iomem __force *)dst_virt,
1523 src_virt, loop_len);
1524 else
1525 memcpy_fromio(dst_virt,
1526 (void __iomem __force *)src_virt,
1527 loop_len);
1528 }
1529 if (src_window->type == SCIF_WINDOW_PEER)
1530 iounmap_remote(src_virt, loop_len, work);
1531
1532 if (dst_window->type == SCIF_WINDOW_PEER)
1533 iounmap_remote(dst_virt, loop_len, work);
1534
1535 src_offset += loop_len;
1536 dst_offset += loop_len;
1537 remaining_len -= loop_len;
1538 if (remaining_len) {
1539 end_src_offset = src_window->offset +
1540 (src_window->nr_pages << PAGE_SHIFT);
1541 end_dst_offset = dst_window->offset +
1542 (dst_window->nr_pages << PAGE_SHIFT);
1543 if (src_offset == end_src_offset) {
1544 src_window = list_next_entry(src_window, list);
1545 scif_init_window_iter(src_window,
1546 &src_win_iter);
1547 }
1548 if (dst_offset == end_dst_offset) {
1549 dst_window = list_next_entry(dst_window, list);
1550 scif_init_window_iter(dst_window,
1551 &dst_win_iter);
1552 }
1553 }
1554 }
1555error:
1556 return ret;
1557}
1558
1559static int scif_rma_list_dma_copy_wrapper(struct scif_endpt *epd,
1560 struct scif_copy_work *work,
1561 struct dma_chan *chan, off_t loffset)
1562{
1563 int src_cache_off, dst_cache_off;
1564 s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1565 u8 *temp = NULL;
1566 bool src_local = true, dst_local = false;
1567 struct scif_dma_comp_cb *comp_cb;
1568 dma_addr_t src_dma_addr, dst_dma_addr;
1569 int err;
1570
1571 if (is_dma_copy_aligned(chan->device, 1, 1, 1))
1572 return _scif_rma_list_dma_copy_aligned(work, chan);
1573
1574 src_cache_off = src_offset & (L1_CACHE_BYTES - 1);
1575 dst_cache_off = dst_offset & (L1_CACHE_BYTES - 1);
1576
1577 if (dst_cache_off == src_cache_off)
1578 return scif_rma_list_dma_copy_aligned(work, chan);
1579
1580 if (work->loopback)
1581 return scif_rma_list_cpu_copy(work);
1582 src_dma_addr = __scif_off_to_dma_addr(work->src_window, src_offset);
1583 dst_dma_addr = __scif_off_to_dma_addr(work->dst_window, dst_offset);
1584 src_local = work->src_window->type == SCIF_WINDOW_SELF;
1585 dst_local = work->dst_window->type == SCIF_WINDOW_SELF;
1586
1587 dst_local = dst_local;
1588 /* Allocate dma_completion cb */
1589 comp_cb = kzalloc(sizeof(*comp_cb), GFP_KERNEL);
1590 if (!comp_cb)
1591 goto error;
1592
1593 work->comp_cb = comp_cb;
1594 comp_cb->cb_cookie = comp_cb;
1595 comp_cb->dma_completion_func = &scif_rma_completion_cb;
1596
1597 if (work->len + (L1_CACHE_BYTES << 1) < SCIF_KMEM_UNALIGNED_BUF_SIZE) {
1598 comp_cb->is_cache = false;
1599 /* Allocate padding bytes to align to a cache line */
1600 temp = kmalloc(work->len + (L1_CACHE_BYTES << 1),
1601 GFP_KERNEL);
1602 if (!temp)
1603 goto free_comp_cb;
1604 comp_cb->temp_buf_to_free = temp;
1605 /* kmalloc(..) does not guarantee cache line alignment */
1606 if (!IS_ALIGNED((u64)temp, L1_CACHE_BYTES))
1607 temp = PTR_ALIGN(temp, L1_CACHE_BYTES);
1608 } else {
1609 comp_cb->is_cache = true;
1610 temp = kmem_cache_alloc(unaligned_cache, GFP_KERNEL);
1611 if (!temp)
1612 goto free_comp_cb;
1613 comp_cb->temp_buf_to_free = temp;
1614 }
1615
1616 if (src_local) {
1617 temp += dst_cache_off;
1618 scif_rma_local_cpu_copy(work->src_offset, work->src_window,
1619 temp, work->len, true);
1620 } else {
1621 comp_cb->dst_window = work->dst_window;
1622 comp_cb->dst_offset = work->dst_offset;
1623 work->src_offset = work->src_offset - src_cache_off;
1624 comp_cb->len = work->len;
1625 work->len = ALIGN(work->len + src_cache_off, L1_CACHE_BYTES);
1626 comp_cb->header_padding = src_cache_off;
1627 }
1628 comp_cb->temp_buf = temp;
1629
1630 err = scif_map_single(&comp_cb->temp_phys, temp,
1631 work->remote_dev, SCIF_KMEM_UNALIGNED_BUF_SIZE);
1632 if (err)
1633 goto free_temp_buf;
1634 comp_cb->sdev = work->remote_dev;
1635 if (scif_rma_list_dma_copy_unaligned(work, temp, chan, src_local) < 0)
1636 goto free_temp_buf;
1637 if (!src_local)
1638 work->fence_type = SCIF_DMA_INTR;
1639 return 0;
1640free_temp_buf:
1641 if (comp_cb->is_cache)
1642 kmem_cache_free(unaligned_cache, comp_cb->temp_buf_to_free);
1643 else
1644 kfree(comp_cb->temp_buf_to_free);
1645free_comp_cb:
1646 kfree(comp_cb);
1647error:
1648 return -ENOMEM;
1649}
1650
1651/**
1652 * scif_rma_copy:
1653 * @epd: end point descriptor.
1654 * @loffset: offset in local registered address space to/from which to copy
1655 * @addr: user virtual address to/from which to copy
1656 * @len: length of range to copy
1657 * @roffset: offset in remote registered address space to/from which to copy
1658 * @flags: flags
1659 * @dir: LOCAL->REMOTE or vice versa.
1660 * @last_chunk: true if this is the last chunk of a larger transfer
1661 *
1662 * Validate parameters, check if src/dst registered ranges requested for copy
1663 * are valid and initiate either CPU or DMA copy.
1664 */
1665static int scif_rma_copy(scif_epd_t epd, off_t loffset, unsigned long addr,
1666 size_t len, off_t roffset, int flags,
1667 enum scif_rma_dir dir, bool last_chunk)
1668{
1669 struct scif_endpt *ep = (struct scif_endpt *)epd;
1670 struct scif_rma_req remote_req;
1671 struct scif_rma_req req;
1672 struct scif_window *local_window = NULL;
1673 struct scif_window *remote_window = NULL;
1674 struct scif_copy_work copy_work;
1675 bool loopback;
1676 int err = 0;
1677 struct dma_chan *chan;
1678 struct scif_mmu_notif *mmn = NULL;
1679 bool cache = false;
1680 struct device *spdev;
1681
1682 err = scif_verify_epd(ep);
1683 if (err)
1684 return err;
1685
1686 if (flags && !(flags & (SCIF_RMA_USECPU | SCIF_RMA_USECACHE |
1687 SCIF_RMA_SYNC | SCIF_RMA_ORDERED)))
1688 return -EINVAL;
1689
1690 loopback = scifdev_self(ep->remote_dev) ? true : false;
1691 copy_work.fence_type = ((flags & SCIF_RMA_SYNC) && last_chunk) ?
1692 SCIF_DMA_POLL : 0;
1693 copy_work.ordered = !!((flags & SCIF_RMA_ORDERED) && last_chunk);
1694
1695 /* Use CPU for Mgmt node <-> Mgmt node copies */
1696 if (loopback && scif_is_mgmt_node()) {
1697 flags |= SCIF_RMA_USECPU;
1698 copy_work.fence_type = 0x0;
1699 }
1700
1701 cache = scif_is_set_reg_cache(flags);
1702
1703 remote_req.out_window = &remote_window;
1704 remote_req.offset = roffset;
1705 remote_req.nr_bytes = len;
1706 /*
1707 * If transfer is from local to remote then the remote window
1708 * must be writeable and vice versa.
1709 */
1710 remote_req.prot = dir == SCIF_LOCAL_TO_REMOTE ? VM_WRITE : VM_READ;
1711 remote_req.type = SCIF_WINDOW_PARTIAL;
1712 remote_req.head = &ep->rma_info.remote_reg_list;
1713
1714 spdev = scif_get_peer_dev(ep->remote_dev);
1715 if (IS_ERR(spdev)) {
1716 err = PTR_ERR(spdev);
1717 return err;
1718 }
1719
1720 if (addr && cache) {
1721 mutex_lock(&ep->rma_info.mmn_lock);
1722 mmn = scif_find_mmu_notifier(current->mm, &ep->rma_info);
1723 if (!mmn)
1724 mmn = scif_add_mmu_notifier(current->mm, ep);
1725 mutex_unlock(&ep->rma_info.mmn_lock);
1726 if (IS_ERR(mmn)) {
1727 scif_put_peer_dev(spdev);
1728 return PTR_ERR(mmn);
1729 }
1730 cache = cache && !scif_rma_tc_can_cache(ep, len);
1731 }
1732 mutex_lock(&ep->rma_info.rma_lock);
1733 if (addr) {
1734 req.out_window = &local_window;
1735 req.nr_bytes = ALIGN(len + (addr & ~PAGE_MASK),
1736 PAGE_SIZE);
1737 req.va_for_temp = addr & PAGE_MASK;
1738 req.prot = (dir == SCIF_LOCAL_TO_REMOTE ?
1739 VM_READ : VM_WRITE | VM_READ);
1740 /* Does a valid local window exist? */
1741 if (mmn) {
1742 spin_lock(&ep->rma_info.tc_lock);
1743 req.head = &mmn->tc_reg_list;
1744 err = scif_query_tcw(ep, &req);
1745 spin_unlock(&ep->rma_info.tc_lock);
1746 }
1747 if (!mmn || err) {
1748 err = scif_register_temp(epd, req.va_for_temp,
1749 req.nr_bytes, req.prot,
1750 &loffset, &local_window);
1751 if (err) {
1752 mutex_unlock(&ep->rma_info.rma_lock);
1753 goto error;
1754 }
1755 if (!cache)
1756 goto skip_cache;
1757 atomic_inc(&ep->rma_info.tcw_refcount);
1758 atomic_add_return(local_window->nr_pages,
1759 &ep->rma_info.tcw_total_pages);
1760 if (mmn) {
1761 spin_lock(&ep->rma_info.tc_lock);
1762 scif_insert_tcw(local_window,
1763 &mmn->tc_reg_list);
1764 spin_unlock(&ep->rma_info.tc_lock);
1765 }
1766 }
1767skip_cache:
1768 loffset = local_window->offset +
1769 (addr - local_window->va_for_temp);
1770 } else {
1771 req.out_window = &local_window;
1772 req.offset = loffset;
1773 /*
1774 * If transfer is from local to remote then the self window
1775 * must be readable and vice versa.
1776 */
1777 req.prot = dir == SCIF_LOCAL_TO_REMOTE ? VM_READ : VM_WRITE;
1778 req.nr_bytes = len;
1779 req.type = SCIF_WINDOW_PARTIAL;
1780 req.head = &ep->rma_info.reg_list;
1781 /* Does a valid local window exist? */
1782 err = scif_query_window(&req);
1783 if (err) {
1784 mutex_unlock(&ep->rma_info.rma_lock);
1785 goto error;
1786 }
1787 }
1788
1789 /* Does a valid remote window exist? */
1790 err = scif_query_window(&remote_req);
1791 if (err) {
1792 mutex_unlock(&ep->rma_info.rma_lock);
1793 goto error;
1794 }
1795
1796 /*
1797 * Prepare copy_work for submitting work to the DMA kernel thread
1798 * or CPU copy routine.
1799 */
1800 copy_work.len = len;
1801 copy_work.loopback = loopback;
1802 copy_work.remote_dev = ep->remote_dev;
1803 if (dir == SCIF_LOCAL_TO_REMOTE) {
1804 copy_work.src_offset = loffset;
1805 copy_work.src_window = local_window;
1806 copy_work.dst_offset = roffset;
1807 copy_work.dst_window = remote_window;
1808 } else {
1809 copy_work.src_offset = roffset;
1810 copy_work.src_window = remote_window;
1811 copy_work.dst_offset = loffset;
1812 copy_work.dst_window = local_window;
1813 }
1814
1815 if (flags & SCIF_RMA_USECPU) {
1816 scif_rma_list_cpu_copy(©_work);
1817 } else {
1818 chan = ep->rma_info.dma_chan;
1819 err = scif_rma_list_dma_copy_wrapper(epd, ©_work,
1820 chan, loffset);
1821 }
1822 if (addr && !cache)
1823 atomic_inc(&ep->rma_info.tw_refcount);
1824
1825 mutex_unlock(&ep->rma_info.rma_lock);
1826
1827 if (last_chunk) {
1828 struct scif_dev *rdev = ep->remote_dev;
1829
1830 if (copy_work.fence_type == SCIF_DMA_POLL)
1831 err = scif_drain_dma_poll(rdev->sdev,
1832 ep->rma_info.dma_chan);
1833 else if (copy_work.fence_type == SCIF_DMA_INTR)
1834 err = scif_drain_dma_intr(rdev->sdev,
1835 ep->rma_info.dma_chan);
1836 }
1837
1838 if (addr && !cache)
1839 scif_queue_for_cleanup(local_window, &scif_info.rma);
1840 scif_put_peer_dev(spdev);
1841 return err;
1842error:
1843 if (err) {
1844 if (addr && local_window && !cache)
1845 scif_destroy_window(ep, local_window);
1846 dev_err(scif_info.mdev.this_device,
1847 "%s %d err %d len 0x%lx\n",
1848 __func__, __LINE__, err, len);
1849 }
1850 scif_put_peer_dev(spdev);
1851 return err;
1852}
1853
1854int scif_readfrom(scif_epd_t epd, off_t loffset, size_t len,
1855 off_t roffset, int flags)
1856{
1857 int err;
1858
1859 dev_dbg(scif_info.mdev.this_device,
1860 "SCIFAPI readfrom: ep %p loffset 0x%lx len 0x%lx offset 0x%lx flags 0x%x\n",
1861 epd, loffset, len, roffset, flags);
1862 if (scif_unaligned(loffset, roffset)) {
1863 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1864 err = scif_rma_copy(epd, loffset, 0x0,
1865 SCIF_MAX_UNALIGNED_BUF_SIZE,
1866 roffset, flags,
1867 SCIF_REMOTE_TO_LOCAL, false);
1868 if (err)
1869 goto readfrom_err;
1870 loffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1871 roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1872 len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1873 }
1874 }
1875 err = scif_rma_copy(epd, loffset, 0x0, len,
1876 roffset, flags, SCIF_REMOTE_TO_LOCAL, true);
1877readfrom_err:
1878 return err;
1879}
1880EXPORT_SYMBOL_GPL(scif_readfrom);
1881
1882int scif_writeto(scif_epd_t epd, off_t loffset, size_t len,
1883 off_t roffset, int flags)
1884{
1885 int err;
1886
1887 dev_dbg(scif_info.mdev.this_device,
1888 "SCIFAPI writeto: ep %p loffset 0x%lx len 0x%lx roffset 0x%lx flags 0x%x\n",
1889 epd, loffset, len, roffset, flags);
1890 if (scif_unaligned(loffset, roffset)) {
1891 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1892 err = scif_rma_copy(epd, loffset, 0x0,
1893 SCIF_MAX_UNALIGNED_BUF_SIZE,
1894 roffset, flags,
1895 SCIF_LOCAL_TO_REMOTE, false);
1896 if (err)
1897 goto writeto_err;
1898 loffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1899 roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1900 len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1901 }
1902 }
1903 err = scif_rma_copy(epd, loffset, 0x0, len,
1904 roffset, flags, SCIF_LOCAL_TO_REMOTE, true);
1905writeto_err:
1906 return err;
1907}
1908EXPORT_SYMBOL_GPL(scif_writeto);
1909
1910int scif_vreadfrom(scif_epd_t epd, void *addr, size_t len,
1911 off_t roffset, int flags)
1912{
1913 int err;
1914
1915 dev_dbg(scif_info.mdev.this_device,
1916 "SCIFAPI vreadfrom: ep %p addr %p len 0x%lx roffset 0x%lx flags 0x%x\n",
1917 epd, addr, len, roffset, flags);
1918 if (scif_unaligned((off_t __force)addr, roffset)) {
1919 if (len > SCIF_MAX_UNALIGNED_BUF_SIZE)
1920 flags &= ~SCIF_RMA_USECACHE;
1921
1922 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1923 err = scif_rma_copy(epd, 0, (u64)addr,
1924 SCIF_MAX_UNALIGNED_BUF_SIZE,
1925 roffset, flags,
1926 SCIF_REMOTE_TO_LOCAL, false);
1927 if (err)
1928 goto vreadfrom_err;
1929 addr += SCIF_MAX_UNALIGNED_BUF_SIZE;
1930 roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1931 len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1932 }
1933 }
1934 err = scif_rma_copy(epd, 0, (u64)addr, len,
1935 roffset, flags, SCIF_REMOTE_TO_LOCAL, true);
1936vreadfrom_err:
1937 return err;
1938}
1939EXPORT_SYMBOL_GPL(scif_vreadfrom);
1940
1941int scif_vwriteto(scif_epd_t epd, void *addr, size_t len,
1942 off_t roffset, int flags)
1943{
1944 int err;
1945
1946 dev_dbg(scif_info.mdev.this_device,
1947 "SCIFAPI vwriteto: ep %p addr %p len 0x%lx roffset 0x%lx flags 0x%x\n",
1948 epd, addr, len, roffset, flags);
1949 if (scif_unaligned((off_t __force)addr, roffset)) {
1950 if (len > SCIF_MAX_UNALIGNED_BUF_SIZE)
1951 flags &= ~SCIF_RMA_USECACHE;
1952
1953 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1954 err = scif_rma_copy(epd, 0, (u64)addr,
1955 SCIF_MAX_UNALIGNED_BUF_SIZE,
1956 roffset, flags,
1957 SCIF_LOCAL_TO_REMOTE, false);
1958 if (err)
1959 goto vwriteto_err;
1960 addr += SCIF_MAX_UNALIGNED_BUF_SIZE;
1961 roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1962 len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1963 }
1964 }
1965 err = scif_rma_copy(epd, 0, (u64)addr, len,
1966 roffset, flags, SCIF_LOCAL_TO_REMOTE, true);
1967vwriteto_err:
1968 return err;
1969}
1970EXPORT_SYMBOL_GPL(scif_vwriteto);