Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Intel MIC Platform Software Stack (MPSS)
4 *
5 * Copyright(c) 2015 Intel Corporation.
6 *
7 * Intel SCIF driver.
8 */
9
10#include "scif_main.h"
11
12/**
13 * scif_recv_mark: Handle SCIF_MARK request
14 * @msg: Interrupt message
15 *
16 * The peer has requested a mark.
17 */
18void scif_recv_mark(struct scif_dev *scifdev, struct scifmsg *msg)
19{
20 struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
21 int mark = 0;
22 int err;
23
24 err = _scif_fence_mark(ep, &mark);
25 if (err)
26 msg->uop = SCIF_MARK_NACK;
27 else
28 msg->uop = SCIF_MARK_ACK;
29 msg->payload[0] = ep->remote_ep;
30 msg->payload[2] = mark;
31 scif_nodeqp_send(ep->remote_dev, msg);
32}
33
34/**
35 * scif_recv_mark_resp: Handle SCIF_MARK_(N)ACK messages.
36 * @msg: Interrupt message
37 *
38 * The peer has responded to a SCIF_MARK message.
39 */
40void scif_recv_mark_resp(struct scif_dev *scifdev, struct scifmsg *msg)
41{
42 struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
43 struct scif_fence_info *fence_req =
44 (struct scif_fence_info *)msg->payload[1];
45
46 mutex_lock(&ep->rma_info.rma_lock);
47 if (msg->uop == SCIF_MARK_ACK) {
48 fence_req->state = OP_COMPLETED;
49 fence_req->dma_mark = (int)msg->payload[2];
50 } else {
51 fence_req->state = OP_FAILED;
52 }
53 mutex_unlock(&ep->rma_info.rma_lock);
54 complete(&fence_req->comp);
55}
56
57/**
58 * scif_recv_wait: Handle SCIF_WAIT request
59 * @msg: Interrupt message
60 *
61 * The peer has requested waiting on a fence.
62 */
63void scif_recv_wait(struct scif_dev *scifdev, struct scifmsg *msg)
64{
65 struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
66 struct scif_remote_fence_info *fence;
67
68 /*
69 * Allocate structure for remote fence information and
70 * send a NACK if the allocation failed. The peer will
71 * return ENOMEM upon receiving a NACK.
72 */
73 fence = kmalloc(sizeof(*fence), GFP_KERNEL);
74 if (!fence) {
75 msg->payload[0] = ep->remote_ep;
76 msg->uop = SCIF_WAIT_NACK;
77 scif_nodeqp_send(ep->remote_dev, msg);
78 return;
79 }
80
81 /* Prepare the fence request */
82 memcpy(&fence->msg, msg, sizeof(struct scifmsg));
83 INIT_LIST_HEAD(&fence->list);
84
85 /* Insert to the global remote fence request list */
86 mutex_lock(&scif_info.fencelock);
87 atomic_inc(&ep->rma_info.fence_refcount);
88 list_add_tail(&fence->list, &scif_info.fence);
89 mutex_unlock(&scif_info.fencelock);
90
91 schedule_work(&scif_info.misc_work);
92}
93
94/**
95 * scif_recv_wait_resp: Handle SCIF_WAIT_(N)ACK messages.
96 * @msg: Interrupt message
97 *
98 * The peer has responded to a SCIF_WAIT message.
99 */
100void scif_recv_wait_resp(struct scif_dev *scifdev, struct scifmsg *msg)
101{
102 struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
103 struct scif_fence_info *fence_req =
104 (struct scif_fence_info *)msg->payload[1];
105
106 mutex_lock(&ep->rma_info.rma_lock);
107 if (msg->uop == SCIF_WAIT_ACK)
108 fence_req->state = OP_COMPLETED;
109 else
110 fence_req->state = OP_FAILED;
111 mutex_unlock(&ep->rma_info.rma_lock);
112 complete(&fence_req->comp);
113}
114
115/**
116 * scif_recv_sig_local: Handle SCIF_SIG_LOCAL request
117 * @msg: Interrupt message
118 *
119 * The peer has requested a signal on a local offset.
120 */
121void scif_recv_sig_local(struct scif_dev *scifdev, struct scifmsg *msg)
122{
123 struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
124 int err;
125
126 err = scif_prog_signal(ep, msg->payload[1], msg->payload[2],
127 SCIF_WINDOW_SELF);
128 if (err)
129 msg->uop = SCIF_SIG_NACK;
130 else
131 msg->uop = SCIF_SIG_ACK;
132 msg->payload[0] = ep->remote_ep;
133 scif_nodeqp_send(ep->remote_dev, msg);
134}
135
136/**
137 * scif_recv_sig_remote: Handle SCIF_SIGNAL_REMOTE request
138 * @msg: Interrupt message
139 *
140 * The peer has requested a signal on a remote offset.
141 */
142void scif_recv_sig_remote(struct scif_dev *scifdev, struct scifmsg *msg)
143{
144 struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
145 int err;
146
147 err = scif_prog_signal(ep, msg->payload[1], msg->payload[2],
148 SCIF_WINDOW_PEER);
149 if (err)
150 msg->uop = SCIF_SIG_NACK;
151 else
152 msg->uop = SCIF_SIG_ACK;
153 msg->payload[0] = ep->remote_ep;
154 scif_nodeqp_send(ep->remote_dev, msg);
155}
156
157/**
158 * scif_recv_sig_resp: Handle SCIF_SIG_(N)ACK messages.
159 * @msg: Interrupt message
160 *
161 * The peer has responded to a signal request.
162 */
163void scif_recv_sig_resp(struct scif_dev *scifdev, struct scifmsg *msg)
164{
165 struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
166 struct scif_fence_info *fence_req =
167 (struct scif_fence_info *)msg->payload[3];
168
169 mutex_lock(&ep->rma_info.rma_lock);
170 if (msg->uop == SCIF_SIG_ACK)
171 fence_req->state = OP_COMPLETED;
172 else
173 fence_req->state = OP_FAILED;
174 mutex_unlock(&ep->rma_info.rma_lock);
175 complete(&fence_req->comp);
176}
177
178static inline void *scif_get_local_va(off_t off, struct scif_window *window)
179{
180 struct page **pages = window->pinned_pages->pages;
181 int page_nr = (off - window->offset) >> PAGE_SHIFT;
182 off_t page_off = off & ~PAGE_MASK;
183
184 return page_address(pages[page_nr]) + page_off;
185}
186
187static void scif_prog_signal_cb(void *arg)
188{
189 struct scif_cb_arg *cb_arg = arg;
190
191 dma_pool_free(cb_arg->ep->remote_dev->signal_pool, cb_arg->status,
192 cb_arg->src_dma_addr);
193 kfree(cb_arg);
194}
195
196static int _scif_prog_signal(scif_epd_t epd, dma_addr_t dst, u64 val)
197{
198 struct scif_endpt *ep = (struct scif_endpt *)epd;
199 struct dma_chan *chan = ep->rma_info.dma_chan;
200 struct dma_device *ddev = chan->device;
201 bool x100 = !is_dma_copy_aligned(chan->device, 1, 1, 1);
202 struct dma_async_tx_descriptor *tx;
203 struct scif_status *status = NULL;
204 struct scif_cb_arg *cb_arg = NULL;
205 dma_addr_t src;
206 dma_cookie_t cookie;
207 int err;
208
209 tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, DMA_PREP_FENCE);
210 if (!tx) {
211 err = -ENOMEM;
212 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
213 __func__, __LINE__, err);
214 goto alloc_fail;
215 }
216 cookie = tx->tx_submit(tx);
217 if (dma_submit_error(cookie)) {
218 err = (int)cookie;
219 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
220 __func__, __LINE__, err);
221 goto alloc_fail;
222 }
223 dma_async_issue_pending(chan);
224 if (x100) {
225 /*
226 * For X100 use the status descriptor to write the value to
227 * the destination.
228 */
229 tx = ddev->device_prep_dma_imm_data(chan, dst, val, 0);
230 } else {
231 status = dma_pool_alloc(ep->remote_dev->signal_pool, GFP_KERNEL,
232 &src);
233 if (!status) {
234 err = -ENOMEM;
235 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
236 __func__, __LINE__, err);
237 goto alloc_fail;
238 }
239 status->val = val;
240 status->src_dma_addr = src;
241 status->ep = ep;
242 src += offsetof(struct scif_status, val);
243 tx = ddev->device_prep_dma_memcpy(chan, dst, src, sizeof(val),
244 DMA_PREP_INTERRUPT);
245 }
246 if (!tx) {
247 err = -ENOMEM;
248 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
249 __func__, __LINE__, err);
250 goto dma_fail;
251 }
252 if (!x100) {
253 cb_arg = kmalloc(sizeof(*cb_arg), GFP_KERNEL);
254 if (!cb_arg) {
255 err = -ENOMEM;
256 goto dma_fail;
257 }
258 cb_arg->src_dma_addr = src;
259 cb_arg->status = status;
260 cb_arg->ep = ep;
261 tx->callback = scif_prog_signal_cb;
262 tx->callback_param = cb_arg;
263 }
264 cookie = tx->tx_submit(tx);
265 if (dma_submit_error(cookie)) {
266 err = -EIO;
267 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
268 __func__, __LINE__, err);
269 goto dma_fail;
270 }
271 dma_async_issue_pending(chan);
272 return 0;
273dma_fail:
274 if (!x100) {
275 dma_pool_free(ep->remote_dev->signal_pool, status,
276 src - offsetof(struct scif_status, val));
277 kfree(cb_arg);
278 }
279alloc_fail:
280 return err;
281}
282
283/*
284 * scif_prog_signal:
285 * @epd - Endpoint Descriptor
286 * @offset - registered address to write @val to
287 * @val - Value to be written at @offset
288 * @type - Type of the window.
289 *
290 * Arrange to write a value to the registered offset after ensuring that the
291 * offset provided is indeed valid.
292 */
293int scif_prog_signal(scif_epd_t epd, off_t offset, u64 val,
294 enum scif_window_type type)
295{
296 struct scif_endpt *ep = (struct scif_endpt *)epd;
297 struct scif_window *window = NULL;
298 struct scif_rma_req req;
299 dma_addr_t dst_dma_addr;
300 int err;
301
302 mutex_lock(&ep->rma_info.rma_lock);
303 req.out_window = &window;
304 req.offset = offset;
305 req.nr_bytes = sizeof(u64);
306 req.prot = SCIF_PROT_WRITE;
307 req.type = SCIF_WINDOW_SINGLE;
308 if (type == SCIF_WINDOW_SELF)
309 req.head = &ep->rma_info.reg_list;
310 else
311 req.head = &ep->rma_info.remote_reg_list;
312 /* Does a valid window exist? */
313 err = scif_query_window(&req);
314 if (err) {
315 dev_err(scif_info.mdev.this_device,
316 "%s %d err %d\n", __func__, __LINE__, err);
317 goto unlock_ret;
318 }
319
320 if (scif_is_mgmt_node() && scifdev_self(ep->remote_dev)) {
321 u64 *dst_virt;
322
323 if (type == SCIF_WINDOW_SELF)
324 dst_virt = scif_get_local_va(offset, window);
325 else
326 dst_virt =
327 scif_get_local_va(offset, (struct scif_window *)
328 window->peer_window);
329 *dst_virt = val;
330 } else {
331 dst_dma_addr = __scif_off_to_dma_addr(window, offset);
332 err = _scif_prog_signal(epd, dst_dma_addr, val);
333 }
334unlock_ret:
335 mutex_unlock(&ep->rma_info.rma_lock);
336 return err;
337}
338
339static int _scif_fence_wait(scif_epd_t epd, int mark)
340{
341 struct scif_endpt *ep = (struct scif_endpt *)epd;
342 dma_cookie_t cookie = mark & ~SCIF_REMOTE_FENCE;
343 int err;
344
345 /* Wait for DMA callback in scif_fence_mark_cb(..) */
346 err = wait_event_interruptible_timeout(ep->rma_info.markwq,
347 dma_async_is_tx_complete(
348 ep->rma_info.dma_chan,
349 cookie, NULL, NULL) ==
350 DMA_COMPLETE,
351 SCIF_NODE_ALIVE_TIMEOUT);
352 if (!err)
353 err = -ETIMEDOUT;
354 else if (err > 0)
355 err = 0;
356 return err;
357}
358
359/**
360 * scif_rma_handle_remote_fences:
361 *
362 * This routine services remote fence requests.
363 */
364void scif_rma_handle_remote_fences(void)
365{
366 struct list_head *item, *tmp;
367 struct scif_remote_fence_info *fence;
368 struct scif_endpt *ep;
369 int mark, err;
370
371 might_sleep();
372 mutex_lock(&scif_info.fencelock);
373 list_for_each_safe(item, tmp, &scif_info.fence) {
374 fence = list_entry(item, struct scif_remote_fence_info,
375 list);
376 /* Remove fence from global list */
377 list_del(&fence->list);
378
379 /* Initiate the fence operation */
380 ep = (struct scif_endpt *)fence->msg.payload[0];
381 mark = fence->msg.payload[2];
382 err = _scif_fence_wait(ep, mark);
383 if (err)
384 fence->msg.uop = SCIF_WAIT_NACK;
385 else
386 fence->msg.uop = SCIF_WAIT_ACK;
387 fence->msg.payload[0] = ep->remote_ep;
388 scif_nodeqp_send(ep->remote_dev, &fence->msg);
389 kfree(fence);
390 if (!atomic_sub_return(1, &ep->rma_info.fence_refcount))
391 schedule_work(&scif_info.misc_work);
392 }
393 mutex_unlock(&scif_info.fencelock);
394}
395
396static int _scif_send_fence(scif_epd_t epd, int uop, int mark, int *out_mark)
397{
398 int err;
399 struct scifmsg msg;
400 struct scif_fence_info *fence_req;
401 struct scif_endpt *ep = (struct scif_endpt *)epd;
402
403 fence_req = kmalloc(sizeof(*fence_req), GFP_KERNEL);
404 if (!fence_req) {
405 err = -ENOMEM;
406 goto error;
407 }
408
409 fence_req->state = OP_IN_PROGRESS;
410 init_completion(&fence_req->comp);
411
412 msg.src = ep->port;
413 msg.uop = uop;
414 msg.payload[0] = ep->remote_ep;
415 msg.payload[1] = (u64)fence_req;
416 if (uop == SCIF_WAIT)
417 msg.payload[2] = mark;
418 spin_lock(&ep->lock);
419 if (ep->state == SCIFEP_CONNECTED)
420 err = scif_nodeqp_send(ep->remote_dev, &msg);
421 else
422 err = -ENOTCONN;
423 spin_unlock(&ep->lock);
424 if (err)
425 goto error_free;
426retry:
427 /* Wait for a SCIF_WAIT_(N)ACK message */
428 err = wait_for_completion_timeout(&fence_req->comp,
429 SCIF_NODE_ALIVE_TIMEOUT);
430 if (!err && scifdev_alive(ep))
431 goto retry;
432 if (!err)
433 err = -ENODEV;
434 if (err > 0)
435 err = 0;
436 mutex_lock(&ep->rma_info.rma_lock);
437 if (err < 0) {
438 if (fence_req->state == OP_IN_PROGRESS)
439 fence_req->state = OP_FAILED;
440 }
441 if (fence_req->state == OP_FAILED && !err)
442 err = -ENOMEM;
443 if (uop == SCIF_MARK && fence_req->state == OP_COMPLETED)
444 *out_mark = SCIF_REMOTE_FENCE | fence_req->dma_mark;
445 mutex_unlock(&ep->rma_info.rma_lock);
446error_free:
447 kfree(fence_req);
448error:
449 return err;
450}
451
452/**
453 * scif_send_fence_mark:
454 * @epd: end point descriptor.
455 * @out_mark: Output DMA mark reported by peer.
456 *
457 * Send a remote fence mark request.
458 */
459static int scif_send_fence_mark(scif_epd_t epd, int *out_mark)
460{
461 return _scif_send_fence(epd, SCIF_MARK, 0, out_mark);
462}
463
464/**
465 * scif_send_fence_wait:
466 * @epd: end point descriptor.
467 * @mark: DMA mark to wait for.
468 *
469 * Send a remote fence wait request.
470 */
471static int scif_send_fence_wait(scif_epd_t epd, int mark)
472{
473 return _scif_send_fence(epd, SCIF_WAIT, mark, NULL);
474}
475
476static int _scif_send_fence_signal_wait(struct scif_endpt *ep,
477 struct scif_fence_info *fence_req)
478{
479 int err;
480
481retry:
482 /* Wait for a SCIF_SIG_(N)ACK message */
483 err = wait_for_completion_timeout(&fence_req->comp,
484 SCIF_NODE_ALIVE_TIMEOUT);
485 if (!err && scifdev_alive(ep))
486 goto retry;
487 if (!err)
488 err = -ENODEV;
489 if (err > 0)
490 err = 0;
491 if (err < 0) {
492 mutex_lock(&ep->rma_info.rma_lock);
493 if (fence_req->state == OP_IN_PROGRESS)
494 fence_req->state = OP_FAILED;
495 mutex_unlock(&ep->rma_info.rma_lock);
496 }
497 if (fence_req->state == OP_FAILED && !err)
498 err = -ENXIO;
499 return err;
500}
501
502/**
503 * scif_send_fence_signal:
504 * @epd - endpoint descriptor
505 * @loff - local offset
506 * @lval - local value to write to loffset
507 * @roff - remote offset
508 * @rval - remote value to write to roffset
509 * @flags - flags
510 *
511 * Sends a remote fence signal request
512 */
513static int scif_send_fence_signal(scif_epd_t epd, off_t roff, u64 rval,
514 off_t loff, u64 lval, int flags)
515{
516 int err = 0;
517 struct scifmsg msg;
518 struct scif_fence_info *fence_req;
519 struct scif_endpt *ep = (struct scif_endpt *)epd;
520
521 fence_req = kmalloc(sizeof(*fence_req), GFP_KERNEL);
522 if (!fence_req) {
523 err = -ENOMEM;
524 goto error;
525 }
526
527 fence_req->state = OP_IN_PROGRESS;
528 init_completion(&fence_req->comp);
529 msg.src = ep->port;
530 if (flags & SCIF_SIGNAL_LOCAL) {
531 msg.uop = SCIF_SIG_LOCAL;
532 msg.payload[0] = ep->remote_ep;
533 msg.payload[1] = roff;
534 msg.payload[2] = rval;
535 msg.payload[3] = (u64)fence_req;
536 spin_lock(&ep->lock);
537 if (ep->state == SCIFEP_CONNECTED)
538 err = scif_nodeqp_send(ep->remote_dev, &msg);
539 else
540 err = -ENOTCONN;
541 spin_unlock(&ep->lock);
542 if (err)
543 goto error_free;
544 err = _scif_send_fence_signal_wait(ep, fence_req);
545 if (err)
546 goto error_free;
547 }
548 fence_req->state = OP_IN_PROGRESS;
549
550 if (flags & SCIF_SIGNAL_REMOTE) {
551 msg.uop = SCIF_SIG_REMOTE;
552 msg.payload[0] = ep->remote_ep;
553 msg.payload[1] = loff;
554 msg.payload[2] = lval;
555 msg.payload[3] = (u64)fence_req;
556 spin_lock(&ep->lock);
557 if (ep->state == SCIFEP_CONNECTED)
558 err = scif_nodeqp_send(ep->remote_dev, &msg);
559 else
560 err = -ENOTCONN;
561 spin_unlock(&ep->lock);
562 if (err)
563 goto error_free;
564 err = _scif_send_fence_signal_wait(ep, fence_req);
565 }
566error_free:
567 kfree(fence_req);
568error:
569 return err;
570}
571
572static void scif_fence_mark_cb(void *arg)
573{
574 struct scif_endpt *ep = (struct scif_endpt *)arg;
575
576 wake_up_interruptible(&ep->rma_info.markwq);
577 atomic_dec(&ep->rma_info.fence_refcount);
578}
579
580/*
581 * _scif_fence_mark:
582 *
583 * @epd - endpoint descriptor
584 * Set up a mark for this endpoint and return the value of the mark.
585 */
586int _scif_fence_mark(scif_epd_t epd, int *mark)
587{
588 struct scif_endpt *ep = (struct scif_endpt *)epd;
589 struct dma_chan *chan = ep->rma_info.dma_chan;
590 struct dma_device *ddev = chan->device;
591 struct dma_async_tx_descriptor *tx;
592 dma_cookie_t cookie;
593 int err;
594
595 tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, DMA_PREP_FENCE);
596 if (!tx) {
597 err = -ENOMEM;
598 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
599 __func__, __LINE__, err);
600 return err;
601 }
602 cookie = tx->tx_submit(tx);
603 if (dma_submit_error(cookie)) {
604 err = (int)cookie;
605 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
606 __func__, __LINE__, err);
607 return err;
608 }
609 dma_async_issue_pending(chan);
610 tx = ddev->device_prep_dma_interrupt(chan, DMA_PREP_INTERRUPT);
611 if (!tx) {
612 err = -ENOMEM;
613 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
614 __func__, __LINE__, err);
615 return err;
616 }
617 tx->callback = scif_fence_mark_cb;
618 tx->callback_param = ep;
619 *mark = cookie = tx->tx_submit(tx);
620 if (dma_submit_error(cookie)) {
621 err = (int)cookie;
622 dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
623 __func__, __LINE__, err);
624 return err;
625 }
626 atomic_inc(&ep->rma_info.fence_refcount);
627 dma_async_issue_pending(chan);
628 return 0;
629}
630
631#define SCIF_LOOPB_MAGIC_MARK 0xdead
632
633int scif_fence_mark(scif_epd_t epd, int flags, int *mark)
634{
635 struct scif_endpt *ep = (struct scif_endpt *)epd;
636 int err = 0;
637
638 dev_dbg(scif_info.mdev.this_device,
639 "SCIFAPI fence_mark: ep %p flags 0x%x mark 0x%x\n",
640 ep, flags, *mark);
641 err = scif_verify_epd(ep);
642 if (err)
643 return err;
644
645 /* Invalid flags? */
646 if (flags & ~(SCIF_FENCE_INIT_SELF | SCIF_FENCE_INIT_PEER))
647 return -EINVAL;
648
649 /* At least one of init self or peer RMA should be set */
650 if (!(flags & (SCIF_FENCE_INIT_SELF | SCIF_FENCE_INIT_PEER)))
651 return -EINVAL;
652
653 /* Exactly one of init self or peer RMA should be set but not both */
654 if ((flags & SCIF_FENCE_INIT_SELF) && (flags & SCIF_FENCE_INIT_PEER))
655 return -EINVAL;
656
657 /*
658 * Management node loopback does not need to use DMA.
659 * Return a valid mark to be symmetric.
660 */
661 if (scifdev_self(ep->remote_dev) && scif_is_mgmt_node()) {
662 *mark = SCIF_LOOPB_MAGIC_MARK;
663 return 0;
664 }
665
666 if (flags & SCIF_FENCE_INIT_SELF)
667 err = _scif_fence_mark(epd, mark);
668 else
669 err = scif_send_fence_mark(ep, mark);
670
671 if (err)
672 dev_err(scif_info.mdev.this_device,
673 "%s %d err %d\n", __func__, __LINE__, err);
674 dev_dbg(scif_info.mdev.this_device,
675 "SCIFAPI fence_mark: ep %p flags 0x%x mark 0x%x err %d\n",
676 ep, flags, *mark, err);
677 return err;
678}
679EXPORT_SYMBOL_GPL(scif_fence_mark);
680
681int scif_fence_wait(scif_epd_t epd, int mark)
682{
683 struct scif_endpt *ep = (struct scif_endpt *)epd;
684 int err = 0;
685
686 dev_dbg(scif_info.mdev.this_device,
687 "SCIFAPI fence_wait: ep %p mark 0x%x\n",
688 ep, mark);
689 err = scif_verify_epd(ep);
690 if (err)
691 return err;
692 /*
693 * Management node loopback does not need to use DMA.
694 * The only valid mark provided is 0 so simply
695 * return success if the mark is valid.
696 */
697 if (scifdev_self(ep->remote_dev) && scif_is_mgmt_node()) {
698 if (mark == SCIF_LOOPB_MAGIC_MARK)
699 return 0;
700 else
701 return -EINVAL;
702 }
703 if (mark & SCIF_REMOTE_FENCE)
704 err = scif_send_fence_wait(epd, mark);
705 else
706 err = _scif_fence_wait(epd, mark);
707 if (err < 0)
708 dev_err(scif_info.mdev.this_device,
709 "%s %d err %d\n", __func__, __LINE__, err);
710 return err;
711}
712EXPORT_SYMBOL_GPL(scif_fence_wait);
713
714int scif_fence_signal(scif_epd_t epd, off_t loff, u64 lval,
715 off_t roff, u64 rval, int flags)
716{
717 struct scif_endpt *ep = (struct scif_endpt *)epd;
718 int err = 0;
719
720 dev_dbg(scif_info.mdev.this_device,
721 "SCIFAPI fence_signal: ep %p loff 0x%lx lval 0x%llx roff 0x%lx rval 0x%llx flags 0x%x\n",
722 ep, loff, lval, roff, rval, flags);
723 err = scif_verify_epd(ep);
724 if (err)
725 return err;
726
727 /* Invalid flags? */
728 if (flags & ~(SCIF_FENCE_INIT_SELF | SCIF_FENCE_INIT_PEER |
729 SCIF_SIGNAL_LOCAL | SCIF_SIGNAL_REMOTE))
730 return -EINVAL;
731
732 /* At least one of init self or peer RMA should be set */
733 if (!(flags & (SCIF_FENCE_INIT_SELF | SCIF_FENCE_INIT_PEER)))
734 return -EINVAL;
735
736 /* Exactly one of init self or peer RMA should be set but not both */
737 if ((flags & SCIF_FENCE_INIT_SELF) && (flags & SCIF_FENCE_INIT_PEER))
738 return -EINVAL;
739
740 /* At least one of SCIF_SIGNAL_LOCAL or SCIF_SIGNAL_REMOTE required */
741 if (!(flags & (SCIF_SIGNAL_LOCAL | SCIF_SIGNAL_REMOTE)))
742 return -EINVAL;
743
744 /* Only Dword offsets allowed */
745 if ((flags & SCIF_SIGNAL_LOCAL) && (loff & (sizeof(u32) - 1)))
746 return -EINVAL;
747
748 /* Only Dword aligned offsets allowed */
749 if ((flags & SCIF_SIGNAL_REMOTE) && (roff & (sizeof(u32) - 1)))
750 return -EINVAL;
751
752 if (flags & SCIF_FENCE_INIT_PEER) {
753 err = scif_send_fence_signal(epd, roff, rval, loff,
754 lval, flags);
755 } else {
756 /* Local Signal in Local RAS */
757 if (flags & SCIF_SIGNAL_LOCAL) {
758 err = scif_prog_signal(epd, loff, lval,
759 SCIF_WINDOW_SELF);
760 if (err)
761 goto error_ret;
762 }
763
764 /* Signal in Remote RAS */
765 if (flags & SCIF_SIGNAL_REMOTE)
766 err = scif_prog_signal(epd, roff,
767 rval, SCIF_WINDOW_PEER);
768 }
769error_ret:
770 if (err)
771 dev_err(scif_info.mdev.this_device,
772 "%s %d err %d\n", __func__, __LINE__, err);
773 return err;
774}
775EXPORT_SYMBOL_GPL(scif_fence_signal);