1// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
  2/*
  3 * Copyright (c) 2020 Intel Corporation. All rights reserved.
  4 */
  5
  6#include <linux/dma-buf.h>
  7#include <linux/dma-resv.h>
  8#include <linux/dma-mapping.h>
  9#include <linux/module.h>
 10
 11#include "uverbs.h"
 12
 13MODULE_IMPORT_NS(DMA_BUF);
 14
 15int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf)
 16{
 17	struct sg_table *sgt;
 18	struct scatterlist *sg;
 19	unsigned long start, end, cur = 0;
 20	unsigned int nmap = 0;
 21	long ret;
 22	int i;
 23
 24	dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv);
 25
 26	if (umem_dmabuf->sgt)
 27		goto wait_fence;
 28
 29	sgt = dma_buf_map_attachment(umem_dmabuf->attach,
 30				     DMA_BIDIRECTIONAL);
 31	if (IS_ERR(sgt))
 32		return PTR_ERR(sgt);
 33
 34	/* modify the sg list in-place to match umem address and length */
 35
 36	start = ALIGN_DOWN(umem_dmabuf->umem.address, PAGE_SIZE);
 37	end = ALIGN(umem_dmabuf->umem.address + umem_dmabuf->umem.length,
 38		    PAGE_SIZE);
 39	for_each_sgtable_dma_sg(sgt, sg, i) {
 40		if (start < cur + sg_dma_len(sg) && cur < end)
 41			nmap++;
 42		if (cur <= start && start < cur + sg_dma_len(sg)) {
 43			unsigned long offset = start - cur;
 44
 45			umem_dmabuf->first_sg = sg;
 46			umem_dmabuf->first_sg_offset = offset;
 47			sg_dma_address(sg) += offset;
 48			sg_dma_len(sg) -= offset;
 49			cur += offset;
 50		}
 51		if (cur < end && end <= cur + sg_dma_len(sg)) {
 52			unsigned long trim = cur + sg_dma_len(sg) - end;
 53
 54			umem_dmabuf->last_sg = sg;
 55			umem_dmabuf->last_sg_trim = trim;
 56			sg_dma_len(sg) -= trim;
 57			break;
 58		}
 59		cur += sg_dma_len(sg);
 60	}
 61
 62	umem_dmabuf->umem.sgt_append.sgt.sgl = umem_dmabuf->first_sg;
 63	umem_dmabuf->umem.sgt_append.sgt.nents = nmap;
 64	umem_dmabuf->sgt = sgt;
 65
 66wait_fence:
 67	/*
 68	 * Although the sg list is valid now, the content of the pages
 69	 * may be not up-to-date. Wait for the exporter to finish
 70	 * the migration.
 71	 */
 72	ret = dma_resv_wait_timeout(umem_dmabuf->attach->dmabuf->resv,
 73				     DMA_RESV_USAGE_KERNEL,
 74				     false, MAX_SCHEDULE_TIMEOUT);
 75	if (ret < 0)
 76		return ret;
 77	if (ret == 0)
 78		return -ETIMEDOUT;
 79	return 0;
 80}
 81EXPORT_SYMBOL(ib_umem_dmabuf_map_pages);
 82
 83void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf)
 84{
 85	dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv);
 86
 87	if (!umem_dmabuf->sgt)
 88		return;
 89
 90	/* retore the original sg list */
 91	if (umem_dmabuf->first_sg) {
 92		sg_dma_address(umem_dmabuf->first_sg) -=
 93			umem_dmabuf->first_sg_offset;
 94		sg_dma_len(umem_dmabuf->first_sg) +=
 95			umem_dmabuf->first_sg_offset;
 96		umem_dmabuf->first_sg = NULL;
 97		umem_dmabuf->first_sg_offset = 0;
 98	}
 99	if (umem_dmabuf->last_sg) {
100		sg_dma_len(umem_dmabuf->last_sg) +=
101			umem_dmabuf->last_sg_trim;
102		umem_dmabuf->last_sg = NULL;
103		umem_dmabuf->last_sg_trim = 0;
104	}
105
106	dma_buf_unmap_attachment(umem_dmabuf->attach, umem_dmabuf->sgt,
107				 DMA_BIDIRECTIONAL);
108
109	umem_dmabuf->sgt = NULL;
110}
111EXPORT_SYMBOL(ib_umem_dmabuf_unmap_pages);
112
113struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device,
114					  unsigned long offset, size_t size,
115					  int fd, int access,
116					  const struct dma_buf_attach_ops *ops)
117{
118	struct dma_buf *dmabuf;
119	struct ib_umem_dmabuf *umem_dmabuf;
120	struct ib_umem *umem;
121	unsigned long end;
122	struct ib_umem_dmabuf *ret = ERR_PTR(-EINVAL);
123
124	if (check_add_overflow(offset, (unsigned long)size, &end))
125		return ret;
126
127	if (unlikely(!ops || !ops->move_notify))
128		return ret;
129
130	dmabuf = dma_buf_get(fd);
131	if (IS_ERR(dmabuf))
132		return ERR_CAST(dmabuf);
133
134	if (dmabuf->size < end)
135		goto out_release_dmabuf;
136
137	umem_dmabuf = kzalloc(sizeof(*umem_dmabuf), GFP_KERNEL);
138	if (!umem_dmabuf) {
139		ret = ERR_PTR(-ENOMEM);
140		goto out_release_dmabuf;
141	}
142
143	umem = &umem_dmabuf->umem;
144	umem->ibdev = device;
145	umem->length = size;
146	umem->address = offset;
147	umem->writable = ib_access_writable(access);
148	umem->is_dmabuf = 1;
149
150	if (!ib_umem_num_pages(umem))
151		goto out_free_umem;
152
153	umem_dmabuf->attach = dma_buf_dynamic_attach(
154					dmabuf,
155					device->dma_device,
156					ops,
157					umem_dmabuf);
158	if (IS_ERR(umem_dmabuf->attach)) {
159		ret = ERR_CAST(umem_dmabuf->attach);
160		goto out_free_umem;
161	}
162	return umem_dmabuf;
163
164out_free_umem:
165	kfree(umem_dmabuf);
166
167out_release_dmabuf:
168	dma_buf_put(dmabuf);
169	return ret;
170}
171EXPORT_SYMBOL(ib_umem_dmabuf_get);
172
173static void
174ib_umem_dmabuf_unsupported_move_notify(struct dma_buf_attachment *attach)
175{
176	struct ib_umem_dmabuf *umem_dmabuf = attach->importer_priv;
177
178	ibdev_warn_ratelimited(umem_dmabuf->umem.ibdev,
179			       "Invalidate callback should not be called when memory is pinned\n");
180}
181
182static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = {
183	.allow_peer2peer = true,
184	.move_notify = ib_umem_dmabuf_unsupported_move_notify,
185};
186
187struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device,
188						 unsigned long offset,
189						 size_t size, int fd,
190						 int access)
191{
192	struct ib_umem_dmabuf *umem_dmabuf;
193	int err;
194
195	umem_dmabuf = ib_umem_dmabuf_get(device, offset, size, fd, access,
196					 &ib_umem_dmabuf_attach_pinned_ops);
197	if (IS_ERR(umem_dmabuf))
198		return umem_dmabuf;
199
200	dma_resv_lock(umem_dmabuf->attach->dmabuf->resv, NULL);
201	err = dma_buf_pin(umem_dmabuf->attach);
202	if (err)
203		goto err_release;
204	umem_dmabuf->pinned = 1;
205
206	err = ib_umem_dmabuf_map_pages(umem_dmabuf);
207	if (err)
208		goto err_unpin;
209	dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv);
210
211	return umem_dmabuf;
212
213err_unpin:
214	dma_buf_unpin(umem_dmabuf->attach);
215err_release:
216	dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv);
217	ib_umem_release(&umem_dmabuf->umem);
218	return ERR_PTR(err);
219}
220EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned);
221
222void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf)
223{
224	struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf;
225
226	dma_resv_lock(dmabuf->resv, NULL);
227	ib_umem_dmabuf_unmap_pages(umem_dmabuf);
228	if (umem_dmabuf->pinned)
229		dma_buf_unpin(umem_dmabuf->attach);
230	dma_resv_unlock(dmabuf->resv);
231
232	dma_buf_detach(dmabuf, umem_dmabuf->attach);
233	dma_buf_put(dmabuf);
234	kfree(umem_dmabuf);
235}