1// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
  2/*
  3 * Copyright (c) 2020 Intel Corporation. All rights reserved.
  4 */
  5
  6#include <linux/dma-buf.h>
  7#include <linux/dma-resv.h>
  8#include <linux/dma-mapping.h>
  9
 10#include "uverbs.h"
 11
 12int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf)
 13{
 14	struct sg_table *sgt;
 15	struct scatterlist *sg;
 16	struct dma_fence *fence;
 17	unsigned long start, end, cur = 0;
 18	unsigned int nmap = 0;
 19	int i;
 20
 21	dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv);
 22
 23	if (umem_dmabuf->sgt)
 24		goto wait_fence;
 25
 26	sgt = dma_buf_map_attachment(umem_dmabuf->attach, DMA_BIDIRECTIONAL);
 27	if (IS_ERR(sgt))
 28		return PTR_ERR(sgt);
 29
 30	/* modify the sg list in-place to match umem address and length */
 31
 32	start = ALIGN_DOWN(umem_dmabuf->umem.address, PAGE_SIZE);
 33	end = ALIGN(umem_dmabuf->umem.address + umem_dmabuf->umem.length,
 34		    PAGE_SIZE);
 35	for_each_sgtable_dma_sg(sgt, sg, i) {
 36		if (start < cur + sg_dma_len(sg) && cur < end)
 37			nmap++;
 38		if (cur <= start && start < cur + sg_dma_len(sg)) {
 39			unsigned long offset = start - cur;
 40
 41			umem_dmabuf->first_sg = sg;
 42			umem_dmabuf->first_sg_offset = offset;
 43			sg_dma_address(sg) += offset;
 44			sg_dma_len(sg) -= offset;
 45			cur += offset;
 46		}
 47		if (cur < end && end <= cur + sg_dma_len(sg)) {
 48			unsigned long trim = cur + sg_dma_len(sg) - end;
 49
 50			umem_dmabuf->last_sg = sg;
 51			umem_dmabuf->last_sg_trim = trim;
 52			sg_dma_len(sg) -= trim;
 53			break;
 54		}
 55		cur += sg_dma_len(sg);
 56	}
 57
 58	umem_dmabuf->umem.sg_head.sgl = umem_dmabuf->first_sg;
 59	umem_dmabuf->umem.sg_head.nents = nmap;
 60	umem_dmabuf->umem.nmap = nmap;
 61	umem_dmabuf->sgt = sgt;
 62
 63wait_fence:
 64	/*
 65	 * Although the sg list is valid now, the content of the pages
 66	 * may be not up-to-date. Wait for the exporter to finish
 67	 * the migration.
 68	 */
 69	fence = dma_resv_excl_fence(umem_dmabuf->attach->dmabuf->resv);
 70	if (fence)
 71		return dma_fence_wait(fence, false);
 72
 73	return 0;
 74}
 75EXPORT_SYMBOL(ib_umem_dmabuf_map_pages);
 76
 77void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf)
 78{
 79	dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv);
 80
 81	if (!umem_dmabuf->sgt)
 82		return;
 83
 84	/* retore the original sg list */
 85	if (umem_dmabuf->first_sg) {
 86		sg_dma_address(umem_dmabuf->first_sg) -=
 87			umem_dmabuf->first_sg_offset;
 88		sg_dma_len(umem_dmabuf->first_sg) +=
 89			umem_dmabuf->first_sg_offset;
 90		umem_dmabuf->first_sg = NULL;
 91		umem_dmabuf->first_sg_offset = 0;
 92	}
 93	if (umem_dmabuf->last_sg) {
 94		sg_dma_len(umem_dmabuf->last_sg) +=
 95			umem_dmabuf->last_sg_trim;
 96		umem_dmabuf->last_sg = NULL;
 97		umem_dmabuf->last_sg_trim = 0;
 98	}
 99
100	dma_buf_unmap_attachment(umem_dmabuf->attach, umem_dmabuf->sgt,
101				 DMA_BIDIRECTIONAL);
102
103	umem_dmabuf->sgt = NULL;
104}
105EXPORT_SYMBOL(ib_umem_dmabuf_unmap_pages);
106
107struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device,
108					  unsigned long offset, size_t size,
109					  int fd, int access,
110					  const struct dma_buf_attach_ops *ops)
111{
112	struct dma_buf *dmabuf;
113	struct ib_umem_dmabuf *umem_dmabuf;
114	struct ib_umem *umem;
115	unsigned long end;
116	struct ib_umem_dmabuf *ret = ERR_PTR(-EINVAL);
117
118	if (check_add_overflow(offset, (unsigned long)size, &end))
119		return ret;
120
121	if (unlikely(!ops || !ops->move_notify))
122		return ret;
123
124	dmabuf = dma_buf_get(fd);
125	if (IS_ERR(dmabuf))
126		return ERR_CAST(dmabuf);
127
128	if (dmabuf->size < end)
129		goto out_release_dmabuf;
130
131	umem_dmabuf = kzalloc(sizeof(*umem_dmabuf), GFP_KERNEL);
132	if (!umem_dmabuf) {
133		ret = ERR_PTR(-ENOMEM);
134		goto out_release_dmabuf;
135	}
136
137	umem = &umem_dmabuf->umem;
138	umem->ibdev = device;
139	umem->length = size;
140	umem->address = offset;
141	umem->writable = ib_access_writable(access);
142	umem->is_dmabuf = 1;
143
144	if (!ib_umem_num_pages(umem))
145		goto out_free_umem;
146
147	umem_dmabuf->attach = dma_buf_dynamic_attach(
148					dmabuf,
149					device->dma_device,
150					ops,
151					umem_dmabuf);
152	if (IS_ERR(umem_dmabuf->attach)) {
153		ret = ERR_CAST(umem_dmabuf->attach);
154		goto out_free_umem;
155	}
156	return umem_dmabuf;
157
158out_free_umem:
159	kfree(umem_dmabuf);
160
161out_release_dmabuf:
162	dma_buf_put(dmabuf);
163	return ret;
164}
165EXPORT_SYMBOL(ib_umem_dmabuf_get);
166
167void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf)
168{
169	struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf;
170
171	dma_resv_lock(dmabuf->resv, NULL);
172	ib_umem_dmabuf_unmap_pages(umem_dmabuf);
173	dma_resv_unlock(dmabuf->resv);
174
175	dma_buf_detach(dmabuf, umem_dmabuf->attach);
176	dma_buf_put(dmabuf);
177	kfree(umem_dmabuf);
178}