1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Intel MIC Platform Software Stack (MPSS)
  4 *
  5 * Copyright(c) 2015 Intel Corporation.
  6 *
  7 * Intel SCIF driver.
  8 */
  9#include "scif_main.h"
 10#include <linux/mmu_notifier.h>
 11#include <linux/highmem.h>
 12
 13/*
 14 * scif_insert_tcw:
 15 *
 16 * Insert a temp window to the temp registration list sorted by va_for_temp.
 17 * RMA lock must be held.
 18 */
 19void scif_insert_tcw(struct scif_window *window, struct list_head *head)
 20{
 21	struct scif_window *curr = NULL;
 22	struct scif_window *prev = list_entry(head, struct scif_window, list);
 23	struct list_head *item;
 24
 25	INIT_LIST_HEAD(&window->list);
 26	/* Compare with tail and if the entry is new tail add it to the end */
 27	if (!list_empty(head)) {
 28		curr = list_entry(head->prev, struct scif_window, list);
 29		if (curr->va_for_temp < window->va_for_temp) {
 30			list_add_tail(&window->list, head);
 31			return;
 32		}
 33	}
 34	list_for_each(item, head) {
 35		curr = list_entry(item, struct scif_window, list);
 36		if (curr->va_for_temp > window->va_for_temp)
 37			break;
 38		prev = curr;
 39	}
 40	list_add(&window->list, &prev->list);
 41}
 42
 43/*
 44 * scif_insert_window:
 45 *
 46 * Insert a window to the self registration list sorted by offset.
 47 * RMA lock must be held.
 48 */
 49void scif_insert_window(struct scif_window *window, struct list_head *head)
 50{
 51	struct scif_window *curr = NULL, *prev = NULL;
 52	struct list_head *item;
 53
 54	INIT_LIST_HEAD(&window->list);
 55	list_for_each(item, head) {
 56		curr = list_entry(item, struct scif_window, list);
 57		if (curr->offset > window->offset)
 58			break;
 59		prev = curr;
 60	}
 61	if (!prev)
 62		list_add(&window->list, head);
 63	else
 64		list_add(&window->list, &prev->list);
 65	scif_set_window_ref(window, window->nr_pages);
 66}
 67
 68/*
 69 * scif_query_tcw:
 70 *
 71 * Query the temp cached registration list of ep for an overlapping window
 72 * in case of permission mismatch, destroy the previous window. if permissions
 73 * match and overlap is partial, destroy the window but return the new range
 74 * RMA lock must be held.
 75 */
 76int scif_query_tcw(struct scif_endpt *ep, struct scif_rma_req *req)
 77{
 78	struct list_head *item, *temp, *head = req->head;
 79	struct scif_window *window;
 80	u64 start_va_window, start_va_req = req->va_for_temp;
 81	u64 end_va_window, end_va_req = start_va_req + req->nr_bytes;
 82
 83	if (!req->nr_bytes)
 84		return -EINVAL;
 85	/*
 86	 * Avoid traversing the entire list to find out that there
 87	 * is no entry that matches
 88	 */
 89	if (!list_empty(head)) {
 90		window = list_last_entry(head, struct scif_window, list);
 91		end_va_window = window->va_for_temp +
 92			(window->nr_pages << PAGE_SHIFT);
 93		if (start_va_req > end_va_window)
 94			return -ENXIO;
 95	}
 96	list_for_each_safe(item, temp, head) {
 97		window = list_entry(item, struct scif_window, list);
 98		start_va_window = window->va_for_temp;
 99		end_va_window = window->va_for_temp +
100			(window->nr_pages << PAGE_SHIFT);
101		if (start_va_req < start_va_window &&
102		    end_va_req < start_va_window)
103			break;
104		if (start_va_req >= end_va_window)
105			continue;
106		if ((window->prot & req->prot) == req->prot) {
107			if (start_va_req >= start_va_window &&
108			    end_va_req <= end_va_window) {
109				*req->out_window = window;
110				return 0;
111			}
112			/* expand window */
113			if (start_va_req < start_va_window) {
114				req->nr_bytes +=
115					start_va_window - start_va_req;
116				req->va_for_temp = start_va_window;
117			}
118			if (end_va_req >= end_va_window)
119				req->nr_bytes += end_va_window - end_va_req;
120		}
121		/* Destroy the old window to create a new one */
122		__scif_rma_destroy_tcw_helper(window);
123		break;
124	}
125	return -ENXIO;
126}
127
128/*
129 * scif_query_window:
130 *
131 * Query the registration list and check if a valid contiguous
132 * range of windows exist.
133 * RMA lock must be held.
134 */
135int scif_query_window(struct scif_rma_req *req)
136{
137	struct list_head *item;
138	struct scif_window *window;
139	s64 end_offset, offset = req->offset;
140	u64 tmp_min, nr_bytes_left = req->nr_bytes;
141
142	if (!req->nr_bytes)
143		return -EINVAL;
144
145	list_for_each(item, req->head) {
146		window = list_entry(item, struct scif_window, list);
147		end_offset = window->offset +
148			(window->nr_pages << PAGE_SHIFT);
149		if (offset < window->offset)
150			/* Offset not found! */
151			return -ENXIO;
152		if (offset >= end_offset)
153			continue;
154		/* Check read/write protections. */
155		if ((window->prot & req->prot) != req->prot)
156			return -EPERM;
157		if (nr_bytes_left == req->nr_bytes)
158			/* Store the first window */
159			*req->out_window = window;
160		tmp_min = min((u64)end_offset - offset, nr_bytes_left);
161		nr_bytes_left -= tmp_min;
162		offset += tmp_min;
163		/*
164		 * Range requested encompasses
165		 * multiple windows contiguously.
166		 */
167		if (!nr_bytes_left) {
168			/* Done for partial window */
169			if (req->type == SCIF_WINDOW_PARTIAL ||
170			    req->type == SCIF_WINDOW_SINGLE)
171				return 0;
172			/* Extra logic for full windows */
173			if (offset == end_offset)
174				/* Spanning multiple whole windows */
175				return 0;
176				/* Not spanning multiple whole windows */
177			return -ENXIO;
178		}
179		if (req->type == SCIF_WINDOW_SINGLE)
180			break;
181	}
182	dev_err(scif_info.mdev.this_device,
183		"%s %d ENXIO\n", __func__, __LINE__);
184	return -ENXIO;
185}
186
187/*
188 * scif_rma_list_unregister:
189 *
190 * Traverse the self registration list starting from window:
191 * 1) Call scif_unregister_window(..)
192 * RMA lock must be held.
193 */
194int scif_rma_list_unregister(struct scif_window *window,
195			     s64 offset, int nr_pages)
196{
197	struct scif_endpt *ep = (struct scif_endpt *)window->ep;
198	struct list_head *head = &ep->rma_info.reg_list;
199	s64 end_offset;
200	int err = 0;
201	int loop_nr_pages;
202	struct scif_window *_window;
203
204	list_for_each_entry_safe_from(window, _window, head, list) {
205		end_offset = window->offset + (window->nr_pages << PAGE_SHIFT);
206		loop_nr_pages = min((int)((end_offset - offset) >> PAGE_SHIFT),
207				    nr_pages);
208		err = scif_unregister_window(window);
209		if (err)
210			return err;
211		nr_pages -= loop_nr_pages;
212		offset += (loop_nr_pages << PAGE_SHIFT);
213		if (!nr_pages)
214			break;
215	}
216	return 0;
217}
218
219/*
220 * scif_unmap_all_window:
221 *
222 * Traverse all the windows in the self registration list and:
223 * 1) Delete any DMA mappings created
224 */
225void scif_unmap_all_windows(scif_epd_t epd)
226{
227	struct list_head *item, *tmp;
228	struct scif_window *window;
229	struct scif_endpt *ep = (struct scif_endpt *)epd;
230	struct list_head *head = &ep->rma_info.reg_list;
231
232	mutex_lock(&ep->rma_info.rma_lock);
233	list_for_each_safe(item, tmp, head) {
234		window = list_entry(item, struct scif_window, list);
235		scif_unmap_window(ep->remote_dev, window);
236	}
237	mutex_unlock(&ep->rma_info.rma_lock);
238}
239
240/*
241 * scif_unregister_all_window:
242 *
243 * Traverse all the windows in the self registration list and:
244 * 1) Call scif_unregister_window(..)
245 * RMA lock must be held.
246 */
247int scif_unregister_all_windows(scif_epd_t epd)
248{
249	struct list_head *item, *tmp;
250	struct scif_window *window;
251	struct scif_endpt *ep = (struct scif_endpt *)epd;
252	struct list_head *head = &ep->rma_info.reg_list;
253	int err = 0;
254
255	mutex_lock(&ep->rma_info.rma_lock);
256retry:
257	item = NULL;
258	tmp = NULL;
259	list_for_each_safe(item, tmp, head) {
260		window = list_entry(item, struct scif_window, list);
261		ep->rma_info.async_list_del = 0;
262		err = scif_unregister_window(window);
263		if (err)
264			dev_err(scif_info.mdev.this_device,
265				"%s %d err %d\n",
266				__func__, __LINE__, err);
267		/*
268		 * Need to restart list traversal if there has been
269		 * an asynchronous list entry deletion.
270		 */
271		if (READ_ONCE(ep->rma_info.async_list_del))
272			goto retry;
273	}
274	mutex_unlock(&ep->rma_info.rma_lock);
275	if (!list_empty(&ep->rma_info.mmn_list)) {
276		spin_lock(&scif_info.rmalock);
277		list_add_tail(&ep->mmu_list, &scif_info.mmu_notif_cleanup);
278		spin_unlock(&scif_info.rmalock);
279		schedule_work(&scif_info.mmu_notif_work);
280	}
281	return err;
282}