1/*
  2 * Copyright (c) 2017, Mellanox Technologies inc.  All rights reserved.
  3 *
  4 * This software is available to you under a choice of one of two
  5 * licenses.  You may choose to be licensed under the terms of the GNU
  6 * General Public License (GPL) Version 2, available from the file
  7 * COPYING in the main directory of this source tree, or the
  8 * OpenIB.org BSD license below:
  9 *
 10 *     Redistribution and use in source and binary forms, with or
 11 *     without modification, are permitted provided that the following
 12 *     conditions are met:
 13 *
 14 *      - Redistributions of source code must retain the above
 15 *        copyright notice, this list of conditions and the following
 16 *        disclaimer.
 17 *
 18 *      - Redistributions in binary form must reproduce the above
 19 *        copyright notice, this list of conditions and the following
 20 *        disclaimer in the documentation and/or other materials
 21 *        provided with the distribution.
 22 *
 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 30 * SOFTWARE.
 31 */
 32
 33#include <rdma/rdma_user_ioctl.h>
 34#include <rdma/uverbs_ioctl.h>
 35#include "rdma_core.h"
 36#include "uverbs.h"
 37
 38struct bundle_alloc_head {
 39	struct bundle_alloc_head *next;
 40	u8 data[];
 41};
 42
 43struct bundle_priv {
 44	/* Must be first */
 45	struct bundle_alloc_head alloc_head;
 46	struct bundle_alloc_head *allocated_mem;
 47	size_t internal_avail;
 48	size_t internal_used;
 49
 50	struct radix_tree_root *radix;
 51	const struct uverbs_api_ioctl_method *method_elm;
 52	void __rcu **radix_slots;
 53	unsigned long radix_slots_len;
 54	u32 method_key;
 55
 56	struct ib_uverbs_attr __user *user_attrs;
 57	struct ib_uverbs_attr *uattrs;
 58
 59	DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
 60	DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);
 61	DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN);
 62
 63	/*
 64	 * Must be last. bundle ends in a flex array which overlaps
 65	 * internal_buffer.
 66	 */
 67	struct uverbs_attr_bundle bundle;
 68	u64 internal_buffer[32];
 69};
 70
 71/*
 72 * Each method has an absolute minimum amount of memory it needs to allocate,
 73 * precompute that amount and determine if the onstack memory can be used or
 74 * if allocation is need.
 75 */
 76void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
 77			      unsigned int num_attrs)
 78{
 79	struct bundle_priv *pbundle;
 80	size_t bundle_size =
 81		offsetof(struct bundle_priv, internal_buffer) +
 82		sizeof(*pbundle->bundle.attrs) * method_elm->key_bitmap_len +
 83		sizeof(*pbundle->uattrs) * num_attrs;
 84
 85	method_elm->use_stack = bundle_size <= sizeof(*pbundle);
 86	method_elm->bundle_size =
 87		ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer));
 88
 89	/* Do not want order-2 allocations for this. */
 90	WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE);
 91}
 92
 93/**
 94 * _uverbs_alloc() - Quickly allocate memory for use with a bundle
 95 * @bundle: The bundle
 96 * @size: Number of bytes to allocate
 97 * @flags: Allocator flags
 98 *
 99 * The bundle allocator is intended for allocations that are connected with
100 * processing the system call related to the bundle. The allocated memory is
101 * always freed once the system call completes, and cannot be freed any other
102 * way.
103 *
104 * This tries to use a small pool of pre-allocated memory for performance.
105 */
106__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
107			     gfp_t flags)
108{
109	struct bundle_priv *pbundle =
110		container_of(bundle, struct bundle_priv, bundle);
111	size_t new_used;
112	void *res;
113
114	if (check_add_overflow(size, pbundle->internal_used, &new_used))
115		return ERR_PTR(-EOVERFLOW);
116
117	if (new_used > pbundle->internal_avail) {
118		struct bundle_alloc_head *buf;
119
120		buf = kvmalloc(struct_size(buf, data, size), flags);
121		if (!buf)
122			return ERR_PTR(-ENOMEM);
123		buf->next = pbundle->allocated_mem;
124		pbundle->allocated_mem = buf;
125		return buf->data;
126	}
127
128	res = (void *)pbundle->internal_buffer + pbundle->internal_used;
129	pbundle->internal_used =
130		ALIGN(new_used, sizeof(*pbundle->internal_buffer));
131	if (want_init_on_alloc(flags))
132		memset(res, 0, size);
133	return res;
134}
135EXPORT_SYMBOL(_uverbs_alloc);
136
137static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
138				   u16 len)
139{
140	if (uattr->len > sizeof_field(struct ib_uverbs_attr, data))
141		return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len,
142					    uattr->len - len);
143
144	return !memchr_inv((const void *)&uattr->data + len,
145			   0, uattr->len - len);
146}
147
148static int uverbs_set_output(const struct uverbs_attr_bundle *bundle,
149			     const struct uverbs_attr *attr)
150{
151	struct bundle_priv *pbundle =
152		container_of(bundle, struct bundle_priv, bundle);
153	u16 flags;
154
155	flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags |
156		UVERBS_ATTR_F_VALID_OUTPUT;
157	if (put_user(flags,
158		     &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags))
159		return -EFAULT;
160	return 0;
161}
162
163static int uverbs_process_idrs_array(struct bundle_priv *pbundle,
164				     const struct uverbs_api_attr *attr_uapi,
165				     struct uverbs_objs_arr_attr *attr,
166				     struct ib_uverbs_attr *uattr,
167				     u32 attr_bkey)
168{
169	const struct uverbs_attr_spec *spec = &attr_uapi->spec;
170	size_t array_len;
171	u32 *idr_vals;
172	int ret = 0;
173	size_t i;
174
175	if (uattr->attr_data.reserved)
176		return -EINVAL;
177
178	if (uattr->len % sizeof(u32))
179		return -EINVAL;
180
181	array_len = uattr->len / sizeof(u32);
182	if (array_len < spec->u2.objs_arr.min_len ||
183	    array_len > spec->u2.objs_arr.max_len)
184		return -EINVAL;
185
186	attr->uobjects =
187		uverbs_alloc(&pbundle->bundle,
188			     array_size(array_len, sizeof(*attr->uobjects)));
189	if (IS_ERR(attr->uobjects))
190		return PTR_ERR(attr->uobjects);
191
192	/*
193	 * Since idr is 4B and *uobjects is >= 4B, we can use attr->uobjects
194	 * to store idrs array and avoid additional memory allocation. The
195	 * idrs array is offset to the end of the uobjects array so we will be
196	 * able to read idr and replace with a pointer.
197	 */
198	idr_vals = (u32 *)(attr->uobjects + array_len) - array_len;
199
200	if (uattr->len > sizeof(uattr->data)) {
201		ret = copy_from_user(idr_vals, u64_to_user_ptr(uattr->data),
202				     uattr->len);
203		if (ret)
204			return -EFAULT;
205	} else {
206		memcpy(idr_vals, &uattr->data, uattr->len);
207	}
208
209	for (i = 0; i != array_len; i++) {
210		attr->uobjects[i] = uverbs_get_uobject_from_file(
211			spec->u2.objs_arr.obj_type, spec->u2.objs_arr.access,
212			idr_vals[i], &pbundle->bundle);
213		if (IS_ERR(attr->uobjects[i])) {
214			ret = PTR_ERR(attr->uobjects[i]);
215			break;
216		}
217	}
218
219	attr->len = i;
220	__set_bit(attr_bkey, pbundle->spec_finalize);
221	return ret;
222}
223
224static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,
225				   struct uverbs_objs_arr_attr *attr,
226				   bool commit,
227				   struct uverbs_attr_bundle *attrs)
228{
229	const struct uverbs_attr_spec *spec = &attr_uapi->spec;
 
 
230	size_t i;
231
232	for (i = 0; i != attr->len; i++)
233		uverbs_finalize_object(attr->uobjects[i],
234				       spec->u2.objs_arr.access, false, commit,
235				       attrs);
 
 
 
 
 
236}
237
238static int uverbs_process_attr(struct bundle_priv *pbundle,
239			       const struct uverbs_api_attr *attr_uapi,
240			       struct ib_uverbs_attr *uattr, u32 attr_bkey)
241{
242	const struct uverbs_attr_spec *spec = &attr_uapi->spec;
243	struct uverbs_attr *e = &pbundle->bundle.attrs[attr_bkey];
244	const struct uverbs_attr_spec *val_spec = spec;
245	struct uverbs_obj_attr *o_attr;
246
247	switch (spec->type) {
248	case UVERBS_ATTR_TYPE_ENUM_IN:
249		if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems)
250			return -EOPNOTSUPP;
251
252		if (uattr->attr_data.enum_data.reserved)
253			return -EINVAL;
254
255		val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id];
256
257		/* Currently we only support PTR_IN based enums */
258		if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN)
259			return -EOPNOTSUPP;
260
261		e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id;
262		fallthrough;
263	case UVERBS_ATTR_TYPE_PTR_IN:
264		/* Ensure that any data provided by userspace beyond the known
265		 * struct is zero. Userspace that knows how to use some future
266		 * longer struct will fail here if used with an old kernel and
267		 * non-zero content, making ABI compat/discovery simpler.
268		 */
269		if (uattr->len > val_spec->u.ptr.len &&
270		    val_spec->zero_trailing &&
271		    !uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len))
272			return -EOPNOTSUPP;
273
274		fallthrough;
275	case UVERBS_ATTR_TYPE_PTR_OUT:
276		if (uattr->len < val_spec->u.ptr.min_len ||
277		    (!val_spec->zero_trailing &&
278		     uattr->len > val_spec->u.ptr.len))
279			return -EINVAL;
280
281		if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN &&
282		    uattr->attr_data.reserved)
283			return -EINVAL;
284
285		e->ptr_attr.uattr_idx = uattr - pbundle->uattrs;
286		e->ptr_attr.len = uattr->len;
287
288		if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) {
289			void *p;
290
291			p = uverbs_alloc(&pbundle->bundle, uattr->len);
292			if (IS_ERR(p))
293				return PTR_ERR(p);
294
295			e->ptr_attr.ptr = p;
296
297			if (copy_from_user(p, u64_to_user_ptr(uattr->data),
298					   uattr->len))
299				return -EFAULT;
300		} else {
301			e->ptr_attr.data = uattr->data;
302		}
303		break;
304
305	case UVERBS_ATTR_TYPE_IDR:
306	case UVERBS_ATTR_TYPE_FD:
307		if (uattr->attr_data.reserved)
308			return -EINVAL;
309
310		if (uattr->len != 0)
311			return -EINVAL;
312
313		o_attr = &e->obj_attr;
314		o_attr->attr_elm = attr_uapi;
315
316		/*
317		 * The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and
318		 * s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64
319		 * here without caring about truncation as we know that the
320		 * IDR implementation today rejects negative IDs
321		 */
322		o_attr->uobject = uverbs_get_uobject_from_file(
323			spec->u.obj.obj_type, spec->u.obj.access,
324			uattr->data_s64, &pbundle->bundle);
325		if (IS_ERR(o_attr->uobject))
326			return PTR_ERR(o_attr->uobject);
327		__set_bit(attr_bkey, pbundle->uobj_finalize);
328
329		if (spec->u.obj.access == UVERBS_ACCESS_NEW) {
330			unsigned int uattr_idx = uattr - pbundle->uattrs;
331			s64 id = o_attr->uobject->id;
332
333			/* Copy the allocated id to the user-space */
334			if (put_user(id, &pbundle->user_attrs[uattr_idx].data))
335				return -EFAULT;
336		}
337
338		break;
339
340	case UVERBS_ATTR_TYPE_RAW_FD:
341		if (uattr->attr_data.reserved || uattr->len != 0 ||
342		    uattr->data_s64 < INT_MIN || uattr->data_s64 > INT_MAX)
343			return -EINVAL;
344		/* _uverbs_get_const_signed() is the accessor */
345		e->ptr_attr.data = uattr->data_s64;
346		break;
347
348	case UVERBS_ATTR_TYPE_IDRS_ARRAY:
349		return uverbs_process_idrs_array(pbundle, attr_uapi,
350						 &e->objs_arr_attr, uattr,
351						 attr_bkey);
352	default:
353		return -EOPNOTSUPP;
354	}
355
356	return 0;
357}
358
359/*
360 * We search the radix tree with the method prefix and now we want to fast
361 * search the suffix bits to get a particular attribute pointer. It is not
362 * totally clear to me if this breaks the radix tree encasulation or not, but
363 * it uses the iter data to determine if the method iter points at the same
364 * chunk that will store the attribute, if so it just derefs it directly. By
365 * construction in most kernel configs the method and attrs will all fit in a
366 * single radix chunk, so in most cases this will have no search. Other cases
367 * this falls back to a full search.
368 */
369static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle,
370					     u32 attr_key)
371{
372	void __rcu **slot;
373
374	if (likely(attr_key < pbundle->radix_slots_len)) {
375		void *entry;
376
377		slot = pbundle->radix_slots + attr_key;
378		entry = rcu_dereference_raw(*slot);
379		if (likely(!radix_tree_is_internal_node(entry) && entry))
380			return slot;
381	}
382
383	return radix_tree_lookup_slot(pbundle->radix,
384				      pbundle->method_key | attr_key);
385}
386
387static int uverbs_set_attr(struct bundle_priv *pbundle,
388			   struct ib_uverbs_attr *uattr)
389{
390	u32 attr_key = uapi_key_attr(uattr->attr_id);
391	u32 attr_bkey = uapi_bkey_attr(attr_key);
392	const struct uverbs_api_attr *attr;
393	void __rcu **slot;
394	int ret;
395
396	slot = uapi_get_attr_for_method(pbundle, attr_key);
397	if (!slot) {
398		/*
399		 * Kernel does not support the attribute but user-space says it
400		 * is mandatory
401		 */
402		if (uattr->flags & UVERBS_ATTR_F_MANDATORY)
403			return -EPROTONOSUPPORT;
404		return 0;
405	}
406	attr = rcu_dereference_protected(*slot, true);
407
408	/* Reject duplicate attributes from user-space */
409	if (test_bit(attr_bkey, pbundle->bundle.attr_present))
410		return -EINVAL;
411
412	ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey);
413	if (ret)
414		return ret;
415
416	__set_bit(attr_bkey, pbundle->bundle.attr_present);
417
418	return 0;
419}
420
421static int ib_uverbs_run_method(struct bundle_priv *pbundle,
422				unsigned int num_attrs)
423{
424	int (*handler)(struct uverbs_attr_bundle *attrs);
425	size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs);
426	unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey;
427	unsigned int i;
428	int ret;
429
430	/* See uverbs_disassociate_api() */
431	handler = srcu_dereference(
432		pbundle->method_elm->handler,
433		&pbundle->bundle.ufile->device->disassociate_srcu);
434	if (!handler)
435		return -EIO;
436
437	pbundle->uattrs = uverbs_alloc(&pbundle->bundle, uattrs_size);
438	if (IS_ERR(pbundle->uattrs))
439		return PTR_ERR(pbundle->uattrs);
440	if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size))
441		return -EFAULT;
442
443	for (i = 0; i != num_attrs; i++) {
444		ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]);
445		if (unlikely(ret))
446			return ret;
447	}
448
449	/* User space did not provide all the mandatory attributes */
450	if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory,
451				    pbundle->bundle.attr_present,
452				    pbundle->method_elm->key_bitmap_len)))
453		return -EINVAL;
454
455	if (pbundle->method_elm->has_udata)
456		uverbs_fill_udata(&pbundle->bundle,
457				  &pbundle->bundle.driver_udata,
458				  UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT);
459	else
460		pbundle->bundle.driver_udata = (struct ib_udata){};
461
462	if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) {
463		struct uverbs_obj_attr *destroy_attr =
464			&pbundle->bundle.attrs[destroy_bkey].obj_attr;
465
466		ret = uobj_destroy(destroy_attr->uobject, &pbundle->bundle);
467		if (ret)
468			return ret;
469		__clear_bit(destroy_bkey, pbundle->uobj_finalize);
470
471		ret = handler(&pbundle->bundle);
472		uobj_put_destroy(destroy_attr->uobject);
473	} else {
474		ret = handler(&pbundle->bundle);
475	}
476
477	/*
478	 * Until the drivers are revised to use the bundle directly we have to
479	 * assume that the driver wrote to its UHW_OUT and flag userspace
480	 * appropriately.
481	 */
482	if (!ret && pbundle->method_elm->has_udata) {
483		const struct uverbs_attr *attr =
484			uverbs_attr_get(&pbundle->bundle, UVERBS_ATTR_UHW_OUT);
485
486		if (!IS_ERR(attr))
487			ret = uverbs_set_output(&pbundle->bundle, attr);
488	}
489
490	/*
491	 * EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can
492	 * not invoke the method because the request is not supported.  No
493	 * other cases should return this code.
494	 */
495	if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT))
496		return -EINVAL;
497
498	return ret;
499}
500
501static void bundle_destroy(struct bundle_priv *pbundle, bool commit)
502{
503	unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len;
504	struct bundle_alloc_head *memblock;
505	unsigned int i;
 
506
507	/* fast path for simple uobjects */
508	i = -1;
509	while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len,
510				  i + 1)) < key_bitmap_len) {
511		struct uverbs_attr *attr = &pbundle->bundle.attrs[i];
 
512
513		uverbs_finalize_object(
514			attr->obj_attr.uobject,
515			attr->obj_attr.attr_elm->spec.u.obj.access,
516			test_bit(i, pbundle->uobj_hw_obj_valid),
517			commit,
518			&pbundle->bundle);
 
 
519	}
520
521	i = -1;
522	while ((i = find_next_bit(pbundle->spec_finalize, key_bitmap_len,
523				  i + 1)) < key_bitmap_len) {
524		struct uverbs_attr *attr = &pbundle->bundle.attrs[i];
525		const struct uverbs_api_attr *attr_uapi;
526		void __rcu **slot;
 
527
528		slot = uapi_get_attr_for_method(
529			pbundle,
530			pbundle->method_key | uapi_bkey_to_key_attr(i));
531		if (WARN_ON(!slot))
532			continue;
533
534		attr_uapi = rcu_dereference_protected(*slot, true);
535
536		if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
537			uverbs_free_idrs_array(attr_uapi, &attr->objs_arr_attr,
538					       commit, &pbundle->bundle);
 
 
 
539		}
540	}
541
542	for (memblock = pbundle->allocated_mem; memblock;) {
543		struct bundle_alloc_head *tmp = memblock;
544
545		memblock = memblock->next;
546		kvfree(tmp);
547	}
 
 
548}
549
550static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
551			       struct ib_uverbs_ioctl_hdr *hdr,
552			       struct ib_uverbs_attr __user *user_attrs)
553{
554	const struct uverbs_api_ioctl_method *method_elm;
555	struct uverbs_api *uapi = ufile->device->uapi;
556	struct radix_tree_iter attrs_iter;
557	struct bundle_priv *pbundle;
558	struct bundle_priv onstack;
559	void __rcu **slot;
 
560	int ret;
561
562	if (unlikely(hdr->driver_id != uapi->driver_id))
563		return -EINVAL;
564
565	slot = radix_tree_iter_lookup(
566		&uapi->radix, &attrs_iter,
567		uapi_key_obj(hdr->object_id) |
568			uapi_key_ioctl_method(hdr->method_id));
569	if (unlikely(!slot))
570		return -EPROTONOSUPPORT;
571	method_elm = rcu_dereference_protected(*slot, true);
572
573	if (!method_elm->use_stack) {
574		pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL);
575		if (!pbundle)
576			return -ENOMEM;
577		pbundle->internal_avail =
578			method_elm->bundle_size -
579			offsetof(struct bundle_priv, internal_buffer);
580		pbundle->alloc_head.next = NULL;
581		pbundle->allocated_mem = &pbundle->alloc_head;
582	} else {
583		pbundle = &onstack;
584		pbundle->internal_avail = sizeof(pbundle->internal_buffer);
585		pbundle->allocated_mem = NULL;
586	}
587
588	/* Space for the pbundle->bundle.attrs flex array */
589	pbundle->method_elm = method_elm;
590	pbundle->method_key = attrs_iter.index;
591	pbundle->bundle.ufile = ufile;
592	pbundle->bundle.context = NULL; /* only valid if bundle has uobject */
593	pbundle->radix = &uapi->radix;
594	pbundle->radix_slots = slot;
595	pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter);
596	pbundle->user_attrs = user_attrs;
597
598	pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len *
599					       sizeof(*pbundle->bundle.attrs),
600				       sizeof(*pbundle->internal_buffer));
601	memset(pbundle->bundle.attr_present, 0,
602	       sizeof(pbundle->bundle.attr_present));
603	memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));
604	memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize));
605	memset(pbundle->uobj_hw_obj_valid, 0,
606	       sizeof(pbundle->uobj_hw_obj_valid));
607
608	ret = ib_uverbs_run_method(pbundle, hdr->num_attrs);
609	bundle_destroy(pbundle, ret == 0);
 
 
 
610	return ret;
611}
612
613long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
614{
615	struct ib_uverbs_file *file = filp->private_data;
616	struct ib_uverbs_ioctl_hdr __user *user_hdr =
617		(struct ib_uverbs_ioctl_hdr __user *)arg;
618	struct ib_uverbs_ioctl_hdr hdr;
619	int srcu_key;
620	int err;
621
622	if (unlikely(cmd != RDMA_VERBS_IOCTL))
623		return -ENOIOCTLCMD;
624
625	err = copy_from_user(&hdr, user_hdr, sizeof(hdr));
626	if (err)
627		return -EFAULT;
628
629	if (hdr.length > PAGE_SIZE ||
630	    hdr.length != struct_size(&hdr, attrs, hdr.num_attrs))
631		return -EINVAL;
632
633	if (hdr.reserved1 || hdr.reserved2)
634		return -EPROTONOSUPPORT;
635
636	srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
637	err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs);
638	srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
639	return err;
640}
641
642int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
643		       size_t idx, u64 allowed_bits)
644{
645	const struct uverbs_attr *attr;
646	u64 flags;
647
648	attr = uverbs_attr_get(attrs_bundle, idx);
649	/* Missing attribute means 0 flags */
650	if (IS_ERR(attr)) {
651		*to = 0;
652		return 0;
653	}
654
655	/*
656	 * New userspace code should use 8 bytes to pass flags, but we
657	 * transparently support old userspaces that were using 4 bytes as
658	 * well.
659	 */
660	if (attr->ptr_attr.len == 8)
661		flags = attr->ptr_attr.data;
662	else if (attr->ptr_attr.len == 4)
663		flags = *(u32 *)&attr->ptr_attr.data;
664	else
665		return -EINVAL;
666
667	if (flags & ~allowed_bits)
668		return -EINVAL;
669
670	*to = flags;
671	return 0;
672}
673EXPORT_SYMBOL(uverbs_get_flags64);
674
675int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
676		       size_t idx, u64 allowed_bits)
677{
678	u64 flags;
679	int ret;
680
681	ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits);
682	if (ret)
683		return ret;
684
685	if (flags > U32_MAX)
686		return -EINVAL;
687	*to = flags;
688
689	return 0;
690}
691EXPORT_SYMBOL(uverbs_get_flags32);
692
693/*
694 * Fill a ib_udata struct (core or uhw) using the given attribute IDs.
695 * This is primarily used to convert the UVERBS_ATTR_UHW() into the
696 * ib_udata format used by the drivers.
697 */
698void uverbs_fill_udata(struct uverbs_attr_bundle *bundle,
699		       struct ib_udata *udata, unsigned int attr_in,
700		       unsigned int attr_out)
701{
702	struct bundle_priv *pbundle =
703		container_of(bundle, struct bundle_priv, bundle);
704	const struct uverbs_attr *in =
705		uverbs_attr_get(&pbundle->bundle, attr_in);
706	const struct uverbs_attr *out =
707		uverbs_attr_get(&pbundle->bundle, attr_out);
708
709	if (!IS_ERR(in)) {
710		udata->inlen = in->ptr_attr.len;
711		if (uverbs_attr_ptr_is_inline(in))
712			udata->inbuf =
713				&pbundle->user_attrs[in->ptr_attr.uattr_idx]
714					 .data;
715		else
716			udata->inbuf = u64_to_user_ptr(in->ptr_attr.data);
717	} else {
718		udata->inbuf = NULL;
719		udata->inlen = 0;
720	}
721
722	if (!IS_ERR(out)) {
723		udata->outbuf = u64_to_user_ptr(out->ptr_attr.data);
724		udata->outlen = out->ptr_attr.len;
725	} else {
726		udata->outbuf = NULL;
727		udata->outlen = 0;
728	}
729}
730
731int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx,
732		   const void *from, size_t size)
733{
734	const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
735	size_t min_size;
736
737	if (IS_ERR(attr))
738		return PTR_ERR(attr);
739
740	min_size = min_t(size_t, attr->ptr_attr.len, size);
741	if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size))
742		return -EFAULT;
743
744	return uverbs_set_output(bundle, attr);
745}
746EXPORT_SYMBOL(uverbs_copy_to);
747
748
749/*
750 * This is only used if the caller has directly used copy_to_use to write the
751 * data.  It signals to user space that the buffer is filled in.
752 */
753int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx)
754{
755	const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
756
757	if (IS_ERR(attr))
758		return PTR_ERR(attr);
759
760	return uverbs_set_output(bundle, attr);
761}
762
763int _uverbs_get_const_signed(s64 *to,
764			     const struct uverbs_attr_bundle *attrs_bundle,
765			     size_t idx, s64 lower_bound, u64 upper_bound,
766			     s64  *def_val)
767{
768	const struct uverbs_attr *attr;
769
770	attr = uverbs_attr_get(attrs_bundle, idx);
771	if (IS_ERR(attr)) {
772		if ((PTR_ERR(attr) != -ENOENT) || !def_val)
773			return PTR_ERR(attr);
774
775		*to = *def_val;
776	} else {
777		*to = attr->ptr_attr.data;
778	}
779
780	if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound))
781		return -EINVAL;
782
783	return 0;
784}
785EXPORT_SYMBOL(_uverbs_get_const_signed);
786
787int _uverbs_get_const_unsigned(u64 *to,
788			       const struct uverbs_attr_bundle *attrs_bundle,
789			       size_t idx, u64 upper_bound, u64 *def_val)
790{
791	const struct uverbs_attr *attr;
792
793	attr = uverbs_attr_get(attrs_bundle, idx);
794	if (IS_ERR(attr)) {
795		if ((PTR_ERR(attr) != -ENOENT) || !def_val)
796			return PTR_ERR(attr);
797
798		*to = *def_val;
799	} else {
800		*to = attr->ptr_attr.data;
801	}
802
803	if (*to > upper_bound)
804		return -EINVAL;
805
806	return 0;
807}
808EXPORT_SYMBOL(_uverbs_get_const_unsigned);
809
810int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
811				  size_t idx, const void *from, size_t size)
812{
813	const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
814
815	if (IS_ERR(attr))
816		return PTR_ERR(attr);
817
818	if (size < attr->ptr_attr.len) {
819		if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size,
820			       attr->ptr_attr.len - size))
821			return -EFAULT;
822	}
823	return uverbs_copy_to(bundle, idx, from, size);
824}
825EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero);
826
827/* Once called an abort will call through to the type's destroy_hw() */
828void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
829				 u16 idx)
830{
831	struct bundle_priv *pbundle =
832		container_of(bundle, struct bundle_priv, bundle);
833
834	__set_bit(uapi_bkey_attr(uapi_key_attr(idx)),
835		  pbundle->uobj_hw_obj_valid);
836}
837EXPORT_SYMBOL(uverbs_finalize_uobj_create);