1/*
  2 * Copyright (c) 2017, Mellanox Technologies inc.  All rights reserved.
  3 *
  4 * This software is available to you under a choice of one of two
  5 * licenses.  You may choose to be licensed under the terms of the GNU
  6 * General Public License (GPL) Version 2, available from the file
  7 * COPYING in the main directory of this source tree, or the
  8 * OpenIB.org BSD license below:
  9 *
 10 *     Redistribution and use in source and binary forms, with or
 11 *     without modification, are permitted provided that the following
 12 *     conditions are met:
 13 *
 14 *      - Redistributions of source code must retain the above
 15 *        copyright notice, this list of conditions and the following
 16 *        disclaimer.
 17 *
 18 *      - Redistributions in binary form must reproduce the above
 19 *        copyright notice, this list of conditions and the following
 20 *        disclaimer in the documentation and/or other materials
 21 *        provided with the distribution.
 22 *
 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 30 * SOFTWARE.
 31 */
 32
 33#include <rdma/rdma_user_ioctl.h>
 34#include <rdma/uverbs_ioctl.h>
 35#include "rdma_core.h"
 36#include "uverbs.h"
 37
 38struct bundle_alloc_head {
 39	struct_group_tagged(bundle_alloc_head_hdr, hdr,
 40		struct bundle_alloc_head *next;
 41	);
 42	u8 data[];
 43};
 44
 45struct bundle_priv {
 46	/* Must be first */
 47	struct bundle_alloc_head_hdr alloc_head;
 48	struct bundle_alloc_head *allocated_mem;
 49	size_t internal_avail;
 50	size_t internal_used;
 51
 52	struct radix_tree_root *radix;
 53	const struct uverbs_api_ioctl_method *method_elm;
 54	void __rcu **radix_slots;
 55	unsigned long radix_slots_len;
 56	u32 method_key;
 57
 58	struct ib_uverbs_attr __user *user_attrs;
 59	struct ib_uverbs_attr *uattrs;
 60
 61	DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
 62	DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);
 63	DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN);
 64
 65	/*
 66	 * Must be last. bundle ends in a flex array which overlaps
 67	 * internal_buffer.
 68	 */
 69	struct uverbs_attr_bundle_hdr bundle;
 70	u64 internal_buffer[32];
 71};
 72
 73/*
 74 * Each method has an absolute minimum amount of memory it needs to allocate,
 75 * precompute that amount and determine if the onstack memory can be used or
 76 * if allocation is need.
 77 */
 78void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
 79			      unsigned int num_attrs)
 80{
 81	struct bundle_priv *pbundle;
 82	struct uverbs_attr_bundle *bundle;
 83	size_t bundle_size =
 84		offsetof(struct bundle_priv, internal_buffer) +
 85		sizeof(*bundle->attrs) * method_elm->key_bitmap_len +
 86		sizeof(*pbundle->uattrs) * num_attrs;
 87
 88	method_elm->use_stack = bundle_size <= sizeof(*pbundle);
 89	method_elm->bundle_size =
 90		ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer));
 91
 92	/* Do not want order-2 allocations for this. */
 93	WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE);
 94}
 95
 96/**
 97 * _uverbs_alloc() - Quickly allocate memory for use with a bundle
 98 * @bundle: The bundle
 99 * @size: Number of bytes to allocate
100 * @flags: Allocator flags
101 *
102 * The bundle allocator is intended for allocations that are connected with
103 * processing the system call related to the bundle. The allocated memory is
104 * always freed once the system call completes, and cannot be freed any other
105 * way.
106 *
107 * This tries to use a small pool of pre-allocated memory for performance.
108 */
109__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
110			     gfp_t flags)
111{
112	struct bundle_priv *pbundle =
113		container_of(&bundle->hdr, struct bundle_priv, bundle);
114	size_t new_used;
115	void *res;
116
117	if (check_add_overflow(size, pbundle->internal_used, &new_used))
118		return ERR_PTR(-EOVERFLOW);
119
120	if (new_used > pbundle->internal_avail) {
121		struct bundle_alloc_head *buf;
122
123		buf = kvmalloc(struct_size(buf, data, size), flags);
124		if (!buf)
125			return ERR_PTR(-ENOMEM);
126		buf->next = pbundle->allocated_mem;
127		pbundle->allocated_mem = buf;
128		return buf->data;
129	}
130
131	res = (void *)pbundle->internal_buffer + pbundle->internal_used;
132	pbundle->internal_used =
133		ALIGN(new_used, sizeof(*pbundle->internal_buffer));
134	if (want_init_on_alloc(flags))
135		memset(res, 0, size);
136	return res;
137}
138EXPORT_SYMBOL(_uverbs_alloc);
139
140static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
141				   u16 len)
142{
143	if (uattr->len > sizeof_field(struct ib_uverbs_attr, data))
144		return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len,
145					    uattr->len - len);
146
147	return !memchr_inv((const void *)&uattr->data + len,
148			   0, uattr->len - len);
149}
150
151static int uverbs_set_output(const struct uverbs_attr_bundle *bundle,
152			     const struct uverbs_attr *attr)
153{
154	struct bundle_priv *pbundle =
155		container_of(&bundle->hdr, struct bundle_priv, bundle);
156	u16 flags;
157
158	flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags |
159		UVERBS_ATTR_F_VALID_OUTPUT;
160	if (put_user(flags,
161		     &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags))
162		return -EFAULT;
163	return 0;
164}
165
166static int uverbs_process_idrs_array(struct bundle_priv *pbundle,
167				     const struct uverbs_api_attr *attr_uapi,
168				     struct uverbs_objs_arr_attr *attr,
169				     struct ib_uverbs_attr *uattr,
170				     u32 attr_bkey)
171{
172	struct uverbs_attr_bundle *bundle =
173		container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
174	const struct uverbs_attr_spec *spec = &attr_uapi->spec;
175	size_t array_len;
176	u32 *idr_vals;
177	int ret = 0;
178	size_t i;
179
180	if (uattr->attr_data.reserved)
181		return -EINVAL;
182
183	if (uattr->len % sizeof(u32))
184		return -EINVAL;
185
186	array_len = uattr->len / sizeof(u32);
187	if (array_len < spec->u2.objs_arr.min_len ||
188	    array_len > spec->u2.objs_arr.max_len)
189		return -EINVAL;
190
191	attr->uobjects =
192		uverbs_alloc(bundle,
193			     array_size(array_len, sizeof(*attr->uobjects)));
194	if (IS_ERR(attr->uobjects))
195		return PTR_ERR(attr->uobjects);
196
197	/*
198	 * Since idr is 4B and *uobjects is >= 4B, we can use attr->uobjects
199	 * to store idrs array and avoid additional memory allocation. The
200	 * idrs array is offset to the end of the uobjects array so we will be
201	 * able to read idr and replace with a pointer.
202	 */
203	idr_vals = (u32 *)(attr->uobjects + array_len) - array_len;
204
205	if (uattr->len > sizeof(uattr->data)) {
206		ret = copy_from_user(idr_vals, u64_to_user_ptr(uattr->data),
207				     uattr->len);
208		if (ret)
209			return -EFAULT;
210	} else {
211		memcpy(idr_vals, &uattr->data, uattr->len);
212	}
213
214	for (i = 0; i != array_len; i++) {
215		attr->uobjects[i] = uverbs_get_uobject_from_file(
216			spec->u2.objs_arr.obj_type, spec->u2.objs_arr.access,
217			idr_vals[i], bundle);
218		if (IS_ERR(attr->uobjects[i])) {
219			ret = PTR_ERR(attr->uobjects[i]);
220			break;
221		}
222	}
223
224	attr->len = i;
225	__set_bit(attr_bkey, pbundle->spec_finalize);
226	return ret;
227}
228
229static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,
230				   struct uverbs_objs_arr_attr *attr,
231				   bool commit,
232				   struct uverbs_attr_bundle *attrs)
233{
234	const struct uverbs_attr_spec *spec = &attr_uapi->spec;
235	size_t i;
236
237	for (i = 0; i != attr->len; i++)
238		uverbs_finalize_object(attr->uobjects[i],
239				       spec->u2.objs_arr.access, false, commit,
240				       attrs);
241}
242
243static int uverbs_process_attr(struct bundle_priv *pbundle,
244			       const struct uverbs_api_attr *attr_uapi,
245			       struct ib_uverbs_attr *uattr, u32 attr_bkey)
246{
247	const struct uverbs_attr_spec *spec = &attr_uapi->spec;
248	struct uverbs_attr_bundle *bundle =
249		container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
250	struct uverbs_attr *e = &bundle->attrs[attr_bkey];
251	const struct uverbs_attr_spec *val_spec = spec;
252	struct uverbs_obj_attr *o_attr;
253
254	switch (spec->type) {
255	case UVERBS_ATTR_TYPE_ENUM_IN:
256		if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems)
257			return -EOPNOTSUPP;
258
259		if (uattr->attr_data.enum_data.reserved)
260			return -EINVAL;
261
262		val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id];
263
264		/* Currently we only support PTR_IN based enums */
265		if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN)
266			return -EOPNOTSUPP;
267
268		e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id;
269		fallthrough;
270	case UVERBS_ATTR_TYPE_PTR_IN:
271		/* Ensure that any data provided by userspace beyond the known
272		 * struct is zero. Userspace that knows how to use some future
273		 * longer struct will fail here if used with an old kernel and
274		 * non-zero content, making ABI compat/discovery simpler.
275		 */
276		if (uattr->len > val_spec->u.ptr.len &&
277		    val_spec->zero_trailing &&
278		    !uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len))
279			return -EOPNOTSUPP;
280
281		fallthrough;
282	case UVERBS_ATTR_TYPE_PTR_OUT:
283		if (uattr->len < val_spec->u.ptr.min_len ||
284		    (!val_spec->zero_trailing &&
285		     uattr->len > val_spec->u.ptr.len))
286			return -EINVAL;
287
288		if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN &&
289		    uattr->attr_data.reserved)
290			return -EINVAL;
291
292		e->ptr_attr.uattr_idx = uattr - pbundle->uattrs;
293		e->ptr_attr.len = uattr->len;
294
295		if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) {
296			void *p;
297
298			p = uverbs_alloc(bundle, uattr->len);
299			if (IS_ERR(p))
300				return PTR_ERR(p);
301
302			e->ptr_attr.ptr = p;
303
304			if (copy_from_user(p, u64_to_user_ptr(uattr->data),
305					   uattr->len))
306				return -EFAULT;
307		} else {
308			e->ptr_attr.data = uattr->data;
309		}
310		break;
311
312	case UVERBS_ATTR_TYPE_IDR:
 
 
 
313	case UVERBS_ATTR_TYPE_FD:
314		if (uattr->attr_data.reserved)
315			return -EINVAL;
316
317		if (uattr->len != 0)
318			return -EINVAL;
319
320		o_attr = &e->obj_attr;
321		o_attr->attr_elm = attr_uapi;
 
 
 
 
 
 
 
 
 
 
322
323		/*
324		 * The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and
325		 * s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64
326		 * here without caring about truncation as we know that the
327		 * IDR implementation today rejects negative IDs
328		 */
329		o_attr->uobject = uverbs_get_uobject_from_file(
330			spec->u.obj.obj_type, spec->u.obj.access,
331			uattr->data_s64, bundle);
332		if (IS_ERR(o_attr->uobject))
333			return PTR_ERR(o_attr->uobject);
334		__set_bit(attr_bkey, pbundle->uobj_finalize);
335
336		if (spec->u.obj.access == UVERBS_ACCESS_NEW) {
337			unsigned int uattr_idx = uattr - pbundle->uattrs;
338			s64 id = o_attr->uobject->id;
339
340			/* Copy the allocated id to the user-space */
341			if (put_user(id, &pbundle->user_attrs[uattr_idx].data))
 
 
 
342				return -EFAULT;
 
343		}
344
345		break;
346
347	case UVERBS_ATTR_TYPE_RAW_FD:
348		if (uattr->attr_data.reserved || uattr->len != 0 ||
349		    uattr->data_s64 < INT_MIN || uattr->data_s64 > INT_MAX)
350			return -EINVAL;
351		/* _uverbs_get_const_signed() is the accessor */
352		e->ptr_attr.data = uattr->data_s64;
353		break;
354
355	case UVERBS_ATTR_TYPE_IDRS_ARRAY:
356		return uverbs_process_idrs_array(pbundle, attr_uapi,
357						 &e->objs_arr_attr, uattr,
358						 attr_bkey);
359	default:
360		return -EOPNOTSUPP;
361	}
362
 
363	return 0;
364}
365
366/*
367 * We search the radix tree with the method prefix and now we want to fast
368 * search the suffix bits to get a particular attribute pointer. It is not
369 * totally clear to me if this breaks the radix tree encasulation or not, but
370 * it uses the iter data to determine if the method iter points at the same
371 * chunk that will store the attribute, if so it just derefs it directly. By
372 * construction in most kernel configs the method and attrs will all fit in a
373 * single radix chunk, so in most cases this will have no search. Other cases
374 * this falls back to a full search.
375 */
376static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle,
377					     u32 attr_key)
378{
379	void __rcu **slot;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
380
381	if (likely(attr_key < pbundle->radix_slots_len)) {
382		void *entry;
 
 
 
 
383
384		slot = pbundle->radix_slots + attr_key;
385		entry = rcu_dereference_raw(*slot);
386		if (likely(!radix_tree_is_internal_node(entry) && entry))
387			return slot;
 
 
 
 
 
 
 
388	}
389
390	return radix_tree_lookup_slot(pbundle->radix,
391				      pbundle->method_key | attr_key);
392}
393
394static int uverbs_set_attr(struct bundle_priv *pbundle,
395			   struct ib_uverbs_attr *uattr)
396{
397	u32 attr_key = uapi_key_attr(uattr->attr_id);
398	u32 attr_bkey = uapi_bkey_attr(attr_key);
399	const struct uverbs_api_attr *attr;
400	void __rcu **slot;
401	int ret;
402
403	slot = uapi_get_attr_for_method(pbundle, attr_key);
404	if (!slot) {
405		/*
406		 * Kernel does not support the attribute but user-space says it
407		 * is mandatory
408		 */
409		if (uattr->flags & UVERBS_ATTR_F_MANDATORY)
410			return -EPROTONOSUPPORT;
411		return 0;
412	}
413	attr = rcu_dereference_protected(*slot, true);
414
415	/* Reject duplicate attributes from user-space */
416	if (test_bit(attr_bkey, pbundle->bundle.attr_present))
417		return -EINVAL;
418
419	ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey);
420	if (ret)
421		return ret;
422
423	__set_bit(attr_bkey, pbundle->bundle.attr_present);
 
 
 
424
425	return 0;
426}
427
428static int ib_uverbs_run_method(struct bundle_priv *pbundle,
429				unsigned int num_attrs)
 
 
 
 
 
430{
431	int (*handler)(struct uverbs_attr_bundle *attrs);
432	struct uverbs_attr_bundle *bundle =
433		container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
434	size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs);
435	unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey;
436	unsigned int i;
437	int ret;
 
 
438
439	/* See uverbs_disassociate_api() */
440	handler = srcu_dereference(
441		pbundle->method_elm->handler,
442		&pbundle->bundle.ufile->device->disassociate_srcu);
443	if (!handler)
444		return -EIO;
445
446	pbundle->uattrs = uverbs_alloc(bundle, uattrs_size);
447	if (IS_ERR(pbundle->uattrs))
448		return PTR_ERR(pbundle->uattrs);
449	if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size))
450		return -EFAULT;
451
452	for (i = 0; i != num_attrs; i++) {
453		ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]);
454		if (unlikely(ret))
455			return ret;
456	}
457
458	/* User space did not provide all the mandatory attributes */
459	if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory,
460				    pbundle->bundle.attr_present,
461				    pbundle->method_elm->key_bitmap_len)))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
462		return -EINVAL;
463
464	if (pbundle->method_elm->has_udata)
465		uverbs_fill_udata(bundle, &pbundle->bundle.driver_udata,
466				  UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT);
467	else
468		pbundle->bundle.driver_udata = (struct ib_udata){};
469
470	if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) {
471		struct uverbs_obj_attr *destroy_attr = &bundle->attrs[destroy_bkey].obj_attr;
 
472
473		ret = uobj_destroy(destroy_attr->uobject, bundle);
474		if (ret)
475			return ret;
476		__clear_bit(destroy_bkey, pbundle->uobj_finalize);
477
478		ret = handler(bundle);
479		uobj_put_destroy(destroy_attr->uobject);
480	} else {
481		ret = handler(bundle);
482	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
483
484	/*
485	 * Until the drivers are revised to use the bundle directly we have to
486	 * assume that the driver wrote to its UHW_OUT and flag userspace
487	 * appropriately.
488	 */
489	if (!ret && pbundle->method_elm->has_udata) {
490		const struct uverbs_attr *attr =
491			uverbs_attr_get(bundle, UVERBS_ATTR_UHW_OUT);
492
493		if (!IS_ERR(attr))
494			ret = uverbs_set_output(bundle, attr);
 
 
 
 
 
 
 
 
 
 
 
495	}
496
 
 
 
497	/*
498	 * EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can
499	 * not invoke the method because the request is not supported.  No
500	 * other cases should return this code.
501	 */
502	if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT))
503		return -EINVAL;
504
505	return ret;
506}
507
508static void bundle_destroy(struct bundle_priv *pbundle, bool commit)
509{
510	unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len;
511	struct uverbs_attr_bundle *bundle =
512		container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
513	struct bundle_alloc_head *memblock;
514	unsigned int i;
515
516	/* fast path for simple uobjects */
517	i = -1;
518	while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len,
519				  i + 1)) < key_bitmap_len) {
520		struct uverbs_attr *attr = &bundle->attrs[i];
521
522		uverbs_finalize_object(
523			attr->obj_attr.uobject,
524			attr->obj_attr.attr_elm->spec.u.obj.access,
525			test_bit(i, pbundle->uobj_hw_obj_valid),
526			commit, bundle);
527	}
528
529	i = -1;
530	while ((i = find_next_bit(pbundle->spec_finalize, key_bitmap_len,
531				  i + 1)) < key_bitmap_len) {
532		struct uverbs_attr *attr = &bundle->attrs[i];
533		const struct uverbs_api_attr *attr_uapi;
534		void __rcu **slot;
535
536		slot = uapi_get_attr_for_method(
537			pbundle,
538			pbundle->method_key | uapi_bkey_to_key_attr(i));
539		if (WARN_ON(!slot))
540			continue;
541
542		attr_uapi = rcu_dereference_protected(*slot, true);
543
544		if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
545			uverbs_free_idrs_array(attr_uapi, &attr->objs_arr_attr,
546					       commit, bundle);
547		}
548	}
549
550	for (memblock = pbundle->allocated_mem; memblock;) {
551		struct bundle_alloc_head *tmp = memblock;
552
553		memblock = memblock->next;
554		kvfree(tmp);
555	}
556}
557
558static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
559			       struct ib_uverbs_ioctl_hdr *hdr,
560			       struct ib_uverbs_attr __user *user_attrs)
561{
562	const struct uverbs_api_ioctl_method *method_elm;
563	struct uverbs_api *uapi = ufile->device->uapi;
564	struct radix_tree_iter attrs_iter;
565	struct bundle_priv *pbundle;
566	struct bundle_priv onstack;
567	void __rcu **slot;
568	int ret;
569
570	if (unlikely(hdr->driver_id != uapi->driver_id))
571		return -EINVAL;
572
573	slot = radix_tree_iter_lookup(
574		&uapi->radix, &attrs_iter,
575		uapi_key_obj(hdr->object_id) |
576			uapi_key_ioctl_method(hdr->method_id));
577	if (unlikely(!slot))
578		return -EPROTONOSUPPORT;
579	method_elm = rcu_dereference_protected(*slot, true);
580
581	if (!method_elm->use_stack) {
582		pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL);
583		if (!pbundle)
584			return -ENOMEM;
585		pbundle->internal_avail =
586			method_elm->bundle_size -
587			offsetof(struct bundle_priv, internal_buffer);
588		pbundle->alloc_head.next = NULL;
589		pbundle->allocated_mem = container_of(&pbundle->alloc_head,
590						struct bundle_alloc_head, hdr);
591	} else {
592		pbundle = &onstack;
593		pbundle->internal_avail = sizeof(pbundle->internal_buffer);
594		pbundle->allocated_mem = NULL;
595	}
596
597	/* Space for the pbundle->bundle.attrs flex array */
598	pbundle->method_elm = method_elm;
599	pbundle->method_key = attrs_iter.index;
600	pbundle->bundle.ufile = ufile;
601	pbundle->bundle.context = NULL; /* only valid if bundle has uobject */
602	pbundle->radix = &uapi->radix;
603	pbundle->radix_slots = slot;
604	pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter);
605	pbundle->user_attrs = user_attrs;
606
607	pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len *
608					       sizeof(*container_of(&pbundle->bundle,
609							struct uverbs_attr_bundle, hdr)->attrs),
610					       sizeof(*pbundle->internal_buffer));
611	memset(pbundle->bundle.attr_present, 0,
612	       sizeof(pbundle->bundle.attr_present));
613	memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));
614	memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize));
615	memset(pbundle->uobj_hw_obj_valid, 0,
616	       sizeof(pbundle->uobj_hw_obj_valid));
617
618	ret = ib_uverbs_run_method(pbundle, hdr->num_attrs);
619	bundle_destroy(pbundle, ret == 0);
620	return ret;
621}
622
623long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
624{
625	struct ib_uverbs_file *file = filp->private_data;
626	struct ib_uverbs_ioctl_hdr __user *user_hdr =
627		(struct ib_uverbs_ioctl_hdr __user *)arg;
628	struct ib_uverbs_ioctl_hdr hdr;
 
629	int srcu_key;
630	int err;
631
632	if (unlikely(cmd != RDMA_VERBS_IOCTL))
633		return -ENOIOCTLCMD;
634
635	err = copy_from_user(&hdr, user_hdr, sizeof(hdr));
636	if (err)
637		return -EFAULT;
638
639	if (hdr.length > PAGE_SIZE ||
640	    hdr.length != struct_size(&hdr, attrs, hdr.num_attrs))
641		return -EINVAL;
642
643	if (hdr.reserved1 || hdr.reserved2)
644		return -EPROTONOSUPPORT;
645
646	srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
647	err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs);
648	srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
649	return err;
650}
651
652int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
653		       size_t idx, u64 allowed_bits)
654{
655	const struct uverbs_attr *attr;
656	u64 flags;
657
658	attr = uverbs_attr_get(attrs_bundle, idx);
659	/* Missing attribute means 0 flags */
660	if (IS_ERR(attr)) {
661		*to = 0;
662		return 0;
663	}
664
665	/*
666	 * New userspace code should use 8 bytes to pass flags, but we
667	 * transparently support old userspaces that were using 4 bytes as
668	 * well.
669	 */
670	if (attr->ptr_attr.len == 8)
671		flags = attr->ptr_attr.data;
672	else if (attr->ptr_attr.len == 4)
673		flags = *(u32 *)&attr->ptr_attr.data;
674	else
675		return -EINVAL;
676
677	if (flags & ~allowed_bits)
678		return -EINVAL;
679
680	*to = flags;
681	return 0;
682}
683EXPORT_SYMBOL(uverbs_get_flags64);
684
685int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
686		       size_t idx, u64 allowed_bits)
687{
688	u64 flags;
689	int ret;
690
691	ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits);
692	if (ret)
693		return ret;
694
695	if (flags > U32_MAX)
696		return -EINVAL;
697	*to = flags;
698
699	return 0;
700}
701EXPORT_SYMBOL(uverbs_get_flags32);
702
703/*
704 * Fill a ib_udata struct (core or uhw) using the given attribute IDs.
705 * This is primarily used to convert the UVERBS_ATTR_UHW() into the
706 * ib_udata format used by the drivers.
707 */
708void uverbs_fill_udata(struct uverbs_attr_bundle *bundle,
709		       struct ib_udata *udata, unsigned int attr_in,
710		       unsigned int attr_out)
711{
712	struct bundle_priv *pbundle =
713		container_of(&bundle->hdr, struct bundle_priv, bundle);
714	struct uverbs_attr_bundle *bundle_aux =
715		container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
716	const struct uverbs_attr *in =
717		uverbs_attr_get(bundle_aux, attr_in);
718	const struct uverbs_attr *out =
719		uverbs_attr_get(bundle_aux, attr_out);
720
721	if (!IS_ERR(in)) {
722		udata->inlen = in->ptr_attr.len;
723		if (uverbs_attr_ptr_is_inline(in))
724			udata->inbuf =
725				&pbundle->user_attrs[in->ptr_attr.uattr_idx]
726					 .data;
727		else
728			udata->inbuf = u64_to_user_ptr(in->ptr_attr.data);
729	} else {
730		udata->inbuf = NULL;
731		udata->inlen = 0;
732	}
733
734	if (!IS_ERR(out)) {
735		udata->outbuf = u64_to_user_ptr(out->ptr_attr.data);
736		udata->outlen = out->ptr_attr.len;
737	} else {
738		udata->outbuf = NULL;
739		udata->outlen = 0;
740	}
741}
742
743int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx,
744		   const void *from, size_t size)
745{
746	const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
747	size_t min_size;
748
749	if (IS_ERR(attr))
750		return PTR_ERR(attr);
751
752	min_size = min_t(size_t, attr->ptr_attr.len, size);
753	if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size))
754		return -EFAULT;
755
756	return uverbs_set_output(bundle, attr);
757}
758EXPORT_SYMBOL(uverbs_copy_to);
759
760
761/*
762 * This is only used if the caller has directly used copy_to_use to write the
763 * data.  It signals to user space that the buffer is filled in.
764 */
765int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx)
766{
767	const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
768
769	if (IS_ERR(attr))
770		return PTR_ERR(attr);
771
772	return uverbs_set_output(bundle, attr);
773}
774
775int _uverbs_get_const_signed(s64 *to,
776			     const struct uverbs_attr_bundle *attrs_bundle,
777			     size_t idx, s64 lower_bound, u64 upper_bound,
778			     s64  *def_val)
779{
780	const struct uverbs_attr *attr;
781
782	attr = uverbs_attr_get(attrs_bundle, idx);
783	if (IS_ERR(attr)) {
784		if ((PTR_ERR(attr) != -ENOENT) || !def_val)
785			return PTR_ERR(attr);
786
787		*to = *def_val;
788	} else {
789		*to = attr->ptr_attr.data;
790	}
791
792	if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound))
793		return -EINVAL;
794
795	return 0;
796}
797EXPORT_SYMBOL(_uverbs_get_const_signed);
798
799int _uverbs_get_const_unsigned(u64 *to,
800			       const struct uverbs_attr_bundle *attrs_bundle,
801			       size_t idx, u64 upper_bound, u64 *def_val)
802{
803	const struct uverbs_attr *attr;
804
805	attr = uverbs_attr_get(attrs_bundle, idx);
806	if (IS_ERR(attr)) {
807		if ((PTR_ERR(attr) != -ENOENT) || !def_val)
808			return PTR_ERR(attr);
809
810		*to = *def_val;
 
811	} else {
812		*to = attr->ptr_attr.data;
813	}
 
 
814
815	if (*to > upper_bound)
816		return -EINVAL;
817
818	return 0;
819}
820EXPORT_SYMBOL(_uverbs_get_const_unsigned);
821
822int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
823				  size_t idx, const void *from, size_t size)
824{
825	const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
826
827	if (IS_ERR(attr))
828		return PTR_ERR(attr);
829
830	if (size < attr->ptr_attr.len) {
831		if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size,
832			       attr->ptr_attr.len - size))
833			return -EFAULT;
834	}
835	return uverbs_copy_to(bundle, idx, from, size);
836}
837EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero);
838
839/* Once called an abort will call through to the type's destroy_hw() */
840void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
841				 u16 idx)
842{
843	struct bundle_priv *pbundle =
844		container_of(&bundle->hdr, struct bundle_priv, bundle);
845
846	__set_bit(uapi_bkey_attr(uapi_key_attr(idx)),
847		  pbundle->uobj_hw_obj_valid);
848}
849EXPORT_SYMBOL(uverbs_finalize_uobj_create);