1/*
  2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
  3 *
  4 * This software is available to you under a choice of one of two
  5 * licenses.  You may choose to be licensed under the terms of the GNU
  6 * General Public License (GPL) Version 2, available from the file
  7 * COPYING in the main directory of this source tree, or the
  8 * OpenIB.org BSD license below:
  9 *
 10 *     Redistribution and use in source and binary forms, with or
 11 *     without modification, are permitted provided that the following
 12 *     conditions are met:
 13 *
 14 *      - Redistributions of source code must retain the above
 15 *        copyright notice, this list of conditions and the following
 16 *        disclaimer.
 17 *
 18 *      - Redistributions in binary form must reproduce the above
 19 *        copyright notice, this list of conditions and the following
 20 *        disclaimer in the documentation and/or other materials
 21 *        provided with the distribution.
 22 *
 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 30 * SOFTWARE.
 31 */
 32
 33#include <linux/module.h>
 34#include <linux/moduleparam.h>
 35#include <rdma/ib_umem.h>
 36#include <linux/atomic.h>
 37#include <rdma/ib_user_verbs.h>
 38
 39#include "iw_cxgb4.h"
 40
 41int use_dsgl = 1;
 42module_param(use_dsgl, int, 0644);
 43MODULE_PARM_DESC(use_dsgl, "Use DSGL for PBL/FastReg (default=1) (DEPRECATED)");
 44
 45#define T4_ULPTX_MIN_IO 32
 46#define C4IW_MAX_INLINE_SIZE 96
 47#define T4_ULPTX_MAX_DMA 1024
 48#define C4IW_INLINE_THRESHOLD 128
 49
 50static int inline_threshold = C4IW_INLINE_THRESHOLD;
 51module_param(inline_threshold, int, 0644);
 52MODULE_PARM_DESC(inline_threshold, "inline vs dsgl threshold (default=128)");
 53
 54static int mr_exceeds_hw_limits(struct c4iw_dev *dev, u64 length)
 55{
 56	return (is_t4(dev->rdev.lldi.adapter_type) ||
 57		is_t5(dev->rdev.lldi.adapter_type)) &&
 58		length >= 8*1024*1024*1024ULL;
 59}
 60
 61static int _c4iw_write_mem_dma_aligned(struct c4iw_rdev *rdev, u32 addr,
 62				       u32 len, dma_addr_t data,
 63				       struct sk_buff *skb,
 64				       struct c4iw_wr_wait *wr_waitp)
 65{
 66	struct ulp_mem_io *req;
 67	struct ulptx_sgl *sgl;
 68	u8 wr_len;
 69	int ret = 0;
 70
 71	addr &= 0x7FFFFFF;
 72
 73	if (wr_waitp)
 74		c4iw_init_wr_wait(wr_waitp);
 75	wr_len = roundup(sizeof(*req) + sizeof(*sgl), 16);
 76
 77	if (!skb) {
 78		skb = alloc_skb(wr_len, GFP_KERNEL | __GFP_NOFAIL);
 79		if (!skb)
 80			return -ENOMEM;
 81	}
 82	set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0);
 83
 84	req = __skb_put_zero(skb, wr_len);
 85	INIT_ULPTX_WR(req, wr_len, 0, 0);
 86	req->wr.wr_hi = cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR) |
 87			(wr_waitp ? FW_WR_COMPL_F : 0));
 88	req->wr.wr_lo = wr_waitp ? (__force __be64)(unsigned long)wr_waitp : 0L;
 89	req->wr.wr_mid = cpu_to_be32(FW_WR_LEN16_V(DIV_ROUND_UP(wr_len, 16)));
 90	req->cmd = cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE) |
 91			       T5_ULP_MEMIO_ORDER_V(1) |
 92			       T5_ULP_MEMIO_FID_V(rdev->lldi.rxq_ids[0]));
 93	req->dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN_V(len>>5));
 94	req->len16 = cpu_to_be32(DIV_ROUND_UP(wr_len-sizeof(req->wr), 16));
 95	req->lock_addr = cpu_to_be32(ULP_MEMIO_ADDR_V(addr));
 96
 97	sgl = (struct ulptx_sgl *)(req + 1);
 98	sgl->cmd_nsge = cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
 99				    ULPTX_NSGE_V(1));
100	sgl->len0 = cpu_to_be32(len);
101	sgl->addr0 = cpu_to_be64(data);
102
103	if (wr_waitp)
104		ret = c4iw_ref_send_wait(rdev, skb, wr_waitp, 0, 0, __func__);
105	else
106		ret = c4iw_ofld_send(rdev, skb);
107	return ret;
108}
109
110static int _c4iw_write_mem_inline(struct c4iw_rdev *rdev, u32 addr, u32 len,
111				  void *data, struct sk_buff *skb,
112				  struct c4iw_wr_wait *wr_waitp)
113{
114	struct ulp_mem_io *req;
115	struct ulptx_idata *sc;
116	u8 wr_len, *to_dp, *from_dp;
117	int copy_len, num_wqe, i, ret = 0;
118	__be32 cmd = cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE));
119
120	if (is_t4(rdev->lldi.adapter_type))
121		cmd |= cpu_to_be32(ULP_MEMIO_ORDER_F);
122	else
123		cmd |= cpu_to_be32(T5_ULP_MEMIO_IMM_F);
124
125	addr &= 0x7FFFFFF;
126	pr_debug("addr 0x%x len %u\n", addr, len);
127	num_wqe = DIV_ROUND_UP(len, C4IW_MAX_INLINE_SIZE);
128	c4iw_init_wr_wait(wr_waitp);
129	for (i = 0; i < num_wqe; i++) {
130
131		copy_len = len > C4IW_MAX_INLINE_SIZE ? C4IW_MAX_INLINE_SIZE :
132			   len;
133		wr_len = roundup(sizeof(*req) + sizeof(*sc) +
134					 roundup(copy_len, T4_ULPTX_MIN_IO),
135				 16);
136
137		if (!skb) {
138			skb = alloc_skb(wr_len, GFP_KERNEL | __GFP_NOFAIL);
139			if (!skb)
140				return -ENOMEM;
141		}
142		set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0);
143
144		req = __skb_put_zero(skb, wr_len);
145		INIT_ULPTX_WR(req, wr_len, 0, 0);
146
147		if (i == (num_wqe-1)) {
148			req->wr.wr_hi = cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR) |
149						    FW_WR_COMPL_F);
150			req->wr.wr_lo = (__force __be64)(unsigned long)wr_waitp;
151		} else
152			req->wr.wr_hi = cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR));
153		req->wr.wr_mid = cpu_to_be32(
154				       FW_WR_LEN16_V(DIV_ROUND_UP(wr_len, 16)));
155
156		req->cmd = cmd;
157		req->dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN_V(
158				DIV_ROUND_UP(copy_len, T4_ULPTX_MIN_IO)));
159		req->len16 = cpu_to_be32(DIV_ROUND_UP(wr_len-sizeof(req->wr),
160						      16));
161		req->lock_addr = cpu_to_be32(ULP_MEMIO_ADDR_V(addr + i * 3));
162
163		sc = (struct ulptx_idata *)(req + 1);
164		sc->cmd_more = cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_IMM));
165		sc->len = cpu_to_be32(roundup(copy_len, T4_ULPTX_MIN_IO));
166
167		to_dp = (u8 *)(sc + 1);
168		from_dp = (u8 *)data + i * C4IW_MAX_INLINE_SIZE;
169		if (data)
170			memcpy(to_dp, from_dp, copy_len);
171		else
172			memset(to_dp, 0, copy_len);
173		if (copy_len % T4_ULPTX_MIN_IO)
174			memset(to_dp + copy_len, 0, T4_ULPTX_MIN_IO -
175			       (copy_len % T4_ULPTX_MIN_IO));
176		if (i == (num_wqe-1))
177			ret = c4iw_ref_send_wait(rdev, skb, wr_waitp, 0, 0,
178						 __func__);
179		else
180			ret = c4iw_ofld_send(rdev, skb);
181		if (ret)
182			break;
183		skb = NULL;
184		len -= C4IW_MAX_INLINE_SIZE;
185	}
186
187	return ret;
188}
189
190static int _c4iw_write_mem_dma(struct c4iw_rdev *rdev, u32 addr, u32 len,
191			       void *data, struct sk_buff *skb,
192			       struct c4iw_wr_wait *wr_waitp)
193{
194	u32 remain = len;
195	u32 dmalen;
196	int ret = 0;
197	dma_addr_t daddr;
198	dma_addr_t save;
199
200	daddr = dma_map_single(&rdev->lldi.pdev->dev, data, len, DMA_TO_DEVICE);
201	if (dma_mapping_error(&rdev->lldi.pdev->dev, daddr))
202		return -1;
203	save = daddr;
204
205	while (remain > inline_threshold) {
206		if (remain < T4_ULPTX_MAX_DMA) {
207			if (remain & ~T4_ULPTX_MIN_IO)
208				dmalen = remain & ~(T4_ULPTX_MIN_IO-1);
209			else
210				dmalen = remain;
211		} else
212			dmalen = T4_ULPTX_MAX_DMA;
213		remain -= dmalen;
214		ret = _c4iw_write_mem_dma_aligned(rdev, addr, dmalen, daddr,
215						 skb, remain ? NULL : wr_waitp);
216		if (ret)
217			goto out;
218		addr += dmalen >> 5;
219		data += dmalen;
220		daddr += dmalen;
221	}
222	if (remain)
223		ret = _c4iw_write_mem_inline(rdev, addr, remain, data, skb,
224					     wr_waitp);
225out:
226	dma_unmap_single(&rdev->lldi.pdev->dev, save, len, DMA_TO_DEVICE);
227	return ret;
228}
229
230/*
231 * write len bytes of data into addr (32B aligned address)
232 * If data is NULL, clear len byte of memory to zero.
233 */
234static int write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len,
235			     void *data, struct sk_buff *skb,
236			     struct c4iw_wr_wait *wr_waitp)
237{
238	int ret;
239
240	if (!rdev->lldi.ulptx_memwrite_dsgl || !use_dsgl) {
241		ret = _c4iw_write_mem_inline(rdev, addr, len, data, skb,
242					      wr_waitp);
243		goto out;
244	}
245
246	if (len <= inline_threshold) {
247		ret = _c4iw_write_mem_inline(rdev, addr, len, data, skb,
248					      wr_waitp);
249		goto out;
250	}
251
252	ret = _c4iw_write_mem_dma(rdev, addr, len, data, skb, wr_waitp);
253	if (ret) {
254		pr_warn_ratelimited("%s: dma map failure (non fatal)\n",
255				    pci_name(rdev->lldi.pdev));
256		ret = _c4iw_write_mem_inline(rdev, addr, len, data, skb,
257					      wr_waitp);
258	}
259out:
260	return ret;
261
262}
263
264/*
265 * Build and write a TPT entry.
266 * IN: stag key, pdid, perm, bind_enabled, zbva, to, len, page_size,
267 *     pbl_size and pbl_addr
268 * OUT: stag index
269 */
270static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry,
271			   u32 *stag, u8 stag_state, u32 pdid,
272			   enum fw_ri_stag_type type, enum fw_ri_mem_perms perm,
273			   int bind_enabled, u32 zbva, u64 to,
274			   u64 len, u8 page_size, u32 pbl_size, u32 pbl_addr,
275			   struct sk_buff *skb, struct c4iw_wr_wait *wr_waitp)
276{
277	int err;
278	struct fw_ri_tpte *tpt;
279	u32 stag_idx;
280	static atomic_t key;
281
282	if (c4iw_fatal_error(rdev))
283		return -EIO;
284
285	tpt = kmalloc(sizeof(*tpt), GFP_KERNEL);
286	if (!tpt)
287		return -ENOMEM;
288
289	stag_state = stag_state > 0;
290	stag_idx = (*stag) >> 8;
291
292	if ((!reset_tpt_entry) && (*stag == T4_STAG_UNSET)) {
293		stag_idx = c4iw_get_resource(&rdev->resource.tpt_table);
294		if (!stag_idx) {
295			mutex_lock(&rdev->stats.lock);
296			rdev->stats.stag.fail++;
297			mutex_unlock(&rdev->stats.lock);
298			kfree(tpt);
299			return -ENOMEM;
300		}
301		mutex_lock(&rdev->stats.lock);
302		rdev->stats.stag.cur += 32;
303		if (rdev->stats.stag.cur > rdev->stats.stag.max)
304			rdev->stats.stag.max = rdev->stats.stag.cur;
305		mutex_unlock(&rdev->stats.lock);
306		*stag = (stag_idx << 8) | (atomic_inc_return(&key) & 0xff);
307	}
308	pr_debug("stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x\n",
309		 stag_state, type, pdid, stag_idx);
310
311	/* write TPT entry */
312	if (reset_tpt_entry)
313		memset(tpt, 0, sizeof(*tpt));
314	else {
315		tpt->valid_to_pdid = cpu_to_be32(FW_RI_TPTE_VALID_F |
316			FW_RI_TPTE_STAGKEY_V((*stag & FW_RI_TPTE_STAGKEY_M)) |
317			FW_RI_TPTE_STAGSTATE_V(stag_state) |
318			FW_RI_TPTE_STAGTYPE_V(type) | FW_RI_TPTE_PDID_V(pdid));
319		tpt->locread_to_qpid = cpu_to_be32(FW_RI_TPTE_PERM_V(perm) |
320			(bind_enabled ? FW_RI_TPTE_MWBINDEN_F : 0) |
321			FW_RI_TPTE_ADDRTYPE_V((zbva ? FW_RI_ZERO_BASED_TO :
322						      FW_RI_VA_BASED_TO))|
323			FW_RI_TPTE_PS_V(page_size));
324		tpt->nosnoop_pbladdr = !pbl_size ? 0 : cpu_to_be32(
325			FW_RI_TPTE_PBLADDR_V(PBL_OFF(rdev, pbl_addr)>>3));
326		tpt->len_lo = cpu_to_be32((u32)(len & 0xffffffffUL));
327		tpt->va_hi = cpu_to_be32((u32)(to >> 32));
328		tpt->va_lo_fbo = cpu_to_be32((u32)(to & 0xffffffffUL));
329		tpt->dca_mwbcnt_pstag = cpu_to_be32(0);
330		tpt->len_hi = cpu_to_be32((u32)(len >> 32));
331	}
332	err = write_adapter_mem(rdev, stag_idx +
333				(rdev->lldi.vr->stag.start >> 5),
334				sizeof(*tpt), tpt, skb, wr_waitp);
335
336	if (reset_tpt_entry) {
337		c4iw_put_resource(&rdev->resource.tpt_table, stag_idx);
338		mutex_lock(&rdev->stats.lock);
339		rdev->stats.stag.cur -= 32;
340		mutex_unlock(&rdev->stats.lock);
341	}
342	kfree(tpt);
343	return err;
344}
345
346static int write_pbl(struct c4iw_rdev *rdev, __be64 *pbl,
347		     u32 pbl_addr, u32 pbl_size, struct c4iw_wr_wait *wr_waitp)
348{
349	int err;
350
351	pr_debug("*pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d\n",
352		 pbl_addr, rdev->lldi.vr->pbl.start,
353		 pbl_size);
354
355	err = write_adapter_mem(rdev, pbl_addr >> 5, pbl_size << 3, pbl, NULL,
356				wr_waitp);
357	return err;
358}
359
360static int dereg_mem(struct c4iw_rdev *rdev, u32 stag, u32 pbl_size,
361		     u32 pbl_addr, struct sk_buff *skb,
362		     struct c4iw_wr_wait *wr_waitp)
363{
364	return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0,
365			       pbl_size, pbl_addr, skb, wr_waitp);
366}
367
368static int allocate_stag(struct c4iw_rdev *rdev, u32 *stag, u32 pdid,
369			 u32 pbl_size, u32 pbl_addr,
370			 struct c4iw_wr_wait *wr_waitp)
371{
372	*stag = T4_STAG_UNSET;
373	return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_NSMR, 0, 0, 0,
374			       0UL, 0, 0, pbl_size, pbl_addr, NULL, wr_waitp);
375}
376
377static int finish_mem_reg(struct c4iw_mr *mhp, u32 stag)
378{
379	u32 mmid;
380
381	mhp->attr.state = 1;
382	mhp->attr.stag = stag;
383	mmid = stag >> 8;
384	mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
385	mhp->ibmr.length = mhp->attr.len;
386	mhp->ibmr.page_size = 1U << (mhp->attr.page_size + 12);
387	pr_debug("mmid 0x%x mhp %p\n", mmid, mhp);
388	return xa_insert_irq(&mhp->rhp->mrs, mmid, mhp, GFP_KERNEL);
389}
390
391static int register_mem(struct c4iw_dev *rhp, struct c4iw_pd *php,
392		      struct c4iw_mr *mhp, int shift)
393{
394	u32 stag = T4_STAG_UNSET;
395	int ret;
396
397	ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, mhp->attr.pdid,
398			      FW_RI_STAG_NSMR, mhp->attr.len ?
399			      mhp->attr.perms : 0,
400			      mhp->attr.mw_bind_enable, mhp->attr.zbva,
401			      mhp->attr.va_fbo, mhp->attr.len ?
402			      mhp->attr.len : -1, shift - 12,
403			      mhp->attr.pbl_size, mhp->attr.pbl_addr, NULL,
404			      mhp->wr_waitp);
405	if (ret)
406		return ret;
407
408	ret = finish_mem_reg(mhp, stag);
409	if (ret) {
410		dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
411			  mhp->attr.pbl_addr, mhp->dereg_skb, mhp->wr_waitp);
412		mhp->dereg_skb = NULL;
413	}
414	return ret;
415}
416
417static int alloc_pbl(struct c4iw_mr *mhp, int npages)
418{
419	mhp->attr.pbl_addr = c4iw_pblpool_alloc(&mhp->rhp->rdev,
420						    npages << 3);
421
422	if (!mhp->attr.pbl_addr)
423		return -ENOMEM;
424
425	mhp->attr.pbl_size = npages;
426
427	return 0;
428}
429
430struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc)
431{
432	struct c4iw_dev *rhp;
433	struct c4iw_pd *php;
434	struct c4iw_mr *mhp;
435	int ret;
436	u32 stag = T4_STAG_UNSET;
437
438	pr_debug("ib_pd %p\n", pd);
439	php = to_c4iw_pd(pd);
440	rhp = php->rhp;
441
442	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
443	if (!mhp)
444		return ERR_PTR(-ENOMEM);
445	mhp->wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL);
446	if (!mhp->wr_waitp) {
447		ret = -ENOMEM;
448		goto err_free_mhp;
449	}
450	c4iw_init_wr_wait(mhp->wr_waitp);
451
452	mhp->dereg_skb = alloc_skb(SGE_MAX_WR_LEN, GFP_KERNEL);
453	if (!mhp->dereg_skb) {
454		ret = -ENOMEM;
455		goto err_free_wr_wait;
456	}
457
458	mhp->rhp = rhp;
459	mhp->attr.pdid = php->pdid;
460	mhp->attr.perms = c4iw_ib_to_tpt_access(acc);
461	mhp->attr.mw_bind_enable = (acc&IB_ACCESS_MW_BIND) == IB_ACCESS_MW_BIND;
462	mhp->attr.zbva = 0;
463	mhp->attr.va_fbo = 0;
464	mhp->attr.page_size = 0;
465	mhp->attr.len = ~0ULL;
466	mhp->attr.pbl_size = 0;
467
468	ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, php->pdid,
469			      FW_RI_STAG_NSMR, mhp->attr.perms,
470			      mhp->attr.mw_bind_enable, 0, 0, ~0ULL, 0, 0, 0,
471			      NULL, mhp->wr_waitp);
472	if (ret)
473		goto err_free_skb;
474
475	ret = finish_mem_reg(mhp, stag);
476	if (ret)
477		goto err_dereg_mem;
478	return &mhp->ibmr;
479err_dereg_mem:
480	dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
481		  mhp->attr.pbl_addr, mhp->dereg_skb, mhp->wr_waitp);
482err_free_skb:
483	kfree_skb(mhp->dereg_skb);
484err_free_wr_wait:
485	c4iw_put_wr_wait(mhp->wr_waitp);
486err_free_mhp:
487	kfree(mhp);
488	return ERR_PTR(ret);
489}
490
491struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
492			       u64 virt, int acc, struct ib_udata *udata)
493{
494	__be64 *pages;
495	int shift, n, i;
496	int err = -ENOMEM;
497	struct ib_block_iter biter;
498	struct c4iw_dev *rhp;
499	struct c4iw_pd *php;
500	struct c4iw_mr *mhp;
501
502	pr_debug("ib_pd %p\n", pd);
503
504	if (length == ~0ULL)
505		return ERR_PTR(-EINVAL);
506
507	if ((length + start) < start)
508		return ERR_PTR(-EINVAL);
509
510	php = to_c4iw_pd(pd);
511	rhp = php->rhp;
512
513	if (mr_exceeds_hw_limits(rhp, length))
514		return ERR_PTR(-EINVAL);
515
516	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
517	if (!mhp)
518		return ERR_PTR(-ENOMEM);
519	mhp->wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL);
520	if (!mhp->wr_waitp)
521		goto err_free_mhp;
522
523	mhp->dereg_skb = alloc_skb(SGE_MAX_WR_LEN, GFP_KERNEL);
524	if (!mhp->dereg_skb)
525		goto err_free_wr_wait;
526
527	mhp->rhp = rhp;
528
529	mhp->umem = ib_umem_get(pd->device, start, length, acc);
530	if (IS_ERR(mhp->umem))
531		goto err_free_skb;
532
533	shift = PAGE_SHIFT;
534
535	n = ib_umem_num_dma_blocks(mhp->umem, 1 << shift);
536	err = alloc_pbl(mhp, n);
537	if (err)
538		goto err_umem_release;
539
540	pages = (__be64 *) __get_free_page(GFP_KERNEL);
541	if (!pages) {
542		err = -ENOMEM;
543		goto err_pbl_free;
544	}
545
546	i = n = 0;
547
548	rdma_umem_for_each_dma_block(mhp->umem, &biter, 1 << shift) {
549		pages[i++] = cpu_to_be64(rdma_block_iter_dma_address(&biter));
550		if (i == PAGE_SIZE / sizeof(*pages)) {
551			err = write_pbl(&mhp->rhp->rdev, pages,
552					mhp->attr.pbl_addr + (n << 3), i,
553					mhp->wr_waitp);
554			if (err)
555				goto pbl_done;
556			n += i;
557			i = 0;
558		}
559	}
560
561	if (i)
562		err = write_pbl(&mhp->rhp->rdev, pages,
563				mhp->attr.pbl_addr + (n << 3), i,
564				mhp->wr_waitp);
565
566pbl_done:
567	free_page((unsigned long) pages);
568	if (err)
569		goto err_pbl_free;
570
571	mhp->attr.pdid = php->pdid;
572	mhp->attr.zbva = 0;
573	mhp->attr.perms = c4iw_ib_to_tpt_access(acc);
574	mhp->attr.va_fbo = virt;
575	mhp->attr.page_size = shift - 12;
576	mhp->attr.len = length;
577
578	err = register_mem(rhp, php, mhp, shift);
579	if (err)
580		goto err_pbl_free;
581
582	return &mhp->ibmr;
583
584err_pbl_free:
585	c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr,
586			      mhp->attr.pbl_size << 3);
587err_umem_release:
588	ib_umem_release(mhp->umem);
589err_free_skb:
590	kfree_skb(mhp->dereg_skb);
591err_free_wr_wait:
592	c4iw_put_wr_wait(mhp->wr_waitp);
593err_free_mhp:
594	kfree(mhp);
595	return ERR_PTR(err);
596}
597
598struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
599			    u32 max_num_sg)
600{
601	struct c4iw_dev *rhp;
602	struct c4iw_pd *php;
603	struct c4iw_mr *mhp;
604	u32 mmid;
605	u32 stag = 0;
606	int ret = 0;
607	int length = roundup(max_num_sg * sizeof(u64), 32);
608
609	php = to_c4iw_pd(pd);
610	rhp = php->rhp;
611
612	if (mr_type != IB_MR_TYPE_MEM_REG ||
613	    max_num_sg > t4_max_fr_depth(rhp->rdev.lldi.ulptx_memwrite_dsgl &&
614					 use_dsgl))
615		return ERR_PTR(-EINVAL);
616
617	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
618	if (!mhp) {
619		ret = -ENOMEM;
620		goto err;
621	}
622
623	mhp->wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL);
624	if (!mhp->wr_waitp) {
625		ret = -ENOMEM;
626		goto err_free_mhp;
627	}
628	c4iw_init_wr_wait(mhp->wr_waitp);
629
630	mhp->mpl = dma_alloc_coherent(&rhp->rdev.lldi.pdev->dev,
631				      length, &mhp->mpl_addr, GFP_KERNEL);
632	if (!mhp->mpl) {
633		ret = -ENOMEM;
634		goto err_free_wr_wait;
635	}
636	mhp->max_mpl_len = length;
637
638	mhp->rhp = rhp;
639	ret = alloc_pbl(mhp, max_num_sg);
640	if (ret)
641		goto err_free_dma;
642	mhp->attr.pbl_size = max_num_sg;
643	ret = allocate_stag(&rhp->rdev, &stag, php->pdid,
644			    mhp->attr.pbl_size, mhp->attr.pbl_addr,
645			    mhp->wr_waitp);
646	if (ret)
647		goto err_free_pbl;
648	mhp->attr.pdid = php->pdid;
649	mhp->attr.type = FW_RI_STAG_NSMR;
650	mhp->attr.stag = stag;
651	mhp->attr.state = 0;
652	mmid = (stag) >> 8;
653	mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
654	if (xa_insert_irq(&rhp->mrs, mmid, mhp, GFP_KERNEL)) {
655		ret = -ENOMEM;
656		goto err_dereg;
657	}
658
659	pr_debug("mmid 0x%x mhp %p stag 0x%x\n", mmid, mhp, stag);
660	return &(mhp->ibmr);
661err_dereg:
662	dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size,
663		  mhp->attr.pbl_addr, mhp->dereg_skb, mhp->wr_waitp);
664err_free_pbl:
665	c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr,
666			      mhp->attr.pbl_size << 3);
667err_free_dma:
668	dma_free_coherent(&mhp->rhp->rdev.lldi.pdev->dev,
669			  mhp->max_mpl_len, mhp->mpl, mhp->mpl_addr);
670err_free_wr_wait:
671	c4iw_put_wr_wait(mhp->wr_waitp);
672err_free_mhp:
673	kfree(mhp);
674err:
675	return ERR_PTR(ret);
676}
677
678static int c4iw_set_page(struct ib_mr *ibmr, u64 addr)
679{
680	struct c4iw_mr *mhp = to_c4iw_mr(ibmr);
681
682	if (unlikely(mhp->mpl_len == mhp->attr.pbl_size))
683		return -ENOMEM;
684
685	mhp->mpl[mhp->mpl_len++] = addr;
686
687	return 0;
688}
689
690int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
691		   unsigned int *sg_offset)
692{
693	struct c4iw_mr *mhp = to_c4iw_mr(ibmr);
694
695	mhp->mpl_len = 0;
696
697	return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, c4iw_set_page);
698}
699
700int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
701{
702	struct c4iw_dev *rhp;
703	struct c4iw_mr *mhp;
704	u32 mmid;
705
706	pr_debug("ib_mr %p\n", ib_mr);
707
708	mhp = to_c4iw_mr(ib_mr);
709	rhp = mhp->rhp;
710	mmid = mhp->attr.stag >> 8;
711	xa_erase_irq(&rhp->mrs, mmid);
712	if (mhp->mpl)
713		dma_free_coherent(&mhp->rhp->rdev.lldi.pdev->dev,
714				  mhp->max_mpl_len, mhp->mpl, mhp->mpl_addr);
715	dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
716		  mhp->attr.pbl_addr, mhp->dereg_skb, mhp->wr_waitp);
717	if (mhp->attr.pbl_size)
718		c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr,
719				  mhp->attr.pbl_size << 3);
720	if (mhp->kva)
721		kfree((void *) (unsigned long) mhp->kva);
722	ib_umem_release(mhp->umem);
723	pr_debug("mmid 0x%x ptr %p\n", mmid, mhp);
724	c4iw_put_wr_wait(mhp->wr_waitp);
725	kfree(mhp);
726	return 0;
727}
728
729void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey)
730{
731	struct c4iw_mr *mhp;
732	unsigned long flags;
733
734	xa_lock_irqsave(&rhp->mrs, flags);
735	mhp = xa_load(&rhp->mrs, rkey >> 8);
736	if (mhp)
737		mhp->attr.state = 0;
738	xa_unlock_irqrestore(&rhp->mrs, flags);
739}