1/*
   2 * Copyright (c) 2007, 2008, 2009 QLogic Corporation. All rights reserved.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and/or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32#include <linux/mm.h>
  33#include <linux/types.h>
  34#include <linux/device.h>
  35#include <linux/dmapool.h>
  36#include <linux/slab.h>
  37#include <linux/list.h>
  38#include <linux/highmem.h>
  39#include <linux/io.h>
  40#include <linux/uio.h>
  41#include <linux/rbtree.h>
  42#include <linux/spinlock.h>
  43#include <linux/delay.h>
  44
  45#include "qib.h"
  46#include "qib_user_sdma.h"
  47
  48/* minimum size of header */
  49#define QIB_USER_SDMA_MIN_HEADER_LENGTH 64
  50/* expected size of headers (for dma_pool) */
  51#define QIB_USER_SDMA_EXP_HEADER_LENGTH 64
  52/* attempt to drain the queue for 5secs */
  53#define QIB_USER_SDMA_DRAIN_TIMEOUT 250
  54
  55/*
  56 * track how many times a process open this driver.
  57 */
  58static struct rb_root qib_user_sdma_rb_root = RB_ROOT;
  59
  60struct qib_user_sdma_rb_node {
  61	struct rb_node node;
  62	int refcount;
  63	pid_t pid;
  64};
  65
  66struct qib_user_sdma_pkt {
  67	struct list_head list;  /* list element */
  68
  69	u8  tiddma;		/* if this is NEW tid-sdma */
  70	u8  largepkt;		/* this is large pkt from kmalloc */
  71	u16 frag_size;		/* frag size used by PSM */
  72	u16 index;              /* last header index or push index */
  73	u16 naddr;              /* dimension of addr (1..3) ... */
  74	u16 addrlimit;		/* addr array size */
  75	u16 tidsmidx;		/* current tidsm index */
  76	u16 tidsmcount;		/* tidsm array item count */
  77	u16 payload_size;	/* payload size so far for header */
  78	u32 bytes_togo;		/* bytes for processing */
  79	u32 counter;            /* sdma pkts queued counter for this entry */
  80	struct qib_tid_session_member *tidsm;	/* tid session member array */
  81	struct qib_user_sdma_queue *pq;	/* which pq this pkt belongs to */
  82	u64 added;              /* global descq number of entries */
  83
  84	struct {
  85		u16 offset;                     /* offset for kvaddr, addr */
  86		u16 length;                     /* length in page */
  87		u16 first_desc;			/* first desc */
  88		u16 last_desc;			/* last desc */
  89		u16 put_page;                   /* should we put_page? */
  90		u16 dma_mapped;                 /* is page dma_mapped? */
  91		u16 dma_length;			/* for dma_unmap_page() */
  92		u16 padding;
  93		struct page *page;              /* may be NULL (coherent mem) */
  94		void *kvaddr;                   /* FIXME: only for pio hack */
  95		dma_addr_t addr;
  96	} addr[4];   /* max pages, any more and we coalesce */
  97};
  98
  99struct qib_user_sdma_queue {
 100	/*
 101	 * pkts sent to dma engine are queued on this
 102	 * list head.  the type of the elements of this
 103	 * list are struct qib_user_sdma_pkt...
 104	 */
 105	struct list_head sent;
 106
 107	/*
 108	 * Because above list will be accessed by both process and
 109	 * signal handler, we need a spinlock for it.
 110	 */
 111	spinlock_t sent_lock ____cacheline_aligned_in_smp;
 112
 113	/* headers with expected length are allocated from here... */
 114	char header_cache_name[64];
 115	struct dma_pool *header_cache;
 116
 117	/* packets are allocated from the slab cache... */
 118	char pkt_slab_name[64];
 119	struct kmem_cache *pkt_slab;
 120
 121	/* as packets go on the queued queue, they are counted... */
 122	u32 counter;
 123	u32 sent_counter;
 124	/* pending packets, not sending yet */
 125	u32 num_pending;
 126	/* sending packets, not complete yet */
 127	u32 num_sending;
 128	/* global descq number of entry of last sending packet */
 129	u64 added;
 130
 131	/* dma page table */
 132	struct rb_root dma_pages_root;
 133
 134	struct qib_user_sdma_rb_node *sdma_rb_node;
 135
 136	/* protect everything above... */
 137	struct mutex lock;
 138};
 139
 140static struct qib_user_sdma_rb_node *
 141qib_user_sdma_rb_search(struct rb_root *root, pid_t pid)
 142{
 143	struct qib_user_sdma_rb_node *sdma_rb_node;
 144	struct rb_node *node = root->rb_node;
 145
 146	while (node) {
 147		sdma_rb_node = rb_entry(node, struct qib_user_sdma_rb_node,
 148					node);
 149		if (pid < sdma_rb_node->pid)
 150			node = node->rb_left;
 151		else if (pid > sdma_rb_node->pid)
 152			node = node->rb_right;
 153		else
 154			return sdma_rb_node;
 155	}
 156	return NULL;
 157}
 158
 159static int
 160qib_user_sdma_rb_insert(struct rb_root *root, struct qib_user_sdma_rb_node *new)
 161{
 162	struct rb_node **node = &(root->rb_node);
 163	struct rb_node *parent = NULL;
 164	struct qib_user_sdma_rb_node *got;
 165
 166	while (*node) {
 167		got = rb_entry(*node, struct qib_user_sdma_rb_node, node);
 168		parent = *node;
 169		if (new->pid < got->pid)
 170			node = &((*node)->rb_left);
 171		else if (new->pid > got->pid)
 172			node = &((*node)->rb_right);
 173		else
 174			return 0;
 175	}
 176
 177	rb_link_node(&new->node, parent, node);
 178	rb_insert_color(&new->node, root);
 179	return 1;
 180}
 181
 182struct qib_user_sdma_queue *
 183qib_user_sdma_queue_create(struct device *dev, int unit, int ctxt, int sctxt)
 184{
 185	struct qib_user_sdma_queue *pq =
 186		kmalloc(sizeof(struct qib_user_sdma_queue), GFP_KERNEL);
 187	struct qib_user_sdma_rb_node *sdma_rb_node;
 188
 189	if (!pq)
 190		goto done;
 191
 192	pq->counter = 0;
 193	pq->sent_counter = 0;
 194	pq->num_pending = 0;
 195	pq->num_sending = 0;
 196	pq->added = 0;
 197	pq->sdma_rb_node = NULL;
 198
 199	INIT_LIST_HEAD(&pq->sent);
 200	spin_lock_init(&pq->sent_lock);
 201	mutex_init(&pq->lock);
 202
 203	snprintf(pq->pkt_slab_name, sizeof(pq->pkt_slab_name),
 204		 "qib-user-sdma-pkts-%u-%02u.%02u", unit, ctxt, sctxt);
 205	pq->pkt_slab = kmem_cache_create(pq->pkt_slab_name,
 206					 sizeof(struct qib_user_sdma_pkt),
 207					 0, 0, NULL);
 208
 209	if (!pq->pkt_slab)
 210		goto err_kfree;
 211
 212	snprintf(pq->header_cache_name, sizeof(pq->header_cache_name),
 213		 "qib-user-sdma-headers-%u-%02u.%02u", unit, ctxt, sctxt);
 214	pq->header_cache = dma_pool_create(pq->header_cache_name,
 215					   dev,
 216					   QIB_USER_SDMA_EXP_HEADER_LENGTH,
 217					   4, 0);
 218	if (!pq->header_cache)
 219		goto err_slab;
 220
 221	pq->dma_pages_root = RB_ROOT;
 222
 223	sdma_rb_node = qib_user_sdma_rb_search(&qib_user_sdma_rb_root,
 224					current->pid);
 225	if (sdma_rb_node) {
 226		sdma_rb_node->refcount++;
 227	} else {
 228		sdma_rb_node = kmalloc(sizeof(
 229			struct qib_user_sdma_rb_node), GFP_KERNEL);
 230		if (!sdma_rb_node)
 231			goto err_rb;
 232
 233		sdma_rb_node->refcount = 1;
 234		sdma_rb_node->pid = current->pid;
 235
 236		qib_user_sdma_rb_insert(&qib_user_sdma_rb_root, sdma_rb_node);
 237	}
 238	pq->sdma_rb_node = sdma_rb_node;
 239
 240	goto done;
 241
 242err_rb:
 243	dma_pool_destroy(pq->header_cache);
 244err_slab:
 245	kmem_cache_destroy(pq->pkt_slab);
 246err_kfree:
 247	kfree(pq);
 248	pq = NULL;
 249
 250done:
 251	return pq;
 252}
 253
 254static void qib_user_sdma_init_frag(struct qib_user_sdma_pkt *pkt,
 255				    int i, u16 offset, u16 len,
 256				    u16 first_desc, u16 last_desc,
 257				    u16 put_page, u16 dma_mapped,
 258				    struct page *page, void *kvaddr,
 259				    dma_addr_t dma_addr, u16 dma_length)
 260{
 261	pkt->addr[i].offset = offset;
 262	pkt->addr[i].length = len;
 263	pkt->addr[i].first_desc = first_desc;
 264	pkt->addr[i].last_desc = last_desc;
 265	pkt->addr[i].put_page = put_page;
 266	pkt->addr[i].dma_mapped = dma_mapped;
 267	pkt->addr[i].page = page;
 268	pkt->addr[i].kvaddr = kvaddr;
 269	pkt->addr[i].addr = dma_addr;
 270	pkt->addr[i].dma_length = dma_length;
 271}
 272
 273static void *qib_user_sdma_alloc_header(struct qib_user_sdma_queue *pq,
 274				size_t len, dma_addr_t *dma_addr)
 275{
 276	void *hdr;
 277
 278	if (len == QIB_USER_SDMA_EXP_HEADER_LENGTH)
 279		hdr = dma_pool_alloc(pq->header_cache, GFP_KERNEL,
 280					     dma_addr);
 281	else
 282		hdr = NULL;
 283
 284	if (!hdr) {
 285		hdr = kmalloc(len, GFP_KERNEL);
 286		if (!hdr)
 287			return NULL;
 288
 289		*dma_addr = 0;
 290	}
 291
 292	return hdr;
 293}
 294
 295static int qib_user_sdma_page_to_frags(const struct qib_devdata *dd,
 296				       struct qib_user_sdma_queue *pq,
 297				       struct qib_user_sdma_pkt *pkt,
 298				       struct page *page, u16 put,
 299				       u16 offset, u16 len, void *kvaddr)
 300{
 301	__le16 *pbc16;
 302	void *pbcvaddr;
 303	struct qib_message_header *hdr;
 304	u16 newlen, pbclen, lastdesc, dma_mapped;
 305	u32 vcto;
 306	union qib_seqnum seqnum;
 307	dma_addr_t pbcdaddr;
 308	dma_addr_t dma_addr =
 309		dma_map_page(&dd->pcidev->dev,
 310			page, offset, len, DMA_TO_DEVICE);
 311	int ret = 0;
 312
 313	if (dma_mapping_error(&dd->pcidev->dev, dma_addr)) {
 314		/*
 315		 * dma mapping error, pkt has not managed
 316		 * this page yet, return the page here so
 317		 * the caller can ignore this page.
 318		 */
 319		if (put) {
 320			unpin_user_page(page);
 321		} else {
 322			/* coalesce case */
 
 323			__free_page(page);
 324		}
 325		ret = -ENOMEM;
 326		goto done;
 327	}
 328	offset = 0;
 329	dma_mapped = 1;
 330
 331
 332next_fragment:
 333
 334	/*
 335	 * In tid-sdma, the transfer length is restricted by
 336	 * receiver side current tid page length.
 337	 */
 338	if (pkt->tiddma && len > pkt->tidsm[pkt->tidsmidx].length)
 339		newlen = pkt->tidsm[pkt->tidsmidx].length;
 340	else
 341		newlen = len;
 342
 343	/*
 344	 * Then the transfer length is restricted by MTU.
 345	 * the last descriptor flag is determined by:
 346	 * 1. the current packet is at frag size length.
 347	 * 2. the current tid page is done if tid-sdma.
 348	 * 3. there is no more byte togo if sdma.
 349	 */
 350	lastdesc = 0;
 351	if ((pkt->payload_size + newlen) >= pkt->frag_size) {
 352		newlen = pkt->frag_size - pkt->payload_size;
 353		lastdesc = 1;
 354	} else if (pkt->tiddma) {
 355		if (newlen == pkt->tidsm[pkt->tidsmidx].length)
 356			lastdesc = 1;
 357	} else {
 358		if (newlen == pkt->bytes_togo)
 359			lastdesc = 1;
 360	}
 361
 362	/* fill the next fragment in this page */
 363	qib_user_sdma_init_frag(pkt, pkt->naddr, /* index */
 364		offset, newlen,		/* offset, len */
 365		0, lastdesc,		/* first last desc */
 366		put, dma_mapped,	/* put page, dma mapped */
 367		page, kvaddr,		/* struct page, virt addr */
 368		dma_addr, len);		/* dma addr, dma length */
 369	pkt->bytes_togo -= newlen;
 370	pkt->payload_size += newlen;
 371	pkt->naddr++;
 372	if (pkt->naddr == pkt->addrlimit) {
 373		ret = -EFAULT;
 374		goto done;
 375	}
 376
 377	/* If there is no more byte togo. (lastdesc==1) */
 378	if (pkt->bytes_togo == 0) {
 379		/* The packet is done, header is not dma mapped yet.
 380		 * it should be from kmalloc */
 381		if (!pkt->addr[pkt->index].addr) {
 382			pkt->addr[pkt->index].addr =
 383				dma_map_single(&dd->pcidev->dev,
 384					pkt->addr[pkt->index].kvaddr,
 385					pkt->addr[pkt->index].dma_length,
 386					DMA_TO_DEVICE);
 387			if (dma_mapping_error(&dd->pcidev->dev,
 388					pkt->addr[pkt->index].addr)) {
 389				ret = -ENOMEM;
 390				goto done;
 391			}
 392			pkt->addr[pkt->index].dma_mapped = 1;
 393		}
 394
 395		goto done;
 396	}
 397
 398	/* If tid-sdma, advance tid info. */
 399	if (pkt->tiddma) {
 400		pkt->tidsm[pkt->tidsmidx].length -= newlen;
 401		if (pkt->tidsm[pkt->tidsmidx].length) {
 402			pkt->tidsm[pkt->tidsmidx].offset += newlen;
 403		} else {
 404			pkt->tidsmidx++;
 405			if (pkt->tidsmidx == pkt->tidsmcount) {
 406				ret = -EFAULT;
 407				goto done;
 408			}
 409		}
 410	}
 411
 412	/*
 413	 * If this is NOT the last descriptor. (newlen==len)
 414	 * the current packet is not done yet, but the current
 415	 * send side page is done.
 416	 */
 417	if (lastdesc == 0)
 418		goto done;
 419
 420	/*
 421	 * If running this driver under PSM with message size
 422	 * fitting into one transfer unit, it is not possible
 423	 * to pass this line. otherwise, it is a buggggg.
 424	 */
 425
 426	/*
 427	 * Since the current packet is done, and there are more
 428	 * bytes togo, we need to create a new sdma header, copying
 429	 * from previous sdma header and modify both.
 430	 */
 431	pbclen = pkt->addr[pkt->index].length;
 432	pbcvaddr = qib_user_sdma_alloc_header(pq, pbclen, &pbcdaddr);
 433	if (!pbcvaddr) {
 434		ret = -ENOMEM;
 435		goto done;
 436	}
 437	/* Copy the previous sdma header to new sdma header */
 438	pbc16 = (__le16 *)pkt->addr[pkt->index].kvaddr;
 439	memcpy(pbcvaddr, pbc16, pbclen);
 440
 441	/* Modify the previous sdma header */
 442	hdr = (struct qib_message_header *)&pbc16[4];
 443
 444	/* New pbc length */
 445	pbc16[0] = cpu_to_le16(le16_to_cpu(pbc16[0])-(pkt->bytes_togo>>2));
 446
 447	/* New packet length */
 448	hdr->lrh[2] = cpu_to_be16(le16_to_cpu(pbc16[0]));
 449
 450	if (pkt->tiddma) {
 451		/* turn on the header suppression */
 452		hdr->iph.pkt_flags =
 453			cpu_to_le16(le16_to_cpu(hdr->iph.pkt_flags)|0x2);
 454		/* turn off ACK_REQ: 0x04 and EXPECTED_DONE: 0x20 */
 455		hdr->flags &= ~(0x04|0x20);
 456	} else {
 457		/* turn off extra bytes: 20-21 bits */
 458		hdr->bth[0] = cpu_to_be32(be32_to_cpu(hdr->bth[0])&0xFFCFFFFF);
 459		/* turn off ACK_REQ: 0x04 */
 460		hdr->flags &= ~(0x04);
 461	}
 462
 463	/* New kdeth checksum */
 464	vcto = le32_to_cpu(hdr->iph.ver_ctxt_tid_offset);
 465	hdr->iph.chksum = cpu_to_le16(QIB_LRH_BTH +
 466		be16_to_cpu(hdr->lrh[2]) -
 467		((vcto>>16)&0xFFFF) - (vcto&0xFFFF) -
 468		le16_to_cpu(hdr->iph.pkt_flags));
 469
 470	/* The packet is done, header is not dma mapped yet.
 471	 * it should be from kmalloc */
 472	if (!pkt->addr[pkt->index].addr) {
 473		pkt->addr[pkt->index].addr =
 474			dma_map_single(&dd->pcidev->dev,
 475				pkt->addr[pkt->index].kvaddr,
 476				pkt->addr[pkt->index].dma_length,
 477				DMA_TO_DEVICE);
 478		if (dma_mapping_error(&dd->pcidev->dev,
 479				pkt->addr[pkt->index].addr)) {
 480			ret = -ENOMEM;
 481			goto done;
 482		}
 483		pkt->addr[pkt->index].dma_mapped = 1;
 484	}
 485
 486	/* Modify the new sdma header */
 487	pbc16 = (__le16 *)pbcvaddr;
 488	hdr = (struct qib_message_header *)&pbc16[4];
 489
 490	/* New pbc length */
 491	pbc16[0] = cpu_to_le16(le16_to_cpu(pbc16[0])-(pkt->payload_size>>2));
 492
 493	/* New packet length */
 494	hdr->lrh[2] = cpu_to_be16(le16_to_cpu(pbc16[0]));
 495
 496	if (pkt->tiddma) {
 497		/* Set new tid and offset for new sdma header */
 498		hdr->iph.ver_ctxt_tid_offset = cpu_to_le32(
 499			(le32_to_cpu(hdr->iph.ver_ctxt_tid_offset)&0xFF000000) +
 500			(pkt->tidsm[pkt->tidsmidx].tid<<QLOGIC_IB_I_TID_SHIFT) +
 501			(pkt->tidsm[pkt->tidsmidx].offset>>2));
 502	} else {
 503		/* Middle protocol new packet offset */
 504		hdr->uwords[2] += pkt->payload_size;
 505	}
 506
 507	/* New kdeth checksum */
 508	vcto = le32_to_cpu(hdr->iph.ver_ctxt_tid_offset);
 509	hdr->iph.chksum = cpu_to_le16(QIB_LRH_BTH +
 510		be16_to_cpu(hdr->lrh[2]) -
 511		((vcto>>16)&0xFFFF) - (vcto&0xFFFF) -
 512		le16_to_cpu(hdr->iph.pkt_flags));
 513
 514	/* Next sequence number in new sdma header */
 515	seqnum.val = be32_to_cpu(hdr->bth[2]);
 516	if (pkt->tiddma)
 517		seqnum.seq++;
 518	else
 519		seqnum.pkt++;
 520	hdr->bth[2] = cpu_to_be32(seqnum.val);
 521
 522	/* Init new sdma header. */
 523	qib_user_sdma_init_frag(pkt, pkt->naddr, /* index */
 524		0, pbclen,		/* offset, len */
 525		1, 0,			/* first last desc */
 526		0, 0,			/* put page, dma mapped */
 527		NULL, pbcvaddr,		/* struct page, virt addr */
 528		pbcdaddr, pbclen);	/* dma addr, dma length */
 529	pkt->index = pkt->naddr;
 530	pkt->payload_size = 0;
 531	pkt->naddr++;
 532	if (pkt->naddr == pkt->addrlimit) {
 533		ret = -EFAULT;
 534		goto done;
 535	}
 536
 537	/* Prepare for next fragment in this page */
 538	if (newlen != len) {
 539		if (dma_mapped) {
 540			put = 0;
 541			dma_mapped = 0;
 542			page = NULL;
 543			kvaddr = NULL;
 544		}
 545		len -= newlen;
 546		offset += newlen;
 547
 548		goto next_fragment;
 549	}
 550
 551done:
 552	return ret;
 553}
 554
 555/* we've too many pages in the iovec, coalesce to a single page */
 556static int qib_user_sdma_coalesce(const struct qib_devdata *dd,
 557				  struct qib_user_sdma_queue *pq,
 558				  struct qib_user_sdma_pkt *pkt,
 559				  const struct iovec *iov,
 560				  unsigned long niov)
 561{
 562	int ret = 0;
 563	struct page *page = alloc_page(GFP_KERNEL);
 564	void *mpage_save;
 565	char *mpage;
 566	int i;
 567	int len = 0;
 568
 569	if (!page) {
 570		ret = -ENOMEM;
 571		goto done;
 572	}
 573
 574	mpage = page_address(page);
 575	mpage_save = mpage;
 576	for (i = 0; i < niov; i++) {
 577		int cfur;
 578
 579		cfur = copy_from_user(mpage,
 580				      iov[i].iov_base, iov[i].iov_len);
 581		if (cfur) {
 582			ret = -EFAULT;
 583			goto page_free;
 584		}
 585
 586		mpage += iov[i].iov_len;
 587		len += iov[i].iov_len;
 588	}
 589
 590	ret = qib_user_sdma_page_to_frags(dd, pq, pkt,
 591			page, 0, 0, len, mpage_save);
 592	goto done;
 593
 594page_free:
 
 595	__free_page(page);
 596done:
 597	return ret;
 598}
 599
 600/*
 601 * How many pages in this iovec element?
 602 */
 603static size_t qib_user_sdma_num_pages(const struct iovec *iov)
 604{
 605	const unsigned long addr  = (unsigned long) iov->iov_base;
 606	const unsigned long  len  = iov->iov_len;
 607	const unsigned long spage = addr & PAGE_MASK;
 608	const unsigned long epage = (addr + len - 1) & PAGE_MASK;
 609
 610	return 1 + ((epage - spage) >> PAGE_SHIFT);
 611}
 612
 613static void qib_user_sdma_free_pkt_frag(struct device *dev,
 614					struct qib_user_sdma_queue *pq,
 615					struct qib_user_sdma_pkt *pkt,
 616					int frag)
 617{
 618	const int i = frag;
 619
 620	if (pkt->addr[i].page) {
 621		/* only user data has page */
 622		if (pkt->addr[i].dma_mapped)
 623			dma_unmap_page(dev,
 624				       pkt->addr[i].addr,
 625				       pkt->addr[i].dma_length,
 626				       DMA_TO_DEVICE);
 627
 
 
 
 628		if (pkt->addr[i].put_page)
 629			unpin_user_page(pkt->addr[i].page);
 630		else
 631			__free_page(pkt->addr[i].page);
 632	} else if (pkt->addr[i].kvaddr) {
 633		/* for headers */
 634		if (pkt->addr[i].dma_mapped) {
 635			/* from kmalloc & dma mapped */
 636			dma_unmap_single(dev,
 637				       pkt->addr[i].addr,
 638				       pkt->addr[i].dma_length,
 639				       DMA_TO_DEVICE);
 640			kfree(pkt->addr[i].kvaddr);
 641		} else if (pkt->addr[i].addr) {
 642			/* free coherent mem from cache... */
 643			dma_pool_free(pq->header_cache,
 644			      pkt->addr[i].kvaddr, pkt->addr[i].addr);
 645		} else {
 646			/* from kmalloc but not dma mapped */
 647			kfree(pkt->addr[i].kvaddr);
 648		}
 649	}
 650}
 651
 652/* return number of pages pinned... */
 653static int qib_user_sdma_pin_pages(const struct qib_devdata *dd,
 654				   struct qib_user_sdma_queue *pq,
 655				   struct qib_user_sdma_pkt *pkt,
 656				   unsigned long addr, int tlen, size_t npages)
 657{
 658	struct page *pages[8];
 659	int i, j;
 660	int ret = 0;
 661
 662	while (npages) {
 663		if (npages > 8)
 664			j = 8;
 665		else
 666			j = npages;
 667
 668		ret = pin_user_pages_fast(addr, j, FOLL_LONGTERM, pages);
 669		if (ret != j) {
 670			i = 0;
 671			j = ret;
 672			ret = -ENOMEM;
 673			goto free_pages;
 674		}
 675
 676		for (i = 0; i < j; i++) {
 677			/* map the pages... */
 678			unsigned long fofs = addr & ~PAGE_MASK;
 679			int flen = ((fofs + tlen) > PAGE_SIZE) ?
 680				(PAGE_SIZE - fofs) : tlen;
 681
 682			ret = qib_user_sdma_page_to_frags(dd, pq, pkt,
 683				pages[i], 1, fofs, flen, NULL);
 684			if (ret < 0) {
 685				/* current page has beed taken
 686				 * care of inside above call.
 687				 */
 688				i++;
 689				goto free_pages;
 690			}
 691
 692			addr += flen;
 693			tlen -= flen;
 694		}
 695
 696		npages -= j;
 697	}
 698
 699	goto done;
 700
 701	/* if error, return all pages not managed by pkt */
 702free_pages:
 703	while (i < j)
 704		unpin_user_page(pages[i++]);
 705
 706done:
 707	return ret;
 708}
 709
 710static int qib_user_sdma_pin_pkt(const struct qib_devdata *dd,
 711				 struct qib_user_sdma_queue *pq,
 712				 struct qib_user_sdma_pkt *pkt,
 713				 const struct iovec *iov,
 714				 unsigned long niov)
 715{
 716	int ret = 0;
 717	unsigned long idx;
 718
 719	for (idx = 0; idx < niov; idx++) {
 720		const size_t npages = qib_user_sdma_num_pages(iov + idx);
 721		const unsigned long addr = (unsigned long) iov[idx].iov_base;
 722
 723		ret = qib_user_sdma_pin_pages(dd, pq, pkt, addr,
 724					      iov[idx].iov_len, npages);
 725		if (ret < 0)
 726			goto free_pkt;
 727	}
 728
 729	goto done;
 730
 731free_pkt:
 732	/* we need to ignore the first entry here */
 733	for (idx = 1; idx < pkt->naddr; idx++)
 734		qib_user_sdma_free_pkt_frag(&dd->pcidev->dev, pq, pkt, idx);
 735
 736	/* need to dma unmap the first entry, this is to restore to
 737	 * the original state so that caller can free the memory in
 738	 * error condition. Caller does not know if dma mapped or not*/
 739	if (pkt->addr[0].dma_mapped) {
 740		dma_unmap_single(&dd->pcidev->dev,
 741		       pkt->addr[0].addr,
 742		       pkt->addr[0].dma_length,
 743		       DMA_TO_DEVICE);
 744		pkt->addr[0].addr = 0;
 745		pkt->addr[0].dma_mapped = 0;
 746	}
 747
 748done:
 749	return ret;
 750}
 751
 752static int qib_user_sdma_init_payload(const struct qib_devdata *dd,
 753				      struct qib_user_sdma_queue *pq,
 754				      struct qib_user_sdma_pkt *pkt,
 755				      const struct iovec *iov,
 756				      unsigned long niov, int npages)
 757{
 758	int ret = 0;
 759
 760	if (pkt->frag_size == pkt->bytes_togo &&
 761			npages >= ARRAY_SIZE(pkt->addr))
 762		ret = qib_user_sdma_coalesce(dd, pq, pkt, iov, niov);
 763	else
 764		ret = qib_user_sdma_pin_pkt(dd, pq, pkt, iov, niov);
 765
 766	return ret;
 767}
 768
 769/* free a packet list -- return counter value of last packet */
 770static void qib_user_sdma_free_pkt_list(struct device *dev,
 771					struct qib_user_sdma_queue *pq,
 772					struct list_head *list)
 773{
 774	struct qib_user_sdma_pkt *pkt, *pkt_next;
 775
 776	list_for_each_entry_safe(pkt, pkt_next, list, list) {
 777		int i;
 778
 779		for (i = 0; i < pkt->naddr; i++)
 780			qib_user_sdma_free_pkt_frag(dev, pq, pkt, i);
 781
 782		if (pkt->largepkt)
 783			kfree(pkt);
 784		else
 785			kmem_cache_free(pq->pkt_slab, pkt);
 786	}
 787	INIT_LIST_HEAD(list);
 788}
 789
 790/*
 791 * copy headers, coalesce etc -- pq->lock must be held
 792 *
 793 * we queue all the packets to list, returning the
 794 * number of bytes total.  list must be empty initially,
 795 * as, if there is an error we clean it...
 796 */
 797static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd,
 798				    struct qib_pportdata *ppd,
 799				    struct qib_user_sdma_queue *pq,
 800				    const struct iovec *iov,
 801				    unsigned long niov,
 802				    struct list_head *list,
 803				    int *maxpkts, int *ndesc)
 804{
 805	unsigned long idx = 0;
 806	int ret = 0;
 807	int npkts = 0;
 808	__le32 *pbc;
 809	dma_addr_t dma_addr;
 810	struct qib_user_sdma_pkt *pkt = NULL;
 811	size_t len;
 812	size_t nw;
 813	u32 counter = pq->counter;
 814	u16 frag_size;
 815
 816	while (idx < niov && npkts < *maxpkts) {
 817		const unsigned long addr = (unsigned long) iov[idx].iov_base;
 818		const unsigned long idx_save = idx;
 819		unsigned pktnw;
 820		unsigned pktnwc;
 821		int nfrags = 0;
 822		size_t npages = 0;
 823		size_t bytes_togo = 0;
 824		int tiddma = 0;
 825		int cfur;
 826
 827		len = iov[idx].iov_len;
 828		nw = len >> 2;
 829
 830		if (len < QIB_USER_SDMA_MIN_HEADER_LENGTH ||
 831		    len > PAGE_SIZE || len & 3 || addr & 3) {
 832			ret = -EINVAL;
 833			goto free_list;
 834		}
 835
 836		pbc = qib_user_sdma_alloc_header(pq, len, &dma_addr);
 837		if (!pbc) {
 838			ret = -ENOMEM;
 839			goto free_list;
 840		}
 841
 842		cfur = copy_from_user(pbc, iov[idx].iov_base, len);
 843		if (cfur) {
 844			ret = -EFAULT;
 845			goto free_pbc;
 846		}
 847
 848		/*
 849		 * This assignment is a bit strange.  it's because
 850		 * the pbc counts the number of 32 bit words in the full
 851		 * packet _except_ the first word of the pbc itself...
 852		 */
 853		pktnwc = nw - 1;
 854
 855		/*
 856		 * pktnw computation yields the number of 32 bit words
 857		 * that the caller has indicated in the PBC.  note that
 858		 * this is one less than the total number of words that
 859		 * goes to the send DMA engine as the first 32 bit word
 860		 * of the PBC itself is not counted.  Armed with this count,
 861		 * we can verify that the packet is consistent with the
 862		 * iovec lengths.
 863		 */
 864		pktnw = le32_to_cpu(*pbc) & 0xFFFF;
 865		if (pktnw < pktnwc) {
 866			ret = -EINVAL;
 867			goto free_pbc;
 868		}
 869
 870		idx++;
 871		while (pktnwc < pktnw && idx < niov) {
 872			const size_t slen = iov[idx].iov_len;
 873			const unsigned long faddr =
 874				(unsigned long) iov[idx].iov_base;
 875
 876			if (slen & 3 || faddr & 3 || !slen) {
 877				ret = -EINVAL;
 878				goto free_pbc;
 879			}
 880
 881			npages += qib_user_sdma_num_pages(&iov[idx]);
 882
 883			if (check_add_overflow(bytes_togo, slen, &bytes_togo) ||
 884			    bytes_togo > type_max(typeof(pkt->bytes_togo))) {
 885				ret = -EINVAL;
 886				goto free_pbc;
 887			}
 888			pktnwc += slen >> 2;
 889			idx++;
 890			nfrags++;
 891		}
 892
 893		if (pktnwc != pktnw) {
 894			ret = -EINVAL;
 895			goto free_pbc;
 896		}
 897
 898		frag_size = ((le32_to_cpu(*pbc))>>16) & 0xFFFF;
 899		if (((frag_size ? frag_size : bytes_togo) + len) >
 900						ppd->ibmaxlen) {
 901			ret = -EINVAL;
 902			goto free_pbc;
 903		}
 904
 905		if (frag_size) {
 906			size_t tidsmsize, n, pktsize, sz, addrlimit;
 
 907
 908			n = npages*((2*PAGE_SIZE/frag_size)+1);
 909			pktsize = struct_size(pkt, addr, n);
 910
 911			/*
 912			 * Determine if this is tid-sdma or just sdma.
 913			 */
 914			tiddma = (((le32_to_cpu(pbc[7])>>
 915				QLOGIC_IB_I_TID_SHIFT)&
 916				QLOGIC_IB_I_TID_MASK) !=
 917				QLOGIC_IB_I_TID_MASK);
 918
 919			if (tiddma)
 920				tidsmsize = iov[idx].iov_len;
 921			else
 922				tidsmsize = 0;
 923
 924			if (check_add_overflow(pktsize, tidsmsize, &sz)) {
 925				ret = -EINVAL;
 926				goto free_pbc;
 927			}
 928			pkt = kmalloc(sz, GFP_KERNEL);
 929			if (!pkt) {
 930				ret = -ENOMEM;
 931				goto free_pbc;
 932			}
 933			pkt->largepkt = 1;
 934			pkt->frag_size = frag_size;
 935			if (check_add_overflow(n, ARRAY_SIZE(pkt->addr),
 936					       &addrlimit) ||
 937			    addrlimit > type_max(typeof(pkt->addrlimit))) {
 938				ret = -EINVAL;
 939				goto free_pkt;
 940			}
 941			pkt->addrlimit = addrlimit;
 942
 943			if (tiddma) {
 944				char *tidsm = (char *)pkt + pktsize;
 945
 946				cfur = copy_from_user(tidsm,
 947					iov[idx].iov_base, tidsmsize);
 948				if (cfur) {
 949					ret = -EFAULT;
 950					goto free_pkt;
 951				}
 952				pkt->tidsm =
 953					(struct qib_tid_session_member *)tidsm;
 954				pkt->tidsmcount = tidsmsize/
 955					sizeof(struct qib_tid_session_member);
 956				pkt->tidsmidx = 0;
 957				idx++;
 958			}
 959
 960			/*
 961			 * pbc 'fill1' field is borrowed to pass frag size,
 962			 * we need to clear it after picking frag size, the
 963			 * hardware requires this field to be zero.
 964			 */
 965			*pbc = cpu_to_le32(le32_to_cpu(*pbc) & 0x0000FFFF);
 966		} else {
 967			pkt = kmem_cache_alloc(pq->pkt_slab, GFP_KERNEL);
 968			if (!pkt) {
 969				ret = -ENOMEM;
 970				goto free_pbc;
 971			}
 972			pkt->largepkt = 0;
 973			pkt->frag_size = bytes_togo;
 974			pkt->addrlimit = ARRAY_SIZE(pkt->addr);
 975		}
 976		pkt->bytes_togo = bytes_togo;
 977		pkt->payload_size = 0;
 978		pkt->counter = counter;
 979		pkt->tiddma = tiddma;
 980
 981		/* setup the first header */
 982		qib_user_sdma_init_frag(pkt, 0, /* index */
 983			0, len,		/* offset, len */
 984			1, 0,		/* first last desc */
 985			0, 0,		/* put page, dma mapped */
 986			NULL, pbc,	/* struct page, virt addr */
 987			dma_addr, len);	/* dma addr, dma length */
 988		pkt->index = 0;
 989		pkt->naddr = 1;
 990
 991		if (nfrags) {
 992			ret = qib_user_sdma_init_payload(dd, pq, pkt,
 993							 iov + idx_save + 1,
 994							 nfrags, npages);
 995			if (ret < 0)
 996				goto free_pkt;
 997		} else {
 998			/* since there is no payload, mark the
 999			 * header as the last desc. */
1000			pkt->addr[0].last_desc = 1;
1001
1002			if (dma_addr == 0) {
1003				/*
1004				 * the header is not dma mapped yet.
1005				 * it should be from kmalloc.
1006				 */
1007				dma_addr = dma_map_single(&dd->pcidev->dev,
1008					pbc, len, DMA_TO_DEVICE);
1009				if (dma_mapping_error(&dd->pcidev->dev,
1010								dma_addr)) {
1011					ret = -ENOMEM;
1012					goto free_pkt;
1013				}
1014				pkt->addr[0].addr = dma_addr;
1015				pkt->addr[0].dma_mapped = 1;
1016			}
1017		}
1018
1019		counter++;
1020		npkts++;
1021		pkt->pq = pq;
1022		pkt->index = 0; /* reset index for push on hw */
1023		*ndesc += pkt->naddr;
1024
1025		list_add_tail(&pkt->list, list);
1026	}
1027
1028	*maxpkts = npkts;
1029	ret = idx;
1030	goto done;
1031
1032free_pkt:
1033	if (pkt->largepkt)
1034		kfree(pkt);
1035	else
1036		kmem_cache_free(pq->pkt_slab, pkt);
1037free_pbc:
1038	if (dma_addr)
1039		dma_pool_free(pq->header_cache, pbc, dma_addr);
1040	else
1041		kfree(pbc);
1042free_list:
1043	qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, list);
1044done:
1045	return ret;
1046}
1047
1048static void qib_user_sdma_set_complete_counter(struct qib_user_sdma_queue *pq,
1049					       u32 c)
1050{
1051	pq->sent_counter = c;
1052}
1053
1054/* try to clean out queue -- needs pq->lock */
1055static int qib_user_sdma_queue_clean(struct qib_pportdata *ppd,
1056				     struct qib_user_sdma_queue *pq)
1057{
1058	struct qib_devdata *dd = ppd->dd;
1059	struct list_head free_list;
1060	struct qib_user_sdma_pkt *pkt;
1061	struct qib_user_sdma_pkt *pkt_prev;
1062	unsigned long flags;
1063	int ret = 0;
1064
1065	if (!pq->num_sending)
1066		return 0;
1067
1068	INIT_LIST_HEAD(&free_list);
1069
1070	/*
1071	 * We need this spin lock here because interrupt handler
1072	 * might modify this list in qib_user_sdma_send_desc(), also
1073	 * we can not get interrupted, otherwise it is a deadlock.
1074	 */
1075	spin_lock_irqsave(&pq->sent_lock, flags);
1076	list_for_each_entry_safe(pkt, pkt_prev, &pq->sent, list) {
1077		s64 descd = ppd->sdma_descq_removed - pkt->added;
1078
1079		if (descd < 0)
1080			break;
1081
1082		list_move_tail(&pkt->list, &free_list);
1083
1084		/* one more packet cleaned */
1085		ret++;
1086		pq->num_sending--;
1087	}
1088	spin_unlock_irqrestore(&pq->sent_lock, flags);
1089
1090	if (!list_empty(&free_list)) {
1091		u32 counter;
1092
1093		pkt = list_entry(free_list.prev,
1094				 struct qib_user_sdma_pkt, list);
1095		counter = pkt->counter;
1096
1097		qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &free_list);
1098		qib_user_sdma_set_complete_counter(pq, counter);
1099	}
1100
1101	return ret;
1102}
1103
1104void qib_user_sdma_queue_destroy(struct qib_user_sdma_queue *pq)
1105{
1106	if (!pq)
1107		return;
1108
1109	pq->sdma_rb_node->refcount--;
1110	if (pq->sdma_rb_node->refcount == 0) {
1111		rb_erase(&pq->sdma_rb_node->node, &qib_user_sdma_rb_root);
1112		kfree(pq->sdma_rb_node);
1113	}
1114	dma_pool_destroy(pq->header_cache);
1115	kmem_cache_destroy(pq->pkt_slab);
1116	kfree(pq);
1117}
1118
1119/* clean descriptor queue, returns > 0 if some elements cleaned */
1120static int qib_user_sdma_hwqueue_clean(struct qib_pportdata *ppd)
1121{
1122	int ret;
1123	unsigned long flags;
1124
1125	spin_lock_irqsave(&ppd->sdma_lock, flags);
1126	ret = qib_sdma_make_progress(ppd);
1127	spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1128
1129	return ret;
1130}
1131
1132/* we're in close, drain packets so that we can cleanup successfully... */
1133void qib_user_sdma_queue_drain(struct qib_pportdata *ppd,
1134			       struct qib_user_sdma_queue *pq)
1135{
1136	struct qib_devdata *dd = ppd->dd;
1137	unsigned long flags;
1138	int i;
1139
1140	if (!pq)
1141		return;
1142
1143	for (i = 0; i < QIB_USER_SDMA_DRAIN_TIMEOUT; i++) {
1144		mutex_lock(&pq->lock);
1145		if (!pq->num_pending && !pq->num_sending) {
1146			mutex_unlock(&pq->lock);
1147			break;
1148		}
1149		qib_user_sdma_hwqueue_clean(ppd);
1150		qib_user_sdma_queue_clean(ppd, pq);
1151		mutex_unlock(&pq->lock);
1152		msleep(20);
1153	}
1154
1155	if (pq->num_pending || pq->num_sending) {
1156		struct qib_user_sdma_pkt *pkt;
1157		struct qib_user_sdma_pkt *pkt_prev;
1158		struct list_head free_list;
1159
1160		mutex_lock(&pq->lock);
1161		spin_lock_irqsave(&ppd->sdma_lock, flags);
1162		/*
1163		 * Since we hold sdma_lock, it is safe without sent_lock.
1164		 */
1165		if (pq->num_pending) {
1166			list_for_each_entry_safe(pkt, pkt_prev,
1167					&ppd->sdma_userpending, list) {
1168				if (pkt->pq == pq) {
1169					list_move_tail(&pkt->list, &pq->sent);
1170					pq->num_pending--;
1171					pq->num_sending++;
1172				}
1173			}
1174		}
1175		spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1176
1177		qib_dev_err(dd, "user sdma lists not empty: forcing!\n");
1178		INIT_LIST_HEAD(&free_list);
1179		list_splice_init(&pq->sent, &free_list);
1180		pq->num_sending = 0;
1181		qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &free_list);
1182		mutex_unlock(&pq->lock);
1183	}
1184}
1185
1186static inline __le64 qib_sdma_make_desc0(u8 gen,
1187					 u64 addr, u64 dwlen, u64 dwoffset)
1188{
1189	return cpu_to_le64(/* SDmaPhyAddr[31:0] */
1190			   ((addr & 0xfffffffcULL) << 32) |
1191			   /* SDmaGeneration[1:0] */
1192			   ((gen & 3ULL) << 30) |
1193			   /* SDmaDwordCount[10:0] */
1194			   ((dwlen & 0x7ffULL) << 16) |
1195			   /* SDmaBufOffset[12:2] */
1196			   (dwoffset & 0x7ffULL));
1197}
1198
1199static inline __le64 qib_sdma_make_first_desc0(__le64 descq)
1200{
1201	return descq | cpu_to_le64(1ULL << 12);
1202}
1203
1204static inline __le64 qib_sdma_make_last_desc0(__le64 descq)
1205{
1206					      /* last */  /* dma head */
1207	return descq | cpu_to_le64(1ULL << 11 | 1ULL << 13);
1208}
1209
1210static inline __le64 qib_sdma_make_desc1(u64 addr)
1211{
1212	/* SDmaPhyAddr[47:32] */
1213	return cpu_to_le64(addr >> 32);
1214}
1215
1216static void qib_user_sdma_send_frag(struct qib_pportdata *ppd,
1217				    struct qib_user_sdma_pkt *pkt, int idx,
1218				    unsigned ofs, u16 tail, u8 gen)
1219{
1220	const u64 addr = (u64) pkt->addr[idx].addr +
1221		(u64) pkt->addr[idx].offset;
1222	const u64 dwlen = (u64) pkt->addr[idx].length / 4;
1223	__le64 *descqp;
1224	__le64 descq0;
1225
1226	descqp = &ppd->sdma_descq[tail].qw[0];
1227
1228	descq0 = qib_sdma_make_desc0(gen, addr, dwlen, ofs);
1229	if (pkt->addr[idx].first_desc)
1230		descq0 = qib_sdma_make_first_desc0(descq0);
1231	if (pkt->addr[idx].last_desc) {
1232		descq0 = qib_sdma_make_last_desc0(descq0);
1233		if (ppd->sdma_intrequest) {
1234			descq0 |= cpu_to_le64(1ULL << 15);
1235			ppd->sdma_intrequest = 0;
1236		}
1237	}
1238
1239	descqp[0] = descq0;
1240	descqp[1] = qib_sdma_make_desc1(addr);
1241}
1242
1243void qib_user_sdma_send_desc(struct qib_pportdata *ppd,
1244				struct list_head *pktlist)
1245{
1246	struct qib_devdata *dd = ppd->dd;
1247	u16 nfree, nsent;
1248	u16 tail, tail_c;
1249	u8 gen, gen_c;
1250
1251	nfree = qib_sdma_descq_freecnt(ppd);
1252	if (!nfree)
1253		return;
1254
1255retry:
1256	nsent = 0;
1257	tail_c = tail = ppd->sdma_descq_tail;
1258	gen_c = gen = ppd->sdma_generation;
1259	while (!list_empty(pktlist)) {
1260		struct qib_user_sdma_pkt *pkt =
1261			list_entry(pktlist->next, struct qib_user_sdma_pkt,
1262				   list);
1263		int i, j, c = 0;
1264		unsigned ofs = 0;
1265		u16 dtail = tail;
1266
1267		for (i = pkt->index; i < pkt->naddr && nfree; i++) {
1268			qib_user_sdma_send_frag(ppd, pkt, i, ofs, tail, gen);
1269			ofs += pkt->addr[i].length >> 2;
1270
1271			if (++tail == ppd->sdma_descq_cnt) {
1272				tail = 0;
1273				++gen;
1274				ppd->sdma_intrequest = 1;
1275			} else if (tail == (ppd->sdma_descq_cnt>>1)) {
1276				ppd->sdma_intrequest = 1;
1277			}
1278			nfree--;
1279			if (pkt->addr[i].last_desc == 0)
1280				continue;
1281
1282			/*
1283			 * If the packet is >= 2KB mtu equivalent, we
1284			 * have to use the large buffers, and have to
1285			 * mark each descriptor as part of a large
1286			 * buffer packet.
1287			 */
1288			if (ofs > dd->piosize2kmax_dwords) {
1289				for (j = pkt->index; j <= i; j++) {
1290					ppd->sdma_descq[dtail].qw[0] |=
1291						cpu_to_le64(1ULL << 14);
1292					if (++dtail == ppd->sdma_descq_cnt)
1293						dtail = 0;
1294				}
1295			}
1296			c += i + 1 - pkt->index;
1297			pkt->index = i + 1; /* index for next first */
1298			tail_c = dtail = tail;
1299			gen_c = gen;
1300			ofs = 0;  /* reset for next packet */
1301		}
1302
1303		ppd->sdma_descq_added += c;
1304		nsent += c;
1305		if (pkt->index == pkt->naddr) {
1306			pkt->added = ppd->sdma_descq_added;
1307			pkt->pq->added = pkt->added;
1308			pkt->pq->num_pending--;
1309			spin_lock(&pkt->pq->sent_lock);
1310			pkt->pq->num_sending++;
1311			list_move_tail(&pkt->list, &pkt->pq->sent);
1312			spin_unlock(&pkt->pq->sent_lock);
1313		}
1314		if (!nfree || (nsent<<2) > ppd->sdma_descq_cnt)
1315			break;
1316	}
1317
1318	/* advance the tail on the chip if necessary */
1319	if (ppd->sdma_descq_tail != tail_c) {
1320		ppd->sdma_generation = gen_c;
1321		dd->f_sdma_update_tail(ppd, tail_c);
1322	}
1323
1324	if (nfree && !list_empty(pktlist))
1325		goto retry;
1326}
1327
1328/* pq->lock must be held, get packets on the wire... */
1329static int qib_user_sdma_push_pkts(struct qib_pportdata *ppd,
1330				 struct qib_user_sdma_queue *pq,
1331				 struct list_head *pktlist, int count)
1332{
1333	unsigned long flags;
1334
1335	if (unlikely(!(ppd->lflags & QIBL_LINKACTIVE)))
1336		return -ECOMM;
1337
1338	/* non-blocking mode */
1339	if (pq->sdma_rb_node->refcount > 1) {
1340		spin_lock_irqsave(&ppd->sdma_lock, flags);
1341		if (unlikely(!__qib_sdma_running(ppd))) {
1342			spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1343			return -ECOMM;
1344		}
1345		pq->num_pending += count;
1346		list_splice_tail_init(pktlist, &ppd->sdma_userpending);
1347		qib_user_sdma_send_desc(ppd, &ppd->sdma_userpending);
1348		spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1349		return 0;
1350	}
1351
1352	/* In this case, descriptors from this process are not
1353	 * linked to ppd pending queue, interrupt handler
1354	 * won't update this process, it is OK to directly
1355	 * modify without sdma lock.
1356	 */
1357
1358
1359	pq->num_pending += count;
1360	/*
1361	 * Blocking mode for single rail process, we must
1362	 * release/regain sdma_lock to give other process
1363	 * chance to make progress. This is important for
1364	 * performance.
1365	 */
1366	do {
1367		spin_lock_irqsave(&ppd->sdma_lock, flags);
1368		if (unlikely(!__qib_sdma_running(ppd))) {
1369			spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1370			return -ECOMM;
1371		}
1372		qib_user_sdma_send_desc(ppd, pktlist);
1373		if (!list_empty(pktlist))
1374			qib_sdma_make_progress(ppd);
1375		spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1376	} while (!list_empty(pktlist));
1377
1378	return 0;
1379}
1380
1381int qib_user_sdma_writev(struct qib_ctxtdata *rcd,
1382			 struct qib_user_sdma_queue *pq,
1383			 const struct iovec *iov,
1384			 unsigned long dim)
1385{
1386	struct qib_devdata *dd = rcd->dd;
1387	struct qib_pportdata *ppd = rcd->ppd;
1388	int ret = 0;
1389	struct list_head list;
1390	int npkts = 0;
1391
1392	INIT_LIST_HEAD(&list);
1393
1394	mutex_lock(&pq->lock);
1395
1396	/* why not -ECOMM like qib_user_sdma_push_pkts() below? */
1397	if (!qib_sdma_running(ppd))
1398		goto done_unlock;
1399
1400	/* if I have packets not complete yet */
1401	if (pq->added > ppd->sdma_descq_removed)
1402		qib_user_sdma_hwqueue_clean(ppd);
1403	/* if I have complete packets to be freed */
1404	if (pq->num_sending)
1405		qib_user_sdma_queue_clean(ppd, pq);
1406
1407	while (dim) {
1408		int mxp = 1;
1409		int ndesc = 0;
1410
1411		ret = qib_user_sdma_queue_pkts(dd, ppd, pq,
1412				iov, dim, &list, &mxp, &ndesc);
1413		if (ret < 0)
1414			goto done_unlock;
1415		else {
1416			dim -= ret;
1417			iov += ret;
1418		}
1419
1420		/* force packets onto the sdma hw queue... */
1421		if (!list_empty(&list)) {
1422			/*
1423			 * Lazily clean hw queue.
1424			 */
1425			if (qib_sdma_descq_freecnt(ppd) < ndesc) {
1426				qib_user_sdma_hwqueue_clean(ppd);
1427				if (pq->num_sending)
1428					qib_user_sdma_queue_clean(ppd, pq);
1429			}
1430
1431			ret = qib_user_sdma_push_pkts(ppd, pq, &list, mxp);
1432			if (ret < 0)
1433				goto done_unlock;
1434			else {
1435				npkts += mxp;
1436				pq->counter += mxp;
1437			}
1438		}
1439	}
1440
1441done_unlock:
1442	if (!list_empty(&list))
1443		qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &list);
1444	mutex_unlock(&pq->lock);
1445
1446	return (ret < 0) ? ret : npkts;
1447}
1448
1449int qib_user_sdma_make_progress(struct qib_pportdata *ppd,
1450				struct qib_user_sdma_queue *pq)
1451{
1452	int ret = 0;
1453
1454	mutex_lock(&pq->lock);
1455	qib_user_sdma_hwqueue_clean(ppd);
1456	ret = qib_user_sdma_queue_clean(ppd, pq);
1457	mutex_unlock(&pq->lock);
1458
1459	return ret;
1460}
1461
1462u32 qib_user_sdma_complete_counter(const struct qib_user_sdma_queue *pq)
1463{
1464	return pq ? pq->sent_counter : 0;
1465}
1466
1467u32 qib_user_sdma_inflight_counter(struct qib_user_sdma_queue *pq)
1468{
1469	return pq ? pq->counter : 0;
1470}