1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/* Copyright (c) 2015 - 2021 Intel Corporation */
   3#include "main.h"
   4
   5/**
   6 * irdma_arp_table -manage arp table
   7 * @rf: RDMA PCI function
   8 * @ip_addr: ip address for device
   9 * @ipv4: IPv4 flag
  10 * @mac_addr: mac address ptr
  11 * @action: modify, delete or add
  12 */
  13int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4,
  14		    const u8 *mac_addr, u32 action)
  15{
  16	unsigned long flags;
  17	int arp_index;
  18	u32 ip[4] = {};
  19
  20	if (ipv4)
  21		ip[0] = *ip_addr;
  22	else
  23		memcpy(ip, ip_addr, sizeof(ip));
  24
  25	spin_lock_irqsave(&rf->arp_lock, flags);
  26	for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) {
  27		if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip)))
  28			break;
  29	}
  30
  31	switch (action) {
  32	case IRDMA_ARP_ADD:
  33		if (arp_index != rf->arp_table_size) {
  34			arp_index = -1;
  35			break;
  36		}
  37
  38		arp_index = 0;
  39		if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size,
  40				     (u32 *)&arp_index, &rf->next_arp_index)) {
  41			arp_index = -1;
  42			break;
  43		}
  44
  45		memcpy(rf->arp_table[arp_index].ip_addr, ip,
  46		       sizeof(rf->arp_table[arp_index].ip_addr));
  47		ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr);
  48		break;
  49	case IRDMA_ARP_RESOLVE:
  50		if (arp_index == rf->arp_table_size)
  51			arp_index = -1;
  52		break;
  53	case IRDMA_ARP_DELETE:
  54		if (arp_index == rf->arp_table_size) {
  55			arp_index = -1;
  56			break;
  57		}
  58
  59		memset(rf->arp_table[arp_index].ip_addr, 0,
  60		       sizeof(rf->arp_table[arp_index].ip_addr));
  61		eth_zero_addr(rf->arp_table[arp_index].mac_addr);
  62		irdma_free_rsrc(rf, rf->allocated_arps, arp_index);
  63		break;
  64	default:
  65		arp_index = -1;
  66		break;
  67	}
  68
  69	spin_unlock_irqrestore(&rf->arp_lock, flags);
  70	return arp_index;
  71}
  72
  73/**
  74 * irdma_add_arp - add a new arp entry if needed
  75 * @rf: RDMA function
  76 * @ip: IP address
  77 * @ipv4: IPv4 flag
  78 * @mac: MAC address
  79 */
  80int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac)
  81{
  82	int arpidx;
  83
  84	arpidx = irdma_arp_table(rf, &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE);
  85	if (arpidx >= 0) {
  86		if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac))
  87			return arpidx;
  88
  89		irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip,
  90				       ipv4, IRDMA_ARP_DELETE);
  91	}
  92
  93	irdma_manage_arp_cache(rf, mac, ip, ipv4, IRDMA_ARP_ADD);
  94
  95	return irdma_arp_table(rf, ip, ipv4, NULL, IRDMA_ARP_RESOLVE);
  96}
  97
  98/**
  99 * wr32 - write 32 bits to hw register
 100 * @hw: hardware information including registers
 101 * @reg: register offset
 102 * @val: value to write to register
 103 */
 104inline void wr32(struct irdma_hw *hw, u32 reg, u32 val)
 105{
 106	writel(val, hw->hw_addr + reg);
 107}
 108
 109/**
 110 * rd32 - read a 32 bit hw register
 111 * @hw: hardware information including registers
 112 * @reg: register offset
 113 *
 114 * Return value of register content
 115 */
 116inline u32 rd32(struct irdma_hw *hw, u32 reg)
 117{
 118	return readl(hw->hw_addr + reg);
 119}
 120
 121/**
 122 * rd64 - read a 64 bit hw register
 123 * @hw: hardware information including registers
 124 * @reg: register offset
 125 *
 126 * Return value of register content
 127 */
 128inline u64 rd64(struct irdma_hw *hw, u32 reg)
 129{
 130	return readq(hw->hw_addr + reg);
 131}
 132
 133static void irdma_gid_change_event(struct ib_device *ibdev)
 134{
 135	struct ib_event ib_event;
 136
 137	ib_event.event = IB_EVENT_GID_CHANGE;
 138	ib_event.device = ibdev;
 139	ib_event.element.port_num = 1;
 140	ib_dispatch_event(&ib_event);
 141}
 142
 143/**
 144 * irdma_inetaddr_event - system notifier for ipv4 addr events
 145 * @notifier: not used
 146 * @event: event for notifier
 147 * @ptr: if address
 148 */
 149int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event,
 150			 void *ptr)
 151{
 152	struct in_ifaddr *ifa = ptr;
 153	struct net_device *real_dev, *netdev = ifa->ifa_dev->dev;
 154	struct irdma_device *iwdev;
 155	struct ib_device *ibdev;
 156	u32 local_ipaddr;
 157
 158	real_dev = rdma_vlan_dev_real_dev(netdev);
 159	if (!real_dev)
 160		real_dev = netdev;
 161
 162	ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
 163	if (!ibdev)
 164		return NOTIFY_DONE;
 165
 166	iwdev = to_iwdev(ibdev);
 167	local_ipaddr = ntohl(ifa->ifa_address);
 168	ibdev_dbg(&iwdev->ibdev,
 169		  "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", real_dev,
 170		  event, &local_ipaddr, real_dev->dev_addr);
 171	switch (event) {
 172	case NETDEV_DOWN:
 173		irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,
 174				       &local_ipaddr, true, IRDMA_ARP_DELETE);
 175		irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, false);
 176		irdma_gid_change_event(&iwdev->ibdev);
 177		break;
 178	case NETDEV_UP:
 179	case NETDEV_CHANGEADDR:
 180		irdma_add_arp(iwdev->rf, &local_ipaddr, true, real_dev->dev_addr);
 181		irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, true);
 182		irdma_gid_change_event(&iwdev->ibdev);
 183		break;
 184	default:
 185		break;
 186	}
 187
 188	ib_device_put(ibdev);
 189
 190	return NOTIFY_DONE;
 191}
 192
 193/**
 194 * irdma_inet6addr_event - system notifier for ipv6 addr events
 195 * @notifier: not used
 196 * @event: event for notifier
 197 * @ptr: if address
 198 */
 199int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event,
 200			  void *ptr)
 201{
 202	struct inet6_ifaddr *ifa = ptr;
 203	struct net_device *real_dev, *netdev = ifa->idev->dev;
 204	struct irdma_device *iwdev;
 205	struct ib_device *ibdev;
 206	u32 local_ipaddr6[4];
 207
 208	real_dev = rdma_vlan_dev_real_dev(netdev);
 209	if (!real_dev)
 210		real_dev = netdev;
 211
 212	ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
 213	if (!ibdev)
 214		return NOTIFY_DONE;
 215
 216	iwdev = to_iwdev(ibdev);
 217	irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32);
 218	ibdev_dbg(&iwdev->ibdev,
 219		  "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", real_dev,
 220		  event, local_ipaddr6, real_dev->dev_addr);
 221	switch (event) {
 222	case NETDEV_DOWN:
 223		irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,
 224				       local_ipaddr6, false, IRDMA_ARP_DELETE);
 225		irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, false);
 226		irdma_gid_change_event(&iwdev->ibdev);
 227		break;
 228	case NETDEV_UP:
 229	case NETDEV_CHANGEADDR:
 230		irdma_add_arp(iwdev->rf, local_ipaddr6, false,
 231			      real_dev->dev_addr);
 232		irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, true);
 233		irdma_gid_change_event(&iwdev->ibdev);
 234		break;
 235	default:
 236		break;
 237	}
 238
 239	ib_device_put(ibdev);
 240
 241	return NOTIFY_DONE;
 242}
 243
 244/**
 245 * irdma_net_event - system notifier for net events
 246 * @notifier: not used
 247 * @event: event for notifier
 248 * @ptr: neighbor
 249 */
 250int irdma_net_event(struct notifier_block *notifier, unsigned long event,
 251		    void *ptr)
 252{
 253	struct neighbour *neigh = ptr;
 254	struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev;
 255	struct irdma_device *iwdev;
 256	struct ib_device *ibdev;
 257	__be32 *p;
 258	u32 local_ipaddr[4] = {};
 259	bool ipv4 = true;
 260
 261	switch (event) {
 262	case NETEVENT_NEIGH_UPDATE:
 263		real_dev = rdma_vlan_dev_real_dev(netdev);
 264		if (!real_dev)
 265			real_dev = netdev;
 266		ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
 267		if (!ibdev)
 268			return NOTIFY_DONE;
 269
 270		iwdev = to_iwdev(ibdev);
 271		p = (__be32 *)neigh->primary_key;
 272		if (neigh->tbl->family == AF_INET6) {
 273			ipv4 = false;
 274			irdma_copy_ip_ntohl(local_ipaddr, p);
 275		} else {
 276			local_ipaddr[0] = ntohl(*p);
 277		}
 278
 279		ibdev_dbg(&iwdev->ibdev,
 280			  "DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n",
 281			  iwdev->netdev, neigh->nud_state, local_ipaddr,
 282			  neigh->ha);
 283
 284		if (neigh->nud_state & NUD_VALID)
 285			irdma_add_arp(iwdev->rf, local_ipaddr, ipv4, neigh->ha);
 286
 287		else
 288			irdma_manage_arp_cache(iwdev->rf, neigh->ha,
 289					       local_ipaddr, ipv4,
 290					       IRDMA_ARP_DELETE);
 291		ib_device_put(ibdev);
 292		break;
 293	default:
 294		break;
 295	}
 296
 297	return NOTIFY_DONE;
 298}
 299
 300/**
 301 * irdma_netdevice_event - system notifier for netdev events
 302 * @notifier: not used
 303 * @event: event for notifier
 304 * @ptr: netdev
 305 */
 306int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,
 307			  void *ptr)
 308{
 309	struct irdma_device *iwdev;
 310	struct ib_device *ibdev;
 311	struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
 312
 313	ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);
 314	if (!ibdev)
 315		return NOTIFY_DONE;
 316
 317	iwdev = to_iwdev(ibdev);
 318	iwdev->iw_status = 1;
 319	switch (event) {
 320	case NETDEV_DOWN:
 321		iwdev->iw_status = 0;
 322		fallthrough;
 323	case NETDEV_UP:
 324		irdma_port_ibevent(iwdev);
 325		break;
 326	default:
 327		break;
 328	}
 329	ib_device_put(ibdev);
 330
 331	return NOTIFY_DONE;
 332}
 333
 334/**
 335 * irdma_add_ipv6_addr - add ipv6 address to the hw arp table
 336 * @iwdev: irdma device
 337 */
 338static void irdma_add_ipv6_addr(struct irdma_device *iwdev)
 339{
 340	struct net_device *ip_dev;
 341	struct inet6_dev *idev;
 342	struct inet6_ifaddr *ifp, *tmp;
 343	u32 local_ipaddr6[4];
 344
 345	rcu_read_lock();
 346	for_each_netdev_rcu (&init_net, ip_dev) {
 347		if (((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF &&
 348		      rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev) ||
 349		      ip_dev == iwdev->netdev) &&
 350		      (READ_ONCE(ip_dev->flags) & IFF_UP)) {
 351			idev = __in6_dev_get(ip_dev);
 352			if (!idev) {
 353				ibdev_err(&iwdev->ibdev, "ipv6 inet device not found\n");
 354				break;
 355			}
 356			list_for_each_entry_safe (ifp, tmp, &idev->addr_list,
 357						  if_list) {
 358				ibdev_dbg(&iwdev->ibdev,
 359					  "INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n",
 360					  &ifp->addr,
 361					  rdma_vlan_dev_vlan_id(ip_dev),
 362					  ip_dev->dev_addr);
 363
 364				irdma_copy_ip_ntohl(local_ipaddr6,
 365						    ifp->addr.in6_u.u6_addr32);
 366				irdma_manage_arp_cache(iwdev->rf,
 367						       ip_dev->dev_addr,
 368						       local_ipaddr6, false,
 369						       IRDMA_ARP_ADD);
 370			}
 371		}
 372	}
 373	rcu_read_unlock();
 374}
 375
 376/**
 377 * irdma_add_ipv4_addr - add ipv4 address to the hw arp table
 378 * @iwdev: irdma device
 379 */
 380static void irdma_add_ipv4_addr(struct irdma_device *iwdev)
 381{
 382	struct net_device *dev;
 383	struct in_device *idev;
 384	u32 ip_addr;
 385
 386	rcu_read_lock();
 387	for_each_netdev_rcu (&init_net, dev) {
 388		if (((rdma_vlan_dev_vlan_id(dev) < 0xFFFF &&
 389		      rdma_vlan_dev_real_dev(dev) == iwdev->netdev) ||
 390		      dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) {
 391			const struct in_ifaddr *ifa;
 392
 393			idev = __in_dev_get_rcu(dev);
 394			if (!idev)
 395				continue;
 396
 397			in_dev_for_each_ifa_rcu(ifa, idev) {
 398				ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n",
 399					  &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev),
 400					  dev->dev_addr);
 401
 402				ip_addr = ntohl(ifa->ifa_address);
 403				irdma_manage_arp_cache(iwdev->rf, dev->dev_addr,
 404						       &ip_addr, true,
 405						       IRDMA_ARP_ADD);
 406			}
 407		}
 408	}
 409	rcu_read_unlock();
 410}
 411
 412/**
 413 * irdma_add_ip - add ip addresses
 414 * @iwdev: irdma device
 415 *
 416 * Add ipv4/ipv6 addresses to the arp cache
 417 */
 418void irdma_add_ip(struct irdma_device *iwdev)
 419{
 420	irdma_add_ipv4_addr(iwdev);
 421	irdma_add_ipv6_addr(iwdev);
 422}
 423
 424/**
 425 * irdma_alloc_and_get_cqp_request - get cqp struct
 426 * @cqp: device cqp ptr
 427 * @wait: cqp to be used in wait mode
 428 */
 429struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp,
 430							  bool wait)
 431{
 432	struct irdma_cqp_request *cqp_request = NULL;
 433	unsigned long flags;
 434
 435	spin_lock_irqsave(&cqp->req_lock, flags);
 436	if (!list_empty(&cqp->cqp_avail_reqs)) {
 437		cqp_request = list_first_entry(&cqp->cqp_avail_reqs,
 438					       struct irdma_cqp_request, list);
 439		list_del_init(&cqp_request->list);
 440	}
 441	spin_unlock_irqrestore(&cqp->req_lock, flags);
 442	if (!cqp_request) {
 443		cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);
 444		if (cqp_request) {
 445			cqp_request->dynamic = true;
 446			if (wait)
 447				init_waitqueue_head(&cqp_request->waitq);
 448		}
 449	}
 450	if (!cqp_request) {
 451		ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory");
 452		return NULL;
 453	}
 454
 455	cqp_request->waiting = wait;
 456	refcount_set(&cqp_request->refcnt, 1);
 457	memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info));
 458
 459	return cqp_request;
 460}
 461
 462/**
 463 * irdma_get_cqp_request - increase refcount for cqp_request
 464 * @cqp_request: pointer to cqp_request instance
 465 */
 466static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request)
 467{
 468	refcount_inc(&cqp_request->refcnt);
 469}
 470
 471/**
 472 * irdma_free_cqp_request - free cqp request
 473 * @cqp: cqp ptr
 474 * @cqp_request: to be put back in cqp list
 475 */
 476void irdma_free_cqp_request(struct irdma_cqp *cqp,
 477			    struct irdma_cqp_request *cqp_request)
 478{
 479	unsigned long flags;
 480
 481	if (cqp_request->dynamic) {
 482		kfree(cqp_request);
 483	} else {
 484		WRITE_ONCE(cqp_request->request_done, false);
 485		cqp_request->callback_fcn = NULL;
 486		cqp_request->waiting = false;
 487
 488		spin_lock_irqsave(&cqp->req_lock, flags);
 489		list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs);
 490		spin_unlock_irqrestore(&cqp->req_lock, flags);
 491	}
 492	wake_up(&cqp->remove_wq);
 493}
 494
 495/**
 496 * irdma_put_cqp_request - dec ref count and free if 0
 497 * @cqp: cqp ptr
 498 * @cqp_request: to be put back in cqp list
 499 */
 500void irdma_put_cqp_request(struct irdma_cqp *cqp,
 501			   struct irdma_cqp_request *cqp_request)
 502{
 503	if (refcount_dec_and_test(&cqp_request->refcnt))
 504		irdma_free_cqp_request(cqp, cqp_request);
 505}
 506
 507/**
 508 * irdma_free_pending_cqp_request -free pending cqp request objs
 509 * @cqp: cqp ptr
 510 * @cqp_request: to be put back in cqp list
 511 */
 512static void
 513irdma_free_pending_cqp_request(struct irdma_cqp *cqp,
 514			       struct irdma_cqp_request *cqp_request)
 515{
 516	if (cqp_request->waiting) {
 517		cqp_request->compl_info.error = true;
 518		WRITE_ONCE(cqp_request->request_done, true);
 519		wake_up(&cqp_request->waitq);
 520	}
 521	wait_event_timeout(cqp->remove_wq,
 522			   refcount_read(&cqp_request->refcnt) == 1, 1000);
 523	irdma_put_cqp_request(cqp, cqp_request);
 524}
 525
 526/**
 527 * irdma_cleanup_pending_cqp_op - clean-up cqp with no
 528 * completions
 529 * @rf: RDMA PCI function
 530 */
 531void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf)
 532{
 533	struct irdma_sc_dev *dev = &rf->sc_dev;
 534	struct irdma_cqp *cqp = &rf->cqp;
 535	struct irdma_cqp_request *cqp_request = NULL;
 536	struct cqp_cmds_info *pcmdinfo = NULL;
 537	u32 i, pending_work, wqe_idx;
 538
 539	pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring);
 540	wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring);
 541	for (i = 0; i < pending_work; i++) {
 542		cqp_request = (struct irdma_cqp_request *)(unsigned long)
 543				      cqp->scratch_array[wqe_idx];
 544		if (cqp_request)
 545			irdma_free_pending_cqp_request(cqp, cqp_request);
 546		wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring);
 547	}
 548
 549	while (!list_empty(&dev->cqp_cmd_head)) {
 550		pcmdinfo = irdma_remove_cqp_head(dev);
 551		cqp_request =
 552			container_of(pcmdinfo, struct irdma_cqp_request, info);
 553		if (cqp_request)
 554			irdma_free_pending_cqp_request(cqp, cqp_request);
 555	}
 556}
 557
 558/**
 559 * irdma_wait_event - wait for completion
 560 * @rf: RDMA PCI function
 561 * @cqp_request: cqp request to wait
 562 */
 563static int irdma_wait_event(struct irdma_pci_f *rf,
 564			    struct irdma_cqp_request *cqp_request)
 565{
 566	struct irdma_cqp_timeout cqp_timeout = {};
 567	bool cqp_error = false;
 568	int err_code = 0;
 569
 570	cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops);
 571	do {
 572		irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
 573		if (wait_event_timeout(cqp_request->waitq,
 574				       READ_ONCE(cqp_request->request_done),
 575				       msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS)))
 576			break;
 577
 578		irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev);
 579
 580		if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD)
 581			continue;
 582
 583		if (!rf->reset) {
 584			rf->reset = true;
 585			rf->gen_ops.request_reset(rf);
 586		}
 587		return -ETIMEDOUT;
 588	} while (1);
 589
 590	cqp_error = cqp_request->compl_info.error;
 591	if (cqp_error) {
 592		err_code = -EIO;
 593		if (cqp_request->compl_info.maj_err_code == 0xFFFF) {
 594			if (cqp_request->compl_info.min_err_code == 0x8002)
 595				err_code = -EBUSY;
 596			else if (cqp_request->compl_info.min_err_code == 0x8029) {
 597				if (!rf->reset) {
 598					rf->reset = true;
 599					rf->gen_ops.request_reset(rf);
 600				}
 601			}
 602		}
 603	}
 604
 605	return err_code;
 606}
 607
 608static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = {
 609	[IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd",
 610	[IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd",
 611	[IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd",
 612	[IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd",
 613	[IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd",
 614	[IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd",
 615	[IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd",
 616	[IRDMA_OP_QP_MODIFY] = "Modify QP Cmd",
 617	[IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd",
 618	[IRDMA_OP_CQ_CREATE] = "Create CQ Cmd",
 619	[IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd",
 620	[IRDMA_OP_QP_CREATE] = "Create QP Cmd",
 621	[IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd",
 622	[IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd",
 623	[IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd",
 624	[IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd",
 625	[IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd",
 626	[IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd",
 627	[IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd",
 628	[IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd",
 629	[IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd",
 630	[IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",
 631	[IRDMA_OP_SUSPEND] = "Suspend QP Cmd",
 632	[IRDMA_OP_RESUME] = "Resume QP Cmd",
 633	[IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd",
 634	[IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",
 635	[IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",
 636	[IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",
 637	[IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd",
 638	[IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd",
 639	[IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd",
 640	[IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd",
 641	[IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd",
 642	[IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd",
 643	[IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd",
 644	[IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd",
 645	[IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd",
 646	[IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd",
 647	[IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd",
 648	[IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd",
 649	[IRDMA_OP_GEN_AE] = "Generate AE Cmd",
 650	[IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd",
 651	[IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd",
 652	[IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd",
 653	[IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd",
 654	[IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd",
 655};
 656
 657static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {
 658	{0xffff, 0x8002, "Invalid State"},
 659	{0xffff, 0x8006, "Flush No Wqe Pending"},
 660	{0xffff, 0x8007, "Modify QP Bad Close"},
 661	{0xffff, 0x8009, "LLP Closed"},
 662	{0xffff, 0x800a, "Reset Not Sent"}
 663};
 664
 665/**
 666 * irdma_cqp_crit_err - check if CQP error is critical
 667 * @dev: pointer to dev structure
 668 * @cqp_cmd: code for last CQP operation
 669 * @maj_err_code: major error code
 670 * @min_err_code: minot error code
 671 */
 672bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,
 673			u16 maj_err_code, u16 min_err_code)
 674{
 675	int i;
 676
 677	for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) {
 678		if (maj_err_code == irdma_noncrit_err_list[i].maj &&
 679		    min_err_code == irdma_noncrit_err_list[i].min) {
 680			ibdev_dbg(to_ibdev(dev),
 681				  "CQP: [%s Error][%s] maj=0x%x min=0x%x\n",
 682				  irdma_noncrit_err_list[i].desc,
 683				  irdma_cqp_cmd_names[cqp_cmd], maj_err_code,
 684				  min_err_code);
 685			return false;
 686		}
 687	}
 688	return true;
 689}
 690
 691/**
 692 * irdma_handle_cqp_op - process cqp command
 693 * @rf: RDMA PCI function
 694 * @cqp_request: cqp request to process
 695 */
 696int irdma_handle_cqp_op(struct irdma_pci_f *rf,
 697			struct irdma_cqp_request *cqp_request)
 698{
 699	struct irdma_sc_dev *dev = &rf->sc_dev;
 700	struct cqp_cmds_info *info = &cqp_request->info;
 701	int status;
 702	bool put_cqp_request = true;
 703
 704	if (rf->reset)
 705		return -EBUSY;
 706
 707	irdma_get_cqp_request(cqp_request);
 708	status = irdma_process_cqp_cmd(dev, info);
 709	if (status)
 710		goto err;
 711
 712	if (cqp_request->waiting) {
 713		put_cqp_request = false;
 714		status = irdma_wait_event(rf, cqp_request);
 715		if (status)
 716			goto err;
 717	}
 718
 719	return 0;
 720
 721err:
 722	if (irdma_cqp_crit_err(dev, info->cqp_cmd,
 723			       cqp_request->compl_info.maj_err_code,
 724			       cqp_request->compl_info.min_err_code))
 725		ibdev_err(&rf->iwdev->ibdev,
 726			  "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n",
 727			  irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting,
 728			  cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code,
 729			  cqp_request->compl_info.min_err_code);
 730
 731	if (put_cqp_request)
 732		irdma_put_cqp_request(&rf->cqp, cqp_request);
 733
 734	return status;
 735}
 736
 737void irdma_qp_add_ref(struct ib_qp *ibqp)
 738{
 739	struct irdma_qp *iwqp = (struct irdma_qp *)ibqp;
 740
 741	refcount_inc(&iwqp->refcnt);
 742}
 743
 744void irdma_qp_rem_ref(struct ib_qp *ibqp)
 745{
 746	struct irdma_qp *iwqp = to_iwqp(ibqp);
 747	struct irdma_device *iwdev = iwqp->iwdev;
 748	u32 qp_num;
 749	unsigned long flags;
 750
 751	spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
 752	if (!refcount_dec_and_test(&iwqp->refcnt)) {
 753		spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
 754		return;
 755	}
 756
 757	qp_num = iwqp->ibqp.qp_num;
 758	iwdev->rf->qp_table[qp_num] = NULL;
 759	spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
 760	complete(&iwqp->free_qp);
 761}
 762
 763void irdma_cq_add_ref(struct ib_cq *ibcq)
 764{
 765	struct irdma_cq *iwcq = to_iwcq(ibcq);
 766
 767	refcount_inc(&iwcq->refcnt);
 768}
 769
 770void irdma_cq_rem_ref(struct ib_cq *ibcq)
 771{
 772	struct ib_device *ibdev = ibcq->device;
 773	struct irdma_device *iwdev = to_iwdev(ibdev);
 774	struct irdma_cq *iwcq = to_iwcq(ibcq);
 775	unsigned long flags;
 776
 777	spin_lock_irqsave(&iwdev->rf->cqtable_lock, flags);
 778	if (!refcount_dec_and_test(&iwcq->refcnt)) {
 779		spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags);
 780		return;
 781	}
 782
 783	iwdev->rf->cq_table[iwcq->cq_num] = NULL;
 784	spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags);
 785	complete(&iwcq->free_cq);
 786}
 787
 788struct ib_device *to_ibdev(struct irdma_sc_dev *dev)
 789{
 790	return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev;
 791}
 792
 793/**
 794 * irdma_get_qp - get qp address
 795 * @device: iwarp device
 796 * @qpn: qp number
 797 */
 798struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn)
 799{
 800	struct irdma_device *iwdev = to_iwdev(device);
 801
 802	if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp)
 803		return NULL;
 804
 805	return &iwdev->rf->qp_table[qpn]->ibqp;
 806}
 807
 808/**
 809 * irdma_remove_cqp_head - return head entry and remove
 810 * @dev: device
 811 */
 812void *irdma_remove_cqp_head(struct irdma_sc_dev *dev)
 813{
 814	struct list_head *entry;
 815	struct list_head *list = &dev->cqp_cmd_head;
 816
 817	if (list_empty(list))
 818		return NULL;
 819
 820	entry = list->next;
 821	list_del(entry);
 822
 823	return entry;
 824}
 825
 826/**
 827 * irdma_cqp_sds_cmd - create cqp command for sd
 828 * @dev: hardware control device structure
 829 * @sdinfo: information for sd cqp
 830 *
 831 */
 832int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
 833		      struct irdma_update_sds_info *sdinfo)
 834{
 835	struct irdma_cqp_request *cqp_request;
 836	struct cqp_cmds_info *cqp_info;
 837	struct irdma_pci_f *rf = dev_to_rf(dev);
 838	int status;
 839
 840	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
 841	if (!cqp_request)
 842		return -ENOMEM;
 843
 844	cqp_info = &cqp_request->info;
 845	memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo,
 846	       sizeof(cqp_info->in.u.update_pe_sds.info));
 847	cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS;
 848	cqp_info->post_sq = 1;
 849	cqp_info->in.u.update_pe_sds.dev = dev;
 850	cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;
 851
 852	status = irdma_handle_cqp_op(rf, cqp_request);
 853	irdma_put_cqp_request(&rf->cqp, cqp_request);
 854
 855	return status;
 856}
 857
 858/**
 859 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume
 860 * @qp: hardware control qp
 861 * @op: suspend or resume
 862 */
 863int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op)
 864{
 865	struct irdma_sc_dev *dev = qp->dev;
 866	struct irdma_cqp_request *cqp_request;
 867	struct irdma_sc_cqp *cqp = dev->cqp;
 868	struct cqp_cmds_info *cqp_info;
 869	struct irdma_pci_f *rf = dev_to_rf(dev);
 870	int status;
 871
 872	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
 873	if (!cqp_request)
 874		return -ENOMEM;
 875
 876	cqp_info = &cqp_request->info;
 877	cqp_info->cqp_cmd = op;
 878	cqp_info->in.u.suspend_resume.cqp = cqp;
 879	cqp_info->in.u.suspend_resume.qp = qp;
 880	cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request;
 881
 882	status = irdma_handle_cqp_op(rf, cqp_request);
 883	irdma_put_cqp_request(&rf->cqp, cqp_request);
 884
 885	return status;
 886}
 887
 888/**
 889 * irdma_term_modify_qp - modify qp for term message
 890 * @qp: hardware control qp
 891 * @next_state: qp's next state
 892 * @term: terminate code
 893 * @term_len: length
 894 */
 895void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term,
 896			  u8 term_len)
 897{
 898	struct irdma_qp *iwqp;
 899
 900	iwqp = qp->qp_uk.back_qp;
 901	irdma_next_iw_state(iwqp, next_state, 0, term, term_len);
 902};
 903
 904/**
 905 * irdma_terminate_done - after terminate is completed
 906 * @qp: hardware control qp
 907 * @timeout_occurred: indicates if terminate timer expired
 908 */
 909void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred)
 910{
 911	struct irdma_qp *iwqp;
 912	u8 hte = 0;
 913	bool first_time;
 914	unsigned long flags;
 915
 916	iwqp = qp->qp_uk.back_qp;
 917	spin_lock_irqsave(&iwqp->lock, flags);
 918	if (iwqp->hte_added) {
 919		iwqp->hte_added = 0;
 920		hte = 1;
 921	}
 922	first_time = !(qp->term_flags & IRDMA_TERM_DONE);
 923	qp->term_flags |= IRDMA_TERM_DONE;
 924	spin_unlock_irqrestore(&iwqp->lock, flags);
 925	if (first_time) {
 926		if (!timeout_occurred)
 927			irdma_terminate_del_timer(qp);
 928
 929		irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0);
 930		irdma_cm_disconn(iwqp);
 931	}
 932}
 933
 934static void irdma_terminate_timeout(struct timer_list *t)
 935{
 936	struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer);
 937	struct irdma_sc_qp *qp = &iwqp->sc_qp;
 938
 939	irdma_terminate_done(qp, 1);
 940	irdma_qp_rem_ref(&iwqp->ibqp);
 941}
 942
 943/**
 944 * irdma_terminate_start_timer - start terminate timeout
 945 * @qp: hardware control qp
 946 */
 947void irdma_terminate_start_timer(struct irdma_sc_qp *qp)
 948{
 949	struct irdma_qp *iwqp;
 950
 951	iwqp = qp->qp_uk.back_qp;
 952	irdma_qp_add_ref(&iwqp->ibqp);
 953	timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0);
 954	iwqp->terminate_timer.expires = jiffies + HZ;
 955
 956	add_timer(&iwqp->terminate_timer);
 957}
 958
 959/**
 960 * irdma_terminate_del_timer - delete terminate timeout
 961 * @qp: hardware control qp
 962 */
 963void irdma_terminate_del_timer(struct irdma_sc_qp *qp)
 964{
 965	struct irdma_qp *iwqp;
 966	int ret;
 967
 968	iwqp = qp->qp_uk.back_qp;
 969	ret = del_timer(&iwqp->terminate_timer);
 970	if (ret)
 971		irdma_qp_rem_ref(&iwqp->ibqp);
 972}
 973
 974/**
 975 * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm
 976 * @dev: function device struct
 977 * @val_mem: buffer for fpm
 978 * @hmc_fn_id: function id for fpm
 979 */
 980int irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
 981				struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
 982{
 983	struct irdma_cqp_request *cqp_request;
 984	struct cqp_cmds_info *cqp_info;
 985	struct irdma_pci_f *rf = dev_to_rf(dev);
 986	int status;
 987
 988	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
 989	if (!cqp_request)
 990		return -ENOMEM;
 991
 992	cqp_info = &cqp_request->info;
 993	cqp_request->param = NULL;
 994	cqp_info->in.u.query_fpm_val.cqp = dev->cqp;
 995	cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa;
 996	cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va;
 997	cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id;
 998	cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL;
 999	cqp_info->post_sq = 1;
1000	cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request;
1001
1002	status = irdma_handle_cqp_op(rf, cqp_request);
1003	irdma_put_cqp_request(&rf->cqp, cqp_request);
1004
1005	return status;
1006}
1007
1008/**
1009 * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw
1010 * @dev: hardware control device structure
1011 * @val_mem: buffer with fpm values
1012 * @hmc_fn_id: function id for fpm
1013 */
1014int irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
1015				 struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
1016{
1017	struct irdma_cqp_request *cqp_request;
1018	struct cqp_cmds_info *cqp_info;
1019	struct irdma_pci_f *rf = dev_to_rf(dev);
1020	int status;
1021
1022	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1023	if (!cqp_request)
1024		return -ENOMEM;
1025
1026	cqp_info = &cqp_request->info;
1027	cqp_request->param = NULL;
1028	cqp_info->in.u.commit_fpm_val.cqp = dev->cqp;
1029	cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa;
1030	cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va;
1031	cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id;
1032	cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL;
1033	cqp_info->post_sq = 1;
1034	cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request;
1035
1036	status = irdma_handle_cqp_op(rf, cqp_request);
1037	irdma_put_cqp_request(&rf->cqp, cqp_request);
1038
1039	return status;
1040}
1041
1042/**
1043 * irdma_cqp_cq_create_cmd - create a cq for the cqp
1044 * @dev: device pointer
1045 * @cq: pointer to created cq
1046 */
1047int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
1048{
1049	struct irdma_pci_f *rf = dev_to_rf(dev);
1050	struct irdma_cqp *iwcqp = &rf->cqp;
1051	struct irdma_cqp_request *cqp_request;
1052	struct cqp_cmds_info *cqp_info;
1053	int status;
1054
1055	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1056	if (!cqp_request)
1057		return -ENOMEM;
1058
1059	cqp_info = &cqp_request->info;
1060	cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
1061	cqp_info->post_sq = 1;
1062	cqp_info->in.u.cq_create.cq = cq;
1063	cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
1064
1065	status = irdma_handle_cqp_op(rf, cqp_request);
1066	irdma_put_cqp_request(iwcqp, cqp_request);
1067
1068	return status;
1069}
1070
1071/**
1072 * irdma_cqp_qp_create_cmd - create a qp for the cqp
1073 * @dev: device pointer
1074 * @qp: pointer to created qp
1075 */
1076int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1077{
1078	struct irdma_pci_f *rf = dev_to_rf(dev);
1079	struct irdma_cqp *iwcqp = &rf->cqp;
1080	struct irdma_cqp_request *cqp_request;
1081	struct cqp_cmds_info *cqp_info;
1082	struct irdma_create_qp_info *qp_info;
1083	int status;
1084
1085	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1086	if (!cqp_request)
1087		return -ENOMEM;
1088
1089	cqp_info = &cqp_request->info;
1090	qp_info = &cqp_request->info.in.u.qp_create.info;
1091	memset(qp_info, 0, sizeof(*qp_info));
1092	qp_info->cq_num_valid = true;
1093	qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS;
1094	cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE;
1095	cqp_info->post_sq = 1;
1096	cqp_info->in.u.qp_create.qp = qp;
1097	cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
1098
1099	status = irdma_handle_cqp_op(rf, cqp_request);
1100	irdma_put_cqp_request(iwcqp, cqp_request);
1101
1102	return status;
1103}
1104
1105/**
1106 * irdma_dealloc_push_page - free a push page for qp
1107 * @rf: RDMA PCI function
1108 * @qp: hardware control qp
1109 */
1110static void irdma_dealloc_push_page(struct irdma_pci_f *rf,
1111				    struct irdma_sc_qp *qp)
1112{
1113	struct irdma_cqp_request *cqp_request;
1114	struct cqp_cmds_info *cqp_info;
1115	int status;
1116
1117	if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)
1118		return;
1119
1120	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
1121	if (!cqp_request)
1122		return;
1123
1124	cqp_info = &cqp_request->info;
1125	cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;
1126	cqp_info->post_sq = 1;
1127	cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
1128	cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
1129	cqp_info->in.u.manage_push_page.info.free_page = 1;
1130	cqp_info->in.u.manage_push_page.info.push_page_type = 0;
1131	cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp;
1132	cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
1133	status = irdma_handle_cqp_op(rf, cqp_request);
1134	if (!status)
1135		qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
1136	irdma_put_cqp_request(&rf->cqp, cqp_request);
1137}
1138
1139/**
1140 * irdma_free_qp_rsrc - free up memory resources for qp
1141 * @iwqp: qp ptr (user or kernel)
1142 */
1143void irdma_free_qp_rsrc(struct irdma_qp *iwqp)
1144{
1145	struct irdma_device *iwdev = iwqp->iwdev;
1146	struct irdma_pci_f *rf = iwdev->rf;
1147	u32 qp_num = iwqp->ibqp.qp_num;
1148
1149	irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
1150	irdma_dealloc_push_page(rf, &iwqp->sc_qp);
1151	if (iwqp->sc_qp.vsi) {
1152		irdma_qp_rem_qos(&iwqp->sc_qp);
1153		iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,
1154					   iwqp->sc_qp.user_pri);
1155	}
1156
1157	if (qp_num > 2)
1158		irdma_free_rsrc(rf, rf->allocated_qps, qp_num);
1159	dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size,
1160			  iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa);
1161	iwqp->q2_ctx_mem.va = NULL;
1162	dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size,
1163			  iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa);
1164	iwqp->kqp.dma_mem.va = NULL;
1165	kfree(iwqp->kqp.sq_wrid_mem);
1166	kfree(iwqp->kqp.rq_wrid_mem);
1167}
1168
1169/**
1170 * irdma_cq_wq_destroy - send cq destroy cqp
1171 * @rf: RDMA PCI function
1172 * @cq: hardware control cq
1173 */
1174void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)
1175{
1176	struct irdma_cqp_request *cqp_request;
1177	struct cqp_cmds_info *cqp_info;
1178
1179	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1180	if (!cqp_request)
1181		return;
1182
1183	cqp_info = &cqp_request->info;
1184	cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY;
1185	cqp_info->post_sq = 1;
1186	cqp_info->in.u.cq_destroy.cq = cq;
1187	cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;
1188
1189	irdma_handle_cqp_op(rf, cqp_request);
1190	irdma_put_cqp_request(&rf->cqp, cqp_request);
1191}
1192
1193/**
1194 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait
1195 * @cqp_request: modify QP completion
1196 */
1197static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request)
1198{
1199	struct cqp_cmds_info *cqp_info;
1200	struct irdma_qp *iwqp;
1201
1202	cqp_info = &cqp_request->info;
1203	iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp;
1204	atomic_dec(&iwqp->hw_mod_qp_pend);
1205	wake_up(&iwqp->mod_qp_waitq);
1206}
1207
1208/**
1209 * irdma_hw_modify_qp - setup cqp for modify qp
1210 * @iwdev: RDMA device
1211 * @iwqp: qp ptr (user or kernel)
1212 * @info: info for modify qp
1213 * @wait: flag to wait or not for modify qp completion
1214 */
1215int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,
1216		       struct irdma_modify_qp_info *info, bool wait)
1217{
1218	int status;
1219	struct irdma_pci_f *rf = iwdev->rf;
1220	struct irdma_cqp_request *cqp_request;
1221	struct cqp_cmds_info *cqp_info;
1222	struct irdma_modify_qp_info *m_info;
1223
1224	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
1225	if (!cqp_request)
1226		return -ENOMEM;
1227
1228	if (!wait) {
1229		cqp_request->callback_fcn = irdma_hw_modify_qp_callback;
1230		atomic_inc(&iwqp->hw_mod_qp_pend);
1231	}
1232	cqp_info = &cqp_request->info;
1233	m_info = &cqp_info->in.u.qp_modify.info;
1234	memcpy(m_info, info, sizeof(*m_info));
1235	cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1236	cqp_info->post_sq = 1;
1237	cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1238	cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1239	status = irdma_handle_cqp_op(rf, cqp_request);
1240	irdma_put_cqp_request(&rf->cqp, cqp_request);
1241	if (status) {
1242		if (rdma_protocol_roce(&iwdev->ibdev, 1))
1243			return status;
1244
1245		switch (m_info->next_iwarp_state) {
1246			struct irdma_gen_ae_info ae_info;
1247
1248		case IRDMA_QP_STATE_RTS:
1249		case IRDMA_QP_STATE_IDLE:
1250		case IRDMA_QP_STATE_TERMINATE:
1251		case IRDMA_QP_STATE_CLOSING:
1252			if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE)
1253				irdma_send_reset(iwqp->cm_node);
1254			else
1255				iwqp->sc_qp.term_flags = IRDMA_TERM_DONE;
1256			if (!wait) {
1257				ae_info.ae_code = IRDMA_AE_BAD_CLOSE;
1258				ae_info.ae_src = 0;
1259				irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false);
1260			} else {
1261				cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp,
1262									      wait);
1263				if (!cqp_request)
1264					return -ENOMEM;
1265
1266				cqp_info = &cqp_request->info;
1267				m_info = &cqp_info->in.u.qp_modify.info;
1268				memcpy(m_info, info, sizeof(*m_info));
1269				cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1270				cqp_info->post_sq = 1;
1271				cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1272				cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1273				m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR;
1274				m_info->reset_tcp_conn = true;
1275				irdma_handle_cqp_op(rf, cqp_request);
1276				irdma_put_cqp_request(&rf->cqp, cqp_request);
1277			}
1278			break;
1279		case IRDMA_QP_STATE_ERROR:
1280		default:
1281			break;
1282		}
1283	}
1284
1285	return status;
1286}
1287
1288/**
1289 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq
1290 * @dev: device pointer
1291 * @cq: pointer to cq
1292 */
1293void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
1294{
1295	struct irdma_pci_f *rf = dev_to_rf(dev);
1296
1297	irdma_cq_wq_destroy(rf, cq);
1298}
1299
1300/**
1301 * irdma_cqp_qp_destroy_cmd - destroy the cqp
1302 * @dev: device pointer
1303 * @qp: pointer to qp
1304 */
1305int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1306{
1307	struct irdma_pci_f *rf = dev_to_rf(dev);
1308	struct irdma_cqp *iwcqp = &rf->cqp;
1309	struct irdma_cqp_request *cqp_request;
1310	struct cqp_cmds_info *cqp_info;
1311	int status;
1312
1313	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1314	if (!cqp_request)
1315		return -ENOMEM;
1316
1317	cqp_info = &cqp_request->info;
1318	memset(cqp_info, 0, sizeof(*cqp_info));
1319	cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY;
1320	cqp_info->post_sq = 1;
1321	cqp_info->in.u.qp_destroy.qp = qp;
1322	cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
1323	cqp_info->in.u.qp_destroy.remove_hash_idx = true;
1324
1325	status = irdma_handle_cqp_op(rf, cqp_request);
1326	irdma_put_cqp_request(&rf->cqp, cqp_request);
1327
1328	return status;
1329}
1330
1331/**
1332 * irdma_ieq_mpa_crc_ae - generate AE for crc error
1333 * @dev: hardware control device structure
1334 * @qp: hardware control qp
1335 */
1336void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1337{
1338	struct irdma_gen_ae_info info = {};
1339	struct irdma_pci_f *rf = dev_to_rf(dev);
1340
1341	ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n");
1342	info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR;
1343	info.ae_src = IRDMA_AE_SOURCE_RQ;
1344	irdma_gen_ae(rf, qp, &info, false);
1345}
1346
1347/**
1348 * irdma_init_hash_desc - initialize hash for crc calculation
1349 * @desc: cryption type
1350 */
1351int irdma_init_hash_desc(struct shash_desc **desc)
1352{
1353	struct crypto_shash *tfm;
1354	struct shash_desc *tdesc;
1355
1356	tfm = crypto_alloc_shash("crc32c", 0, 0);
1357	if (IS_ERR(tfm))
1358		return -EINVAL;
1359
1360	tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm),
1361			GFP_KERNEL);
1362	if (!tdesc) {
1363		crypto_free_shash(tfm);
1364		return -EINVAL;
1365	}
1366
1367	tdesc->tfm = tfm;
1368	*desc = tdesc;
1369
1370	return 0;
1371}
1372
1373/**
1374 * irdma_free_hash_desc - free hash desc
1375 * @desc: to be freed
1376 */
1377void irdma_free_hash_desc(struct shash_desc *desc)
1378{
1379	if (desc) {
1380		crypto_free_shash(desc->tfm);
1381		kfree(desc);
1382	}
1383}
1384
1385/**
1386 * irdma_ieq_check_mpacrc - check if mpa crc is OK
1387 * @desc: desc for hash
1388 * @addr: address of buffer for crc
1389 * @len: length of buffer
1390 * @val: value to be compared
1391 */
1392int irdma_ieq_check_mpacrc(struct shash_desc *desc, void *addr, u32 len,
1393			   u32 val)
1394{
1395	u32 crc = 0;
 
 
1396
1397	crypto_shash_digest(desc, addr, len, (u8 *)&crc);
 
 
 
1398	if (crc != val)
1399		return -EINVAL;
1400
1401	return 0;
1402}
1403
1404/**
1405 * irdma_ieq_get_qp - get qp based on quad in puda buffer
1406 * @dev: hardware control device structure
1407 * @buf: receive puda buffer on exception q
1408 */
1409struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,
1410				     struct irdma_puda_buf *buf)
1411{
1412	struct irdma_qp *iwqp;
1413	struct irdma_cm_node *cm_node;
1414	struct irdma_device *iwdev = buf->vsi->back_vsi;
1415	u32 loc_addr[4] = {};
1416	u32 rem_addr[4] = {};
1417	u16 loc_port, rem_port;
1418	struct ipv6hdr *ip6h;
1419	struct iphdr *iph = (struct iphdr *)buf->iph;
1420	struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1421
1422	if (iph->version == 4) {
1423		loc_addr[0] = ntohl(iph->daddr);
1424		rem_addr[0] = ntohl(iph->saddr);
1425	} else {
1426		ip6h = (struct ipv6hdr *)buf->iph;
1427		irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32);
1428		irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32);
1429	}
1430	loc_port = ntohs(tcph->dest);
1431	rem_port = ntohs(tcph->source);
1432	cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port,
1433				  loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF);
1434	if (!cm_node)
1435		return NULL;
1436
1437	iwqp = cm_node->iwqp;
1438	irdma_rem_ref_cm_node(cm_node);
1439
1440	return &iwqp->sc_qp;
1441}
1442
1443/**
1444 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs
1445 * @qp: qp ptr
1446 */
1447void irdma_send_ieq_ack(struct irdma_sc_qp *qp)
1448{
1449	struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node;
1450	struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf;
1451	struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1452
1453	cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum;
1454	cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
1455
1456	irdma_send_ack(cm_node);
1457}
1458
1459/**
1460 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer
1461 * @qp: qp pointer
1462 * @ah_info: AH info pointer
1463 */
1464void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,
1465				struct irdma_ah_info *ah_info)
1466{
1467	struct irdma_puda_buf *buf = qp->pfpdu.ah_buf;
1468	struct iphdr *iph;
1469	struct ipv6hdr *ip6h;
1470
1471	memset(ah_info, 0, sizeof(*ah_info));
1472	ah_info->do_lpbk = true;
1473	ah_info->vlan_tag = buf->vlan_id;
1474	ah_info->insert_vlan_tag = buf->vlan_valid;
1475	ah_info->ipv4_valid = buf->ipv4;
1476	ah_info->vsi = qp->vsi;
1477
1478	if (buf->smac_valid)
1479		ether_addr_copy(ah_info->mac_addr, buf->smac);
1480
1481	if (buf->ipv4) {
1482		ah_info->ipv4_valid = true;
1483		iph = (struct iphdr *)buf->iph;
1484		ah_info->hop_ttl = iph->ttl;
1485		ah_info->tc_tos = iph->tos;
1486		ah_info->dest_ip_addr[0] = ntohl(iph->daddr);
1487		ah_info->src_ip_addr[0] = ntohl(iph->saddr);
1488	} else {
1489		ip6h = (struct ipv6hdr *)buf->iph;
1490		ah_info->hop_ttl = ip6h->hop_limit;
1491		ah_info->tc_tos = ip6h->priority;
1492		irdma_copy_ip_ntohl(ah_info->dest_ip_addr,
1493				    ip6h->daddr.in6_u.u6_addr32);
1494		irdma_copy_ip_ntohl(ah_info->src_ip_addr,
1495				    ip6h->saddr.in6_u.u6_addr32);
1496	}
1497
1498	ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev),
1499						ah_info->dest_ip_addr,
1500						ah_info->ipv4_valid,
1501						NULL, IRDMA_ARP_RESOLVE);
1502}
1503
1504/**
1505 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer
1506 * @buf: puda to update
1507 * @len: length of buffer
1508 * @seqnum: seq number for tcp
1509 */
1510static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf,
1511					     u16 len, u32 seqnum)
1512{
1513	struct tcphdr *tcph;
1514	struct iphdr *iph;
1515	u16 iphlen;
1516	u16 pktsize;
1517	u8 *addr = buf->mem.va;
1518
1519	iphlen = (buf->ipv4) ? 20 : 40;
1520	iph = (struct iphdr *)(addr + buf->maclen);
1521	tcph = (struct tcphdr *)(addr + buf->maclen + iphlen);
1522	pktsize = len + buf->tcphlen + iphlen;
1523	iph->tot_len = htons(pktsize);
1524	tcph->seq = htonl(seqnum);
1525}
1526
1527/**
1528 * irdma_ieq_update_tcpip_info - update tcpip in the buffer
1529 * @buf: puda to update
1530 * @len: length of buffer
1531 * @seqnum: seq number for tcp
1532 */
1533void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,
1534				 u32 seqnum)
1535{
1536	struct tcphdr *tcph;
1537	u8 *addr;
1538
1539	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1540		return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum);
1541
1542	addr = buf->mem.va;
1543	tcph = (struct tcphdr *)addr;
1544	tcph->seq = htonl(seqnum);
1545}
1546
1547/**
1548 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda
1549 * buffer
1550 * @info: to get information
1551 * @buf: puda buffer
1552 */
1553static int irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1554					  struct irdma_puda_buf *buf)
1555{
1556	struct iphdr *iph;
1557	struct ipv6hdr *ip6h;
1558	struct tcphdr *tcph;
1559	u16 iphlen;
1560	u16 pkt_len;
1561	u8 *mem = buf->mem.va;
1562	struct ethhdr *ethh = buf->mem.va;
1563
1564	if (ethh->h_proto == htons(0x8100)) {
1565		info->vlan_valid = true;
1566		buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) &
1567			       VLAN_VID_MASK;
1568	}
1569
1570	buf->maclen = (info->vlan_valid) ? 18 : 14;
1571	iphlen = (info->l3proto) ? 40 : 20;
1572	buf->ipv4 = (info->l3proto) ? false : true;
1573	buf->iph = mem + buf->maclen;
1574	iph = (struct iphdr *)buf->iph;
1575	buf->tcph = buf->iph + iphlen;
1576	tcph = (struct tcphdr *)buf->tcph;
1577
1578	if (buf->ipv4) {
1579		pkt_len = ntohs(iph->tot_len);
1580	} else {
1581		ip6h = (struct ipv6hdr *)buf->iph;
1582		pkt_len = ntohs(ip6h->payload_len) + iphlen;
1583	}
1584
1585	buf->totallen = pkt_len + buf->maclen;
1586
1587	if (info->payload_len < buf->totallen) {
1588		ibdev_dbg(to_ibdev(buf->vsi->dev),
1589			  "ERR: payload_len = 0x%x totallen expected0x%x\n",
1590			  info->payload_len, buf->totallen);
1591		return -EINVAL;
1592	}
1593
1594	buf->tcphlen = tcph->doff << 2;
1595	buf->datalen = pkt_len - iphlen - buf->tcphlen;
1596	buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1597	buf->hdrlen = buf->maclen + iphlen + buf->tcphlen;
1598	buf->seqnum = ntohl(tcph->seq);
1599
1600	return 0;
1601}
1602
1603/**
1604 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer
1605 * @info: to get information
1606 * @buf: puda buffer
1607 */
1608int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1609			      struct irdma_puda_buf *buf)
1610{
1611	struct tcphdr *tcph;
1612	u32 pkt_len;
1613	u8 *mem;
1614
1615	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1616		return irdma_gen1_puda_get_tcpip_info(info, buf);
1617
1618	mem = buf->mem.va;
1619	buf->vlan_valid = info->vlan_valid;
1620	if (info->vlan_valid)
1621		buf->vlan_id = info->vlan;
1622
1623	buf->ipv4 = info->ipv4;
1624	if (buf->ipv4)
1625		buf->iph = mem + IRDMA_IPV4_PAD;
1626	else
1627		buf->iph = mem;
1628
1629	buf->tcph = mem + IRDMA_TCP_OFFSET;
1630	tcph = (struct tcphdr *)buf->tcph;
1631	pkt_len = info->payload_len;
1632	buf->totallen = pkt_len;
1633	buf->tcphlen = tcph->doff << 2;
1634	buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen;
1635	buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1636	buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen;
1637	buf->seqnum = ntohl(tcph->seq);
1638
1639	if (info->smac_valid) {
1640		ether_addr_copy(buf->smac, info->smac);
1641		buf->smac_valid = true;
1642	}
1643
1644	return 0;
1645}
1646
1647/**
1648 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats
1649 * @t: timer_list pointer
1650 */
1651static void irdma_hw_stats_timeout(struct timer_list *t)
1652{
1653	struct irdma_vsi_pestat *pf_devstat =
1654		from_timer(pf_devstat, t, stats_timer);
1655	struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi;
1656
1657	if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
1658		irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false);
1659	else
1660		irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat);
 
 
1661
1662	mod_timer(&pf_devstat->stats_timer,
1663		  jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1664}
1665
1666/**
1667 * irdma_hw_stats_start_timer - Start periodic stats timer
1668 * @vsi: vsi structure pointer
1669 */
1670void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi)
1671{
1672	struct irdma_vsi_pestat *devstat = vsi->pestat;
1673
1674	timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0);
1675	mod_timer(&devstat->stats_timer,
1676		  jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1677}
1678
1679/**
1680 * irdma_hw_stats_stop_timer - Delete periodic stats timer
1681 * @vsi: pointer to vsi structure
1682 */
1683void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi)
1684{
1685	struct irdma_vsi_pestat *devstat = vsi->pestat;
1686
1687	del_timer_sync(&devstat->stats_timer);
1688}
1689
1690/**
1691 * irdma_process_stats - Checking for wrap and update stats
1692 * @pestat: stats structure pointer
1693 */
1694static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat)
1695{
1696	sc_vsi_update_stats(pestat->vsi);
1697}
1698
1699/**
1700 * irdma_cqp_gather_stats_gen1 - Gather stats
1701 * @dev: pointer to device structure
1702 * @pestat: statistics structure
1703 */
1704void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,
1705				 struct irdma_vsi_pestat *pestat)
1706{
1707	struct irdma_gather_stats *gather_stats =
1708		pestat->gather_info.gather_stats_va;
1709	const struct irdma_hw_stat_map *map = dev->hw_stats_map;
1710	u16 max_stats_idx = dev->hw_attrs.max_stat_idx;
1711	u32 stats_inst_offset_32;
1712	u32 stats_inst_offset_64;
1713	u64 new_val;
1714	u16 i;
1715
1716	stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ?
1717				pestat->gather_info.stats_inst_index :
1718				pestat->hw->hmc.hmc_fn_id;
1719	stats_inst_offset_32 *= 4;
1720	stats_inst_offset_64 = stats_inst_offset_32 * 2;
1721
1722	for (i = 0; i < max_stats_idx; i++) {
1723		if (map[i].bitmask <= IRDMA_MAX_STATS_32)
1724			new_val = rd32(dev->hw,
1725				       dev->hw_stats_regs[i] + stats_inst_offset_32);
1726		else
1727			new_val = rd64(dev->hw,
1728				       dev->hw_stats_regs[i] + stats_inst_offset_64);
1729		gather_stats->val[map[i].byteoff / sizeof(u64)] = new_val;
1730	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1731
1732	irdma_process_stats(pestat);
1733}
1734
1735/**
1736 * irdma_process_cqp_stats - Checking for wrap and update stats
1737 * @cqp_request: cqp_request structure pointer
1738 */
1739static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)
1740{
1741	struct irdma_vsi_pestat *pestat = cqp_request->param;
1742
1743	irdma_process_stats(pestat);
1744}
1745
1746/**
1747 * irdma_cqp_gather_stats_cmd - Gather stats
1748 * @dev: pointer to device structure
1749 * @pestat: pointer to stats info
1750 * @wait: flag to wait or not wait for stats
1751 */
1752int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
1753			       struct irdma_vsi_pestat *pestat, bool wait)
1754
1755{
1756	struct irdma_pci_f *rf = dev_to_rf(dev);
1757	struct irdma_cqp *iwcqp = &rf->cqp;
1758	struct irdma_cqp_request *cqp_request;
1759	struct cqp_cmds_info *cqp_info;
1760	int status;
1761
1762	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1763	if (!cqp_request)
1764		return -ENOMEM;
1765
1766	cqp_info = &cqp_request->info;
1767	memset(cqp_info, 0, sizeof(*cqp_info));
1768	cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER;
1769	cqp_info->post_sq = 1;
1770	cqp_info->in.u.stats_gather.info = pestat->gather_info;
1771	cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request;
1772	cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp;
1773	cqp_request->param = pestat;
1774	if (!wait)
1775		cqp_request->callback_fcn = irdma_process_cqp_stats;
1776	status = irdma_handle_cqp_op(rf, cqp_request);
1777	if (wait)
1778		irdma_process_stats(pestat);
1779	irdma_put_cqp_request(&rf->cqp, cqp_request);
1780
1781	return status;
1782}
1783
1784/**
1785 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance
1786 * @vsi: pointer to vsi structure
1787 * @cmd: command to allocate or free
1788 * @stats_info: pointer to allocate stats info
1789 */
1790int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
1791			     struct irdma_stats_inst_info *stats_info)
1792{
1793	struct irdma_pci_f *rf = dev_to_rf(vsi->dev);
1794	struct irdma_cqp *iwcqp = &rf->cqp;
1795	struct irdma_cqp_request *cqp_request;
1796	struct cqp_cmds_info *cqp_info;
1797	int status;
1798	bool wait = false;
1799
1800	if (cmd == IRDMA_OP_STATS_ALLOCATE)
1801		wait = true;
1802	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1803	if (!cqp_request)
1804		return -ENOMEM;
1805
1806	cqp_info = &cqp_request->info;
1807	memset(cqp_info, 0, sizeof(*cqp_info));
1808	cqp_info->cqp_cmd = cmd;
1809	cqp_info->post_sq = 1;
1810	cqp_info->in.u.stats_manage.info = *stats_info;
1811	cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request;
1812	cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp;
1813	status = irdma_handle_cqp_op(rf, cqp_request);
1814	if (wait)
1815		stats_info->stats_idx = cqp_request->compl_info.op_ret_val;
1816	irdma_put_cqp_request(iwcqp, cqp_request);
1817
1818	return status;
1819}
1820
1821/**
1822 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0
1823 * @dev: pointer to device info
1824 * @sc_ceq: pointer to ceq structure
1825 * @op: Create or Destroy
1826 */
1827int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,
1828		      u8 op)
1829{
1830	struct irdma_cqp_request *cqp_request;
1831	struct cqp_cmds_info *cqp_info;
1832	struct irdma_pci_f *rf = dev_to_rf(dev);
1833	int status;
1834
1835	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1836	if (!cqp_request)
1837		return -ENOMEM;
1838
1839	cqp_info = &cqp_request->info;
1840	cqp_info->post_sq = 1;
1841	cqp_info->cqp_cmd = op;
1842	cqp_info->in.u.ceq_create.ceq = sc_ceq;
1843	cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request;
1844
1845	status = irdma_handle_cqp_op(rf, cqp_request);
1846	irdma_put_cqp_request(&rf->cqp, cqp_request);
1847
1848	return status;
1849}
1850
1851/**
1852 * irdma_cqp_aeq_cmd - Create/Destroy AEQ
1853 * @dev: pointer to device info
1854 * @sc_aeq: pointer to aeq structure
1855 * @op: Create or Destroy
1856 */
1857int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq,
1858		      u8 op)
1859{
1860	struct irdma_cqp_request *cqp_request;
1861	struct cqp_cmds_info *cqp_info;
1862	struct irdma_pci_f *rf = dev_to_rf(dev);
1863	int status;
1864
1865	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1866	if (!cqp_request)
1867		return -ENOMEM;
1868
1869	cqp_info = &cqp_request->info;
1870	cqp_info->post_sq = 1;
1871	cqp_info->cqp_cmd = op;
1872	cqp_info->in.u.aeq_create.aeq = sc_aeq;
1873	cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request;
1874
1875	status = irdma_handle_cqp_op(rf, cqp_request);
1876	irdma_put_cqp_request(&rf->cqp, cqp_request);
1877
1878	return status;
1879}
1880
1881/**
1882 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node
1883 * @dev: pointer to device structure
1884 * @cmd: Add, modify or delete
1885 * @node_info: pointer to ws node info
1886 */
1887int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
1888			  struct irdma_ws_node_info *node_info)
1889{
1890	struct irdma_pci_f *rf = dev_to_rf(dev);
1891	struct irdma_cqp *iwcqp = &rf->cqp;
1892	struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp;
1893	struct irdma_cqp_request *cqp_request;
1894	struct cqp_cmds_info *cqp_info;
1895	int status;
1896	bool poll;
1897
1898	if (!rf->sc_dev.ceq_valid)
1899		poll = true;
1900	else
1901		poll = false;
1902
1903	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll);
1904	if (!cqp_request)
1905		return -ENOMEM;
1906
1907	cqp_info = &cqp_request->info;
1908	memset(cqp_info, 0, sizeof(*cqp_info));
1909	cqp_info->cqp_cmd = cmd;
1910	cqp_info->post_sq = 1;
1911	cqp_info->in.u.ws_node.info = *node_info;
1912	cqp_info->in.u.ws_node.cqp = cqp;
1913	cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request;
1914	status = irdma_handle_cqp_op(rf, cqp_request);
1915	if (status)
1916		goto exit;
1917
1918	if (poll) {
1919		struct irdma_ccq_cqe_info compl_info;
1920
1921		status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE,
1922						       &compl_info);
1923		node_info->qs_handle = compl_info.op_ret_val;
1924		ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n",
1925			  compl_info.op_code, compl_info.op_ret_val);
1926	} else {
1927		node_info->qs_handle = cqp_request->compl_info.op_ret_val;
1928	}
1929
1930exit:
1931	irdma_put_cqp_request(&rf->cqp, cqp_request);
1932
1933	return status;
1934}
1935
1936/**
1937 * irdma_ah_cqp_op - perform an AH cqp operation
1938 * @rf: RDMA PCI function
1939 * @sc_ah: address handle
1940 * @cmd: AH operation
1941 * @wait: wait if true
1942 * @callback_fcn: Callback function on CQP op completion
1943 * @cb_param: parameter for callback function
1944 *
1945 * returns errno
1946 */
1947int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
1948		    bool wait,
1949		    void (*callback_fcn)(struct irdma_cqp_request *),
1950		    void *cb_param)
1951{
1952	struct irdma_cqp_request *cqp_request;
1953	struct cqp_cmds_info *cqp_info;
1954	int status;
1955
1956	if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY)
1957		return -EINVAL;
1958
1959	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
1960	if (!cqp_request)
1961		return -ENOMEM;
1962
1963	cqp_info = &cqp_request->info;
1964	cqp_info->cqp_cmd = cmd;
1965	cqp_info->post_sq = 1;
1966	if (cmd == IRDMA_OP_AH_CREATE) {
1967		cqp_info->in.u.ah_create.info = sc_ah->ah_info;
1968		cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;
1969		cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp;
1970	} else if (cmd == IRDMA_OP_AH_DESTROY) {
1971		cqp_info->in.u.ah_destroy.info = sc_ah->ah_info;
1972		cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request;
1973		cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp;
1974	}
1975
1976	if (!wait) {
1977		cqp_request->callback_fcn = callback_fcn;
1978		cqp_request->param = cb_param;
1979	}
1980	status = irdma_handle_cqp_op(rf, cqp_request);
1981	irdma_put_cqp_request(&rf->cqp, cqp_request);
1982
1983	if (status)
1984		return -ENOMEM;
1985
1986	if (wait)
1987		sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE);
1988
1989	return 0;
1990}
1991
1992/**
1993 * irdma_ieq_ah_cb - callback after creation of AH for IEQ
1994 * @cqp_request: pointer to cqp_request of create AH
1995 */
1996static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request)
1997{
1998	struct irdma_sc_qp *qp = cqp_request->param;
1999	struct irdma_sc_ah *sc_ah = qp->pfpdu.ah;
2000	unsigned long flags;
2001
2002	spin_lock_irqsave(&qp->pfpdu.lock, flags);
2003	if (!cqp_request->compl_info.op_ret_val) {
2004		sc_ah->ah_info.ah_valid = true;
2005		irdma_ieq_process_fpdus(qp, qp->vsi->ieq);
2006	} else {
2007		sc_ah->ah_info.ah_valid = false;
2008		irdma_ieq_cleanup_qp(qp->vsi->ieq, qp);
2009	}
2010	spin_unlock_irqrestore(&qp->pfpdu.lock, flags);
2011}
2012
2013/**
2014 * irdma_ilq_ah_cb - callback after creation of AH for ILQ
2015 * @cqp_request: pointer to cqp_request of create AH
2016 */
2017static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request)
2018{
2019	struct irdma_cm_node *cm_node = cqp_request->param;
2020	struct irdma_sc_ah *sc_ah = cm_node->ah;
2021
2022	sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val;
2023	irdma_add_conn_est_qh(cm_node);
2024}
2025
2026/**
2027 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's
2028 * @dev: device pointer
2029 * @ah_info: Address handle info
2030 * @wait: When true will wait for operation to complete
2031 * @type: ILQ/IEQ
2032 * @cb_param: Callback param when not waiting
2033 * @ah_ret: Returned pointer to address handle if created
2034 *
2035 */
2036int irdma_puda_create_ah(struct irdma_sc_dev *dev,
2037			 struct irdma_ah_info *ah_info, bool wait,
2038			 enum puda_rsrc_type type, void *cb_param,
2039			 struct irdma_sc_ah **ah_ret)
2040{
2041	struct irdma_sc_ah *ah;
2042	struct irdma_pci_f *rf = dev_to_rf(dev);
2043	int err;
2044
2045	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2046	*ah_ret = ah;
2047	if (!ah)
2048		return -ENOMEM;
2049
2050	err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah,
2051			       &ah_info->ah_idx, &rf->next_ah);
2052	if (err)
2053		goto err_free;
2054
2055	ah->dev = dev;
2056	ah->ah_info = *ah_info;
2057
2058	if (type == IRDMA_PUDA_RSRC_TYPE_ILQ)
2059		err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
2060				      irdma_ilq_ah_cb, cb_param);
2061	else
2062		err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
2063				      irdma_ieq_ah_cb, cb_param);
2064
2065	if (err)
2066		goto error;
2067	return 0;
2068
2069error:
2070	irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
2071err_free:
2072	kfree(ah);
2073	*ah_ret = NULL;
2074	return -ENOMEM;
2075}
2076
2077/**
2078 * irdma_puda_free_ah - free a puda address handle
2079 * @dev: device pointer
2080 * @ah: The address handle to free
2081 */
2082void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)
2083{
2084	struct irdma_pci_f *rf = dev_to_rf(dev);
2085
2086	if (!ah)
2087		return;
2088
2089	if (ah->ah_info.ah_valid) {
2090		irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL);
2091		irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
2092	}
2093
2094	kfree(ah);
2095}
2096
2097/**
2098 * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP
2099 * @cqp_request: pointer to cqp_request of create AH
2100 */
2101void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request)
2102{
2103	struct irdma_sc_ah *sc_ah = cqp_request->param;
2104
2105	if (!cqp_request->compl_info.op_ret_val)
2106		sc_ah->ah_info.ah_valid = true;
2107	else
2108		sc_ah->ah_info.ah_valid = false;
2109}
2110
2111/**
2112 * irdma_prm_add_pble_mem - add moemory to pble resources
2113 * @pprm: pble resource manager
2114 * @pchunk: chunk of memory to add
2115 */
2116int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
2117			   struct irdma_chunk *pchunk)
2118{
2119	u64 sizeofbitmap;
2120
2121	if (pchunk->size & 0xfff)
2122		return -EINVAL;
2123
2124	sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;
2125
2126	pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL);
2127	if (!pchunk->bitmapbuf)
2128		return -ENOMEM;
2129
2130	pchunk->sizeofbitmap = sizeofbitmap;
2131	/* each pble is 8 bytes hence shift by 3 */
2132	pprm->total_pble_alloc += pchunk->size >> 3;
2133	pprm->free_pble_cnt += pchunk->size >> 3;
2134
2135	return 0;
2136}
2137
2138/**
2139 * irdma_prm_get_pbles - get pble's from prm
2140 * @pprm: pble resource manager
2141 * @chunkinfo: nformation about chunk where pble's were acquired
2142 * @mem_size: size of pble memory needed
2143 * @vaddr: returns virtual address of pble memory
2144 * @fpm_addr: returns fpm address of pble memory
2145 */
2146int irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
2147			struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,
2148			u64 **vaddr, u64 *fpm_addr)
2149{
2150	u64 bits_needed;
2151	u64 bit_idx = PBLE_INVALID_IDX;
2152	struct irdma_chunk *pchunk = NULL;
2153	struct list_head *chunk_entry = pprm->clist.next;
2154	u32 offset;
2155	unsigned long flags;
2156	*vaddr = NULL;
2157	*fpm_addr = 0;
2158
2159	bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift));
2160
2161	spin_lock_irqsave(&pprm->prm_lock, flags);
2162	while (chunk_entry != &pprm->clist) {
2163		pchunk = (struct irdma_chunk *)chunk_entry;
2164		bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf,
2165						     pchunk->sizeofbitmap, 0,
2166						     bits_needed, 0);
2167		if (bit_idx < pchunk->sizeofbitmap)
2168			break;
2169
2170		/* list.next used macro */
2171		chunk_entry = pchunk->list.next;
2172	}
2173
2174	if (!pchunk || bit_idx >= pchunk->sizeofbitmap) {
2175		spin_unlock_irqrestore(&pprm->prm_lock, flags);
2176		return -ENOMEM;
2177	}
2178
2179	bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed);
2180	offset = bit_idx << pprm->pble_shift;
2181	*vaddr = pchunk->vaddr + offset;
2182	*fpm_addr = pchunk->fpm_addr + offset;
2183
2184	chunkinfo->pchunk = pchunk;
2185	chunkinfo->bit_idx = bit_idx;
2186	chunkinfo->bits_used = bits_needed;
2187	/* 3 is sizeof pble divide */
2188	pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3);
2189	spin_unlock_irqrestore(&pprm->prm_lock, flags);
2190
2191	return 0;
2192}
2193
2194/**
2195 * irdma_prm_return_pbles - return pbles back to prm
2196 * @pprm: pble resource manager
2197 * @chunkinfo: chunk where pble's were acquired and to be freed
2198 */
2199void irdma_prm_return_pbles(struct irdma_pble_prm *pprm,
2200			    struct irdma_pble_chunkinfo *chunkinfo)
2201{
2202	unsigned long flags;
2203
2204	spin_lock_irqsave(&pprm->prm_lock, flags);
2205	pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3);
2206	bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx,
2207		     chunkinfo->bits_used);
2208	spin_unlock_irqrestore(&pprm->prm_lock, flags);
2209}
2210
2211int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma,
2212			   u32 pg_cnt)
2213{
2214	struct page *vm_page;
2215	int i;
2216	u8 *addr;
2217
2218	addr = (u8 *)(uintptr_t)va;
2219	for (i = 0; i < pg_cnt; i++) {
2220		vm_page = vmalloc_to_page(addr);
2221		if (!vm_page)
2222			goto err;
2223
2224		pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE,
2225					 DMA_BIDIRECTIONAL);
2226		if (dma_mapping_error(hw->device, pg_dma[i]))
2227			goto err;
2228
2229		addr += PAGE_SIZE;
2230	}
2231
2232	return 0;
2233
2234err:
2235	irdma_unmap_vm_page_list(hw, pg_dma, i);
2236	return -ENOMEM;
2237}
2238
2239void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt)
2240{
2241	int i;
2242
2243	for (i = 0; i < pg_cnt; i++)
2244		dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL);
2245}
2246
2247/**
2248 * irdma_pble_free_paged_mem - free virtual paged memory
2249 * @chunk: chunk to free with paged memory
2250 */
2251void irdma_pble_free_paged_mem(struct irdma_chunk *chunk)
2252{
2253	if (!chunk->pg_cnt)
2254		goto done;
2255
2256	irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs,
2257				 chunk->pg_cnt);
2258
2259done:
2260	kfree(chunk->dmainfo.dmaaddrs);
2261	chunk->dmainfo.dmaaddrs = NULL;
2262	vfree(chunk->vaddr);
2263	chunk->vaddr = NULL;
2264	chunk->type = 0;
2265}
2266
2267/**
2268 * irdma_pble_get_paged_mem -allocate paged memory for pbles
2269 * @chunk: chunk to add for paged memory
2270 * @pg_cnt: number of pages needed
2271 */
2272int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt)
2273{
2274	u32 size;
2275	void *va;
2276
2277	chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
2278	if (!chunk->dmainfo.dmaaddrs)
2279		return -ENOMEM;
2280
2281	size = PAGE_SIZE * pg_cnt;
2282	va = vmalloc(size);
2283	if (!va)
2284		goto err;
2285
2286	if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs,
2287				   pg_cnt)) {
2288		vfree(va);
2289		goto err;
2290	}
2291	chunk->vaddr = va;
2292	chunk->size = size;
2293	chunk->pg_cnt = pg_cnt;
2294	chunk->type = PBLE_SD_PAGED;
2295
2296	return 0;
2297err:
2298	kfree(chunk->dmainfo.dmaaddrs);
2299	chunk->dmainfo.dmaaddrs = NULL;
2300
2301	return -ENOMEM;
2302}
2303
2304/**
2305 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID
2306 * @dev: device pointer
2307 */
2308u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev)
2309{
2310	struct irdma_pci_f *rf = dev_to_rf(dev);
2311	u32 next = 1;
2312	u32 node_id;
2313
2314	if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id,
2315			     &node_id, &next))
2316		return IRDMA_WS_NODE_INVALID;
2317
2318	return (u16)node_id;
2319}
2320
2321/**
2322 * irdma_free_ws_node_id - Free a tx scheduler node ID
2323 * @dev: device pointer
2324 * @node_id: Work scheduler node ID
2325 */
2326void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id)
2327{
2328	struct irdma_pci_f *rf = dev_to_rf(dev);
2329
2330	irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id);
2331}
2332
2333/**
2334 * irdma_modify_qp_to_err - Modify a QP to error
2335 * @sc_qp: qp structure
2336 */
2337void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp)
2338{
2339	struct irdma_qp *qp = sc_qp->qp_uk.back_qp;
2340	struct ib_qp_attr attr;
2341
2342	if (qp->iwdev->rf->reset)
2343		return;
2344	attr.qp_state = IB_QPS_ERR;
2345
2346	if (rdma_protocol_roce(qp->ibqp.device, 1))
2347		irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2348	else
2349		irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2350}
2351
2352void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event)
2353{
2354	struct ib_event ibevent;
2355
2356	if (!iwqp->ibqp.event_handler)
2357		return;
2358
2359	switch (event) {
2360	case IRDMA_QP_EVENT_CATASTROPHIC:
2361		ibevent.event = IB_EVENT_QP_FATAL;
2362		break;
2363	case IRDMA_QP_EVENT_ACCESS_ERR:
2364		ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2365		break;
2366	case IRDMA_QP_EVENT_REQ_ERR:
2367		ibevent.event = IB_EVENT_QP_REQ_ERR;
2368		break;
2369	}
2370	ibevent.device = iwqp->ibqp.device;
2371	ibevent.element.qp = &iwqp->ibqp;
2372	iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context);
2373}
2374
2375bool irdma_cq_empty(struct irdma_cq *iwcq)
2376{
2377	struct irdma_cq_uk *ukcq;
2378	u64 qword3;
2379	__le64 *cqe;
2380	u8 polarity;
2381
2382	ukcq  = &iwcq->sc_cq.cq_uk;
2383	cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq);
2384	get_64bit_val(cqe, 24, &qword3);
2385	polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
2386
2387	return polarity != ukcq->polarity;
2388}
2389
2390void irdma_remove_cmpls_list(struct irdma_cq *iwcq)
2391{
2392	struct irdma_cmpl_gen *cmpl_node;
2393	struct list_head *tmp_node, *list_node;
2394
2395	list_for_each_safe (list_node, tmp_node, &iwcq->cmpl_generated) {
2396		cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list);
2397		list_del(&cmpl_node->list);
2398		kfree(cmpl_node);
2399	}
2400}
2401
2402int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info)
2403{
2404	struct irdma_cmpl_gen *cmpl;
2405
2406	if (list_empty(&iwcq->cmpl_generated))
2407		return -ENOENT;
2408	cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list);
2409	list_del(&cmpl->list);
2410	memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info));
2411	kfree(cmpl);
2412
2413	ibdev_dbg(iwcq->ibcq.device,
2414		  "VERBS: %s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%llx\n",
2415		  __func__, cq_poll_info->qp_id, cq_poll_info->op_type,
2416		  cq_poll_info->wr_id);
2417
2418	return 0;
2419}
2420
2421/**
2422 * irdma_set_cpi_common_values - fill in values for polling info struct
2423 * @cpi: resulting structure of cq_poll_info type
2424 * @qp: QPair
2425 * @qp_num: id of the QP
2426 */
2427static void irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi,
2428					struct irdma_qp_uk *qp, u32 qp_num)
2429{
2430	cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED;
2431	cpi->error = true;
2432	cpi->major_err = IRDMA_FLUSH_MAJOR_ERR;
2433	cpi->minor_err = FLUSH_GENERAL_ERR;
2434	cpi->qp_handle = (irdma_qp_handle)(uintptr_t)qp;
2435	cpi->qp_id = qp_num;
2436}
2437
2438static inline void irdma_comp_handler(struct irdma_cq *cq)
2439{
2440	if (!cq->ibcq.comp_handler)
2441		return;
2442	if (atomic_cmpxchg(&cq->armed, 1, 0))
2443		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
2444}
2445
2446void irdma_generate_flush_completions(struct irdma_qp *iwqp)
2447{
2448	struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk;
2449	struct irdma_ring *sq_ring = &qp->sq_ring;
2450	struct irdma_ring *rq_ring = &qp->rq_ring;
2451	struct irdma_cmpl_gen *cmpl;
2452	__le64 *sw_wqe;
2453	u64 wqe_qword;
2454	u32 wqe_idx;
2455	bool compl_generated = false;
2456	unsigned long flags1;
2457
2458	spin_lock_irqsave(&iwqp->iwscq->lock, flags1);
2459	if (irdma_cq_empty(iwqp->iwscq)) {
2460		unsigned long flags2;
2461
2462		spin_lock_irqsave(&iwqp->lock, flags2);
2463		while (IRDMA_RING_MORE_WORK(*sq_ring)) {
2464			cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2465			if (!cmpl) {
2466				spin_unlock_irqrestore(&iwqp->lock, flags2);
2467				spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2468				return;
2469			}
2470
2471			wqe_idx = sq_ring->tail;
2472			irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);
2473
2474			cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
2475			sw_wqe = qp->sq_base[wqe_idx].elem;
2476			get_64bit_val(sw_wqe, 24, &wqe_qword);
2477			cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE);
2478			cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ;
2479			/* remove the SQ WR by moving SQ tail*/
2480			IRDMA_RING_SET_TAIL(*sq_ring,
2481				sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta);
2482			if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) {
2483				kfree(cmpl);
2484				continue;
2485			}
2486			ibdev_dbg(iwqp->iwscq->ibcq.device,
2487				  "DEV: %s: adding wr_id = 0x%llx SQ Completion to list qp_id=%d\n",
2488				  __func__, cmpl->cpi.wr_id, qp->qp_id);
2489			list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated);
2490			compl_generated = true;
2491		}
2492		spin_unlock_irqrestore(&iwqp->lock, flags2);
2493		spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2494		if (compl_generated)
2495			irdma_comp_handler(iwqp->iwscq);
2496	} else {
2497		spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2498		mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,
2499				 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
2500	}
2501
2502	spin_lock_irqsave(&iwqp->iwrcq->lock, flags1);
2503	if (irdma_cq_empty(iwqp->iwrcq)) {
2504		unsigned long flags2;
2505
2506		spin_lock_irqsave(&iwqp->lock, flags2);
2507		while (IRDMA_RING_MORE_WORK(*rq_ring)) {
2508			cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2509			if (!cmpl) {
2510				spin_unlock_irqrestore(&iwqp->lock, flags2);
2511				spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2512				return;
2513			}
2514
2515			wqe_idx = rq_ring->tail;
2516			irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);
2517
2518			cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx];
2519			cmpl->cpi.op_type = IRDMA_OP_TYPE_REC;
2520			cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ;
2521			/* remove the RQ WR by moving RQ tail */
2522			IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1);
2523			ibdev_dbg(iwqp->iwrcq->ibcq.device,
2524				  "DEV: %s: adding wr_id = 0x%llx RQ Completion to list qp_id=%d, wqe_idx=%d\n",
2525				  __func__, cmpl->cpi.wr_id, qp->qp_id,
2526				  wqe_idx);
2527			list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated);
2528
2529			compl_generated = true;
2530		}
2531		spin_unlock_irqrestore(&iwqp->lock, flags2);
2532		spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2533		if (compl_generated)
2534			irdma_comp_handler(iwqp->iwrcq);
2535	} else {
2536		spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2537		mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,
2538				 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
2539	}
2540}