Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 | /* * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef __T4_H__ #define __T4_H__ #include "t4_hw.h" #include "t4_regs.h" #include "t4_values.h" #include "t4_msg.h" #include "t4_tcb.h" #include "t4fw_ri_api.h" #define T4_MAX_NUM_PD 65536 #define T4_MAX_MR_SIZE (~0ULL) #define T4_PAGESIZE_MASK 0xffff000 /* 4KB-128MB */ #define T4_STAG_UNSET 0xffffffff #define T4_FW_MAJ 0 #define PCIE_MA_SYNC_A 0x30b4 struct t4_status_page { __be32 rsvd1; /* flit 0 - hw owns */ __be16 rsvd2; __be16 qid; __be16 cidx; __be16 pidx; u8 qp_err; /* flit 1 - sw owns */ u8 db_off; u8 pad[2]; u16 host_wq_pidx; u16 host_cidx; u16 host_pidx; u16 pad2; u32 srqidx; }; #define T4_RQT_ENTRY_SHIFT 6 #define T4_RQT_ENTRY_SIZE BIT(T4_RQT_ENTRY_SHIFT) #define T4_EQ_ENTRY_SIZE 64 #define T4_SQ_NUM_SLOTS 5 #define T4_SQ_NUM_BYTES (T4_EQ_ENTRY_SIZE * T4_SQ_NUM_SLOTS) #define T4_MAX_SEND_SGE ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_send_wr) - \ sizeof(struct fw_ri_isgl)) / sizeof(struct fw_ri_sge)) #define T4_MAX_SEND_INLINE ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_send_wr) - \ sizeof(struct fw_ri_immd))) #define T4_MAX_WRITE_INLINE ((T4_SQ_NUM_BYTES - \ sizeof(struct fw_ri_rdma_write_wr) - \ sizeof(struct fw_ri_immd))) #define T4_MAX_WRITE_SGE ((T4_SQ_NUM_BYTES - \ sizeof(struct fw_ri_rdma_write_wr) - \ sizeof(struct fw_ri_isgl)) / sizeof(struct fw_ri_sge)) #define T4_MAX_FR_IMMD ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_fr_nsmr_wr) - \ sizeof(struct fw_ri_immd)) & ~31UL) #define T4_MAX_FR_IMMD_DEPTH (T4_MAX_FR_IMMD / sizeof(u64)) #define T4_MAX_FR_DSGL 1024 #define T4_MAX_FR_DSGL_DEPTH (T4_MAX_FR_DSGL / sizeof(u64)) static inline int t4_max_fr_depth(int use_dsgl) { return use_dsgl ? T4_MAX_FR_DSGL_DEPTH : T4_MAX_FR_IMMD_DEPTH; } #define T4_RQ_NUM_SLOTS 2 #define T4_RQ_NUM_BYTES (T4_EQ_ENTRY_SIZE * T4_RQ_NUM_SLOTS) #define T4_MAX_RECV_SGE 4 #define T4_WRITE_CMPL_MAX_SGL 4 #define T4_WRITE_CMPL_MAX_CQE 16 union t4_wr { struct fw_ri_res_wr res; struct fw_ri_wr ri; struct fw_ri_rdma_write_wr write; struct fw_ri_send_wr send; struct fw_ri_rdma_read_wr read; struct fw_ri_bind_mw_wr bind; struct fw_ri_fr_nsmr_wr fr; struct fw_ri_fr_nsmr_tpte_wr fr_tpte; struct fw_ri_inv_lstag_wr inv; struct fw_ri_rdma_write_cmpl_wr write_cmpl; struct t4_status_page status; __be64 flits[T4_EQ_ENTRY_SIZE / sizeof(__be64) * T4_SQ_NUM_SLOTS]; }; union t4_recv_wr { struct fw_ri_recv_wr recv; struct t4_status_page status; __be64 flits[T4_EQ_ENTRY_SIZE / sizeof(__be64) * T4_RQ_NUM_SLOTS]; }; static inline void init_wr_hdr(union t4_wr *wqe, u16 wrid, enum fw_wr_opcodes opcode, u8 flags, u8 len16) { wqe->send.opcode = (u8)opcode; wqe->send.flags = flags; wqe->send.wrid = wrid; wqe->send.r1[0] = 0; wqe->send.r1[1] = 0; wqe->send.r1[2] = 0; wqe->send.len16 = len16; } /* CQE/AE status codes */ #define T4_ERR_SUCCESS 0x0 #define T4_ERR_STAG 0x1 /* STAG invalid: either the */ /* STAG is offlimt, being 0, */ /* or STAG_key mismatch */ #define T4_ERR_PDID 0x2 /* PDID mismatch */ #define T4_ERR_QPID 0x3 /* QPID mismatch */ #define T4_ERR_ACCESS 0x4 /* Invalid access right */ #define T4_ERR_WRAP 0x5 /* Wrap error */ #define T4_ERR_BOUND 0x6 /* base and bounds voilation */ #define T4_ERR_INVALIDATE_SHARED_MR 0x7 /* attempt to invalidate a */ /* shared memory region */ #define T4_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8 /* attempt to invalidate a */ /* shared memory region */ #define T4_ERR_ECC 0x9 /* ECC error detected */ #define T4_ERR_ECC_PSTAG 0xA /* ECC error detected when */ /* reading PSTAG for a MW */ /* Invalidate */ #define T4_ERR_PBL_ADDR_BOUND 0xB /* pbl addr out of bounds: */ /* software error */ #define T4_ERR_SWFLUSH 0xC /* SW FLUSHED */ #define T4_ERR_CRC 0x10 /* CRC error */ #define T4_ERR_MARKER 0x11 /* Marker error */ #define T4_ERR_PDU_LEN_ERR 0x12 /* invalid PDU length */ #define T4_ERR_OUT_OF_RQE 0x13 /* out of RQE */ #define T4_ERR_DDP_VERSION 0x14 /* wrong DDP version */ #define T4_ERR_RDMA_VERSION 0x15 /* wrong RDMA version */ #define T4_ERR_OPCODE 0x16 /* invalid rdma opcode */ #define T4_ERR_DDP_QUEUE_NUM 0x17 /* invalid ddp queue number */ #define T4_ERR_MSN 0x18 /* MSN error */ #define T4_ERR_TBIT 0x19 /* tag bit not set correctly */ #define T4_ERR_MO 0x1A /* MO not 0 for TERMINATE */ /* or READ_REQ */ #define T4_ERR_MSN_GAP 0x1B #define T4_ERR_MSN_RANGE 0x1C #define T4_ERR_IRD_OVERFLOW 0x1D #define T4_ERR_RQE_ADDR_BOUND 0x1E /* RQE addr out of bounds: */ /* software error */ #define T4_ERR_INTERNAL_ERR 0x1F /* internal error (opcode */ /* mismatch) */ /* * CQE defs */ struct t4_cqe { __be32 header; __be32 len; union { struct { __be32 stag; __be32 msn; } rcqe; struct { __be32 stag; u16 nada2; u16 cidx; } scqe; struct { __be32 wrid_hi; __be32 wrid_low; } gen; struct { __be32 stag; __be32 msn; __be32 reserved; __be32 abs_rqe_idx; } srcqe; struct { __be32 mo; __be32 msn; /* * Use union for immediate data to be consistent with * stack's 32 bit data and iWARP spec's 64 bit data. */ union { struct { __be32 imm_data32; u32 reserved; } ib_imm_data; __be64 imm_data64; } iw_imm_data; } imm_data_rcqe; u64 drain_cookie; __be64 flits[3]; } u; __be64 reserved[3]; __be64 bits_type_ts; }; /* macros for flit 0 of the cqe */ #define CQE_QPID_S 12 #define CQE_QPID_M 0xFFFFF #define CQE_QPID_G(x) ((((x) >> CQE_QPID_S)) & CQE_QPID_M) #define CQE_QPID_V(x) ((x)<<CQE_QPID_S) #define CQE_SWCQE_S 11 #define CQE_SWCQE_M 0x1 #define CQE_SWCQE_G(x) ((((x) >> CQE_SWCQE_S)) & CQE_SWCQE_M) #define CQE_SWCQE_V(x) ((x)<<CQE_SWCQE_S) #define CQE_DRAIN_S 10 #define CQE_DRAIN_M 0x1 #define CQE_DRAIN_G(x) ((((x) >> CQE_DRAIN_S)) & CQE_DRAIN_M) #define CQE_DRAIN_V(x) ((x)<<CQE_DRAIN_S) #define CQE_STATUS_S 5 #define CQE_STATUS_M 0x1F #define CQE_STATUS_G(x) ((((x) >> CQE_STATUS_S)) & CQE_STATUS_M) #define CQE_STATUS_V(x) ((x)<<CQE_STATUS_S) #define CQE_TYPE_S 4 #define CQE_TYPE_M 0x1 #define CQE_TYPE_G(x) ((((x) >> CQE_TYPE_S)) & CQE_TYPE_M) #define CQE_TYPE_V(x) ((x)<<CQE_TYPE_S) #define CQE_OPCODE_S 0 #define CQE_OPCODE_M 0xF #define CQE_OPCODE_G(x) ((((x) >> CQE_OPCODE_S)) & CQE_OPCODE_M) #define CQE_OPCODE_V(x) ((x)<<CQE_OPCODE_S) #define SW_CQE(x) (CQE_SWCQE_G(be32_to_cpu((x)->header))) #define DRAIN_CQE(x) (CQE_DRAIN_G(be32_to_cpu((x)->header))) #define CQE_QPID(x) (CQE_QPID_G(be32_to_cpu((x)->header))) #define CQE_TYPE(x) (CQE_TYPE_G(be32_to_cpu((x)->header))) #define SQ_TYPE(x) (CQE_TYPE((x))) #define RQ_TYPE(x) (!CQE_TYPE((x))) #define CQE_STATUS(x) (CQE_STATUS_G(be32_to_cpu((x)->header))) #define CQE_OPCODE(x) (CQE_OPCODE_G(be32_to_cpu((x)->header))) #define CQE_SEND_OPCODE(x)( \ (CQE_OPCODE_G(be32_to_cpu((x)->header)) == FW_RI_SEND) || \ (CQE_OPCODE_G(be32_to_cpu((x)->header)) == FW_RI_SEND_WITH_SE) || \ (CQE_OPCODE_G(be32_to_cpu((x)->header)) == FW_RI_SEND_WITH_INV) || \ (CQE_OPCODE_G(be32_to_cpu((x)->header)) == FW_RI_SEND_WITH_SE_INV)) #define CQE_LEN(x) (be32_to_cpu((x)->len)) /* used for RQ completion processing */ #define CQE_WRID_STAG(x) (be32_to_cpu((x)->u.rcqe.stag)) #define CQE_WRID_MSN(x) (be32_to_cpu((x)->u.rcqe.msn)) #define CQE_ABS_RQE_IDX(x) (be32_to_cpu((x)->u.srcqe.abs_rqe_idx)) #define CQE_IMM_DATA(x)( \ (x)->u.imm_data_rcqe.iw_imm_data.ib_imm_data.imm_data32) /* used for SQ completion processing */ #define CQE_WRID_SQ_IDX(x) ((x)->u.scqe.cidx) #define CQE_WRID_FR_STAG(x) (be32_to_cpu((x)->u.scqe.stag)) /* generic accessor macros */ #define CQE_WRID_HI(x) (be32_to_cpu((x)->u.gen.wrid_hi)) #define CQE_WRID_LOW(x) (be32_to_cpu((x)->u.gen.wrid_low)) #define CQE_DRAIN_COOKIE(x) ((x)->u.drain_cookie) /* macros for flit 3 of the cqe */ #define CQE_GENBIT_S 63 #define CQE_GENBIT_M 0x1 #define CQE_GENBIT_G(x) (((x) >> CQE_GENBIT_S) & CQE_GENBIT_M) #define CQE_GENBIT_V(x) ((x)<<CQE_GENBIT_S) #define CQE_OVFBIT_S 62 #define CQE_OVFBIT_M 0x1 #define CQE_OVFBIT_G(x) ((((x) >> CQE_OVFBIT_S)) & CQE_OVFBIT_M) #define CQE_IQTYPE_S 60 #define CQE_IQTYPE_M 0x3 #define CQE_IQTYPE_G(x) ((((x) >> CQE_IQTYPE_S)) & CQE_IQTYPE_M) #define CQE_TS_M 0x0fffffffffffffffULL #define CQE_TS_G(x) ((x) & CQE_TS_M) #define CQE_OVFBIT(x) ((unsigned)CQE_OVFBIT_G(be64_to_cpu((x)->bits_type_ts))) #define CQE_GENBIT(x) ((unsigned)CQE_GENBIT_G(be64_to_cpu((x)->bits_type_ts))) #define CQE_TS(x) (CQE_TS_G(be64_to_cpu((x)->bits_type_ts))) struct t4_swsqe { u64 wr_id; struct t4_cqe cqe; int read_len; int opcode; int complete; int signaled; u16 idx; int flushed; ktime_t host_time; u64 sge_ts; }; static inline pgprot_t t4_pgprot_wc(pgprot_t prot) { #if defined(__i386__) || defined(__x86_64__) || defined(CONFIG_PPC64) return pgprot_writecombine(prot); #else return pgprot_noncached(prot); #endif } enum { T4_SQ_ONCHIP = (1<<0), }; struct t4_sq { union t4_wr *queue; dma_addr_t dma_addr; DEFINE_DMA_UNMAP_ADDR(mapping); unsigned long phys_addr; struct t4_swsqe *sw_sq; struct t4_swsqe *oldest_read; void __iomem *bar2_va; u64 bar2_pa; size_t memsize; u32 bar2_qid; u32 qid; u16 in_use; u16 size; u16 cidx; u16 pidx; u16 wq_pidx; u16 wq_pidx_inc; u16 flags; short flush_cidx; }; struct t4_swrqe { u64 wr_id; ktime_t host_time; u64 sge_ts; int valid; }; struct t4_rq { union t4_recv_wr *queue; dma_addr_t dma_addr; DEFINE_DMA_UNMAP_ADDR(mapping); struct t4_swrqe *sw_rq; void __iomem *bar2_va; u64 bar2_pa; size_t memsize; u32 bar2_qid; u32 qid; u32 msn; u32 rqt_hwaddr; u16 rqt_size; u16 in_use; u16 size; u16 cidx; u16 pidx; u16 wq_pidx; u16 wq_pidx_inc; }; struct t4_wq { struct t4_sq sq; struct t4_rq rq; void __iomem *db; struct c4iw_rdev *rdev; int flushed; u8 *qp_errp; u32 *srqidxp; }; struct t4_srq_pending_wr { u64 wr_id; union t4_recv_wr wqe; u8 len16; }; struct t4_srq { union t4_recv_wr *queue; dma_addr_t dma_addr; DEFINE_DMA_UNMAP_ADDR(mapping); struct t4_swrqe *sw_rq; void __iomem *bar2_va; u64 bar2_pa; size_t memsize; u32 bar2_qid; u32 qid; u32 msn; u32 rqt_hwaddr; u32 rqt_abs_idx; u16 rqt_size; u16 size; u16 cidx; u16 pidx; u16 wq_pidx; u16 wq_pidx_inc; u16 in_use; struct t4_srq_pending_wr *pending_wrs; u16 pending_cidx; u16 pending_pidx; u16 pending_in_use; u16 ooo_count; }; static inline u32 t4_srq_avail(struct t4_srq *srq) { return srq->size - 1 - srq->in_use; } static inline void t4_srq_produce(struct t4_srq *srq, u8 len16) { srq->in_use++; if (++srq->pidx == srq->size) srq->pidx = 0; srq->wq_pidx += DIV_ROUND_UP(len16 * 16, T4_EQ_ENTRY_SIZE); if (srq->wq_pidx >= srq->size * T4_RQ_NUM_SLOTS) srq->wq_pidx %= srq->size * T4_RQ_NUM_SLOTS; srq->queue[srq->size].status.host_pidx = srq->pidx; } static inline void t4_srq_produce_pending_wr(struct t4_srq *srq) { srq->pending_in_use++; srq->in_use++; if (++srq->pending_pidx == srq->size) srq->pending_pidx = 0; } static inline void t4_srq_consume_pending_wr(struct t4_srq *srq) { srq->pending_in_use--; srq->in_use--; if (++srq->pending_cidx == srq->size) srq->pending_cidx = 0; } static inline void t4_srq_produce_ooo(struct t4_srq *srq) { srq->in_use--; srq->ooo_count++; } static inline void t4_srq_consume_ooo(struct t4_srq *srq) { srq->cidx++; if (srq->cidx == srq->size) srq->cidx = 0; srq->queue[srq->size].status.host_cidx = srq->cidx; srq->ooo_count--; } static inline void t4_srq_consume(struct t4_srq *srq) { srq->in_use--; if (++srq->cidx == srq->size) srq->cidx = 0; srq->queue[srq->size].status.host_cidx = srq->cidx; } static inline int t4_rqes_posted(struct t4_wq *wq) { return wq->rq.in_use; } static inline int t4_rq_empty(struct t4_wq *wq) { return wq->rq.in_use == 0; } static inline u32 t4_rq_avail(struct t4_wq *wq) { return wq->rq.size - 1 - wq->rq.in_use; } static inline void t4_rq_produce(struct t4_wq *wq, u8 len16) { wq->rq.in_use++; if (++wq->rq.pidx == wq->rq.size) wq->rq.pidx = 0; wq->rq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); if (wq->rq.wq_pidx >= wq->rq.size * T4_RQ_NUM_SLOTS) wq->rq.wq_pidx %= wq->rq.size * T4_RQ_NUM_SLOTS; } static inline void t4_rq_consume(struct t4_wq *wq) { wq->rq.in_use--; if (++wq->rq.cidx == wq->rq.size) wq->rq.cidx = 0; } static inline u16 t4_rq_host_wq_pidx(struct t4_wq *wq) { return wq->rq.queue[wq->rq.size].status.host_wq_pidx; } static inline u16 t4_rq_wq_size(struct t4_wq *wq) { return wq->rq.size * T4_RQ_NUM_SLOTS; } static inline int t4_sq_onchip(struct t4_sq *sq) { return sq->flags & T4_SQ_ONCHIP; } static inline int t4_sq_empty(struct t4_wq *wq) { return wq->sq.in_use == 0; } static inline u32 t4_sq_avail(struct t4_wq *wq) { return wq->sq.size - 1 - wq->sq.in_use; } static inline void t4_sq_produce(struct t4_wq *wq, u8 len16) { wq->sq.in_use++; if (++wq->sq.pidx == wq->sq.size) wq->sq.pidx = 0; wq->sq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); if (wq->sq.wq_pidx >= wq->sq.size * T4_SQ_NUM_SLOTS) wq->sq.wq_pidx %= wq->sq.size * T4_SQ_NUM_SLOTS; } static inline void t4_sq_consume(struct t4_wq *wq) { if (wq->sq.cidx == wq->sq.flush_cidx) wq->sq.flush_cidx = -1; wq->sq.in_use--; if (++wq->sq.cidx == wq->sq.size) wq->sq.cidx = 0; } static inline u16 t4_sq_host_wq_pidx(struct t4_wq *wq) { return wq->sq.queue[wq->sq.size].status.host_wq_pidx; } static inline u16 t4_sq_wq_size(struct t4_wq *wq) { return wq->sq.size * T4_SQ_NUM_SLOTS; } /* This function copies 64 byte coalesced work request to memory * mapped BAR2 space. For coalesced WRs, the SGE fetches data * from the FIFO instead of from Host. */ static inline void pio_copy(u64 __iomem *dst, u64 *src) { int count = 8; while (count) { writeq(*src, dst); src++; dst++; count--; } } static inline void t4_ring_srq_db(struct t4_srq *srq, u16 inc, u8 len16, union t4_recv_wr *wqe) { /* Flush host queue memory writes. */ wmb(); if (inc == 1 && srq->bar2_qid == 0 && wqe) { pr_debug("%s : WC srq->pidx = %d; len16=%d\n", __func__, srq->pidx, len16); pio_copy(srq->bar2_va + SGE_UDB_WCDOORBELL, (u64 *)wqe); } else { pr_debug("%s: DB srq->pidx = %d; len16=%d\n", __func__, srq->pidx, len16); writel(PIDX_T5_V(inc) | QID_V(srq->bar2_qid), srq->bar2_va + SGE_UDB_KDOORBELL); } /* Flush user doorbell area writes. */ wmb(); } static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc, union t4_wr *wqe) { /* Flush host queue memory writes. */ wmb(); if (wq->sq.bar2_va) { if (inc == 1 && wq->sq.bar2_qid == 0 && wqe) { pr_debug("WC wq->sq.pidx = %d\n", wq->sq.pidx); pio_copy((u64 __iomem *) (wq->sq.bar2_va + SGE_UDB_WCDOORBELL), (u64 *)wqe); } else { pr_debug("DB wq->sq.pidx = %d\n", wq->sq.pidx); writel(PIDX_T5_V(inc) | QID_V(wq->sq.bar2_qid), wq->sq.bar2_va + SGE_UDB_KDOORBELL); } /* Flush user doorbell area writes. */ wmb(); return; } writel(QID_V(wq->sq.qid) | PIDX_V(inc), wq->db); } static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc, union t4_recv_wr *wqe) { /* Flush host queue memory writes. */ wmb(); if (wq->rq.bar2_va) { if (inc == 1 && wq->rq.bar2_qid == 0 && wqe) { pr_debug("WC wq->rq.pidx = %d\n", wq->rq.pidx); pio_copy((u64 __iomem *) (wq->rq.bar2_va + SGE_UDB_WCDOORBELL), (void *)wqe); } else { pr_debug("DB wq->rq.pidx = %d\n", wq->rq.pidx); writel(PIDX_T5_V(inc) | QID_V(wq->rq.bar2_qid), wq->rq.bar2_va + SGE_UDB_KDOORBELL); } /* Flush user doorbell area writes. */ wmb(); return; } writel(QID_V(wq->rq.qid) | PIDX_V(inc), wq->db); } static inline int t4_wq_in_error(struct t4_wq *wq) { return *wq->qp_errp; } static inline void t4_set_wq_in_error(struct t4_wq *wq, u32 srqidx) { if (srqidx) *wq->srqidxp = srqidx; *wq->qp_errp = 1; } static inline void t4_disable_wq_db(struct t4_wq *wq) { wq->rq.queue[wq->rq.size].status.db_off = 1; } static inline void t4_enable_wq_db(struct t4_wq *wq) { wq->rq.queue[wq->rq.size].status.db_off = 0; } enum t4_cq_flags { CQ_ARMED = 1, }; struct t4_cq { struct t4_cqe *queue; dma_addr_t dma_addr; DEFINE_DMA_UNMAP_ADDR(mapping); struct t4_cqe *sw_queue; void __iomem *gts; void __iomem *bar2_va; u64 bar2_pa; u32 bar2_qid; struct c4iw_rdev *rdev; size_t memsize; __be64 bits_type_ts; u32 cqid; u32 qid_mask; int vector; u16 size; /* including status page */ u16 cidx; u16 sw_pidx; u16 sw_cidx; u16 sw_in_use; u16 cidx_inc; u8 gen; u8 error; u8 *qp_errp; unsigned long flags; }; static inline void write_gts(struct t4_cq *cq, u32 val) { if (cq->bar2_va) writel(val | INGRESSQID_V(cq->bar2_qid), cq->bar2_va + SGE_UDB_GTS); else writel(val | INGRESSQID_V(cq->cqid), cq->gts); } static inline int t4_clear_cq_armed(struct t4_cq *cq) { return test_and_clear_bit(CQ_ARMED, &cq->flags); } static inline int t4_arm_cq(struct t4_cq *cq, int se) { u32 val; set_bit(CQ_ARMED, &cq->flags); while (cq->cidx_inc > CIDXINC_M) { val = SEINTARM_V(0) | CIDXINC_V(CIDXINC_M) | TIMERREG_V(7); write_gts(cq, val); cq->cidx_inc -= CIDXINC_M; } val = SEINTARM_V(se) | CIDXINC_V(cq->cidx_inc) | TIMERREG_V(6); write_gts(cq, val); cq->cidx_inc = 0; return 0; } static inline void t4_swcq_produce(struct t4_cq *cq) { cq->sw_in_use++; if (cq->sw_in_use == cq->size) { pr_warn("%s cxgb4 sw cq overflow cqid %u\n", __func__, cq->cqid); cq->error = 1; cq->sw_in_use--; return; } if (++cq->sw_pidx == cq->size) cq->sw_pidx = 0; } static inline void t4_swcq_consume(struct t4_cq *cq) { cq->sw_in_use--; if (++cq->sw_cidx == cq->size) cq->sw_cidx = 0; } static inline void t4_hwcq_consume(struct t4_cq *cq) { cq->bits_type_ts = cq->queue[cq->cidx].bits_type_ts; if (++cq->cidx_inc == (cq->size >> 4) || cq->cidx_inc == CIDXINC_M) { u32 val; val = SEINTARM_V(0) | CIDXINC_V(cq->cidx_inc) | TIMERREG_V(7); write_gts(cq, val); cq->cidx_inc = 0; } if (++cq->cidx == cq->size) { cq->cidx = 0; cq->gen ^= 1; } } static inline int t4_valid_cqe(struct t4_cq *cq, struct t4_cqe *cqe) { return (CQE_GENBIT(cqe) == cq->gen); } static inline int t4_cq_notempty(struct t4_cq *cq) { return cq->sw_in_use || t4_valid_cqe(cq, &cq->queue[cq->cidx]); } static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe) { int ret; u16 prev_cidx; if (cq->cidx == 0) prev_cidx = cq->size - 1; else prev_cidx = cq->cidx - 1; if (cq->queue[prev_cidx].bits_type_ts != cq->bits_type_ts) { ret = -EOVERFLOW; cq->error = 1; pr_err("cq overflow cqid %u\n", cq->cqid); } else if (t4_valid_cqe(cq, &cq->queue[cq->cidx])) { /* Ensure CQE is flushed to memory */ rmb(); *cqe = &cq->queue[cq->cidx]; ret = 0; } else ret = -ENODATA; return ret; } static inline int t4_next_cqe(struct t4_cq *cq, struct t4_cqe **cqe) { int ret = 0; if (cq->error) ret = -ENODATA; else if (cq->sw_in_use) *cqe = &cq->sw_queue[cq->sw_cidx]; else ret = t4_next_hw_cqe(cq, cqe); return ret; } static inline void t4_set_cq_in_error(struct t4_cq *cq) { *cq->qp_errp = 1; } #endif struct t4_dev_status_page { u8 db_off; u8 write_cmpl_supported; u16 pad2; u32 pad3; u64 qp_start; u64 qp_size; u64 cq_start; u64 cq_size; }; |