Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
4 */
5
6#include "mana_ib.h"
7
8void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
9 u32 port)
10{
11 struct gdma_dev *gd = dev->gdma_dev;
12 struct mana_port_context *mpc;
13 struct net_device *ndev;
14 struct mana_context *mc;
15
16 mc = gd->driver_data;
17 ndev = mc->ports[port];
18 mpc = netdev_priv(ndev);
19
20 mutex_lock(&pd->vport_mutex);
21
22 pd->vport_use_count--;
23 WARN_ON(pd->vport_use_count < 0);
24
25 if (!pd->vport_use_count)
26 mana_uncfg_vport(mpc);
27
28 mutex_unlock(&pd->vport_mutex);
29}
30
31int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port, struct mana_ib_pd *pd,
32 u32 doorbell_id)
33{
34 struct gdma_dev *mdev = dev->gdma_dev;
35 struct mana_port_context *mpc;
36 struct mana_context *mc;
37 struct net_device *ndev;
38 int err;
39
40 mc = mdev->driver_data;
41 ndev = mc->ports[port];
42 mpc = netdev_priv(ndev);
43
44 mutex_lock(&pd->vport_mutex);
45
46 pd->vport_use_count++;
47 if (pd->vport_use_count > 1) {
48 ibdev_dbg(&dev->ib_dev,
49 "Skip as this PD is already configured vport\n");
50 mutex_unlock(&pd->vport_mutex);
51 return 0;
52 }
53
54 err = mana_cfg_vport(mpc, pd->pdn, doorbell_id);
55 if (err) {
56 pd->vport_use_count--;
57 mutex_unlock(&pd->vport_mutex);
58
59 ibdev_dbg(&dev->ib_dev, "Failed to configure vPort %d\n", err);
60 return err;
61 }
62
63 mutex_unlock(&pd->vport_mutex);
64
65 pd->tx_shortform_allowed = mpc->tx_shortform_allowed;
66 pd->tx_vp_offset = mpc->tx_vp_offset;
67
68 ibdev_dbg(&dev->ib_dev, "vport handle %llx pdid %x doorbell_id %x\n",
69 mpc->port_handle, pd->pdn, doorbell_id);
70
71 return 0;
72}
73
74int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
75{
76 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
77 struct ib_device *ibdev = ibpd->device;
78 struct gdma_create_pd_resp resp = {};
79 struct gdma_create_pd_req req = {};
80 enum gdma_pd_flags flags = 0;
81 struct mana_ib_dev *dev;
82 struct gdma_dev *mdev;
83 int err;
84
85 dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
86 mdev = dev->gdma_dev;
87
88 mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req),
89 sizeof(resp));
90
91 req.flags = flags;
92 err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
93 sizeof(resp), &resp);
94
95 if (err || resp.hdr.status) {
96 ibdev_dbg(&dev->ib_dev,
97 "Failed to get pd_id err %d status %u\n", err,
98 resp.hdr.status);
99 if (!err)
100 err = -EPROTO;
101
102 return err;
103 }
104
105 pd->pd_handle = resp.pd_handle;
106 pd->pdn = resp.pd_id;
107 ibdev_dbg(&dev->ib_dev, "pd_handle 0x%llx pd_id %d\n",
108 pd->pd_handle, pd->pdn);
109
110 mutex_init(&pd->vport_mutex);
111 pd->vport_use_count = 0;
112 return 0;
113}
114
115int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
116{
117 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
118 struct ib_device *ibdev = ibpd->device;
119 struct gdma_destory_pd_resp resp = {};
120 struct gdma_destroy_pd_req req = {};
121 struct mana_ib_dev *dev;
122 struct gdma_dev *mdev;
123 int err;
124
125 dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
126 mdev = dev->gdma_dev;
127
128 mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req),
129 sizeof(resp));
130
131 req.pd_handle = pd->pd_handle;
132 err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req,
133 sizeof(resp), &resp);
134
135 if (err || resp.hdr.status) {
136 ibdev_dbg(&dev->ib_dev,
137 "Failed to destroy pd_handle 0x%llx err %d status %u",
138 pd->pd_handle, err, resp.hdr.status);
139 if (!err)
140 err = -EPROTO;
141 }
142
143 return err;
144}
145
146static int mana_gd_destroy_doorbell_page(struct gdma_context *gc,
147 int doorbell_page)
148{
149 struct gdma_destroy_resource_range_req req = {};
150 struct gdma_resp_hdr resp = {};
151 int err;
152
153 mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_RESOURCE_RANGE,
154 sizeof(req), sizeof(resp));
155
156 req.resource_type = GDMA_RESOURCE_DOORBELL_PAGE;
157 req.num_resources = 1;
158 req.allocated_resources = doorbell_page;
159
160 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
161 if (err || resp.status) {
162 dev_err(gc->dev,
163 "Failed to destroy doorbell page: ret %d, 0x%x\n",
164 err, resp.status);
165 return err ?: -EPROTO;
166 }
167
168 return 0;
169}
170
171static int mana_gd_allocate_doorbell_page(struct gdma_context *gc,
172 int *doorbell_page)
173{
174 struct gdma_allocate_resource_range_req req = {};
175 struct gdma_allocate_resource_range_resp resp = {};
176 int err;
177
178 mana_gd_init_req_hdr(&req.hdr, GDMA_ALLOCATE_RESOURCE_RANGE,
179 sizeof(req), sizeof(resp));
180
181 req.resource_type = GDMA_RESOURCE_DOORBELL_PAGE;
182 req.num_resources = 1;
183 req.alignment = 1;
184
185 /* Have GDMA start searching from 0 */
186 req.allocated_resources = 0;
187
188 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
189 if (err || resp.hdr.status) {
190 dev_err(gc->dev,
191 "Failed to allocate doorbell page: ret %d, 0x%x\n",
192 err, resp.hdr.status);
193 return err ?: -EPROTO;
194 }
195
196 *doorbell_page = resp.allocated_resources;
197
198 return 0;
199}
200
201int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
202 struct ib_udata *udata)
203{
204 struct mana_ib_ucontext *ucontext =
205 container_of(ibcontext, struct mana_ib_ucontext, ibucontext);
206 struct ib_device *ibdev = ibcontext->device;
207 struct mana_ib_dev *mdev;
208 struct gdma_context *gc;
209 struct gdma_dev *dev;
210 int doorbell_page;
211 int ret;
212
213 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
214 dev = mdev->gdma_dev;
215 gc = dev->gdma_context;
216
217 /* Allocate a doorbell page index */
218 ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page);
219 if (ret) {
220 ibdev_dbg(ibdev, "Failed to allocate doorbell page %d\n", ret);
221 return ret;
222 }
223
224 ibdev_dbg(ibdev, "Doorbell page allocated %d\n", doorbell_page);
225
226 ucontext->doorbell = doorbell_page;
227
228 return 0;
229}
230
231void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
232{
233 struct mana_ib_ucontext *mana_ucontext =
234 container_of(ibcontext, struct mana_ib_ucontext, ibucontext);
235 struct ib_device *ibdev = ibcontext->device;
236 struct mana_ib_dev *mdev;
237 struct gdma_context *gc;
238 int ret;
239
240 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
241 gc = mdev->gdma_dev->gdma_context;
242
243 ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell);
244 if (ret)
245 ibdev_dbg(ibdev, "Failed to destroy doorbell page %d\n", ret);
246}
247
248static int
249mana_ib_gd_first_dma_region(struct mana_ib_dev *dev,
250 struct gdma_context *gc,
251 struct gdma_create_dma_region_req *create_req,
252 size_t num_pages, mana_handle_t *gdma_region)
253{
254 struct gdma_create_dma_region_resp create_resp = {};
255 unsigned int create_req_msg_size;
256 int err;
257
258 create_req_msg_size =
259 struct_size(create_req, page_addr_list, num_pages);
260 create_req->page_addr_list_len = num_pages;
261
262 err = mana_gd_send_request(gc, create_req_msg_size, create_req,
263 sizeof(create_resp), &create_resp);
264 if (err || create_resp.hdr.status) {
265 ibdev_dbg(&dev->ib_dev,
266 "Failed to create DMA region: %d, 0x%x\n",
267 err, create_resp.hdr.status);
268 if (!err)
269 err = -EPROTO;
270
271 return err;
272 }
273
274 *gdma_region = create_resp.dma_region_handle;
275 ibdev_dbg(&dev->ib_dev, "Created DMA region handle 0x%llx\n",
276 *gdma_region);
277
278 return 0;
279}
280
281static int
282mana_ib_gd_add_dma_region(struct mana_ib_dev *dev, struct gdma_context *gc,
283 struct gdma_dma_region_add_pages_req *add_req,
284 unsigned int num_pages, u32 expected_status)
285{
286 unsigned int add_req_msg_size =
287 struct_size(add_req, page_addr_list, num_pages);
288 struct gdma_general_resp add_resp = {};
289 int err;
290
291 mana_gd_init_req_hdr(&add_req->hdr, GDMA_DMA_REGION_ADD_PAGES,
292 add_req_msg_size, sizeof(add_resp));
293 add_req->page_addr_list_len = num_pages;
294
295 err = mana_gd_send_request(gc, add_req_msg_size, add_req,
296 sizeof(add_resp), &add_resp);
297 if (err || add_resp.hdr.status != expected_status) {
298 ibdev_dbg(&dev->ib_dev,
299 "Failed to create DMA region: %d, 0x%x\n",
300 err, add_resp.hdr.status);
301
302 if (!err)
303 err = -EPROTO;
304
305 return err;
306 }
307
308 return 0;
309}
310
311int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
312 mana_handle_t *gdma_region)
313{
314 struct gdma_dma_region_add_pages_req *add_req = NULL;
315 size_t num_pages_processed = 0, num_pages_to_handle;
316 struct gdma_create_dma_region_req *create_req;
317 unsigned int create_req_msg_size;
318 struct hw_channel_context *hwc;
319 struct ib_block_iter biter;
320 size_t max_pgs_add_cmd = 0;
321 size_t max_pgs_create_cmd;
322 struct gdma_context *gc;
323 size_t num_pages_total;
324 struct gdma_dev *mdev;
325 unsigned long page_sz;
326 unsigned int tail = 0;
327 u64 *page_addr_list;
328 void *request_buf;
329 int err;
330
331 mdev = dev->gdma_dev;
332 gc = mdev->gdma_context;
333 hwc = gc->hwc.driver_data;
334
335 /* Hardware requires dma region to align to chosen page size */
336 page_sz = ib_umem_find_best_pgsz(umem, PAGE_SZ_BM, 0);
337 if (!page_sz) {
338 ibdev_dbg(&dev->ib_dev, "failed to find page size.\n");
339 return -ENOMEM;
340 }
341 num_pages_total = ib_umem_num_dma_blocks(umem, page_sz);
342
343 max_pgs_create_cmd =
344 (hwc->max_req_msg_size - sizeof(*create_req)) / sizeof(u64);
345 num_pages_to_handle =
346 min_t(size_t, num_pages_total, max_pgs_create_cmd);
347 create_req_msg_size =
348 struct_size(create_req, page_addr_list, num_pages_to_handle);
349
350 request_buf = kzalloc(hwc->max_req_msg_size, GFP_KERNEL);
351 if (!request_buf)
352 return -ENOMEM;
353
354 create_req = request_buf;
355 mana_gd_init_req_hdr(&create_req->hdr, GDMA_CREATE_DMA_REGION,
356 create_req_msg_size,
357 sizeof(struct gdma_create_dma_region_resp));
358
359 create_req->length = umem->length;
360 create_req->offset_in_page = umem->address & (page_sz - 1);
361 create_req->gdma_page_type = order_base_2(page_sz) - PAGE_SHIFT;
362 create_req->page_count = num_pages_total;
363
364 ibdev_dbg(&dev->ib_dev, "size_dma_region %lu num_pages_total %lu\n",
365 umem->length, num_pages_total);
366
367 ibdev_dbg(&dev->ib_dev, "page_sz %lu offset_in_page %u\n",
368 page_sz, create_req->offset_in_page);
369
370 ibdev_dbg(&dev->ib_dev, "num_pages_to_handle %lu, gdma_page_type %u",
371 num_pages_to_handle, create_req->gdma_page_type);
372
373 page_addr_list = create_req->page_addr_list;
374 rdma_umem_for_each_dma_block(umem, &biter, page_sz) {
375 page_addr_list[tail++] = rdma_block_iter_dma_address(&biter);
376 if (tail < num_pages_to_handle)
377 continue;
378
379 if (!num_pages_processed) {
380 /* First create message */
381 err = mana_ib_gd_first_dma_region(dev, gc, create_req,
382 tail, gdma_region);
383 if (err)
384 goto out;
385
386 max_pgs_add_cmd = (hwc->max_req_msg_size -
387 sizeof(*add_req)) / sizeof(u64);
388
389 add_req = request_buf;
390 add_req->dma_region_handle = *gdma_region;
391 add_req->reserved3 = 0;
392 page_addr_list = add_req->page_addr_list;
393 } else {
394 /* Subsequent create messages */
395 u32 expected_s = 0;
396
397 if (num_pages_processed + num_pages_to_handle <
398 num_pages_total)
399 expected_s = GDMA_STATUS_MORE_ENTRIES;
400
401 err = mana_ib_gd_add_dma_region(dev, gc, add_req, tail,
402 expected_s);
403 if (err)
404 break;
405 }
406
407 num_pages_processed += tail;
408 tail = 0;
409
410 /* The remaining pages to create */
411 num_pages_to_handle =
412 min_t(size_t,
413 num_pages_total - num_pages_processed,
414 max_pgs_add_cmd);
415 }
416
417 if (err)
418 mana_ib_gd_destroy_dma_region(dev, *gdma_region);
419
420out:
421 kfree(request_buf);
422 return err;
423}
424
425int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64 gdma_region)
426{
427 struct gdma_dev *mdev = dev->gdma_dev;
428 struct gdma_context *gc;
429
430 gc = mdev->gdma_context;
431 ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n", gdma_region);
432
433 return mana_gd_destroy_dma_region(gc, gdma_region);
434}
435
436int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
437{
438 struct mana_ib_ucontext *mana_ucontext =
439 container_of(ibcontext, struct mana_ib_ucontext, ibucontext);
440 struct ib_device *ibdev = ibcontext->device;
441 struct mana_ib_dev *mdev;
442 struct gdma_context *gc;
443 phys_addr_t pfn;
444 pgprot_t prot;
445 int ret;
446
447 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
448 gc = mdev->gdma_dev->gdma_context;
449
450 if (vma->vm_pgoff != 0) {
451 ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n", vma->vm_pgoff);
452 return -EINVAL;
453 }
454
455 /* Map to the page indexed by ucontext->doorbell */
456 pfn = (gc->phys_db_page_base +
457 gc->db_page_size * mana_ucontext->doorbell) >>
458 PAGE_SHIFT;
459 prot = pgprot_writecombine(vma->vm_page_prot);
460
461 ret = rdma_user_mmap_io(ibcontext, vma, pfn, gc->db_page_size, prot,
462 NULL);
463 if (ret)
464 ibdev_dbg(ibdev, "can't rdma_user_mmap_io ret %d\n", ret);
465 else
466 ibdev_dbg(ibdev, "mapped I/O pfn 0x%llx page_size %u, ret %d\n",
467 pfn, gc->db_page_size, ret);
468
469 return ret;
470}
471
472int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
473 struct ib_port_immutable *immutable)
474{
475 /*
476 * This version only support RAW_PACKET
477 * other values need to be filled for other types
478 */
479 immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
480
481 return 0;
482}
483
484int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
485 struct ib_udata *uhw)
486{
487 props->max_qp = MANA_MAX_NUM_QUEUES;
488 props->max_qp_wr = MAX_SEND_BUFFERS_PER_QUEUE;
489
490 /*
491 * max_cqe could be potentially much bigger.
492 * As this version of driver only support RAW QP, set it to the same
493 * value as max_qp_wr
494 */
495 props->max_cqe = MAX_SEND_BUFFERS_PER_QUEUE;
496
497 props->max_mr_size = MANA_IB_MAX_MR_SIZE;
498 props->max_mr = MANA_IB_MAX_MR;
499 props->max_send_sge = MAX_TX_WQE_SGL_ENTRIES;
500 props->max_recv_sge = MAX_RX_WQE_SGL_ENTRIES;
501
502 return 0;
503}
504
505int mana_ib_query_port(struct ib_device *ibdev, u32 port,
506 struct ib_port_attr *props)
507{
508 /* This version doesn't return port properties */
509 return 0;
510}
511
512int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
513 union ib_gid *gid)
514{
515 /* This version doesn't return GID properties */
516 return 0;
517}
518
519void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
520{
521}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
4 */
5
6#include "mana_ib.h"
7
8void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
9 u32 port)
10{
11 struct mana_port_context *mpc;
12 struct net_device *ndev;
13
14 ndev = mana_ib_get_netdev(&dev->ib_dev, port);
15 mpc = netdev_priv(ndev);
16
17 mutex_lock(&pd->vport_mutex);
18
19 pd->vport_use_count--;
20 WARN_ON(pd->vport_use_count < 0);
21
22 if (!pd->vport_use_count)
23 mana_uncfg_vport(mpc);
24
25 mutex_unlock(&pd->vport_mutex);
26}
27
28int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port, struct mana_ib_pd *pd,
29 u32 doorbell_id)
30{
31 struct mana_port_context *mpc;
32 struct net_device *ndev;
33 int err;
34
35 ndev = mana_ib_get_netdev(&dev->ib_dev, port);
36 mpc = netdev_priv(ndev);
37
38 mutex_lock(&pd->vport_mutex);
39
40 pd->vport_use_count++;
41 if (pd->vport_use_count > 1) {
42 ibdev_dbg(&dev->ib_dev,
43 "Skip as this PD is already configured vport\n");
44 mutex_unlock(&pd->vport_mutex);
45 return 0;
46 }
47
48 err = mana_cfg_vport(mpc, pd->pdn, doorbell_id);
49 if (err) {
50 pd->vport_use_count--;
51 mutex_unlock(&pd->vport_mutex);
52
53 ibdev_dbg(&dev->ib_dev, "Failed to configure vPort %d\n", err);
54 return err;
55 }
56
57 mutex_unlock(&pd->vport_mutex);
58
59 pd->tx_shortform_allowed = mpc->tx_shortform_allowed;
60 pd->tx_vp_offset = mpc->tx_vp_offset;
61
62 ibdev_dbg(&dev->ib_dev, "vport handle %llx pdid %x doorbell_id %x\n",
63 mpc->port_handle, pd->pdn, doorbell_id);
64
65 return 0;
66}
67
68int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
69{
70 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
71 struct ib_device *ibdev = ibpd->device;
72 struct gdma_create_pd_resp resp = {};
73 struct gdma_create_pd_req req = {};
74 enum gdma_pd_flags flags = 0;
75 struct mana_ib_dev *dev;
76 struct gdma_context *gc;
77 int err;
78
79 dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
80 gc = mdev_to_gc(dev);
81
82 mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req),
83 sizeof(resp));
84
85 req.flags = flags;
86 err = mana_gd_send_request(gc, sizeof(req), &req,
87 sizeof(resp), &resp);
88
89 if (err || resp.hdr.status) {
90 ibdev_dbg(&dev->ib_dev,
91 "Failed to get pd_id err %d status %u\n", err,
92 resp.hdr.status);
93 if (!err)
94 err = -EPROTO;
95
96 return err;
97 }
98
99 pd->pd_handle = resp.pd_handle;
100 pd->pdn = resp.pd_id;
101 ibdev_dbg(&dev->ib_dev, "pd_handle 0x%llx pd_id %d\n",
102 pd->pd_handle, pd->pdn);
103
104 mutex_init(&pd->vport_mutex);
105 pd->vport_use_count = 0;
106 return 0;
107}
108
109int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
110{
111 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
112 struct ib_device *ibdev = ibpd->device;
113 struct gdma_destory_pd_resp resp = {};
114 struct gdma_destroy_pd_req req = {};
115 struct mana_ib_dev *dev;
116 struct gdma_context *gc;
117 int err;
118
119 dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
120 gc = mdev_to_gc(dev);
121
122 mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req),
123 sizeof(resp));
124
125 req.pd_handle = pd->pd_handle;
126 err = mana_gd_send_request(gc, sizeof(req), &req,
127 sizeof(resp), &resp);
128
129 if (err || resp.hdr.status) {
130 ibdev_dbg(&dev->ib_dev,
131 "Failed to destroy pd_handle 0x%llx err %d status %u",
132 pd->pd_handle, err, resp.hdr.status);
133 if (!err)
134 err = -EPROTO;
135 }
136
137 return err;
138}
139
140static int mana_gd_destroy_doorbell_page(struct gdma_context *gc,
141 int doorbell_page)
142{
143 struct gdma_destroy_resource_range_req req = {};
144 struct gdma_resp_hdr resp = {};
145 int err;
146
147 mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_RESOURCE_RANGE,
148 sizeof(req), sizeof(resp));
149
150 req.resource_type = GDMA_RESOURCE_DOORBELL_PAGE;
151 req.num_resources = 1;
152 req.allocated_resources = doorbell_page;
153
154 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
155 if (err || resp.status) {
156 dev_err(gc->dev,
157 "Failed to destroy doorbell page: ret %d, 0x%x\n",
158 err, resp.status);
159 return err ?: -EPROTO;
160 }
161
162 return 0;
163}
164
165static int mana_gd_allocate_doorbell_page(struct gdma_context *gc,
166 int *doorbell_page)
167{
168 struct gdma_allocate_resource_range_req req = {};
169 struct gdma_allocate_resource_range_resp resp = {};
170 int err;
171
172 mana_gd_init_req_hdr(&req.hdr, GDMA_ALLOCATE_RESOURCE_RANGE,
173 sizeof(req), sizeof(resp));
174
175 req.resource_type = GDMA_RESOURCE_DOORBELL_PAGE;
176 req.num_resources = 1;
177 req.alignment = PAGE_SIZE / MANA_PAGE_SIZE;
178
179 /* Have GDMA start searching from 0 */
180 req.allocated_resources = 0;
181
182 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
183 if (err || resp.hdr.status) {
184 dev_err(gc->dev,
185 "Failed to allocate doorbell page: ret %d, 0x%x\n",
186 err, resp.hdr.status);
187 return err ?: -EPROTO;
188 }
189
190 *doorbell_page = resp.allocated_resources;
191
192 return 0;
193}
194
195int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
196 struct ib_udata *udata)
197{
198 struct mana_ib_ucontext *ucontext =
199 container_of(ibcontext, struct mana_ib_ucontext, ibucontext);
200 struct ib_device *ibdev = ibcontext->device;
201 struct mana_ib_dev *mdev;
202 struct gdma_context *gc;
203 int doorbell_page;
204 int ret;
205
206 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
207 gc = mdev_to_gc(mdev);
208
209 /* Allocate a doorbell page index */
210 ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page);
211 if (ret) {
212 ibdev_dbg(ibdev, "Failed to allocate doorbell page %d\n", ret);
213 return ret;
214 }
215
216 ibdev_dbg(ibdev, "Doorbell page allocated %d\n", doorbell_page);
217
218 ucontext->doorbell = doorbell_page;
219
220 return 0;
221}
222
223void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
224{
225 struct mana_ib_ucontext *mana_ucontext =
226 container_of(ibcontext, struct mana_ib_ucontext, ibucontext);
227 struct ib_device *ibdev = ibcontext->device;
228 struct mana_ib_dev *mdev;
229 struct gdma_context *gc;
230 int ret;
231
232 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
233 gc = mdev_to_gc(mdev);
234
235 ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell);
236 if (ret)
237 ibdev_dbg(ibdev, "Failed to destroy doorbell page %d\n", ret);
238}
239
240int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
241 struct mana_ib_queue *queue)
242{
243 struct ib_umem *umem;
244 int err;
245
246 queue->umem = NULL;
247 queue->id = INVALID_QUEUE_ID;
248 queue->gdma_region = GDMA_INVALID_DMA_REGION;
249
250 umem = ib_umem_get(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE);
251 if (IS_ERR(umem)) {
252 err = PTR_ERR(umem);
253 ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %d\n", err);
254 return err;
255 }
256
257 err = mana_ib_create_zero_offset_dma_region(mdev, umem, &queue->gdma_region);
258 if (err) {
259 ibdev_dbg(&mdev->ib_dev, "Failed to create dma region, %d\n", err);
260 goto free_umem;
261 }
262 queue->umem = umem;
263
264 ibdev_dbg(&mdev->ib_dev, "created dma region 0x%llx\n", queue->gdma_region);
265
266 return 0;
267free_umem:
268 ib_umem_release(umem);
269 return err;
270}
271
272void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue)
273{
274 /* Ignore return code as there is not much we can do about it.
275 * The error message is printed inside.
276 */
277 mana_ib_gd_destroy_dma_region(mdev, queue->gdma_region);
278 ib_umem_release(queue->umem);
279}
280
281static int
282mana_ib_gd_first_dma_region(struct mana_ib_dev *dev,
283 struct gdma_context *gc,
284 struct gdma_create_dma_region_req *create_req,
285 size_t num_pages, mana_handle_t *gdma_region,
286 u32 expected_status)
287{
288 struct gdma_create_dma_region_resp create_resp = {};
289 unsigned int create_req_msg_size;
290 int err;
291
292 create_req_msg_size =
293 struct_size(create_req, page_addr_list, num_pages);
294 create_req->page_addr_list_len = num_pages;
295
296 err = mana_gd_send_request(gc, create_req_msg_size, create_req,
297 sizeof(create_resp), &create_resp);
298 if (err || create_resp.hdr.status != expected_status) {
299 ibdev_dbg(&dev->ib_dev,
300 "Failed to create DMA region: %d, 0x%x\n",
301 err, create_resp.hdr.status);
302 if (!err)
303 err = -EPROTO;
304
305 return err;
306 }
307
308 *gdma_region = create_resp.dma_region_handle;
309 ibdev_dbg(&dev->ib_dev, "Created DMA region handle 0x%llx\n",
310 *gdma_region);
311
312 return 0;
313}
314
315static int
316mana_ib_gd_add_dma_region(struct mana_ib_dev *dev, struct gdma_context *gc,
317 struct gdma_dma_region_add_pages_req *add_req,
318 unsigned int num_pages, u32 expected_status)
319{
320 unsigned int add_req_msg_size =
321 struct_size(add_req, page_addr_list, num_pages);
322 struct gdma_general_resp add_resp = {};
323 int err;
324
325 mana_gd_init_req_hdr(&add_req->hdr, GDMA_DMA_REGION_ADD_PAGES,
326 add_req_msg_size, sizeof(add_resp));
327 add_req->page_addr_list_len = num_pages;
328
329 err = mana_gd_send_request(gc, add_req_msg_size, add_req,
330 sizeof(add_resp), &add_resp);
331 if (err || add_resp.hdr.status != expected_status) {
332 ibdev_dbg(&dev->ib_dev,
333 "Failed to create DMA region: %d, 0x%x\n",
334 err, add_resp.hdr.status);
335
336 if (!err)
337 err = -EPROTO;
338
339 return err;
340 }
341
342 return 0;
343}
344
345static int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
346 mana_handle_t *gdma_region, unsigned long page_sz)
347{
348 struct gdma_dma_region_add_pages_req *add_req = NULL;
349 size_t num_pages_processed = 0, num_pages_to_handle;
350 struct gdma_create_dma_region_req *create_req;
351 unsigned int create_req_msg_size;
352 struct hw_channel_context *hwc;
353 struct ib_block_iter biter;
354 size_t max_pgs_add_cmd = 0;
355 size_t max_pgs_create_cmd;
356 struct gdma_context *gc;
357 size_t num_pages_total;
358 unsigned int tail = 0;
359 u64 *page_addr_list;
360 void *request_buf;
361 int err;
362
363 gc = mdev_to_gc(dev);
364 hwc = gc->hwc.driver_data;
365
366 num_pages_total = ib_umem_num_dma_blocks(umem, page_sz);
367
368 max_pgs_create_cmd =
369 (hwc->max_req_msg_size - sizeof(*create_req)) / sizeof(u64);
370 num_pages_to_handle =
371 min_t(size_t, num_pages_total, max_pgs_create_cmd);
372 create_req_msg_size =
373 struct_size(create_req, page_addr_list, num_pages_to_handle);
374
375 request_buf = kzalloc(hwc->max_req_msg_size, GFP_KERNEL);
376 if (!request_buf)
377 return -ENOMEM;
378
379 create_req = request_buf;
380 mana_gd_init_req_hdr(&create_req->hdr, GDMA_CREATE_DMA_REGION,
381 create_req_msg_size,
382 sizeof(struct gdma_create_dma_region_resp));
383
384 create_req->length = umem->length;
385 create_req->offset_in_page = ib_umem_dma_offset(umem, page_sz);
386 create_req->gdma_page_type = order_base_2(page_sz) - MANA_PAGE_SHIFT;
387 create_req->page_count = num_pages_total;
388
389 ibdev_dbg(&dev->ib_dev, "size_dma_region %lu num_pages_total %lu\n",
390 umem->length, num_pages_total);
391
392 ibdev_dbg(&dev->ib_dev, "page_sz %lu offset_in_page %u\n",
393 page_sz, create_req->offset_in_page);
394
395 ibdev_dbg(&dev->ib_dev, "num_pages_to_handle %lu, gdma_page_type %u",
396 num_pages_to_handle, create_req->gdma_page_type);
397
398 page_addr_list = create_req->page_addr_list;
399 rdma_umem_for_each_dma_block(umem, &biter, page_sz) {
400 u32 expected_status = 0;
401
402 page_addr_list[tail++] = rdma_block_iter_dma_address(&biter);
403 if (tail < num_pages_to_handle)
404 continue;
405
406 if (num_pages_processed + num_pages_to_handle <
407 num_pages_total)
408 expected_status = GDMA_STATUS_MORE_ENTRIES;
409
410 if (!num_pages_processed) {
411 /* First create message */
412 err = mana_ib_gd_first_dma_region(dev, gc, create_req,
413 tail, gdma_region,
414 expected_status);
415 if (err)
416 goto out;
417
418 max_pgs_add_cmd = (hwc->max_req_msg_size -
419 sizeof(*add_req)) / sizeof(u64);
420
421 add_req = request_buf;
422 add_req->dma_region_handle = *gdma_region;
423 add_req->reserved3 = 0;
424 page_addr_list = add_req->page_addr_list;
425 } else {
426 /* Subsequent create messages */
427 err = mana_ib_gd_add_dma_region(dev, gc, add_req, tail,
428 expected_status);
429 if (err)
430 break;
431 }
432
433 num_pages_processed += tail;
434 tail = 0;
435
436 /* The remaining pages to create */
437 num_pages_to_handle =
438 min_t(size_t,
439 num_pages_total - num_pages_processed,
440 max_pgs_add_cmd);
441 }
442
443 if (err)
444 mana_ib_gd_destroy_dma_region(dev, *gdma_region);
445
446out:
447 kfree(request_buf);
448 return err;
449}
450
451int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
452 mana_handle_t *gdma_region, u64 virt)
453{
454 unsigned long page_sz;
455
456 page_sz = ib_umem_find_best_pgsz(umem, PAGE_SZ_BM, virt);
457 if (!page_sz) {
458 ibdev_dbg(&dev->ib_dev, "Failed to find page size.\n");
459 return -EINVAL;
460 }
461
462 return mana_ib_gd_create_dma_region(dev, umem, gdma_region, page_sz);
463}
464
465int mana_ib_create_zero_offset_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem,
466 mana_handle_t *gdma_region)
467{
468 unsigned long page_sz;
469
470 /* Hardware requires dma region to align to chosen page size */
471 page_sz = ib_umem_find_best_pgoff(umem, PAGE_SZ_BM, 0);
472 if (!page_sz) {
473 ibdev_dbg(&dev->ib_dev, "Failed to find page size.\n");
474 return -EINVAL;
475 }
476
477 return mana_ib_gd_create_dma_region(dev, umem, gdma_region, page_sz);
478}
479
480int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64 gdma_region)
481{
482 struct gdma_context *gc = mdev_to_gc(dev);
483
484 ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n", gdma_region);
485
486 return mana_gd_destroy_dma_region(gc, gdma_region);
487}
488
489int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
490{
491 struct mana_ib_ucontext *mana_ucontext =
492 container_of(ibcontext, struct mana_ib_ucontext, ibucontext);
493 struct ib_device *ibdev = ibcontext->device;
494 struct mana_ib_dev *mdev;
495 struct gdma_context *gc;
496 phys_addr_t pfn;
497 pgprot_t prot;
498 int ret;
499
500 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
501 gc = mdev_to_gc(mdev);
502
503 if (vma->vm_pgoff != 0) {
504 ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n", vma->vm_pgoff);
505 return -EINVAL;
506 }
507
508 /* Map to the page indexed by ucontext->doorbell */
509 pfn = (gc->phys_db_page_base +
510 gc->db_page_size * mana_ucontext->doorbell) >>
511 PAGE_SHIFT;
512 prot = pgprot_writecombine(vma->vm_page_prot);
513
514 ret = rdma_user_mmap_io(ibcontext, vma, pfn, PAGE_SIZE, prot,
515 NULL);
516 if (ret)
517 ibdev_dbg(ibdev, "can't rdma_user_mmap_io ret %d\n", ret);
518 else
519 ibdev_dbg(ibdev, "mapped I/O pfn 0x%llx page_size %lu, ret %d\n",
520 pfn, PAGE_SIZE, ret);
521
522 return ret;
523}
524
525int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
526 struct ib_port_immutable *immutable)
527{
528 struct ib_port_attr attr;
529 int err;
530
531 err = ib_query_port(ibdev, port_num, &attr);
532 if (err)
533 return err;
534
535 immutable->pkey_tbl_len = attr.pkey_tbl_len;
536 immutable->gid_tbl_len = attr.gid_tbl_len;
537 immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
538 if (port_num == 1)
539 immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
540
541 return 0;
542}
543
544int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
545 struct ib_udata *uhw)
546{
547 struct mana_ib_dev *dev = container_of(ibdev,
548 struct mana_ib_dev, ib_dev);
549
550 memset(props, 0, sizeof(*props));
551 props->max_mr_size = MANA_IB_MAX_MR_SIZE;
552 props->page_size_cap = PAGE_SZ_BM;
553 props->max_qp = dev->adapter_caps.max_qp_count;
554 props->max_qp_wr = dev->adapter_caps.max_qp_wr;
555 props->device_cap_flags = IB_DEVICE_RC_RNR_NAK_GEN;
556 props->max_send_sge = dev->adapter_caps.max_send_sge_count;
557 props->max_recv_sge = dev->adapter_caps.max_recv_sge_count;
558 props->max_sge_rd = dev->adapter_caps.max_recv_sge_count;
559 props->max_cq = dev->adapter_caps.max_cq_count;
560 props->max_cqe = dev->adapter_caps.max_qp_wr;
561 props->max_mr = dev->adapter_caps.max_mr_count;
562 props->max_pd = dev->adapter_caps.max_pd_count;
563 props->max_qp_rd_atom = dev->adapter_caps.max_inbound_read_limit;
564 props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp;
565 props->max_qp_init_rd_atom = dev->adapter_caps.max_outbound_read_limit;
566 props->atomic_cap = IB_ATOMIC_NONE;
567 props->masked_atomic_cap = IB_ATOMIC_NONE;
568 props->max_ah = INT_MAX;
569 props->max_pkeys = 1;
570 props->local_ca_ack_delay = MANA_CA_ACK_DELAY;
571
572 return 0;
573}
574
575int mana_ib_query_port(struct ib_device *ibdev, u32 port,
576 struct ib_port_attr *props)
577{
578 struct net_device *ndev = mana_ib_get_netdev(ibdev, port);
579
580 if (!ndev)
581 return -EINVAL;
582
583 memset(props, 0, sizeof(*props));
584 props->max_mtu = IB_MTU_4096;
585 props->active_mtu = ib_mtu_int_to_enum(ndev->mtu);
586
587 if (netif_carrier_ok(ndev) && netif_running(ndev)) {
588 props->state = IB_PORT_ACTIVE;
589 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
590 } else {
591 props->state = IB_PORT_DOWN;
592 props->phys_state = IB_PORT_PHYS_STATE_DISABLED;
593 }
594
595 props->active_width = IB_WIDTH_4X;
596 props->active_speed = IB_SPEED_EDR;
597 props->pkey_tbl_len = 1;
598 if (port == 1)
599 props->gid_tbl_len = 16;
600
601 return 0;
602}
603
604enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_num)
605{
606 return IB_LINK_LAYER_ETHERNET;
607}
608
609int mana_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
610{
611 if (index != 0)
612 return -EINVAL;
613 *pkey = IB_DEFAULT_PKEY_FULL;
614 return 0;
615}
616
617int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
618 union ib_gid *gid)
619{
620 /* This version doesn't return GID properties */
621 return 0;
622}
623
624void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
625{
626}
627
628int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev)
629{
630 struct mana_ib_adapter_caps *caps = &dev->adapter_caps;
631 struct mana_ib_query_adapter_caps_resp resp = {};
632 struct mana_ib_query_adapter_caps_req req = {};
633 int err;
634
635 mana_gd_init_req_hdr(&req.hdr, MANA_IB_GET_ADAPTER_CAP, sizeof(req),
636 sizeof(resp));
637 req.hdr.resp.msg_version = GDMA_MESSAGE_V3;
638 req.hdr.dev_id = dev->gdma_dev->dev_id;
639
640 err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req),
641 &req, sizeof(resp), &resp);
642
643 if (err) {
644 ibdev_err(&dev->ib_dev,
645 "Failed to query adapter caps err %d", err);
646 return err;
647 }
648
649 caps->max_sq_id = resp.max_sq_id;
650 caps->max_rq_id = resp.max_rq_id;
651 caps->max_cq_id = resp.max_cq_id;
652 caps->max_qp_count = resp.max_qp_count;
653 caps->max_cq_count = resp.max_cq_count;
654 caps->max_mr_count = resp.max_mr_count;
655 caps->max_pd_count = resp.max_pd_count;
656 caps->max_inbound_read_limit = resp.max_inbound_read_limit;
657 caps->max_outbound_read_limit = resp.max_outbound_read_limit;
658 caps->mw_count = resp.mw_count;
659 caps->max_srq_count = resp.max_srq_count;
660 caps->max_qp_wr = min_t(u32,
661 resp.max_requester_sq_size / GDMA_MAX_SQE_SIZE,
662 resp.max_requester_rq_size / GDMA_MAX_RQE_SIZE);
663 caps->max_inline_data_size = resp.max_inline_data_size;
664 caps->max_send_sge_count = resp.max_send_sge_count;
665 caps->max_recv_sge_count = resp.max_recv_sge_count;
666
667 return 0;
668}
669
670static void
671mana_ib_event_handler(void *ctx, struct gdma_queue *q, struct gdma_event *event)
672{
673 struct mana_ib_dev *mdev = (struct mana_ib_dev *)ctx;
674 struct mana_ib_qp *qp;
675 struct ib_event ev;
676 u32 qpn;
677
678 switch (event->type) {
679 case GDMA_EQE_RNIC_QP_FATAL:
680 qpn = event->details[0];
681 qp = mana_get_qp_ref(mdev, qpn);
682 if (!qp)
683 break;
684 if (qp->ibqp.event_handler) {
685 ev.device = qp->ibqp.device;
686 ev.element.qp = &qp->ibqp;
687 ev.event = IB_EVENT_QP_FATAL;
688 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
689 }
690 mana_put_qp_ref(qp);
691 break;
692 default:
693 break;
694 }
695}
696
697int mana_ib_create_eqs(struct mana_ib_dev *mdev)
698{
699 struct gdma_context *gc = mdev_to_gc(mdev);
700 struct gdma_queue_spec spec = {};
701 int err, i;
702
703 spec.type = GDMA_EQ;
704 spec.monitor_avl_buf = false;
705 spec.queue_size = EQ_SIZE;
706 spec.eq.callback = mana_ib_event_handler;
707 spec.eq.context = mdev;
708 spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE;
709 spec.eq.msix_index = 0;
710
711 err = mana_gd_create_mana_eq(&gc->mana_ib, &spec, &mdev->fatal_err_eq);
712 if (err)
713 return err;
714
715 mdev->eqs = kcalloc(mdev->ib_dev.num_comp_vectors, sizeof(struct gdma_queue *),
716 GFP_KERNEL);
717 if (!mdev->eqs) {
718 err = -ENOMEM;
719 goto destroy_fatal_eq;
720 }
721 spec.eq.callback = NULL;
722 for (i = 0; i < mdev->ib_dev.num_comp_vectors; i++) {
723 spec.eq.msix_index = (i + 1) % gc->num_msix_usable;
724 err = mana_gd_create_mana_eq(mdev->gdma_dev, &spec, &mdev->eqs[i]);
725 if (err)
726 goto destroy_eqs;
727 }
728
729 return 0;
730
731destroy_eqs:
732 while (i-- > 0)
733 mana_gd_destroy_queue(gc, mdev->eqs[i]);
734 kfree(mdev->eqs);
735destroy_fatal_eq:
736 mana_gd_destroy_queue(gc, mdev->fatal_err_eq);
737 return err;
738}
739
740void mana_ib_destroy_eqs(struct mana_ib_dev *mdev)
741{
742 struct gdma_context *gc = mdev_to_gc(mdev);
743 int i;
744
745 mana_gd_destroy_queue(gc, mdev->fatal_err_eq);
746
747 for (i = 0; i < mdev->ib_dev.num_comp_vectors; i++)
748 mana_gd_destroy_queue(gc, mdev->eqs[i]);
749
750 kfree(mdev->eqs);
751}
752
753int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev)
754{
755 struct mana_rnic_create_adapter_resp resp = {};
756 struct mana_rnic_create_adapter_req req = {};
757 struct gdma_context *gc = mdev_to_gc(mdev);
758 int err;
759
760 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_ADAPTER, sizeof(req), sizeof(resp));
761 req.hdr.req.msg_version = GDMA_MESSAGE_V2;
762 req.hdr.dev_id = gc->mana_ib.dev_id;
763 req.notify_eq_id = mdev->fatal_err_eq->id;
764
765 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
766 if (err) {
767 ibdev_err(&mdev->ib_dev, "Failed to create RNIC adapter err %d", err);
768 return err;
769 }
770 mdev->adapter_handle = resp.adapter;
771
772 return 0;
773}
774
775int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev)
776{
777 struct mana_rnic_destroy_adapter_resp resp = {};
778 struct mana_rnic_destroy_adapter_req req = {};
779 struct gdma_context *gc;
780 int err;
781
782 gc = mdev_to_gc(mdev);
783 mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_ADAPTER, sizeof(req), sizeof(resp));
784 req.hdr.dev_id = gc->mana_ib.dev_id;
785 req.adapter = mdev->adapter_handle;
786
787 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
788 if (err) {
789 ibdev_err(&mdev->ib_dev, "Failed to destroy RNIC adapter err %d", err);
790 return err;
791 }
792
793 return 0;
794}
795
796int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context)
797{
798 struct mana_ib_dev *mdev = container_of(attr->device, struct mana_ib_dev, ib_dev);
799 enum rdma_network_type ntype = rdma_gid_attr_network_type(attr);
800 struct mana_rnic_config_addr_resp resp = {};
801 struct gdma_context *gc = mdev_to_gc(mdev);
802 struct mana_rnic_config_addr_req req = {};
803 int err;
804
805 if (ntype != RDMA_NETWORK_IPV4 && ntype != RDMA_NETWORK_IPV6) {
806 ibdev_dbg(&mdev->ib_dev, "Unsupported rdma network type %d", ntype);
807 return -EINVAL;
808 }
809
810 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_IP_ADDR, sizeof(req), sizeof(resp));
811 req.hdr.dev_id = gc->mana_ib.dev_id;
812 req.adapter = mdev->adapter_handle;
813 req.op = ADDR_OP_ADD;
814 req.sgid_type = (ntype == RDMA_NETWORK_IPV6) ? SGID_TYPE_IPV6 : SGID_TYPE_IPV4;
815 copy_in_reverse(req.ip_addr, attr->gid.raw, sizeof(union ib_gid));
816
817 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
818 if (err) {
819 ibdev_err(&mdev->ib_dev, "Failed to config IP addr err %d\n", err);
820 return err;
821 }
822
823 return 0;
824}
825
826int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context)
827{
828 struct mana_ib_dev *mdev = container_of(attr->device, struct mana_ib_dev, ib_dev);
829 enum rdma_network_type ntype = rdma_gid_attr_network_type(attr);
830 struct mana_rnic_config_addr_resp resp = {};
831 struct gdma_context *gc = mdev_to_gc(mdev);
832 struct mana_rnic_config_addr_req req = {};
833 int err;
834
835 if (ntype != RDMA_NETWORK_IPV4 && ntype != RDMA_NETWORK_IPV6) {
836 ibdev_dbg(&mdev->ib_dev, "Unsupported rdma network type %d", ntype);
837 return -EINVAL;
838 }
839
840 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_IP_ADDR, sizeof(req), sizeof(resp));
841 req.hdr.dev_id = gc->mana_ib.dev_id;
842 req.adapter = mdev->adapter_handle;
843 req.op = ADDR_OP_REMOVE;
844 req.sgid_type = (ntype == RDMA_NETWORK_IPV6) ? SGID_TYPE_IPV6 : SGID_TYPE_IPV4;
845 copy_in_reverse(req.ip_addr, attr->gid.raw, sizeof(union ib_gid));
846
847 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
848 if (err) {
849 ibdev_err(&mdev->ib_dev, "Failed to config IP addr err %d\n", err);
850 return err;
851 }
852
853 return 0;
854}
855
856int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 *mac)
857{
858 struct mana_rnic_config_mac_addr_resp resp = {};
859 struct mana_rnic_config_mac_addr_req req = {};
860 struct gdma_context *gc = mdev_to_gc(mdev);
861 int err;
862
863 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_MAC_ADDR, sizeof(req), sizeof(resp));
864 req.hdr.dev_id = gc->mana_ib.dev_id;
865 req.adapter = mdev->adapter_handle;
866 req.op = op;
867 copy_in_reverse(req.mac_addr, mac, ETH_ALEN);
868
869 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
870 if (err) {
871 ibdev_err(&mdev->ib_dev, "Failed to config Mac addr err %d", err);
872 return err;
873 }
874
875 return 0;
876}
877
878int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq, u32 doorbell)
879{
880 struct gdma_context *gc = mdev_to_gc(mdev);
881 struct mana_rnic_create_cq_resp resp = {};
882 struct mana_rnic_create_cq_req req = {};
883 int err;
884
885 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_CQ, sizeof(req), sizeof(resp));
886 req.hdr.dev_id = gc->mana_ib.dev_id;
887 req.adapter = mdev->adapter_handle;
888 req.gdma_region = cq->queue.gdma_region;
889 req.eq_id = mdev->eqs[cq->comp_vector]->id;
890 req.doorbell_page = doorbell;
891
892 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
893
894 if (err) {
895 ibdev_err(&mdev->ib_dev, "Failed to create cq err %d", err);
896 return err;
897 }
898
899 cq->queue.id = resp.cq_id;
900 cq->cq_handle = resp.cq_handle;
901 /* The GDMA region is now owned by the CQ handle */
902 cq->queue.gdma_region = GDMA_INVALID_DMA_REGION;
903
904 return 0;
905}
906
907int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
908{
909 struct gdma_context *gc = mdev_to_gc(mdev);
910 struct mana_rnic_destroy_cq_resp resp = {};
911 struct mana_rnic_destroy_cq_req req = {};
912 int err;
913
914 if (cq->cq_handle == INVALID_MANA_HANDLE)
915 return 0;
916
917 mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_CQ, sizeof(req), sizeof(resp));
918 req.hdr.dev_id = gc->mana_ib.dev_id;
919 req.adapter = mdev->adapter_handle;
920 req.cq_handle = cq->cq_handle;
921
922 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
923
924 if (err) {
925 ibdev_err(&mdev->ib_dev, "Failed to destroy cq err %d", err);
926 return err;
927 }
928
929 return 0;
930}
931
932int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
933 struct ib_qp_init_attr *attr, u32 doorbell, u64 flags)
934{
935 struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
936 struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq);
937 struct mana_ib_pd *pd = container_of(qp->ibqp.pd, struct mana_ib_pd, ibpd);
938 struct gdma_context *gc = mdev_to_gc(mdev);
939 struct mana_rnic_create_qp_resp resp = {};
940 struct mana_rnic_create_qp_req req = {};
941 int err, i;
942
943 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_RC_QP, sizeof(req), sizeof(resp));
944 req.hdr.dev_id = gc->mana_ib.dev_id;
945 req.adapter = mdev->adapter_handle;
946 req.pd_handle = pd->pd_handle;
947 req.send_cq_handle = send_cq->cq_handle;
948 req.recv_cq_handle = recv_cq->cq_handle;
949 for (i = 0; i < MANA_RC_QUEUE_TYPE_MAX; i++)
950 req.dma_region[i] = qp->rc_qp.queues[i].gdma_region;
951 req.doorbell_page = doorbell;
952 req.max_send_wr = attr->cap.max_send_wr;
953 req.max_recv_wr = attr->cap.max_recv_wr;
954 req.max_send_sge = attr->cap.max_send_sge;
955 req.max_recv_sge = attr->cap.max_recv_sge;
956 req.flags = flags;
957
958 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
959 if (err) {
960 ibdev_err(&mdev->ib_dev, "Failed to create rc qp err %d", err);
961 return err;
962 }
963 qp->qp_handle = resp.rc_qp_handle;
964 for (i = 0; i < MANA_RC_QUEUE_TYPE_MAX; i++) {
965 qp->rc_qp.queues[i].id = resp.queue_ids[i];
966 /* The GDMA regions are now owned by the RNIC QP handle */
967 qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
968 }
969 return 0;
970}
971
972int mana_ib_gd_destroy_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
973{
974 struct mana_rnic_destroy_rc_qp_resp resp = {0};
975 struct mana_rnic_destroy_rc_qp_req req = {0};
976 struct gdma_context *gc = mdev_to_gc(mdev);
977 int err;
978
979 mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_RC_QP, sizeof(req), sizeof(resp));
980 req.hdr.dev_id = gc->mana_ib.dev_id;
981 req.adapter = mdev->adapter_handle;
982 req.rc_qp_handle = qp->qp_handle;
983 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
984 if (err) {
985 ibdev_err(&mdev->ib_dev, "Failed to destroy rc qp err %d", err);
986 return err;
987 }
988 return 0;
989}