Loading...
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2015 - 2021 Intel Corporation */
3#include "osdep.h"
4#include "hmc.h"
5#include "defs.h"
6#include "type.h"
7#include "protos.h"
8
9/**
10 * irdma_find_sd_index_limit - finds segment descriptor index limit
11 * @hmc_info: pointer to the HMC configuration information structure
12 * @type: type of HMC resources we're searching
13 * @idx: starting index for the object
14 * @cnt: number of objects we're trying to create
15 * @sd_idx: pointer to return index of the segment descriptor in question
16 * @sd_limit: pointer to return the maximum number of segment descriptors
17 *
18 * This function calculates the segment descriptor index and index limit
19 * for the resource defined by irdma_hmc_rsrc_type.
20 */
21
22static void irdma_find_sd_index_limit(struct irdma_hmc_info *hmc_info, u32 type,
23 u32 idx, u32 cnt, u32 *sd_idx,
24 u32 *sd_limit)
25{
26 u64 fpm_addr, fpm_limit;
27
28 fpm_addr = hmc_info->hmc_obj[(type)].base +
29 hmc_info->hmc_obj[type].size * idx;
30 fpm_limit = fpm_addr + hmc_info->hmc_obj[type].size * cnt;
31 *sd_idx = (u32)(fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE);
32 *sd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_DIRECT_BP_SIZE);
33 *sd_limit += 1;
34}
35
36/**
37 * irdma_find_pd_index_limit - finds page descriptor index limit
38 * @hmc_info: pointer to the HMC configuration information struct
39 * @type: HMC resource type we're examining
40 * @idx: starting index for the object
41 * @cnt: number of objects we're trying to create
42 * @pd_idx: pointer to return page descriptor index
43 * @pd_limit: pointer to return page descriptor index limit
44 *
45 * Calculates the page descriptor index and index limit for the resource
46 * defined by irdma_hmc_rsrc_type.
47 */
48
49static void irdma_find_pd_index_limit(struct irdma_hmc_info *hmc_info, u32 type,
50 u32 idx, u32 cnt, u32 *pd_idx,
51 u32 *pd_limit)
52{
53 u64 fpm_adr, fpm_limit;
54
55 fpm_adr = hmc_info->hmc_obj[type].base +
56 hmc_info->hmc_obj[type].size * idx;
57 fpm_limit = fpm_adr + (hmc_info)->hmc_obj[(type)].size * (cnt);
58 *pd_idx = (u32)(fpm_adr / IRDMA_HMC_PAGED_BP_SIZE);
59 *pd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_PAGED_BP_SIZE);
60 *pd_limit += 1;
61}
62
63/**
64 * irdma_set_sd_entry - setup entry for sd programming
65 * @pa: physical addr
66 * @idx: sd index
67 * @type: paged or direct sd
68 * @entry: sd entry ptr
69 */
70static void irdma_set_sd_entry(u64 pa, u32 idx, enum irdma_sd_entry_type type,
71 struct irdma_update_sd_entry *entry)
72{
73 entry->data = pa |
74 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) |
75 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE,
76 type == IRDMA_SD_TYPE_PAGED ? 0 : 1) |
77 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDVALID, 1);
78
79 entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | BIT(15);
80}
81
82/**
83 * irdma_clr_sd_entry - setup entry for sd clear
84 * @idx: sd index
85 * @type: paged or direct sd
86 * @entry: sd entry ptr
87 */
88static void irdma_clr_sd_entry(u32 idx, enum irdma_sd_entry_type type,
89 struct irdma_update_sd_entry *entry)
90{
91 entry->data = FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) |
92 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE,
93 type == IRDMA_SD_TYPE_PAGED ? 0 : 1);
94
95 entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | BIT(15);
96}
97
98/**
99 * irdma_invalidate_pf_hmc_pd - Invalidates the pd cache in the hardware for PF
100 * @dev: pointer to our device struct
101 * @sd_idx: segment descriptor index
102 * @pd_idx: page descriptor index
103 */
104static inline void irdma_invalidate_pf_hmc_pd(struct irdma_sc_dev *dev, u32 sd_idx,
105 u32 pd_idx)
106{
107 u32 val = FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDIDX, sd_idx) |
108 FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDPARTSEL, 1) |
109 FIELD_PREP(IRDMA_PFHMC_PDINV_PMPDIDX, pd_idx);
110
111 writel(val, dev->hw_regs[IRDMA_PFHMC_PDINV]);
112}
113
114/**
115 * irdma_hmc_sd_one - setup 1 sd entry for cqp
116 * @dev: pointer to the device structure
117 * @hmc_fn_id: hmc's function id
118 * @pa: physical addr
119 * @sd_idx: sd index
120 * @type: paged or direct sd
121 * @setsd: flag to set or clear sd
122 */
123int irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id, u64 pa, u32 sd_idx,
124 enum irdma_sd_entry_type type, bool setsd)
125{
126 struct irdma_update_sds_info sdinfo;
127
128 sdinfo.cnt = 1;
129 sdinfo.hmc_fn_id = hmc_fn_id;
130 if (setsd)
131 irdma_set_sd_entry(pa, sd_idx, type, sdinfo.entry);
132 else
133 irdma_clr_sd_entry(sd_idx, type, sdinfo.entry);
134 return dev->cqp->process_cqp_sds(dev, &sdinfo);
135}
136
137/**
138 * irdma_hmc_sd_grp - setup group of sd entries for cqp
139 * @dev: pointer to the device structure
140 * @hmc_info: pointer to the HMC configuration information struct
141 * @sd_index: sd index
142 * @sd_cnt: number of sd entries
143 * @setsd: flag to set or clear sd
144 */
145static int irdma_hmc_sd_grp(struct irdma_sc_dev *dev,
146 struct irdma_hmc_info *hmc_info, u32 sd_index,
147 u32 sd_cnt, bool setsd)
148{
149 struct irdma_hmc_sd_entry *sd_entry;
150 struct irdma_update_sds_info sdinfo = {};
151 u64 pa;
152 u32 i;
153 int ret_code = 0;
154
155 sdinfo.hmc_fn_id = hmc_info->hmc_fn_id;
156 for (i = sd_index; i < sd_index + sd_cnt; i++) {
157 sd_entry = &hmc_info->sd_table.sd_entry[i];
158 if (!sd_entry || (!sd_entry->valid && setsd) ||
159 (sd_entry->valid && !setsd))
160 continue;
161 if (setsd) {
162 pa = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ?
163 sd_entry->u.pd_table.pd_page_addr.pa :
164 sd_entry->u.bp.addr.pa;
165 irdma_set_sd_entry(pa, i, sd_entry->entry_type,
166 &sdinfo.entry[sdinfo.cnt]);
167 } else {
168 irdma_clr_sd_entry(i, sd_entry->entry_type,
169 &sdinfo.entry[sdinfo.cnt]);
170 }
171 sdinfo.cnt++;
172 if (sdinfo.cnt == IRDMA_MAX_SD_ENTRIES) {
173 ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
174 if (ret_code) {
175 ibdev_dbg(to_ibdev(dev),
176 "HMC: sd_programming failed err=%d\n",
177 ret_code);
178 return ret_code;
179 }
180
181 sdinfo.cnt = 0;
182 }
183 }
184 if (sdinfo.cnt)
185 ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
186
187 return ret_code;
188}
189
190/**
191 * irdma_hmc_finish_add_sd_reg - program sd entries for objects
192 * @dev: pointer to the device structure
193 * @info: create obj info
194 */
195static int irdma_hmc_finish_add_sd_reg(struct irdma_sc_dev *dev,
196 struct irdma_hmc_create_obj_info *info)
197{
198 if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
199 return -EINVAL;
200
201 if ((info->start_idx + info->count) >
202 info->hmc_info->hmc_obj[info->rsrc_type].cnt)
203 return -EINVAL;
204
205 if (!info->add_sd_cnt)
206 return 0;
207 return irdma_hmc_sd_grp(dev, info->hmc_info,
208 info->hmc_info->sd_indexes[0], info->add_sd_cnt,
209 true);
210}
211
212/**
213 * irdma_sc_create_hmc_obj - allocate backing store for hmc objects
214 * @dev: pointer to the device structure
215 * @info: pointer to irdma_hmc_create_obj_info struct
216 *
217 * This will allocate memory for PDs and backing pages and populate
218 * the sd and pd entries.
219 */
220int irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,
221 struct irdma_hmc_create_obj_info *info)
222{
223 struct irdma_hmc_sd_entry *sd_entry;
224 u32 sd_idx, sd_lmt;
225 u32 pd_idx = 0, pd_lmt = 0;
226 u32 pd_idx1 = 0, pd_lmt1 = 0;
227 u32 i, j;
228 bool pd_error = false;
229 int ret_code = 0;
230
231 if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
232 return -EINVAL;
233
234 if ((info->start_idx + info->count) >
235 info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
236 ibdev_dbg(to_ibdev(dev),
237 "HMC: error type %u, start = %u, req cnt %u, cnt = %u\n",
238 info->rsrc_type, info->start_idx, info->count,
239 info->hmc_info->hmc_obj[info->rsrc_type].cnt);
240 return -EINVAL;
241 }
242
243 irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type,
244 info->start_idx, info->count, &sd_idx,
245 &sd_lmt);
246 if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
247 sd_lmt > info->hmc_info->sd_table.sd_cnt) {
248 return -EINVAL;
249 }
250
251 irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type,
252 info->start_idx, info->count, &pd_idx,
253 &pd_lmt);
254
255 for (j = sd_idx; j < sd_lmt; j++) {
256 ret_code = irdma_add_sd_table_entry(dev->hw, info->hmc_info, j,
257 info->entry_type,
258 IRDMA_HMC_DIRECT_BP_SIZE);
259 if (ret_code)
260 goto exit_sd_error;
261
262 sd_entry = &info->hmc_info->sd_table.sd_entry[j];
263 if (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED &&
264 (dev->hmc_info == info->hmc_info &&
265 info->rsrc_type != IRDMA_HMC_IW_PBLE)) {
266 pd_idx1 = max(pd_idx, (j * IRDMA_HMC_MAX_BP_COUNT));
267 pd_lmt1 = min(pd_lmt, (j + 1) * IRDMA_HMC_MAX_BP_COUNT);
268 for (i = pd_idx1; i < pd_lmt1; i++) {
269 /* update the pd table entry */
270 ret_code = irdma_add_pd_table_entry(dev,
271 info->hmc_info,
272 i, NULL);
273 if (ret_code) {
274 pd_error = true;
275 break;
276 }
277 }
278 if (pd_error) {
279 while (i && (i > pd_idx1)) {
280 irdma_remove_pd_bp(dev, info->hmc_info,
281 i - 1);
282 i--;
283 }
284 }
285 }
286 if (sd_entry->valid)
287 continue;
288
289 info->hmc_info->sd_indexes[info->add_sd_cnt] = (u16)j;
290 info->add_sd_cnt++;
291 sd_entry->valid = true;
292 }
293 return irdma_hmc_finish_add_sd_reg(dev, info);
294
295exit_sd_error:
296 while (j && (j > sd_idx)) {
297 sd_entry = &info->hmc_info->sd_table.sd_entry[j - 1];
298 switch (sd_entry->entry_type) {
299 case IRDMA_SD_TYPE_PAGED:
300 pd_idx1 = max(pd_idx, (j - 1) * IRDMA_HMC_MAX_BP_COUNT);
301 pd_lmt1 = min(pd_lmt, (j * IRDMA_HMC_MAX_BP_COUNT));
302 for (i = pd_idx1; i < pd_lmt1; i++)
303 irdma_prep_remove_pd_page(info->hmc_info, i);
304 break;
305 case IRDMA_SD_TYPE_DIRECT:
306 irdma_prep_remove_pd_page(info->hmc_info, (j - 1));
307 break;
308 default:
309 ret_code = -EINVAL;
310 break;
311 }
312 j--;
313 }
314
315 return ret_code;
316}
317
318/**
319 * irdma_finish_del_sd_reg - delete sd entries for objects
320 * @dev: pointer to the device structure
321 * @info: dele obj info
322 * @reset: true if called before reset
323 */
324static int irdma_finish_del_sd_reg(struct irdma_sc_dev *dev,
325 struct irdma_hmc_del_obj_info *info,
326 bool reset)
327{
328 struct irdma_hmc_sd_entry *sd_entry;
329 int ret_code = 0;
330 u32 i, sd_idx;
331 struct irdma_dma_mem *mem;
332
333 if (!reset)
334 ret_code = irdma_hmc_sd_grp(dev, info->hmc_info,
335 info->hmc_info->sd_indexes[0],
336 info->del_sd_cnt, false);
337
338 if (ret_code)
339 ibdev_dbg(to_ibdev(dev), "HMC: error cqp sd sd_grp\n");
340 for (i = 0; i < info->del_sd_cnt; i++) {
341 sd_idx = info->hmc_info->sd_indexes[i];
342 sd_entry = &info->hmc_info->sd_table.sd_entry[sd_idx];
343 mem = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ?
344 &sd_entry->u.pd_table.pd_page_addr :
345 &sd_entry->u.bp.addr;
346
347 if (!mem || !mem->va) {
348 ibdev_dbg(to_ibdev(dev), "HMC: error cqp sd mem\n");
349 } else {
350 dma_free_coherent(dev->hw->device, mem->size, mem->va,
351 mem->pa);
352 mem->va = NULL;
353 }
354 }
355
356 return ret_code;
357}
358
359/**
360 * irdma_sc_del_hmc_obj - remove pe hmc objects
361 * @dev: pointer to the device structure
362 * @info: pointer to irdma_hmc_del_obj_info struct
363 * @reset: true if called before reset
364 *
365 * This will de-populate the SDs and PDs. It frees
366 * the memory for PDS and backing storage. After this function is returned,
367 * caller should deallocate memory allocated previously for
368 * book-keeping information about PDs and backing storage.
369 */
370int irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,
371 struct irdma_hmc_del_obj_info *info, bool reset)
372{
373 struct irdma_hmc_pd_table *pd_table;
374 u32 sd_idx, sd_lmt;
375 u32 pd_idx, pd_lmt, rel_pd_idx;
376 u32 i, j;
377 int ret_code = 0;
378
379 if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
380 ibdev_dbg(to_ibdev(dev),
381 "HMC: error start_idx[%04d] >= [type %04d].cnt[%04d]\n",
382 info->start_idx, info->rsrc_type,
383 info->hmc_info->hmc_obj[info->rsrc_type].cnt);
384 return -EINVAL;
385 }
386
387 if ((info->start_idx + info->count) >
388 info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
389 ibdev_dbg(to_ibdev(dev),
390 "HMC: error start_idx[%04d] + count %04d >= [type %04d].cnt[%04d]\n",
391 info->start_idx, info->count, info->rsrc_type,
392 info->hmc_info->hmc_obj[info->rsrc_type].cnt);
393 return -EINVAL;
394 }
395
396 irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type,
397 info->start_idx, info->count, &pd_idx,
398 &pd_lmt);
399
400 for (j = pd_idx; j < pd_lmt; j++) {
401 sd_idx = j / IRDMA_HMC_PD_CNT_IN_SD;
402
403 if (!info->hmc_info->sd_table.sd_entry[sd_idx].valid)
404 continue;
405
406 if (info->hmc_info->sd_table.sd_entry[sd_idx].entry_type !=
407 IRDMA_SD_TYPE_PAGED)
408 continue;
409
410 rel_pd_idx = j % IRDMA_HMC_PD_CNT_IN_SD;
411 pd_table = &info->hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
412 if (pd_table->pd_entry &&
413 pd_table->pd_entry[rel_pd_idx].valid) {
414 ret_code = irdma_remove_pd_bp(dev, info->hmc_info, j);
415 if (ret_code) {
416 ibdev_dbg(to_ibdev(dev),
417 "HMC: remove_pd_bp error\n");
418 return ret_code;
419 }
420 }
421 }
422
423 irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type,
424 info->start_idx, info->count, &sd_idx,
425 &sd_lmt);
426 if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
427 sd_lmt > info->hmc_info->sd_table.sd_cnt) {
428 ibdev_dbg(to_ibdev(dev), "HMC: invalid sd_idx\n");
429 return -EINVAL;
430 }
431
432 for (i = sd_idx; i < sd_lmt; i++) {
433 pd_table = &info->hmc_info->sd_table.sd_entry[i].u.pd_table;
434 if (!info->hmc_info->sd_table.sd_entry[i].valid)
435 continue;
436 switch (info->hmc_info->sd_table.sd_entry[i].entry_type) {
437 case IRDMA_SD_TYPE_DIRECT:
438 ret_code = irdma_prep_remove_sd_bp(info->hmc_info, i);
439 if (!ret_code) {
440 info->hmc_info->sd_indexes[info->del_sd_cnt] =
441 (u16)i;
442 info->del_sd_cnt++;
443 }
444 break;
445 case IRDMA_SD_TYPE_PAGED:
446 ret_code = irdma_prep_remove_pd_page(info->hmc_info, i);
447 if (ret_code)
448 break;
449 if (dev->hmc_info != info->hmc_info &&
450 info->rsrc_type == IRDMA_HMC_IW_PBLE &&
451 pd_table->pd_entry) {
452 kfree(pd_table->pd_entry_virt_mem.va);
453 pd_table->pd_entry = NULL;
454 }
455 info->hmc_info->sd_indexes[info->del_sd_cnt] = (u16)i;
456 info->del_sd_cnt++;
457 break;
458 default:
459 break;
460 }
461 }
462 return irdma_finish_del_sd_reg(dev, info, reset);
463}
464
465/**
466 * irdma_add_sd_table_entry - Adds a segment descriptor to the table
467 * @hw: pointer to our hw struct
468 * @hmc_info: pointer to the HMC configuration information struct
469 * @sd_index: segment descriptor index to manipulate
470 * @type: what type of segment descriptor we're manipulating
471 * @direct_mode_sz: size to alloc in direct mode
472 */
473int irdma_add_sd_table_entry(struct irdma_hw *hw,
474 struct irdma_hmc_info *hmc_info, u32 sd_index,
475 enum irdma_sd_entry_type type, u64 direct_mode_sz)
476{
477 struct irdma_hmc_sd_entry *sd_entry;
478 struct irdma_dma_mem dma_mem;
479 u64 alloc_len;
480
481 sd_entry = &hmc_info->sd_table.sd_entry[sd_index];
482 if (!sd_entry->valid) {
483 if (type == IRDMA_SD_TYPE_PAGED)
484 alloc_len = IRDMA_HMC_PAGED_BP_SIZE;
485 else
486 alloc_len = direct_mode_sz;
487
488 /* allocate a 4K pd page or 2M backing page */
489 dma_mem.size = ALIGN(alloc_len, IRDMA_HMC_PD_BP_BUF_ALIGNMENT);
490 dma_mem.va = dma_alloc_coherent(hw->device, dma_mem.size,
491 &dma_mem.pa, GFP_KERNEL);
492 if (!dma_mem.va)
493 return -ENOMEM;
494 if (type == IRDMA_SD_TYPE_PAGED) {
495 struct irdma_virt_mem *vmem =
496 &sd_entry->u.pd_table.pd_entry_virt_mem;
497
498 vmem->size = sizeof(struct irdma_hmc_pd_entry) * 512;
499 vmem->va = kzalloc(vmem->size, GFP_KERNEL);
500 if (!vmem->va) {
501 dma_free_coherent(hw->device, dma_mem.size,
502 dma_mem.va, dma_mem.pa);
503 dma_mem.va = NULL;
504 return -ENOMEM;
505 }
506 sd_entry->u.pd_table.pd_entry = vmem->va;
507
508 memcpy(&sd_entry->u.pd_table.pd_page_addr, &dma_mem,
509 sizeof(sd_entry->u.pd_table.pd_page_addr));
510 } else {
511 memcpy(&sd_entry->u.bp.addr, &dma_mem,
512 sizeof(sd_entry->u.bp.addr));
513
514 sd_entry->u.bp.sd_pd_index = sd_index;
515 }
516
517 hmc_info->sd_table.sd_entry[sd_index].entry_type = type;
518 hmc_info->sd_table.use_cnt++;
519 }
520 if (sd_entry->entry_type == IRDMA_SD_TYPE_DIRECT)
521 sd_entry->u.bp.use_cnt++;
522
523 return 0;
524}
525
526/**
527 * irdma_add_pd_table_entry - Adds page descriptor to the specified table
528 * @dev: pointer to our device structure
529 * @hmc_info: pointer to the HMC configuration information structure
530 * @pd_index: which page descriptor index to manipulate
531 * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one.
532 *
533 * This function:
534 * 1. Initializes the pd entry
535 * 2. Adds pd_entry in the pd_table
536 * 3. Mark the entry valid in irdma_hmc_pd_entry structure
537 * 4. Initializes the pd_entry's ref count to 1
538 * assumptions:
539 * 1. The memory for pd should be pinned down, physically contiguous and
540 * aligned on 4K boundary and zeroed memory.
541 * 2. It should be 4K in size.
542 */
543int irdma_add_pd_table_entry(struct irdma_sc_dev *dev,
544 struct irdma_hmc_info *hmc_info, u32 pd_index,
545 struct irdma_dma_mem *rsrc_pg)
546{
547 struct irdma_hmc_pd_table *pd_table;
548 struct irdma_hmc_pd_entry *pd_entry;
549 struct irdma_dma_mem mem;
550 struct irdma_dma_mem *page = &mem;
551 u32 sd_idx, rel_pd_idx;
552 u64 *pd_addr;
553 u64 page_desc;
554
555 if (pd_index / IRDMA_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt)
556 return -EINVAL;
557
558 sd_idx = (pd_index / IRDMA_HMC_PD_CNT_IN_SD);
559 if (hmc_info->sd_table.sd_entry[sd_idx].entry_type !=
560 IRDMA_SD_TYPE_PAGED)
561 return 0;
562
563 rel_pd_idx = (pd_index % IRDMA_HMC_PD_CNT_IN_SD);
564 pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
565 pd_entry = &pd_table->pd_entry[rel_pd_idx];
566 if (!pd_entry->valid) {
567 if (rsrc_pg) {
568 pd_entry->rsrc_pg = true;
569 page = rsrc_pg;
570 } else {
571 page->size = ALIGN(IRDMA_HMC_PAGED_BP_SIZE,
572 IRDMA_HMC_PD_BP_BUF_ALIGNMENT);
573 page->va = dma_alloc_coherent(dev->hw->device,
574 page->size, &page->pa,
575 GFP_KERNEL);
576 if (!page->va)
577 return -ENOMEM;
578
579 pd_entry->rsrc_pg = false;
580 }
581
582 memcpy(&pd_entry->bp.addr, page, sizeof(pd_entry->bp.addr));
583 pd_entry->bp.sd_pd_index = pd_index;
584 pd_entry->bp.entry_type = IRDMA_SD_TYPE_PAGED;
585 page_desc = page->pa | 0x1;
586 pd_addr = pd_table->pd_page_addr.va;
587 pd_addr += rel_pd_idx;
588 memcpy(pd_addr, &page_desc, sizeof(*pd_addr));
589 pd_entry->sd_index = sd_idx;
590 pd_entry->valid = true;
591 pd_table->use_cnt++;
592 irdma_invalidate_pf_hmc_pd(dev, sd_idx, rel_pd_idx);
593 }
594 pd_entry->bp.use_cnt++;
595
596 return 0;
597}
598
599/**
600 * irdma_remove_pd_bp - remove a backing page from a page descriptor
601 * @dev: pointer to our HW structure
602 * @hmc_info: pointer to the HMC configuration information structure
603 * @idx: the page index
604 *
605 * This function:
606 * 1. Marks the entry in pd table (for paged address mode) or in sd table
607 * (for direct address mode) invalid.
608 * 2. Write to register PMPDINV to invalidate the backing page in FV cache
609 * 3. Decrement the ref count for the pd _entry
610 * assumptions:
611 * 1. Caller can deallocate the memory used by backing storage after this
612 * function returns.
613 */
614int irdma_remove_pd_bp(struct irdma_sc_dev *dev,
615 struct irdma_hmc_info *hmc_info, u32 idx)
616{
617 struct irdma_hmc_pd_entry *pd_entry;
618 struct irdma_hmc_pd_table *pd_table;
619 struct irdma_hmc_sd_entry *sd_entry;
620 u32 sd_idx, rel_pd_idx;
621 struct irdma_dma_mem *mem;
622 u64 *pd_addr;
623
624 sd_idx = idx / IRDMA_HMC_PD_CNT_IN_SD;
625 rel_pd_idx = idx % IRDMA_HMC_PD_CNT_IN_SD;
626 if (sd_idx >= hmc_info->sd_table.sd_cnt)
627 return -EINVAL;
628
629 sd_entry = &hmc_info->sd_table.sd_entry[sd_idx];
630 if (sd_entry->entry_type != IRDMA_SD_TYPE_PAGED)
631 return -EINVAL;
632
633 pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
634 pd_entry = &pd_table->pd_entry[rel_pd_idx];
635 if (--pd_entry->bp.use_cnt)
636 return 0;
637
638 pd_entry->valid = false;
639 pd_table->use_cnt--;
640 pd_addr = pd_table->pd_page_addr.va;
641 pd_addr += rel_pd_idx;
642 memset(pd_addr, 0, sizeof(u64));
643 irdma_invalidate_pf_hmc_pd(dev, sd_idx, idx);
644
645 if (!pd_entry->rsrc_pg) {
646 mem = &pd_entry->bp.addr;
647 if (!mem || !mem->va)
648 return -EINVAL;
649
650 dma_free_coherent(dev->hw->device, mem->size, mem->va,
651 mem->pa);
652 mem->va = NULL;
653 }
654 if (!pd_table->use_cnt)
655 kfree(pd_table->pd_entry_virt_mem.va);
656
657 return 0;
658}
659
660/**
661 * irdma_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry
662 * @hmc_info: pointer to the HMC configuration information structure
663 * @idx: the page index
664 */
665int irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info, u32 idx)
666{
667 struct irdma_hmc_sd_entry *sd_entry;
668
669 sd_entry = &hmc_info->sd_table.sd_entry[idx];
670 if (--sd_entry->u.bp.use_cnt)
671 return -EBUSY;
672
673 hmc_info->sd_table.use_cnt--;
674 sd_entry->valid = false;
675
676 return 0;
677}
678
679/**
680 * irdma_prep_remove_pd_page - Prepares to remove a PD page from sd entry.
681 * @hmc_info: pointer to the HMC configuration information structure
682 * @idx: segment descriptor index to find the relevant page descriptor
683 */
684int irdma_prep_remove_pd_page(struct irdma_hmc_info *hmc_info, u32 idx)
685{
686 struct irdma_hmc_sd_entry *sd_entry;
687
688 sd_entry = &hmc_info->sd_table.sd_entry[idx];
689
690 if (sd_entry->u.pd_table.use_cnt)
691 return -EBUSY;
692
693 sd_entry->valid = false;
694 hmc_info->sd_table.use_cnt--;
695
696 return 0;
697}
1// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2/* Copyright (c) 2015 - 2021 Intel Corporation */
3#include "osdep.h"
4#include "status.h"
5#include "hmc.h"
6#include "defs.h"
7#include "type.h"
8#include "protos.h"
9
10/**
11 * irdma_find_sd_index_limit - finds segment descriptor index limit
12 * @hmc_info: pointer to the HMC configuration information structure
13 * @type: type of HMC resources we're searching
14 * @idx: starting index for the object
15 * @cnt: number of objects we're trying to create
16 * @sd_idx: pointer to return index of the segment descriptor in question
17 * @sd_limit: pointer to return the maximum number of segment descriptors
18 *
19 * This function calculates the segment descriptor index and index limit
20 * for the resource defined by irdma_hmc_rsrc_type.
21 */
22
23static void irdma_find_sd_index_limit(struct irdma_hmc_info *hmc_info, u32 type,
24 u32 idx, u32 cnt, u32 *sd_idx,
25 u32 *sd_limit)
26{
27 u64 fpm_addr, fpm_limit;
28
29 fpm_addr = hmc_info->hmc_obj[(type)].base +
30 hmc_info->hmc_obj[type].size * idx;
31 fpm_limit = fpm_addr + hmc_info->hmc_obj[type].size * cnt;
32 *sd_idx = (u32)(fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE);
33 *sd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_DIRECT_BP_SIZE);
34 *sd_limit += 1;
35}
36
37/**
38 * irdma_find_pd_index_limit - finds page descriptor index limit
39 * @hmc_info: pointer to the HMC configuration information struct
40 * @type: HMC resource type we're examining
41 * @idx: starting index for the object
42 * @cnt: number of objects we're trying to create
43 * @pd_idx: pointer to return page descriptor index
44 * @pd_limit: pointer to return page descriptor index limit
45 *
46 * Calculates the page descriptor index and index limit for the resource
47 * defined by irdma_hmc_rsrc_type.
48 */
49
50static void irdma_find_pd_index_limit(struct irdma_hmc_info *hmc_info, u32 type,
51 u32 idx, u32 cnt, u32 *pd_idx,
52 u32 *pd_limit)
53{
54 u64 fpm_adr, fpm_limit;
55
56 fpm_adr = hmc_info->hmc_obj[type].base +
57 hmc_info->hmc_obj[type].size * idx;
58 fpm_limit = fpm_adr + (hmc_info)->hmc_obj[(type)].size * (cnt);
59 *pd_idx = (u32)(fpm_adr / IRDMA_HMC_PAGED_BP_SIZE);
60 *pd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_PAGED_BP_SIZE);
61 *pd_limit += 1;
62}
63
64/**
65 * irdma_set_sd_entry - setup entry for sd programming
66 * @pa: physical addr
67 * @idx: sd index
68 * @type: paged or direct sd
69 * @entry: sd entry ptr
70 */
71static void irdma_set_sd_entry(u64 pa, u32 idx, enum irdma_sd_entry_type type,
72 struct irdma_update_sd_entry *entry)
73{
74 entry->data = pa |
75 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) |
76 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE,
77 type == IRDMA_SD_TYPE_PAGED ? 0 : 1) |
78 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDVALID, 1);
79
80 entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | BIT(15);
81}
82
83/**
84 * irdma_clr_sd_entry - setup entry for sd clear
85 * @idx: sd index
86 * @type: paged or direct sd
87 * @entry: sd entry ptr
88 */
89static void irdma_clr_sd_entry(u32 idx, enum irdma_sd_entry_type type,
90 struct irdma_update_sd_entry *entry)
91{
92 entry->data = FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) |
93 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE,
94 type == IRDMA_SD_TYPE_PAGED ? 0 : 1);
95
96 entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | BIT(15);
97}
98
99/**
100 * irdma_invalidate_pf_hmc_pd - Invalidates the pd cache in the hardware for PF
101 * @dev: pointer to our device struct
102 * @sd_idx: segment descriptor index
103 * @pd_idx: page descriptor index
104 */
105static inline void irdma_invalidate_pf_hmc_pd(struct irdma_sc_dev *dev, u32 sd_idx,
106 u32 pd_idx)
107{
108 u32 val = FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDIDX, sd_idx) |
109 FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDPARTSEL, 1) |
110 FIELD_PREP(IRDMA_PFHMC_PDINV_PMPDIDX, pd_idx);
111
112 writel(val, dev->hw_regs[IRDMA_PFHMC_PDINV]);
113}
114
115/**
116 * irdma_hmc_sd_one - setup 1 sd entry for cqp
117 * @dev: pointer to the device structure
118 * @hmc_fn_id: hmc's function id
119 * @pa: physical addr
120 * @sd_idx: sd index
121 * @type: paged or direct sd
122 * @setsd: flag to set or clear sd
123 */
124enum irdma_status_code irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id,
125 u64 pa, u32 sd_idx,
126 enum irdma_sd_entry_type type,
127 bool setsd)
128{
129 struct irdma_update_sds_info sdinfo;
130
131 sdinfo.cnt = 1;
132 sdinfo.hmc_fn_id = hmc_fn_id;
133 if (setsd)
134 irdma_set_sd_entry(pa, sd_idx, type, sdinfo.entry);
135 else
136 irdma_clr_sd_entry(sd_idx, type, sdinfo.entry);
137 return dev->cqp->process_cqp_sds(dev, &sdinfo);
138}
139
140/**
141 * irdma_hmc_sd_grp - setup group of sd entries for cqp
142 * @dev: pointer to the device structure
143 * @hmc_info: pointer to the HMC configuration information struct
144 * @sd_index: sd index
145 * @sd_cnt: number of sd entries
146 * @setsd: flag to set or clear sd
147 */
148static enum irdma_status_code irdma_hmc_sd_grp(struct irdma_sc_dev *dev,
149 struct irdma_hmc_info *hmc_info,
150 u32 sd_index, u32 sd_cnt,
151 bool setsd)
152{
153 struct irdma_hmc_sd_entry *sd_entry;
154 struct irdma_update_sds_info sdinfo = {};
155 u64 pa;
156 u32 i;
157 enum irdma_status_code ret_code = 0;
158
159 sdinfo.hmc_fn_id = hmc_info->hmc_fn_id;
160 for (i = sd_index; i < sd_index + sd_cnt; i++) {
161 sd_entry = &hmc_info->sd_table.sd_entry[i];
162 if (!sd_entry || (!sd_entry->valid && setsd) ||
163 (sd_entry->valid && !setsd))
164 continue;
165 if (setsd) {
166 pa = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ?
167 sd_entry->u.pd_table.pd_page_addr.pa :
168 sd_entry->u.bp.addr.pa;
169 irdma_set_sd_entry(pa, i, sd_entry->entry_type,
170 &sdinfo.entry[sdinfo.cnt]);
171 } else {
172 irdma_clr_sd_entry(i, sd_entry->entry_type,
173 &sdinfo.entry[sdinfo.cnt]);
174 }
175 sdinfo.cnt++;
176 if (sdinfo.cnt == IRDMA_MAX_SD_ENTRIES) {
177 ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
178 if (ret_code) {
179 ibdev_dbg(to_ibdev(dev),
180 "HMC: sd_programming failed err=%d\n",
181 ret_code);
182 return ret_code;
183 }
184
185 sdinfo.cnt = 0;
186 }
187 }
188 if (sdinfo.cnt)
189 ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
190
191 return ret_code;
192}
193
194/**
195 * irdma_hmc_finish_add_sd_reg - program sd entries for objects
196 * @dev: pointer to the device structure
197 * @info: create obj info
198 */
199static enum irdma_status_code
200irdma_hmc_finish_add_sd_reg(struct irdma_sc_dev *dev,
201 struct irdma_hmc_create_obj_info *info)
202{
203 if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
204 return IRDMA_ERR_INVALID_HMC_OBJ_INDEX;
205
206 if ((info->start_idx + info->count) >
207 info->hmc_info->hmc_obj[info->rsrc_type].cnt)
208 return IRDMA_ERR_INVALID_HMC_OBJ_COUNT;
209
210 if (!info->add_sd_cnt)
211 return 0;
212 return irdma_hmc_sd_grp(dev, info->hmc_info,
213 info->hmc_info->sd_indexes[0], info->add_sd_cnt,
214 true);
215}
216
217/**
218 * irdma_sc_create_hmc_obj - allocate backing store for hmc objects
219 * @dev: pointer to the device structure
220 * @info: pointer to irdma_hmc_create_obj_info struct
221 *
222 * This will allocate memory for PDs and backing pages and populate
223 * the sd and pd entries.
224 */
225enum irdma_status_code
226irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,
227 struct irdma_hmc_create_obj_info *info)
228{
229 struct irdma_hmc_sd_entry *sd_entry;
230 u32 sd_idx, sd_lmt;
231 u32 pd_idx = 0, pd_lmt = 0;
232 u32 pd_idx1 = 0, pd_lmt1 = 0;
233 u32 i, j;
234 bool pd_error = false;
235 enum irdma_status_code ret_code = 0;
236
237 if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
238 return IRDMA_ERR_INVALID_HMC_OBJ_INDEX;
239
240 if ((info->start_idx + info->count) >
241 info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
242 ibdev_dbg(to_ibdev(dev),
243 "HMC: error type %u, start = %u, req cnt %u, cnt = %u\n",
244 info->rsrc_type, info->start_idx, info->count,
245 info->hmc_info->hmc_obj[info->rsrc_type].cnt);
246 return IRDMA_ERR_INVALID_HMC_OBJ_COUNT;
247 }
248
249 irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type,
250 info->start_idx, info->count, &sd_idx,
251 &sd_lmt);
252 if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
253 sd_lmt > info->hmc_info->sd_table.sd_cnt) {
254 return IRDMA_ERR_INVALID_SD_INDEX;
255 }
256
257 irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type,
258 info->start_idx, info->count, &pd_idx,
259 &pd_lmt);
260
261 for (j = sd_idx; j < sd_lmt; j++) {
262 ret_code = irdma_add_sd_table_entry(dev->hw, info->hmc_info, j,
263 info->entry_type,
264 IRDMA_HMC_DIRECT_BP_SIZE);
265 if (ret_code)
266 goto exit_sd_error;
267
268 sd_entry = &info->hmc_info->sd_table.sd_entry[j];
269 if (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED &&
270 (dev->hmc_info == info->hmc_info &&
271 info->rsrc_type != IRDMA_HMC_IW_PBLE)) {
272 pd_idx1 = max(pd_idx, (j * IRDMA_HMC_MAX_BP_COUNT));
273 pd_lmt1 = min(pd_lmt, (j + 1) * IRDMA_HMC_MAX_BP_COUNT);
274 for (i = pd_idx1; i < pd_lmt1; i++) {
275 /* update the pd table entry */
276 ret_code = irdma_add_pd_table_entry(dev,
277 info->hmc_info,
278 i, NULL);
279 if (ret_code) {
280 pd_error = true;
281 break;
282 }
283 }
284 if (pd_error) {
285 while (i && (i > pd_idx1)) {
286 irdma_remove_pd_bp(dev, info->hmc_info,
287 i - 1);
288 i--;
289 }
290 }
291 }
292 if (sd_entry->valid)
293 continue;
294
295 info->hmc_info->sd_indexes[info->add_sd_cnt] = (u16)j;
296 info->add_sd_cnt++;
297 sd_entry->valid = true;
298 }
299 return irdma_hmc_finish_add_sd_reg(dev, info);
300
301exit_sd_error:
302 while (j && (j > sd_idx)) {
303 sd_entry = &info->hmc_info->sd_table.sd_entry[j - 1];
304 switch (sd_entry->entry_type) {
305 case IRDMA_SD_TYPE_PAGED:
306 pd_idx1 = max(pd_idx, (j - 1) * IRDMA_HMC_MAX_BP_COUNT);
307 pd_lmt1 = min(pd_lmt, (j * IRDMA_HMC_MAX_BP_COUNT));
308 for (i = pd_idx1; i < pd_lmt1; i++)
309 irdma_prep_remove_pd_page(info->hmc_info, i);
310 break;
311 case IRDMA_SD_TYPE_DIRECT:
312 irdma_prep_remove_pd_page(info->hmc_info, (j - 1));
313 break;
314 default:
315 ret_code = IRDMA_ERR_INVALID_SD_TYPE;
316 break;
317 }
318 j--;
319 }
320
321 return ret_code;
322}
323
324/**
325 * irdma_finish_del_sd_reg - delete sd entries for objects
326 * @dev: pointer to the device structure
327 * @info: dele obj info
328 * @reset: true if called before reset
329 */
330static enum irdma_status_code
331irdma_finish_del_sd_reg(struct irdma_sc_dev *dev,
332 struct irdma_hmc_del_obj_info *info, bool reset)
333{
334 struct irdma_hmc_sd_entry *sd_entry;
335 enum irdma_status_code ret_code = 0;
336 u32 i, sd_idx;
337 struct irdma_dma_mem *mem;
338
339 if (!reset)
340 ret_code = irdma_hmc_sd_grp(dev, info->hmc_info,
341 info->hmc_info->sd_indexes[0],
342 info->del_sd_cnt, false);
343
344 if (ret_code)
345 ibdev_dbg(to_ibdev(dev), "HMC: error cqp sd sd_grp\n");
346 for (i = 0; i < info->del_sd_cnt; i++) {
347 sd_idx = info->hmc_info->sd_indexes[i];
348 sd_entry = &info->hmc_info->sd_table.sd_entry[sd_idx];
349 mem = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ?
350 &sd_entry->u.pd_table.pd_page_addr :
351 &sd_entry->u.bp.addr;
352
353 if (!mem || !mem->va) {
354 ibdev_dbg(to_ibdev(dev), "HMC: error cqp sd mem\n");
355 } else {
356 dma_free_coherent(dev->hw->device, mem->size, mem->va,
357 mem->pa);
358 mem->va = NULL;
359 }
360 }
361
362 return ret_code;
363}
364
365/**
366 * irdma_sc_del_hmc_obj - remove pe hmc objects
367 * @dev: pointer to the device structure
368 * @info: pointer to irdma_hmc_del_obj_info struct
369 * @reset: true if called before reset
370 *
371 * This will de-populate the SDs and PDs. It frees
372 * the memory for PDS and backing storage. After this function is returned,
373 * caller should deallocate memory allocated previously for
374 * book-keeping information about PDs and backing storage.
375 */
376enum irdma_status_code irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,
377 struct irdma_hmc_del_obj_info *info,
378 bool reset)
379{
380 struct irdma_hmc_pd_table *pd_table;
381 u32 sd_idx, sd_lmt;
382 u32 pd_idx, pd_lmt, rel_pd_idx;
383 u32 i, j;
384 enum irdma_status_code ret_code = 0;
385
386 if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
387 ibdev_dbg(to_ibdev(dev),
388 "HMC: error start_idx[%04d] >= [type %04d].cnt[%04d]\n",
389 info->start_idx, info->rsrc_type,
390 info->hmc_info->hmc_obj[info->rsrc_type].cnt);
391 return IRDMA_ERR_INVALID_HMC_OBJ_INDEX;
392 }
393
394 if ((info->start_idx + info->count) >
395 info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
396 ibdev_dbg(to_ibdev(dev),
397 "HMC: error start_idx[%04d] + count %04d >= [type %04d].cnt[%04d]\n",
398 info->start_idx, info->count, info->rsrc_type,
399 info->hmc_info->hmc_obj[info->rsrc_type].cnt);
400 return IRDMA_ERR_INVALID_HMC_OBJ_COUNT;
401 }
402
403 irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type,
404 info->start_idx, info->count, &pd_idx,
405 &pd_lmt);
406
407 for (j = pd_idx; j < pd_lmt; j++) {
408 sd_idx = j / IRDMA_HMC_PD_CNT_IN_SD;
409
410 if (!info->hmc_info->sd_table.sd_entry[sd_idx].valid)
411 continue;
412
413 if (info->hmc_info->sd_table.sd_entry[sd_idx].entry_type !=
414 IRDMA_SD_TYPE_PAGED)
415 continue;
416
417 rel_pd_idx = j % IRDMA_HMC_PD_CNT_IN_SD;
418 pd_table = &info->hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
419 if (pd_table->pd_entry &&
420 pd_table->pd_entry[rel_pd_idx].valid) {
421 ret_code = irdma_remove_pd_bp(dev, info->hmc_info, j);
422 if (ret_code) {
423 ibdev_dbg(to_ibdev(dev),
424 "HMC: remove_pd_bp error\n");
425 return ret_code;
426 }
427 }
428 }
429
430 irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type,
431 info->start_idx, info->count, &sd_idx,
432 &sd_lmt);
433 if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
434 sd_lmt > info->hmc_info->sd_table.sd_cnt) {
435 ibdev_dbg(to_ibdev(dev), "HMC: invalid sd_idx\n");
436 return IRDMA_ERR_INVALID_SD_INDEX;
437 }
438
439 for (i = sd_idx; i < sd_lmt; i++) {
440 pd_table = &info->hmc_info->sd_table.sd_entry[i].u.pd_table;
441 if (!info->hmc_info->sd_table.sd_entry[i].valid)
442 continue;
443 switch (info->hmc_info->sd_table.sd_entry[i].entry_type) {
444 case IRDMA_SD_TYPE_DIRECT:
445 ret_code = irdma_prep_remove_sd_bp(info->hmc_info, i);
446 if (!ret_code) {
447 info->hmc_info->sd_indexes[info->del_sd_cnt] =
448 (u16)i;
449 info->del_sd_cnt++;
450 }
451 break;
452 case IRDMA_SD_TYPE_PAGED:
453 ret_code = irdma_prep_remove_pd_page(info->hmc_info, i);
454 if (ret_code)
455 break;
456 if (dev->hmc_info != info->hmc_info &&
457 info->rsrc_type == IRDMA_HMC_IW_PBLE &&
458 pd_table->pd_entry) {
459 kfree(pd_table->pd_entry_virt_mem.va);
460 pd_table->pd_entry = NULL;
461 }
462 info->hmc_info->sd_indexes[info->del_sd_cnt] = (u16)i;
463 info->del_sd_cnt++;
464 break;
465 default:
466 break;
467 }
468 }
469 return irdma_finish_del_sd_reg(dev, info, reset);
470}
471
472/**
473 * irdma_add_sd_table_entry - Adds a segment descriptor to the table
474 * @hw: pointer to our hw struct
475 * @hmc_info: pointer to the HMC configuration information struct
476 * @sd_index: segment descriptor index to manipulate
477 * @type: what type of segment descriptor we're manipulating
478 * @direct_mode_sz: size to alloc in direct mode
479 */
480enum irdma_status_code irdma_add_sd_table_entry(struct irdma_hw *hw,
481 struct irdma_hmc_info *hmc_info,
482 u32 sd_index,
483 enum irdma_sd_entry_type type,
484 u64 direct_mode_sz)
485{
486 struct irdma_hmc_sd_entry *sd_entry;
487 struct irdma_dma_mem dma_mem;
488 u64 alloc_len;
489
490 sd_entry = &hmc_info->sd_table.sd_entry[sd_index];
491 if (!sd_entry->valid) {
492 if (type == IRDMA_SD_TYPE_PAGED)
493 alloc_len = IRDMA_HMC_PAGED_BP_SIZE;
494 else
495 alloc_len = direct_mode_sz;
496
497 /* allocate a 4K pd page or 2M backing page */
498 dma_mem.size = ALIGN(alloc_len, IRDMA_HMC_PD_BP_BUF_ALIGNMENT);
499 dma_mem.va = dma_alloc_coherent(hw->device, dma_mem.size,
500 &dma_mem.pa, GFP_KERNEL);
501 if (!dma_mem.va)
502 return IRDMA_ERR_NO_MEMORY;
503 if (type == IRDMA_SD_TYPE_PAGED) {
504 struct irdma_virt_mem *vmem =
505 &sd_entry->u.pd_table.pd_entry_virt_mem;
506
507 vmem->size = sizeof(struct irdma_hmc_pd_entry) * 512;
508 vmem->va = kzalloc(vmem->size, GFP_KERNEL);
509 if (!vmem->va) {
510 dma_free_coherent(hw->device, dma_mem.size,
511 dma_mem.va, dma_mem.pa);
512 dma_mem.va = NULL;
513 return IRDMA_ERR_NO_MEMORY;
514 }
515 sd_entry->u.pd_table.pd_entry = vmem->va;
516
517 memcpy(&sd_entry->u.pd_table.pd_page_addr, &dma_mem,
518 sizeof(sd_entry->u.pd_table.pd_page_addr));
519 } else {
520 memcpy(&sd_entry->u.bp.addr, &dma_mem,
521 sizeof(sd_entry->u.bp.addr));
522
523 sd_entry->u.bp.sd_pd_index = sd_index;
524 }
525
526 hmc_info->sd_table.sd_entry[sd_index].entry_type = type;
527 hmc_info->sd_table.use_cnt++;
528 }
529 if (sd_entry->entry_type == IRDMA_SD_TYPE_DIRECT)
530 sd_entry->u.bp.use_cnt++;
531
532 return 0;
533}
534
535/**
536 * irdma_add_pd_table_entry - Adds page descriptor to the specified table
537 * @dev: pointer to our device structure
538 * @hmc_info: pointer to the HMC configuration information structure
539 * @pd_index: which page descriptor index to manipulate
540 * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one.
541 *
542 * This function:
543 * 1. Initializes the pd entry
544 * 2. Adds pd_entry in the pd_table
545 * 3. Mark the entry valid in irdma_hmc_pd_entry structure
546 * 4. Initializes the pd_entry's ref count to 1
547 * assumptions:
548 * 1. The memory for pd should be pinned down, physically contiguous and
549 * aligned on 4K boundary and zeroed memory.
550 * 2. It should be 4K in size.
551 */
552enum irdma_status_code irdma_add_pd_table_entry(struct irdma_sc_dev *dev,
553 struct irdma_hmc_info *hmc_info,
554 u32 pd_index,
555 struct irdma_dma_mem *rsrc_pg)
556{
557 struct irdma_hmc_pd_table *pd_table;
558 struct irdma_hmc_pd_entry *pd_entry;
559 struct irdma_dma_mem mem;
560 struct irdma_dma_mem *page = &mem;
561 u32 sd_idx, rel_pd_idx;
562 u64 *pd_addr;
563 u64 page_desc;
564
565 if (pd_index / IRDMA_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt)
566 return IRDMA_ERR_INVALID_PAGE_DESC_INDEX;
567
568 sd_idx = (pd_index / IRDMA_HMC_PD_CNT_IN_SD);
569 if (hmc_info->sd_table.sd_entry[sd_idx].entry_type !=
570 IRDMA_SD_TYPE_PAGED)
571 return 0;
572
573 rel_pd_idx = (pd_index % IRDMA_HMC_PD_CNT_IN_SD);
574 pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
575 pd_entry = &pd_table->pd_entry[rel_pd_idx];
576 if (!pd_entry->valid) {
577 if (rsrc_pg) {
578 pd_entry->rsrc_pg = true;
579 page = rsrc_pg;
580 } else {
581 page->size = ALIGN(IRDMA_HMC_PAGED_BP_SIZE,
582 IRDMA_HMC_PD_BP_BUF_ALIGNMENT);
583 page->va = dma_alloc_coherent(dev->hw->device,
584 page->size, &page->pa,
585 GFP_KERNEL);
586 if (!page->va)
587 return IRDMA_ERR_NO_MEMORY;
588
589 pd_entry->rsrc_pg = false;
590 }
591
592 memcpy(&pd_entry->bp.addr, page, sizeof(pd_entry->bp.addr));
593 pd_entry->bp.sd_pd_index = pd_index;
594 pd_entry->bp.entry_type = IRDMA_SD_TYPE_PAGED;
595 page_desc = page->pa | 0x1;
596 pd_addr = pd_table->pd_page_addr.va;
597 pd_addr += rel_pd_idx;
598 memcpy(pd_addr, &page_desc, sizeof(*pd_addr));
599 pd_entry->sd_index = sd_idx;
600 pd_entry->valid = true;
601 pd_table->use_cnt++;
602 irdma_invalidate_pf_hmc_pd(dev, sd_idx, rel_pd_idx);
603 }
604 pd_entry->bp.use_cnt++;
605
606 return 0;
607}
608
609/**
610 * irdma_remove_pd_bp - remove a backing page from a page descriptor
611 * @dev: pointer to our HW structure
612 * @hmc_info: pointer to the HMC configuration information structure
613 * @idx: the page index
614 *
615 * This function:
616 * 1. Marks the entry in pd table (for paged address mode) or in sd table
617 * (for direct address mode) invalid.
618 * 2. Write to register PMPDINV to invalidate the backing page in FV cache
619 * 3. Decrement the ref count for the pd _entry
620 * assumptions:
621 * 1. Caller can deallocate the memory used by backing storage after this
622 * function returns.
623 */
624enum irdma_status_code irdma_remove_pd_bp(struct irdma_sc_dev *dev,
625 struct irdma_hmc_info *hmc_info,
626 u32 idx)
627{
628 struct irdma_hmc_pd_entry *pd_entry;
629 struct irdma_hmc_pd_table *pd_table;
630 struct irdma_hmc_sd_entry *sd_entry;
631 u32 sd_idx, rel_pd_idx;
632 struct irdma_dma_mem *mem;
633 u64 *pd_addr;
634
635 sd_idx = idx / IRDMA_HMC_PD_CNT_IN_SD;
636 rel_pd_idx = idx % IRDMA_HMC_PD_CNT_IN_SD;
637 if (sd_idx >= hmc_info->sd_table.sd_cnt)
638 return IRDMA_ERR_INVALID_PAGE_DESC_INDEX;
639
640 sd_entry = &hmc_info->sd_table.sd_entry[sd_idx];
641 if (sd_entry->entry_type != IRDMA_SD_TYPE_PAGED)
642 return IRDMA_ERR_INVALID_SD_TYPE;
643
644 pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
645 pd_entry = &pd_table->pd_entry[rel_pd_idx];
646 if (--pd_entry->bp.use_cnt)
647 return 0;
648
649 pd_entry->valid = false;
650 pd_table->use_cnt--;
651 pd_addr = pd_table->pd_page_addr.va;
652 pd_addr += rel_pd_idx;
653 memset(pd_addr, 0, sizeof(u64));
654 irdma_invalidate_pf_hmc_pd(dev, sd_idx, idx);
655
656 if (!pd_entry->rsrc_pg) {
657 mem = &pd_entry->bp.addr;
658 if (!mem || !mem->va)
659 return IRDMA_ERR_PARAM;
660
661 dma_free_coherent(dev->hw->device, mem->size, mem->va,
662 mem->pa);
663 mem->va = NULL;
664 }
665 if (!pd_table->use_cnt)
666 kfree(pd_table->pd_entry_virt_mem.va);
667
668 return 0;
669}
670
671/**
672 * irdma_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry
673 * @hmc_info: pointer to the HMC configuration information structure
674 * @idx: the page index
675 */
676enum irdma_status_code irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info,
677 u32 idx)
678{
679 struct irdma_hmc_sd_entry *sd_entry;
680
681 sd_entry = &hmc_info->sd_table.sd_entry[idx];
682 if (--sd_entry->u.bp.use_cnt)
683 return IRDMA_ERR_NOT_READY;
684
685 hmc_info->sd_table.use_cnt--;
686 sd_entry->valid = false;
687
688 return 0;
689}
690
691/**
692 * irdma_prep_remove_pd_page - Prepares to remove a PD page from sd entry.
693 * @hmc_info: pointer to the HMC configuration information structure
694 * @idx: segment descriptor index to find the relevant page descriptor
695 */
696enum irdma_status_code
697irdma_prep_remove_pd_page(struct irdma_hmc_info *hmc_info, u32 idx)
698{
699 struct irdma_hmc_sd_entry *sd_entry;
700
701 sd_entry = &hmc_info->sd_table.sd_entry[idx];
702
703 if (sd_entry->u.pd_table.use_cnt)
704 return IRDMA_ERR_NOT_READY;
705
706 sd_entry->valid = false;
707 hmc_info->sd_table.use_cnt--;
708
709 return 0;
710}