Loading...
1/*
2 * Copyright (c) 2016 Hisilicon Limited.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#include "hns_roce_device.h"
35#include "hns_roce_hem.h"
36#include "hns_roce_common.h"
37
38#define HEM_INDEX_BUF BIT(0)
39#define HEM_INDEX_L0 BIT(1)
40#define HEM_INDEX_L1 BIT(2)
41struct hns_roce_hem_index {
42 u64 buf;
43 u64 l0;
44 u64 l1;
45 u32 inited; /* indicate which index is available */
46};
47
48bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
49{
50 int hop_num = 0;
51
52 switch (type) {
53 case HEM_TYPE_QPC:
54 hop_num = hr_dev->caps.qpc_hop_num;
55 break;
56 case HEM_TYPE_MTPT:
57 hop_num = hr_dev->caps.mpt_hop_num;
58 break;
59 case HEM_TYPE_CQC:
60 hop_num = hr_dev->caps.cqc_hop_num;
61 break;
62 case HEM_TYPE_SRQC:
63 hop_num = hr_dev->caps.srqc_hop_num;
64 break;
65 case HEM_TYPE_SCCC:
66 hop_num = hr_dev->caps.sccc_hop_num;
67 break;
68 case HEM_TYPE_QPC_TIMER:
69 hop_num = hr_dev->caps.qpc_timer_hop_num;
70 break;
71 case HEM_TYPE_CQC_TIMER:
72 hop_num = hr_dev->caps.cqc_timer_hop_num;
73 break;
74 case HEM_TYPE_GMV:
75 hop_num = hr_dev->caps.gmv_hop_num;
76 break;
77 default:
78 return false;
79 }
80
81 return hop_num;
82}
83
84static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx,
85 u32 bt_chunk_num, u64 hem_max_num)
86{
87 u64 start_idx = round_down(hem_idx, bt_chunk_num);
88 u64 check_max_num = start_idx + bt_chunk_num;
89 u64 i;
90
91 for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++)
92 if (i != hem_idx && hem[i])
93 return false;
94
95 return true;
96}
97
98static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num)
99{
100 u64 start_idx = round_down(ba_idx, bt_chunk_num);
101 int i;
102
103 for (i = 0; i < bt_chunk_num; i++)
104 if (i != ba_idx && bt[start_idx + i])
105 return false;
106
107 return true;
108}
109
110static int hns_roce_get_bt_num(u32 table_type, u32 hop_num)
111{
112 if (check_whether_bt_num_3(table_type, hop_num))
113 return 3;
114 else if (check_whether_bt_num_2(table_type, hop_num))
115 return 2;
116 else if (check_whether_bt_num_1(table_type, hop_num))
117 return 1;
118 else
119 return 0;
120}
121
122static int get_hem_table_config(struct hns_roce_dev *hr_dev,
123 struct hns_roce_hem_mhop *mhop,
124 u32 type)
125{
126 struct device *dev = hr_dev->dev;
127
128 switch (type) {
129 case HEM_TYPE_QPC:
130 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
131 + PAGE_SHIFT);
132 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
133 + PAGE_SHIFT);
134 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num;
135 mhop->hop_num = hr_dev->caps.qpc_hop_num;
136 break;
137 case HEM_TYPE_MTPT:
138 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
139 + PAGE_SHIFT);
140 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
141 + PAGE_SHIFT);
142 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num;
143 mhop->hop_num = hr_dev->caps.mpt_hop_num;
144 break;
145 case HEM_TYPE_CQC:
146 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
147 + PAGE_SHIFT);
148 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
149 + PAGE_SHIFT);
150 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
151 mhop->hop_num = hr_dev->caps.cqc_hop_num;
152 break;
153 case HEM_TYPE_SCCC:
154 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz
155 + PAGE_SHIFT);
156 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz
157 + PAGE_SHIFT);
158 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num;
159 mhop->hop_num = hr_dev->caps.sccc_hop_num;
160 break;
161 case HEM_TYPE_QPC_TIMER:
162 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz
163 + PAGE_SHIFT);
164 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz
165 + PAGE_SHIFT);
166 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num;
167 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num;
168 break;
169 case HEM_TYPE_CQC_TIMER:
170 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz
171 + PAGE_SHIFT);
172 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz
173 + PAGE_SHIFT);
174 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num;
175 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num;
176 break;
177 case HEM_TYPE_SRQC:
178 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
179 + PAGE_SHIFT);
180 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
181 + PAGE_SHIFT);
182 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
183 mhop->hop_num = hr_dev->caps.srqc_hop_num;
184 break;
185 case HEM_TYPE_GMV:
186 mhop->buf_chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz +
187 PAGE_SHIFT);
188 mhop->bt_chunk_size = 1 << (hr_dev->caps.gmv_ba_pg_sz +
189 PAGE_SHIFT);
190 mhop->ba_l0_num = hr_dev->caps.gmv_bt_num;
191 mhop->hop_num = hr_dev->caps.gmv_hop_num;
192 break;
193 default:
194 dev_err(dev, "table %u not support multi-hop addressing!\n",
195 type);
196 return -EINVAL;
197 }
198
199 return 0;
200}
201
202int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
203 struct hns_roce_hem_table *table, unsigned long *obj,
204 struct hns_roce_hem_mhop *mhop)
205{
206 struct device *dev = hr_dev->dev;
207 u32 chunk_ba_num;
208 u32 chunk_size;
209 u32 table_idx;
210 u32 bt_num;
211
212 if (get_hem_table_config(hr_dev, mhop, table->type))
213 return -EINVAL;
214
215 if (!obj)
216 return 0;
217
218 /*
219 * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages.
220 * MTT/CQE alloc hem for bt pages.
221 */
222 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
223 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
224 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
225 mhop->bt_chunk_size;
226 table_idx = *obj / (chunk_size / table->obj_size);
227 switch (bt_num) {
228 case 3:
229 mhop->l2_idx = table_idx & (chunk_ba_num - 1);
230 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1);
231 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num;
232 break;
233 case 2:
234 mhop->l1_idx = table_idx & (chunk_ba_num - 1);
235 mhop->l0_idx = table_idx / chunk_ba_num;
236 break;
237 case 1:
238 mhop->l0_idx = table_idx;
239 break;
240 default:
241 dev_err(dev, "table %u not support hop_num = %u!\n",
242 table->type, mhop->hop_num);
243 return -EINVAL;
244 }
245 if (mhop->l0_idx >= mhop->ba_l0_num)
246 mhop->l0_idx %= mhop->ba_l0_num;
247
248 return 0;
249}
250
251static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
252 int npages,
253 unsigned long hem_alloc_size,
254 gfp_t gfp_mask)
255{
256 struct hns_roce_hem_chunk *chunk = NULL;
257 struct hns_roce_hem *hem;
258 struct scatterlist *mem;
259 int order;
260 void *buf;
261
262 WARN_ON(gfp_mask & __GFP_HIGHMEM);
263
264 hem = kmalloc(sizeof(*hem),
265 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
266 if (!hem)
267 return NULL;
268
269 INIT_LIST_HEAD(&hem->chunk_list);
270
271 order = get_order(hem_alloc_size);
272
273 while (npages > 0) {
274 if (!chunk) {
275 chunk = kmalloc(sizeof(*chunk),
276 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
277 if (!chunk)
278 goto fail;
279
280 sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
281 chunk->npages = 0;
282 chunk->nsg = 0;
283 memset(chunk->buf, 0, sizeof(chunk->buf));
284 list_add_tail(&chunk->list, &hem->chunk_list);
285 }
286
287 while (1 << order > npages)
288 --order;
289
290 /*
291 * Alloc memory one time. If failed, don't alloc small block
292 * memory, directly return fail.
293 */
294 mem = &chunk->mem[chunk->npages];
295 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order,
296 &sg_dma_address(mem), gfp_mask);
297 if (!buf)
298 goto fail;
299
300 chunk->buf[chunk->npages] = buf;
301 sg_dma_len(mem) = PAGE_SIZE << order;
302
303 ++chunk->npages;
304 ++chunk->nsg;
305 npages -= 1 << order;
306 }
307
308 return hem;
309
310fail:
311 hns_roce_free_hem(hr_dev, hem);
312 return NULL;
313}
314
315void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
316{
317 struct hns_roce_hem_chunk *chunk, *tmp;
318 int i;
319
320 if (!hem)
321 return;
322
323 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
324 for (i = 0; i < chunk->npages; ++i)
325 dma_free_coherent(hr_dev->dev,
326 sg_dma_len(&chunk->mem[i]),
327 chunk->buf[i],
328 sg_dma_address(&chunk->mem[i]));
329 kfree(chunk);
330 }
331
332 kfree(hem);
333}
334
335static int calc_hem_config(struct hns_roce_dev *hr_dev,
336 struct hns_roce_hem_table *table, unsigned long obj,
337 struct hns_roce_hem_mhop *mhop,
338 struct hns_roce_hem_index *index)
339{
340 struct ib_device *ibdev = &hr_dev->ib_dev;
341 unsigned long mhop_obj = obj;
342 u32 l0_idx, l1_idx, l2_idx;
343 u32 chunk_ba_num;
344 u32 bt_num;
345 int ret;
346
347 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop);
348 if (ret)
349 return ret;
350
351 l0_idx = mhop->l0_idx;
352 l1_idx = mhop->l1_idx;
353 l2_idx = mhop->l2_idx;
354 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
355 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
356 switch (bt_num) {
357 case 3:
358 index->l1 = l0_idx * chunk_ba_num + l1_idx;
359 index->l0 = l0_idx;
360 index->buf = l0_idx * chunk_ba_num * chunk_ba_num +
361 l1_idx * chunk_ba_num + l2_idx;
362 break;
363 case 2:
364 index->l0 = l0_idx;
365 index->buf = l0_idx * chunk_ba_num + l1_idx;
366 break;
367 case 1:
368 index->buf = l0_idx;
369 break;
370 default:
371 ibdev_err(ibdev, "table %u not support mhop.hop_num = %u!\n",
372 table->type, mhop->hop_num);
373 return -EINVAL;
374 }
375
376 if (unlikely(index->buf >= table->num_hem)) {
377 ibdev_err(ibdev, "table %u exceed hem limt idx %llu, max %lu!\n",
378 table->type, index->buf, table->num_hem);
379 return -EINVAL;
380 }
381
382 return 0;
383}
384
385static void free_mhop_hem(struct hns_roce_dev *hr_dev,
386 struct hns_roce_hem_table *table,
387 struct hns_roce_hem_mhop *mhop,
388 struct hns_roce_hem_index *index)
389{
390 u32 bt_size = mhop->bt_chunk_size;
391 struct device *dev = hr_dev->dev;
392
393 if (index->inited & HEM_INDEX_BUF) {
394 hns_roce_free_hem(hr_dev, table->hem[index->buf]);
395 table->hem[index->buf] = NULL;
396 }
397
398 if (index->inited & HEM_INDEX_L1) {
399 dma_free_coherent(dev, bt_size, table->bt_l1[index->l1],
400 table->bt_l1_dma_addr[index->l1]);
401 table->bt_l1[index->l1] = NULL;
402 }
403
404 if (index->inited & HEM_INDEX_L0) {
405 dma_free_coherent(dev, bt_size, table->bt_l0[index->l0],
406 table->bt_l0_dma_addr[index->l0]);
407 table->bt_l0[index->l0] = NULL;
408 }
409}
410
411static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,
412 struct hns_roce_hem_table *table,
413 struct hns_roce_hem_mhop *mhop,
414 struct hns_roce_hem_index *index)
415{
416 u32 bt_size = mhop->bt_chunk_size;
417 struct device *dev = hr_dev->dev;
418 struct hns_roce_hem_iter iter;
419 gfp_t flag;
420 u64 bt_ba;
421 u32 size;
422 int ret;
423
424 /* alloc L1 BA's chunk */
425 if ((check_whether_bt_num_3(table->type, mhop->hop_num) ||
426 check_whether_bt_num_2(table->type, mhop->hop_num)) &&
427 !table->bt_l0[index->l0]) {
428 table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size,
429 &table->bt_l0_dma_addr[index->l0],
430 GFP_KERNEL);
431 if (!table->bt_l0[index->l0]) {
432 ret = -ENOMEM;
433 goto out;
434 }
435 index->inited |= HEM_INDEX_L0;
436 }
437
438 /* alloc L2 BA's chunk */
439 if (check_whether_bt_num_3(table->type, mhop->hop_num) &&
440 !table->bt_l1[index->l1]) {
441 table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size,
442 &table->bt_l1_dma_addr[index->l1],
443 GFP_KERNEL);
444 if (!table->bt_l1[index->l1]) {
445 ret = -ENOMEM;
446 goto err_alloc_hem;
447 }
448 index->inited |= HEM_INDEX_L1;
449 *(table->bt_l0[index->l0] + mhop->l1_idx) =
450 table->bt_l1_dma_addr[index->l1];
451 }
452
453 /*
454 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC.
455 * alloc bt space chunk for MTT/CQE.
456 */
457 size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size;
458 flag = GFP_KERNEL | __GFP_NOWARN;
459 table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT,
460 size, flag);
461 if (!table->hem[index->buf]) {
462 ret = -ENOMEM;
463 goto err_alloc_hem;
464 }
465
466 index->inited |= HEM_INDEX_BUF;
467 hns_roce_hem_first(table->hem[index->buf], &iter);
468 bt_ba = hns_roce_hem_addr(&iter);
469 if (table->type < HEM_TYPE_MTT) {
470 if (mhop->hop_num == 2)
471 *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba;
472 else if (mhop->hop_num == 1)
473 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
474 } else if (mhop->hop_num == 2) {
475 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
476 }
477
478 return 0;
479err_alloc_hem:
480 free_mhop_hem(hr_dev, table, mhop, index);
481out:
482 return ret;
483}
484
485static int set_mhop_hem(struct hns_roce_dev *hr_dev,
486 struct hns_roce_hem_table *table, unsigned long obj,
487 struct hns_roce_hem_mhop *mhop,
488 struct hns_roce_hem_index *index)
489{
490 struct ib_device *ibdev = &hr_dev->ib_dev;
491 u32 step_idx;
492 int ret = 0;
493
494 if (index->inited & HEM_INDEX_L0) {
495 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0);
496 if (ret) {
497 ibdev_err(ibdev, "set HEM step 0 failed!\n");
498 goto out;
499 }
500 }
501
502 if (index->inited & HEM_INDEX_L1) {
503 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1);
504 if (ret) {
505 ibdev_err(ibdev, "set HEM step 1 failed!\n");
506 goto out;
507 }
508 }
509
510 if (index->inited & HEM_INDEX_BUF) {
511 if (mhop->hop_num == HNS_ROCE_HOP_NUM_0)
512 step_idx = 0;
513 else
514 step_idx = mhop->hop_num;
515 ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx);
516 if (ret)
517 ibdev_err(ibdev, "set HEM step last failed!\n");
518 }
519out:
520 return ret;
521}
522
523static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
524 struct hns_roce_hem_table *table,
525 unsigned long obj)
526{
527 struct ib_device *ibdev = &hr_dev->ib_dev;
528 struct hns_roce_hem_index index = {};
529 struct hns_roce_hem_mhop mhop = {};
530 int ret;
531
532 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
533 if (ret) {
534 ibdev_err(ibdev, "calc hem config failed!\n");
535 return ret;
536 }
537
538 mutex_lock(&table->mutex);
539 if (table->hem[index.buf]) {
540 refcount_inc(&table->hem[index.buf]->refcount);
541 goto out;
542 }
543
544 ret = alloc_mhop_hem(hr_dev, table, &mhop, &index);
545 if (ret) {
546 ibdev_err(ibdev, "alloc mhop hem failed!\n");
547 goto out;
548 }
549
550 /* set HEM base address to hardware */
551 if (table->type < HEM_TYPE_MTT) {
552 ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index);
553 if (ret) {
554 ibdev_err(ibdev, "set HEM address to HW failed!\n");
555 goto err_alloc;
556 }
557 }
558
559 refcount_set(&table->hem[index.buf]->refcount, 1);
560 goto out;
561
562err_alloc:
563 free_mhop_hem(hr_dev, table, &mhop, &index);
564out:
565 mutex_unlock(&table->mutex);
566 return ret;
567}
568
569int hns_roce_table_get(struct hns_roce_dev *hr_dev,
570 struct hns_roce_hem_table *table, unsigned long obj)
571{
572 struct device *dev = hr_dev->dev;
573 unsigned long i;
574 int ret = 0;
575
576 if (hns_roce_check_whether_mhop(hr_dev, table->type))
577 return hns_roce_table_mhop_get(hr_dev, table, obj);
578
579 i = obj / (table->table_chunk_size / table->obj_size);
580
581 mutex_lock(&table->mutex);
582
583 if (table->hem[i]) {
584 refcount_inc(&table->hem[i]->refcount);
585 goto out;
586 }
587
588 table->hem[i] = hns_roce_alloc_hem(hr_dev,
589 table->table_chunk_size >> PAGE_SHIFT,
590 table->table_chunk_size,
591 GFP_KERNEL | __GFP_NOWARN);
592 if (!table->hem[i]) {
593 ret = -ENOMEM;
594 goto out;
595 }
596
597 /* Set HEM base address(128K/page, pa) to Hardware */
598 ret = hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT);
599 if (ret) {
600 hns_roce_free_hem(hr_dev, table->hem[i]);
601 table->hem[i] = NULL;
602 dev_err(dev, "set HEM base address to HW failed, ret = %d.\n",
603 ret);
604 goto out;
605 }
606
607 refcount_set(&table->hem[i]->refcount, 1);
608out:
609 mutex_unlock(&table->mutex);
610 return ret;
611}
612
613static void clear_mhop_hem(struct hns_roce_dev *hr_dev,
614 struct hns_roce_hem_table *table, unsigned long obj,
615 struct hns_roce_hem_mhop *mhop,
616 struct hns_roce_hem_index *index)
617{
618 struct ib_device *ibdev = &hr_dev->ib_dev;
619 u32 hop_num = mhop->hop_num;
620 u32 chunk_ba_num;
621 u32 step_idx;
622 int ret;
623
624 index->inited = HEM_INDEX_BUF;
625 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
626 if (check_whether_bt_num_2(table->type, hop_num)) {
627 if (hns_roce_check_hem_null(table->hem, index->buf,
628 chunk_ba_num, table->num_hem))
629 index->inited |= HEM_INDEX_L0;
630 } else if (check_whether_bt_num_3(table->type, hop_num)) {
631 if (hns_roce_check_hem_null(table->hem, index->buf,
632 chunk_ba_num, table->num_hem)) {
633 index->inited |= HEM_INDEX_L1;
634 if (hns_roce_check_bt_null(table->bt_l1, index->l1,
635 chunk_ba_num))
636 index->inited |= HEM_INDEX_L0;
637 }
638 }
639
640 if (table->type < HEM_TYPE_MTT) {
641 if (hop_num == HNS_ROCE_HOP_NUM_0)
642 step_idx = 0;
643 else
644 step_idx = hop_num;
645
646 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx);
647 if (ret)
648 ibdev_warn(ibdev, "failed to clear hop%u HEM, ret = %d.\n",
649 hop_num, ret);
650
651 if (index->inited & HEM_INDEX_L1) {
652 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 1);
653 if (ret)
654 ibdev_warn(ibdev, "failed to clear HEM step 1, ret = %d.\n",
655 ret);
656 }
657
658 if (index->inited & HEM_INDEX_L0) {
659 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0);
660 if (ret)
661 ibdev_warn(ibdev, "failed to clear HEM step 0, ret = %d.\n",
662 ret);
663 }
664 }
665}
666
667static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
668 struct hns_roce_hem_table *table,
669 unsigned long obj,
670 int check_refcount)
671{
672 struct ib_device *ibdev = &hr_dev->ib_dev;
673 struct hns_roce_hem_index index = {};
674 struct hns_roce_hem_mhop mhop = {};
675 int ret;
676
677 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
678 if (ret) {
679 ibdev_err(ibdev, "calc hem config failed!\n");
680 return;
681 }
682
683 if (!check_refcount)
684 mutex_lock(&table->mutex);
685 else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount,
686 &table->mutex))
687 return;
688
689 clear_mhop_hem(hr_dev, table, obj, &mhop, &index);
690 free_mhop_hem(hr_dev, table, &mhop, &index);
691
692 mutex_unlock(&table->mutex);
693}
694
695void hns_roce_table_put(struct hns_roce_dev *hr_dev,
696 struct hns_roce_hem_table *table, unsigned long obj)
697{
698 struct device *dev = hr_dev->dev;
699 unsigned long i;
700 int ret;
701
702 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
703 hns_roce_table_mhop_put(hr_dev, table, obj, 1);
704 return;
705 }
706
707 i = obj / (table->table_chunk_size / table->obj_size);
708
709 if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount,
710 &table->mutex))
711 return;
712
713 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT);
714 if (ret)
715 dev_warn(dev, "failed to clear HEM base address, ret = %d.\n",
716 ret);
717
718 hns_roce_free_hem(hr_dev, table->hem[i]);
719 table->hem[i] = NULL;
720
721 mutex_unlock(&table->mutex);
722}
723
724void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
725 struct hns_roce_hem_table *table,
726 unsigned long obj, dma_addr_t *dma_handle)
727{
728 struct hns_roce_hem_chunk *chunk;
729 struct hns_roce_hem_mhop mhop;
730 struct hns_roce_hem *hem;
731 unsigned long mhop_obj = obj;
732 unsigned long obj_per_chunk;
733 unsigned long idx_offset;
734 int offset, dma_offset;
735 void *addr = NULL;
736 u32 hem_idx = 0;
737 int length;
738 int i, j;
739
740 mutex_lock(&table->mutex);
741
742 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {
743 obj_per_chunk = table->table_chunk_size / table->obj_size;
744 hem = table->hem[obj / obj_per_chunk];
745 idx_offset = obj % obj_per_chunk;
746 dma_offset = offset = idx_offset * table->obj_size;
747 } else {
748 u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */
749
750 if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop))
751 goto out;
752 /* mtt mhop */
753 i = mhop.l0_idx;
754 j = mhop.l1_idx;
755 if (mhop.hop_num == 2)
756 hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j;
757 else if (mhop.hop_num == 1 ||
758 mhop.hop_num == HNS_ROCE_HOP_NUM_0)
759 hem_idx = i;
760
761 hem = table->hem[hem_idx];
762 dma_offset = offset = obj * seg_size % mhop.bt_chunk_size;
763 if (mhop.hop_num == 2)
764 dma_offset = offset = 0;
765 }
766
767 if (!hem)
768 goto out;
769
770 list_for_each_entry(chunk, &hem->chunk_list, list) {
771 for (i = 0; i < chunk->npages; ++i) {
772 length = sg_dma_len(&chunk->mem[i]);
773 if (dma_handle && dma_offset >= 0) {
774 if (length > (u32)dma_offset)
775 *dma_handle = sg_dma_address(
776 &chunk->mem[i]) + dma_offset;
777 dma_offset -= length;
778 }
779
780 if (length > (u32)offset) {
781 addr = chunk->buf[i] + offset;
782 goto out;
783 }
784 offset -= length;
785 }
786 }
787
788out:
789 mutex_unlock(&table->mutex);
790 return addr;
791}
792
793int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
794 struct hns_roce_hem_table *table, u32 type,
795 unsigned long obj_size, unsigned long nobj)
796{
797 unsigned long obj_per_chunk;
798 unsigned long num_hem;
799
800 if (!hns_roce_check_whether_mhop(hr_dev, type)) {
801 table->table_chunk_size = hr_dev->caps.chunk_sz;
802 obj_per_chunk = table->table_chunk_size / obj_size;
803 num_hem = DIV_ROUND_UP(nobj, obj_per_chunk);
804
805 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
806 if (!table->hem)
807 return -ENOMEM;
808 } else {
809 struct hns_roce_hem_mhop mhop = {};
810 unsigned long buf_chunk_size;
811 unsigned long bt_chunk_size;
812 unsigned long bt_chunk_num;
813 unsigned long num_bt_l0;
814 u32 hop_num;
815
816 if (get_hem_table_config(hr_dev, &mhop, type))
817 return -EINVAL;
818
819 buf_chunk_size = mhop.buf_chunk_size;
820 bt_chunk_size = mhop.bt_chunk_size;
821 num_bt_l0 = mhop.ba_l0_num;
822 hop_num = mhop.hop_num;
823
824 obj_per_chunk = buf_chunk_size / obj_size;
825 num_hem = DIV_ROUND_UP(nobj, obj_per_chunk);
826 bt_chunk_num = bt_chunk_size / BA_BYTE_LEN;
827
828 if (type >= HEM_TYPE_MTT)
829 num_bt_l0 = bt_chunk_num;
830
831 table->hem = kcalloc(num_hem, sizeof(*table->hem),
832 GFP_KERNEL);
833 if (!table->hem)
834 goto err_kcalloc_hem_buf;
835
836 if (check_whether_bt_num_3(type, hop_num)) {
837 unsigned long num_bt_l1;
838
839 num_bt_l1 = DIV_ROUND_UP(num_hem, bt_chunk_num);
840 table->bt_l1 = kcalloc(num_bt_l1,
841 sizeof(*table->bt_l1),
842 GFP_KERNEL);
843 if (!table->bt_l1)
844 goto err_kcalloc_bt_l1;
845
846 table->bt_l1_dma_addr = kcalloc(num_bt_l1,
847 sizeof(*table->bt_l1_dma_addr),
848 GFP_KERNEL);
849
850 if (!table->bt_l1_dma_addr)
851 goto err_kcalloc_l1_dma;
852 }
853
854 if (check_whether_bt_num_2(type, hop_num) ||
855 check_whether_bt_num_3(type, hop_num)) {
856 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),
857 GFP_KERNEL);
858 if (!table->bt_l0)
859 goto err_kcalloc_bt_l0;
860
861 table->bt_l0_dma_addr = kcalloc(num_bt_l0,
862 sizeof(*table->bt_l0_dma_addr),
863 GFP_KERNEL);
864 if (!table->bt_l0_dma_addr)
865 goto err_kcalloc_l0_dma;
866 }
867 }
868
869 table->type = type;
870 table->num_hem = num_hem;
871 table->obj_size = obj_size;
872 mutex_init(&table->mutex);
873
874 return 0;
875
876err_kcalloc_l0_dma:
877 kfree(table->bt_l0);
878 table->bt_l0 = NULL;
879
880err_kcalloc_bt_l0:
881 kfree(table->bt_l1_dma_addr);
882 table->bt_l1_dma_addr = NULL;
883
884err_kcalloc_l1_dma:
885 kfree(table->bt_l1);
886 table->bt_l1 = NULL;
887
888err_kcalloc_bt_l1:
889 kfree(table->hem);
890 table->hem = NULL;
891
892err_kcalloc_hem_buf:
893 return -ENOMEM;
894}
895
896static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
897 struct hns_roce_hem_table *table)
898{
899 struct hns_roce_hem_mhop mhop;
900 u32 buf_chunk_size;
901 u64 obj;
902 int i;
903
904 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))
905 return;
906 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
907 mhop.bt_chunk_size;
908
909 for (i = 0; i < table->num_hem; ++i) {
910 obj = i * buf_chunk_size / table->obj_size;
911 if (table->hem[i])
912 hns_roce_table_mhop_put(hr_dev, table, obj, 0);
913 }
914
915 kfree(table->hem);
916 table->hem = NULL;
917 kfree(table->bt_l1);
918 table->bt_l1 = NULL;
919 kfree(table->bt_l1_dma_addr);
920 table->bt_l1_dma_addr = NULL;
921 kfree(table->bt_l0);
922 table->bt_l0 = NULL;
923 kfree(table->bt_l0_dma_addr);
924 table->bt_l0_dma_addr = NULL;
925}
926
927void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
928 struct hns_roce_hem_table *table)
929{
930 struct device *dev = hr_dev->dev;
931 unsigned long i;
932 int obj;
933 int ret;
934
935 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
936 hns_roce_cleanup_mhop_hem_table(hr_dev, table);
937 return;
938 }
939
940 for (i = 0; i < table->num_hem; ++i)
941 if (table->hem[i]) {
942 obj = i * table->table_chunk_size / table->obj_size;
943 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0);
944 if (ret)
945 dev_err(dev, "clear HEM base address failed, ret = %d.\n",
946 ret);
947
948 hns_roce_free_hem(hr_dev, table->hem[i]);
949 }
950
951 kfree(table->hem);
952}
953
954void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
955{
956 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
957 hns_roce_cleanup_hem_table(hr_dev,
958 &hr_dev->srq_table.table);
959 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
960 if (hr_dev->caps.qpc_timer_entry_sz)
961 hns_roce_cleanup_hem_table(hr_dev,
962 &hr_dev->qpc_timer_table);
963 if (hr_dev->caps.cqc_timer_entry_sz)
964 hns_roce_cleanup_hem_table(hr_dev,
965 &hr_dev->cqc_timer_table);
966 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)
967 hns_roce_cleanup_hem_table(hr_dev,
968 &hr_dev->qp_table.sccc_table);
969 if (hr_dev->caps.trrl_entry_sz)
970 hns_roce_cleanup_hem_table(hr_dev,
971 &hr_dev->qp_table.trrl_table);
972
973 if (hr_dev->caps.gmv_entry_sz)
974 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->gmv_table);
975
976 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
977 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
978 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
979}
980
981struct hns_roce_hem_item {
982 struct list_head list; /* link all hems in the same bt level */
983 struct list_head sibling; /* link all hems in last hop for mtt */
984 void *addr;
985 dma_addr_t dma_addr;
986 size_t count; /* max ba numbers */
987 int start; /* start buf offset in this hem */
988 int end; /* end buf offset in this hem */
989};
990
991/* All HEM items are linked in a tree structure */
992struct hns_roce_hem_head {
993 struct list_head branch[HNS_ROCE_MAX_BT_REGION];
994 struct list_head root;
995 struct list_head leaf;
996};
997
998static struct hns_roce_hem_item *
999hem_list_alloc_item(struct hns_roce_dev *hr_dev, int start, int end, int count,
1000 bool exist_bt)
1001{
1002 struct hns_roce_hem_item *hem;
1003
1004 hem = kzalloc(sizeof(*hem), GFP_KERNEL);
1005 if (!hem)
1006 return NULL;
1007
1008 if (exist_bt) {
1009 hem->addr = dma_alloc_coherent(hr_dev->dev, count * BA_BYTE_LEN,
1010 &hem->dma_addr, GFP_KERNEL);
1011 if (!hem->addr) {
1012 kfree(hem);
1013 return NULL;
1014 }
1015 }
1016
1017 hem->count = count;
1018 hem->start = start;
1019 hem->end = end;
1020 INIT_LIST_HEAD(&hem->list);
1021 INIT_LIST_HEAD(&hem->sibling);
1022
1023 return hem;
1024}
1025
1026static void hem_list_free_item(struct hns_roce_dev *hr_dev,
1027 struct hns_roce_hem_item *hem, bool exist_bt)
1028{
1029 if (exist_bt)
1030 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN,
1031 hem->addr, hem->dma_addr);
1032 kfree(hem);
1033}
1034
1035static void hem_list_free_all(struct hns_roce_dev *hr_dev,
1036 struct list_head *head, bool exist_bt)
1037{
1038 struct hns_roce_hem_item *hem, *temp_hem;
1039
1040 list_for_each_entry_safe(hem, temp_hem, head, list) {
1041 list_del(&hem->list);
1042 hem_list_free_item(hr_dev, hem, exist_bt);
1043 }
1044}
1045
1046static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr,
1047 u64 table_addr)
1048{
1049 *(u64 *)(base_addr) = table_addr;
1050}
1051
1052/* assign L0 table address to hem from root bt */
1053static void hem_list_assign_bt(struct hns_roce_dev *hr_dev,
1054 struct hns_roce_hem_item *hem, void *cpu_addr,
1055 u64 phy_addr)
1056{
1057 hem->addr = cpu_addr;
1058 hem->dma_addr = (dma_addr_t)phy_addr;
1059}
1060
1061static inline bool hem_list_page_is_in_range(struct hns_roce_hem_item *hem,
1062 int offset)
1063{
1064 return (hem->start <= offset && offset <= hem->end);
1065}
1066
1067static struct hns_roce_hem_item *hem_list_search_item(struct list_head *ba_list,
1068 int page_offset)
1069{
1070 struct hns_roce_hem_item *hem, *temp_hem;
1071 struct hns_roce_hem_item *found = NULL;
1072
1073 list_for_each_entry_safe(hem, temp_hem, ba_list, list) {
1074 if (hem_list_page_is_in_range(hem, page_offset)) {
1075 found = hem;
1076 break;
1077 }
1078 }
1079
1080 return found;
1081}
1082
1083static bool hem_list_is_bottom_bt(int hopnum, int bt_level)
1084{
1085 /*
1086 * hopnum base address table levels
1087 * 0 L0(buf)
1088 * 1 L0 -> buf
1089 * 2 L0 -> L1 -> buf
1090 * 3 L0 -> L1 -> L2 -> buf
1091 */
1092 return bt_level >= (hopnum ? hopnum - 1 : hopnum);
1093}
1094
1095/*
1096 * calc base address entries num
1097 * @hopnum: num of mutihop addressing
1098 * @bt_level: base address table level
1099 * @unit: ba entries per bt page
1100 */
1101static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit)
1102{
1103 u32 step;
1104 int max;
1105 int i;
1106
1107 if (hopnum <= bt_level)
1108 return 0;
1109 /*
1110 * hopnum bt_level range
1111 * 1 0 unit
1112 * ------------
1113 * 2 0 unit * unit
1114 * 2 1 unit
1115 * ------------
1116 * 3 0 unit * unit * unit
1117 * 3 1 unit * unit
1118 * 3 2 unit
1119 */
1120 step = 1;
1121 max = hopnum - bt_level;
1122 for (i = 0; i < max; i++)
1123 step = step * unit;
1124
1125 return step;
1126}
1127
1128/*
1129 * calc the root ba entries which could cover all regions
1130 * @regions: buf region array
1131 * @region_cnt: array size of @regions
1132 * @unit: ba entries per bt page
1133 */
1134int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,
1135 int region_cnt, int unit)
1136{
1137 struct hns_roce_buf_region *r;
1138 int total = 0;
1139 int step;
1140 int i;
1141
1142 for (i = 0; i < region_cnt; i++) {
1143 r = (struct hns_roce_buf_region *)®ions[i];
1144 if (r->hopnum > 1) {
1145 step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1146 if (step > 0)
1147 total += (r->count + step - 1) / step;
1148 } else {
1149 total += r->count;
1150 }
1151 }
1152
1153 return total;
1154}
1155
1156static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev,
1157 const struct hns_roce_buf_region *r, int unit,
1158 int offset, struct list_head *mid_bt,
1159 struct list_head *btm_bt)
1160{
1161 struct hns_roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL };
1162 struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL];
1163 struct hns_roce_hem_item *cur, *pre;
1164 const int hopnum = r->hopnum;
1165 int start_aligned;
1166 int distance;
1167 int ret = 0;
1168 int max_ofs;
1169 int level;
1170 u32 step;
1171 int end;
1172
1173 if (hopnum <= 1)
1174 return 0;
1175
1176 if (hopnum > HNS_ROCE_MAX_BT_LEVEL) {
1177 dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum);
1178 return -EINVAL;
1179 }
1180
1181 if (offset < r->offset) {
1182 dev_err(hr_dev->dev, "invalid offset %d, min %u!\n",
1183 offset, r->offset);
1184 return -EINVAL;
1185 }
1186
1187 distance = offset - r->offset;
1188 max_ofs = r->offset + r->count - 1;
1189 for (level = 0; level < hopnum; level++)
1190 INIT_LIST_HEAD(&temp_list[level]);
1191
1192 /* config L1 bt to last bt and link them to corresponding parent */
1193 for (level = 1; level < hopnum; level++) {
1194 cur = hem_list_search_item(&mid_bt[level], offset);
1195 if (cur) {
1196 hem_ptrs[level] = cur;
1197 continue;
1198 }
1199
1200 step = hem_list_calc_ba_range(hopnum, level, unit);
1201 if (step < 1) {
1202 ret = -EINVAL;
1203 goto err_exit;
1204 }
1205
1206 start_aligned = (distance / step) * step + r->offset;
1207 end = min_t(int, start_aligned + step - 1, max_ofs);
1208 cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit,
1209 true);
1210 if (!cur) {
1211 ret = -ENOMEM;
1212 goto err_exit;
1213 }
1214 hem_ptrs[level] = cur;
1215 list_add(&cur->list, &temp_list[level]);
1216 if (hem_list_is_bottom_bt(hopnum, level))
1217 list_add(&cur->sibling, &temp_list[0]);
1218
1219 /* link bt to parent bt */
1220 if (level > 1) {
1221 pre = hem_ptrs[level - 1];
1222 step = (cur->start - pre->start) / step * BA_BYTE_LEN;
1223 hem_list_link_bt(hr_dev, pre->addr + step,
1224 cur->dma_addr);
1225 }
1226 }
1227
1228 list_splice(&temp_list[0], btm_bt);
1229 for (level = 1; level < hopnum; level++)
1230 list_splice(&temp_list[level], &mid_bt[level]);
1231
1232 return 0;
1233
1234err_exit:
1235 for (level = 1; level < hopnum; level++)
1236 hem_list_free_all(hr_dev, &temp_list[level], true);
1237
1238 return ret;
1239}
1240
1241static struct hns_roce_hem_item *
1242alloc_root_hem(struct hns_roce_dev *hr_dev, int unit, int *max_ba_num,
1243 const struct hns_roce_buf_region *regions, int region_cnt)
1244{
1245 const struct hns_roce_buf_region *r;
1246 struct hns_roce_hem_item *hem;
1247 int ba_num;
1248 int offset;
1249
1250 ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit);
1251 if (ba_num < 1)
1252 return ERR_PTR(-ENOMEM);
1253
1254 if (ba_num > unit)
1255 return ERR_PTR(-ENOBUFS);
1256
1257 offset = regions[0].offset;
1258 /* indicate to last region */
1259 r = ®ions[region_cnt - 1];
1260 hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1,
1261 ba_num, true);
1262 if (!hem)
1263 return ERR_PTR(-ENOMEM);
1264
1265 *max_ba_num = ba_num;
1266
1267 return hem;
1268}
1269
1270static int alloc_fake_root_bt(struct hns_roce_dev *hr_dev, void *cpu_base,
1271 u64 phy_base, const struct hns_roce_buf_region *r,
1272 struct list_head *branch_head,
1273 struct list_head *leaf_head)
1274{
1275 struct hns_roce_hem_item *hem;
1276
1277 hem = hem_list_alloc_item(hr_dev, r->offset, r->offset + r->count - 1,
1278 r->count, false);
1279 if (!hem)
1280 return -ENOMEM;
1281
1282 hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base);
1283 list_add(&hem->list, branch_head);
1284 list_add(&hem->sibling, leaf_head);
1285
1286 return r->count;
1287}
1288
1289static int setup_middle_bt(struct hns_roce_dev *hr_dev, void *cpu_base,
1290 int unit, const struct hns_roce_buf_region *r,
1291 const struct list_head *branch_head)
1292{
1293 struct hns_roce_hem_item *hem, *temp_hem;
1294 int total = 0;
1295 int offset;
1296 int step;
1297
1298 step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1299 if (step < 1)
1300 return -EINVAL;
1301
1302 /* if exist mid bt, link L1 to L0 */
1303 list_for_each_entry_safe(hem, temp_hem, branch_head, list) {
1304 offset = (hem->start - r->offset) / step * BA_BYTE_LEN;
1305 hem_list_link_bt(hr_dev, cpu_base + offset, hem->dma_addr);
1306 total++;
1307 }
1308
1309 return total;
1310}
1311
1312static int
1313setup_root_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list,
1314 int unit, int max_ba_num, struct hns_roce_hem_head *head,
1315 const struct hns_roce_buf_region *regions, int region_cnt)
1316{
1317 const struct hns_roce_buf_region *r;
1318 struct hns_roce_hem_item *root_hem;
1319 void *cpu_base;
1320 u64 phy_base;
1321 int i, total;
1322 int ret;
1323
1324 root_hem = list_first_entry(&head->root,
1325 struct hns_roce_hem_item, list);
1326 if (!root_hem)
1327 return -ENOMEM;
1328
1329 total = 0;
1330 for (i = 0; i < region_cnt && total < max_ba_num; i++) {
1331 r = ®ions[i];
1332 if (!r->count)
1333 continue;
1334
1335 /* all regions's mid[x][0] shared the root_bt's trunk */
1336 cpu_base = root_hem->addr + total * BA_BYTE_LEN;
1337 phy_base = root_hem->dma_addr + total * BA_BYTE_LEN;
1338
1339 /* if hopnum is 0 or 1, cut a new fake hem from the root bt
1340 * which's address share to all regions.
1341 */
1342 if (hem_list_is_bottom_bt(r->hopnum, 0))
1343 ret = alloc_fake_root_bt(hr_dev, cpu_base, phy_base, r,
1344 &head->branch[i], &head->leaf);
1345 else
1346 ret = setup_middle_bt(hr_dev, cpu_base, unit, r,
1347 &hem_list->mid_bt[i][1]);
1348
1349 if (ret < 0)
1350 return ret;
1351
1352 total += ret;
1353 }
1354
1355 list_splice(&head->leaf, &hem_list->btm_bt);
1356 list_splice(&head->root, &hem_list->root_bt);
1357 for (i = 0; i < region_cnt; i++)
1358 list_splice(&head->branch[i], &hem_list->mid_bt[i][0]);
1359
1360 return 0;
1361}
1362
1363static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
1364 struct hns_roce_hem_list *hem_list, int unit,
1365 const struct hns_roce_buf_region *regions,
1366 int region_cnt)
1367{
1368 struct hns_roce_hem_item *root_hem;
1369 struct hns_roce_hem_head head;
1370 int max_ba_num;
1371 int ret;
1372 int i;
1373
1374 root_hem = hem_list_search_item(&hem_list->root_bt, regions[0].offset);
1375 if (root_hem)
1376 return 0;
1377
1378 max_ba_num = 0;
1379 root_hem = alloc_root_hem(hr_dev, unit, &max_ba_num, regions,
1380 region_cnt);
1381 if (IS_ERR(root_hem))
1382 return PTR_ERR(root_hem);
1383
1384 /* List head for storing all allocated HEM items */
1385 INIT_LIST_HEAD(&head.root);
1386 INIT_LIST_HEAD(&head.leaf);
1387 for (i = 0; i < region_cnt; i++)
1388 INIT_LIST_HEAD(&head.branch[i]);
1389
1390 hem_list->root_ba = root_hem->dma_addr;
1391 list_add(&root_hem->list, &head.root);
1392 ret = setup_root_hem(hr_dev, hem_list, unit, max_ba_num, &head, regions,
1393 region_cnt);
1394 if (ret) {
1395 for (i = 0; i < region_cnt; i++)
1396 hem_list_free_all(hr_dev, &head.branch[i], false);
1397
1398 hem_list_free_all(hr_dev, &head.root, true);
1399 }
1400
1401 return ret;
1402}
1403
1404/* construct the base address table and link them by address hop config */
1405int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
1406 struct hns_roce_hem_list *hem_list,
1407 const struct hns_roce_buf_region *regions,
1408 int region_cnt, unsigned int bt_pg_shift)
1409{
1410 const struct hns_roce_buf_region *r;
1411 int ofs, end;
1412 int unit;
1413 int ret;
1414 int i;
1415
1416 if (region_cnt > HNS_ROCE_MAX_BT_REGION) {
1417 dev_err(hr_dev->dev, "invalid region region_cnt %d!\n",
1418 region_cnt);
1419 return -EINVAL;
1420 }
1421
1422 unit = (1 << bt_pg_shift) / BA_BYTE_LEN;
1423 for (i = 0; i < region_cnt; i++) {
1424 r = ®ions[i];
1425 if (!r->count)
1426 continue;
1427
1428 end = r->offset + r->count;
1429 for (ofs = r->offset; ofs < end; ofs += unit) {
1430 ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs,
1431 hem_list->mid_bt[i],
1432 &hem_list->btm_bt);
1433 if (ret) {
1434 dev_err(hr_dev->dev,
1435 "alloc hem trunk fail ret = %d!\n", ret);
1436 goto err_alloc;
1437 }
1438 }
1439 }
1440
1441 ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions,
1442 region_cnt);
1443 if (ret)
1444 dev_err(hr_dev->dev, "alloc hem root fail ret = %d!\n", ret);
1445 else
1446 return 0;
1447
1448err_alloc:
1449 hns_roce_hem_list_release(hr_dev, hem_list);
1450
1451 return ret;
1452}
1453
1454void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,
1455 struct hns_roce_hem_list *hem_list)
1456{
1457 int i, j;
1458
1459 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1460 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1461 hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j],
1462 j != 0);
1463
1464 hem_list_free_all(hr_dev, &hem_list->root_bt, true);
1465 INIT_LIST_HEAD(&hem_list->btm_bt);
1466 hem_list->root_ba = 0;
1467}
1468
1469void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list)
1470{
1471 int i, j;
1472
1473 INIT_LIST_HEAD(&hem_list->root_bt);
1474 INIT_LIST_HEAD(&hem_list->btm_bt);
1475 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1476 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1477 INIT_LIST_HEAD(&hem_list->mid_bt[i][j]);
1478}
1479
1480void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
1481 struct hns_roce_hem_list *hem_list,
1482 int offset, int *mtt_cnt)
1483{
1484 struct list_head *head = &hem_list->btm_bt;
1485 struct hns_roce_hem_item *hem, *temp_hem;
1486 void *cpu_base = NULL;
1487 int nr = 0;
1488
1489 list_for_each_entry_safe(hem, temp_hem, head, sibling) {
1490 if (hem_list_page_is_in_range(hem, offset)) {
1491 nr = offset - hem->start;
1492 cpu_base = hem->addr + nr * BA_BYTE_LEN;
1493 nr = hem->end + 1 - offset;
1494 break;
1495 }
1496 }
1497
1498 if (mtt_cnt)
1499 *mtt_cnt = nr;
1500
1501 return cpu_base;
1502}
1/*
2 * Copyright (c) 2016 Hisilicon Limited.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#include "hns_roce_device.h"
35#include "hns_roce_hem.h"
36#include "hns_roce_common.h"
37
38#define HEM_INDEX_BUF BIT(0)
39#define HEM_INDEX_L0 BIT(1)
40#define HEM_INDEX_L1 BIT(2)
41struct hns_roce_hem_index {
42 u64 buf;
43 u64 l0;
44 u64 l1;
45 u32 inited; /* indicate which index is available */
46};
47
48bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
49{
50 int hop_num = 0;
51
52 switch (type) {
53 case HEM_TYPE_QPC:
54 hop_num = hr_dev->caps.qpc_hop_num;
55 break;
56 case HEM_TYPE_MTPT:
57 hop_num = hr_dev->caps.mpt_hop_num;
58 break;
59 case HEM_TYPE_CQC:
60 hop_num = hr_dev->caps.cqc_hop_num;
61 break;
62 case HEM_TYPE_SRQC:
63 hop_num = hr_dev->caps.srqc_hop_num;
64 break;
65 case HEM_TYPE_SCCC:
66 hop_num = hr_dev->caps.sccc_hop_num;
67 break;
68 case HEM_TYPE_QPC_TIMER:
69 hop_num = hr_dev->caps.qpc_timer_hop_num;
70 break;
71 case HEM_TYPE_CQC_TIMER:
72 hop_num = hr_dev->caps.cqc_timer_hop_num;
73 break;
74 case HEM_TYPE_GMV:
75 hop_num = hr_dev->caps.gmv_hop_num;
76 break;
77 default:
78 return false;
79 }
80
81 return hop_num;
82}
83
84static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx,
85 u32 bt_chunk_num, u64 hem_max_num)
86{
87 u64 start_idx = round_down(hem_idx, bt_chunk_num);
88 u64 check_max_num = start_idx + bt_chunk_num;
89 u64 i;
90
91 for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++)
92 if (i != hem_idx && hem[i])
93 return false;
94
95 return true;
96}
97
98static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num)
99{
100 u64 start_idx = round_down(ba_idx, bt_chunk_num);
101 int i;
102
103 for (i = 0; i < bt_chunk_num; i++)
104 if (i != ba_idx && bt[start_idx + i])
105 return false;
106
107 return true;
108}
109
110static int hns_roce_get_bt_num(u32 table_type, u32 hop_num)
111{
112 if (check_whether_bt_num_3(table_type, hop_num))
113 return 3;
114 else if (check_whether_bt_num_2(table_type, hop_num))
115 return 2;
116 else if (check_whether_bt_num_1(table_type, hop_num))
117 return 1;
118 else
119 return 0;
120}
121
122static int get_hem_table_config(struct hns_roce_dev *hr_dev,
123 struct hns_roce_hem_mhop *mhop,
124 u32 type)
125{
126 struct device *dev = hr_dev->dev;
127
128 switch (type) {
129 case HEM_TYPE_QPC:
130 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
131 + PAGE_SHIFT);
132 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
133 + PAGE_SHIFT);
134 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num;
135 mhop->hop_num = hr_dev->caps.qpc_hop_num;
136 break;
137 case HEM_TYPE_MTPT:
138 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
139 + PAGE_SHIFT);
140 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
141 + PAGE_SHIFT);
142 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num;
143 mhop->hop_num = hr_dev->caps.mpt_hop_num;
144 break;
145 case HEM_TYPE_CQC:
146 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
147 + PAGE_SHIFT);
148 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
149 + PAGE_SHIFT);
150 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
151 mhop->hop_num = hr_dev->caps.cqc_hop_num;
152 break;
153 case HEM_TYPE_SCCC:
154 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz
155 + PAGE_SHIFT);
156 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz
157 + PAGE_SHIFT);
158 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num;
159 mhop->hop_num = hr_dev->caps.sccc_hop_num;
160 break;
161 case HEM_TYPE_QPC_TIMER:
162 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz
163 + PAGE_SHIFT);
164 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz
165 + PAGE_SHIFT);
166 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num;
167 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num;
168 break;
169 case HEM_TYPE_CQC_TIMER:
170 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz
171 + PAGE_SHIFT);
172 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz
173 + PAGE_SHIFT);
174 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num;
175 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num;
176 break;
177 case HEM_TYPE_SRQC:
178 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
179 + PAGE_SHIFT);
180 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
181 + PAGE_SHIFT);
182 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
183 mhop->hop_num = hr_dev->caps.srqc_hop_num;
184 break;
185 case HEM_TYPE_GMV:
186 mhop->buf_chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz +
187 PAGE_SHIFT);
188 mhop->bt_chunk_size = 1 << (hr_dev->caps.gmv_ba_pg_sz +
189 PAGE_SHIFT);
190 mhop->ba_l0_num = hr_dev->caps.gmv_bt_num;
191 mhop->hop_num = hr_dev->caps.gmv_hop_num;
192 break;
193 default:
194 dev_err(dev, "table %u not support multi-hop addressing!\n",
195 type);
196 return -EINVAL;
197 }
198
199 return 0;
200}
201
202int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
203 struct hns_roce_hem_table *table, unsigned long *obj,
204 struct hns_roce_hem_mhop *mhop)
205{
206 struct device *dev = hr_dev->dev;
207 u32 chunk_ba_num;
208 u32 chunk_size;
209 u32 table_idx;
210 u32 bt_num;
211
212 if (get_hem_table_config(hr_dev, mhop, table->type))
213 return -EINVAL;
214
215 if (!obj)
216 return 0;
217
218 /*
219 * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages.
220 * MTT/CQE alloc hem for bt pages.
221 */
222 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
223 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
224 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
225 mhop->bt_chunk_size;
226 table_idx = *obj / (chunk_size / table->obj_size);
227 switch (bt_num) {
228 case 3:
229 mhop->l2_idx = table_idx & (chunk_ba_num - 1);
230 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1);
231 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num;
232 break;
233 case 2:
234 mhop->l1_idx = table_idx & (chunk_ba_num - 1);
235 mhop->l0_idx = table_idx / chunk_ba_num;
236 break;
237 case 1:
238 mhop->l0_idx = table_idx;
239 break;
240 default:
241 dev_err(dev, "table %u not support hop_num = %u!\n",
242 table->type, mhop->hop_num);
243 return -EINVAL;
244 }
245 if (mhop->l0_idx >= mhop->ba_l0_num)
246 mhop->l0_idx %= mhop->ba_l0_num;
247
248 return 0;
249}
250
251static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
252 unsigned long hem_alloc_size,
253 gfp_t gfp_mask)
254{
255 struct hns_roce_hem *hem;
256 int order;
257 void *buf;
258
259 WARN_ON(gfp_mask & __GFP_HIGHMEM);
260
261 order = get_order(hem_alloc_size);
262 if (PAGE_SIZE << order != hem_alloc_size) {
263 dev_err(hr_dev->dev, "invalid hem_alloc_size: %lu!\n",
264 hem_alloc_size);
265 return NULL;
266 }
267
268 hem = kmalloc(sizeof(*hem),
269 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
270 if (!hem)
271 return NULL;
272
273 buf = dma_alloc_coherent(hr_dev->dev, hem_alloc_size,
274 &hem->dma, gfp_mask);
275 if (!buf)
276 goto fail;
277
278 hem->buf = buf;
279 hem->size = hem_alloc_size;
280
281 return hem;
282
283fail:
284 kfree(hem);
285 return NULL;
286}
287
288void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
289{
290 if (!hem)
291 return;
292
293 dma_free_coherent(hr_dev->dev, hem->size, hem->buf, hem->dma);
294
295 kfree(hem);
296}
297
298static int calc_hem_config(struct hns_roce_dev *hr_dev,
299 struct hns_roce_hem_table *table, unsigned long obj,
300 struct hns_roce_hem_mhop *mhop,
301 struct hns_roce_hem_index *index)
302{
303 struct ib_device *ibdev = &hr_dev->ib_dev;
304 unsigned long mhop_obj = obj;
305 u32 l0_idx, l1_idx, l2_idx;
306 u32 chunk_ba_num;
307 u32 bt_num;
308 int ret;
309
310 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop);
311 if (ret)
312 return ret;
313
314 l0_idx = mhop->l0_idx;
315 l1_idx = mhop->l1_idx;
316 l2_idx = mhop->l2_idx;
317 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
318 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
319 switch (bt_num) {
320 case 3:
321 index->l1 = l0_idx * chunk_ba_num + l1_idx;
322 index->l0 = l0_idx;
323 index->buf = l0_idx * chunk_ba_num * chunk_ba_num +
324 l1_idx * chunk_ba_num + l2_idx;
325 break;
326 case 2:
327 index->l0 = l0_idx;
328 index->buf = l0_idx * chunk_ba_num + l1_idx;
329 break;
330 case 1:
331 index->buf = l0_idx;
332 break;
333 default:
334 ibdev_err(ibdev, "table %u not support mhop.hop_num = %u!\n",
335 table->type, mhop->hop_num);
336 return -EINVAL;
337 }
338
339 if (unlikely(index->buf >= table->num_hem)) {
340 ibdev_err(ibdev, "table %u exceed hem limt idx %llu, max %lu!\n",
341 table->type, index->buf, table->num_hem);
342 return -EINVAL;
343 }
344
345 return 0;
346}
347
348static void free_mhop_hem(struct hns_roce_dev *hr_dev,
349 struct hns_roce_hem_table *table,
350 struct hns_roce_hem_mhop *mhop,
351 struct hns_roce_hem_index *index)
352{
353 u32 bt_size = mhop->bt_chunk_size;
354 struct device *dev = hr_dev->dev;
355
356 if (index->inited & HEM_INDEX_BUF) {
357 hns_roce_free_hem(hr_dev, table->hem[index->buf]);
358 table->hem[index->buf] = NULL;
359 }
360
361 if (index->inited & HEM_INDEX_L1) {
362 dma_free_coherent(dev, bt_size, table->bt_l1[index->l1],
363 table->bt_l1_dma_addr[index->l1]);
364 table->bt_l1[index->l1] = NULL;
365 }
366
367 if (index->inited & HEM_INDEX_L0) {
368 dma_free_coherent(dev, bt_size, table->bt_l0[index->l0],
369 table->bt_l0_dma_addr[index->l0]);
370 table->bt_l0[index->l0] = NULL;
371 }
372}
373
374static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,
375 struct hns_roce_hem_table *table,
376 struct hns_roce_hem_mhop *mhop,
377 struct hns_roce_hem_index *index)
378{
379 u32 bt_size = mhop->bt_chunk_size;
380 struct device *dev = hr_dev->dev;
381 gfp_t flag;
382 u64 bt_ba;
383 u32 size;
384 int ret;
385
386 /* alloc L1 BA's chunk */
387 if ((check_whether_bt_num_3(table->type, mhop->hop_num) ||
388 check_whether_bt_num_2(table->type, mhop->hop_num)) &&
389 !table->bt_l0[index->l0]) {
390 table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size,
391 &table->bt_l0_dma_addr[index->l0],
392 GFP_KERNEL);
393 if (!table->bt_l0[index->l0]) {
394 ret = -ENOMEM;
395 goto out;
396 }
397 index->inited |= HEM_INDEX_L0;
398 }
399
400 /* alloc L2 BA's chunk */
401 if (check_whether_bt_num_3(table->type, mhop->hop_num) &&
402 !table->bt_l1[index->l1]) {
403 table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size,
404 &table->bt_l1_dma_addr[index->l1],
405 GFP_KERNEL);
406 if (!table->bt_l1[index->l1]) {
407 ret = -ENOMEM;
408 goto err_alloc_hem;
409 }
410 index->inited |= HEM_INDEX_L1;
411 *(table->bt_l0[index->l0] + mhop->l1_idx) =
412 table->bt_l1_dma_addr[index->l1];
413 }
414
415 /*
416 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC.
417 * alloc bt space chunk for MTT/CQE.
418 */
419 size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size;
420 flag = GFP_KERNEL | __GFP_NOWARN;
421 table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size, flag);
422 if (!table->hem[index->buf]) {
423 ret = -ENOMEM;
424 goto err_alloc_hem;
425 }
426
427 index->inited |= HEM_INDEX_BUF;
428 bt_ba = table->hem[index->buf]->dma;
429
430 if (table->type < HEM_TYPE_MTT) {
431 if (mhop->hop_num == 2)
432 *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba;
433 else if (mhop->hop_num == 1)
434 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
435 } else if (mhop->hop_num == 2) {
436 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
437 }
438
439 return 0;
440err_alloc_hem:
441 free_mhop_hem(hr_dev, table, mhop, index);
442out:
443 return ret;
444}
445
446static int set_mhop_hem(struct hns_roce_dev *hr_dev,
447 struct hns_roce_hem_table *table, unsigned long obj,
448 struct hns_roce_hem_mhop *mhop,
449 struct hns_roce_hem_index *index)
450{
451 struct ib_device *ibdev = &hr_dev->ib_dev;
452 u32 step_idx;
453 int ret = 0;
454
455 if (index->inited & HEM_INDEX_L0) {
456 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0);
457 if (ret) {
458 ibdev_err(ibdev, "set HEM step 0 failed!\n");
459 goto out;
460 }
461 }
462
463 if (index->inited & HEM_INDEX_L1) {
464 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1);
465 if (ret) {
466 ibdev_err(ibdev, "set HEM step 1 failed!\n");
467 goto out;
468 }
469 }
470
471 if (index->inited & HEM_INDEX_BUF) {
472 if (mhop->hop_num == HNS_ROCE_HOP_NUM_0)
473 step_idx = 0;
474 else
475 step_idx = mhop->hop_num;
476 ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx);
477 if (ret)
478 ibdev_err(ibdev, "set HEM step last failed!\n");
479 }
480out:
481 return ret;
482}
483
484static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
485 struct hns_roce_hem_table *table,
486 unsigned long obj)
487{
488 struct ib_device *ibdev = &hr_dev->ib_dev;
489 struct hns_roce_hem_index index = {};
490 struct hns_roce_hem_mhop mhop = {};
491 int ret;
492
493 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
494 if (ret) {
495 ibdev_err(ibdev, "calc hem config failed!\n");
496 return ret;
497 }
498
499 mutex_lock(&table->mutex);
500 if (table->hem[index.buf]) {
501 refcount_inc(&table->hem[index.buf]->refcount);
502 goto out;
503 }
504
505 ret = alloc_mhop_hem(hr_dev, table, &mhop, &index);
506 if (ret) {
507 ibdev_err(ibdev, "alloc mhop hem failed!\n");
508 goto out;
509 }
510
511 /* set HEM base address to hardware */
512 if (table->type < HEM_TYPE_MTT) {
513 ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index);
514 if (ret) {
515 ibdev_err(ibdev, "set HEM address to HW failed!\n");
516 goto err_alloc;
517 }
518 }
519
520 refcount_set(&table->hem[index.buf]->refcount, 1);
521 goto out;
522
523err_alloc:
524 free_mhop_hem(hr_dev, table, &mhop, &index);
525out:
526 mutex_unlock(&table->mutex);
527 return ret;
528}
529
530int hns_roce_table_get(struct hns_roce_dev *hr_dev,
531 struct hns_roce_hem_table *table, unsigned long obj)
532{
533 struct device *dev = hr_dev->dev;
534 unsigned long i;
535 int ret = 0;
536
537 if (hns_roce_check_whether_mhop(hr_dev, table->type))
538 return hns_roce_table_mhop_get(hr_dev, table, obj);
539
540 i = obj / (table->table_chunk_size / table->obj_size);
541
542 mutex_lock(&table->mutex);
543
544 if (table->hem[i]) {
545 refcount_inc(&table->hem[i]->refcount);
546 goto out;
547 }
548
549 table->hem[i] = hns_roce_alloc_hem(hr_dev,
550 table->table_chunk_size,
551 GFP_KERNEL | __GFP_NOWARN);
552 if (!table->hem[i]) {
553 ret = -ENOMEM;
554 goto out;
555 }
556
557 /* Set HEM base address(128K/page, pa) to Hardware */
558 ret = hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT);
559 if (ret) {
560 hns_roce_free_hem(hr_dev, table->hem[i]);
561 table->hem[i] = NULL;
562 dev_err(dev, "set HEM base address to HW failed, ret = %d.\n",
563 ret);
564 goto out;
565 }
566
567 refcount_set(&table->hem[i]->refcount, 1);
568out:
569 mutex_unlock(&table->mutex);
570 return ret;
571}
572
573static void clear_mhop_hem(struct hns_roce_dev *hr_dev,
574 struct hns_roce_hem_table *table, unsigned long obj,
575 struct hns_roce_hem_mhop *mhop,
576 struct hns_roce_hem_index *index)
577{
578 struct ib_device *ibdev = &hr_dev->ib_dev;
579 u32 hop_num = mhop->hop_num;
580 u32 chunk_ba_num;
581 u32 step_idx;
582 int ret;
583
584 index->inited = HEM_INDEX_BUF;
585 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
586 if (check_whether_bt_num_2(table->type, hop_num)) {
587 if (hns_roce_check_hem_null(table->hem, index->buf,
588 chunk_ba_num, table->num_hem))
589 index->inited |= HEM_INDEX_L0;
590 } else if (check_whether_bt_num_3(table->type, hop_num)) {
591 if (hns_roce_check_hem_null(table->hem, index->buf,
592 chunk_ba_num, table->num_hem)) {
593 index->inited |= HEM_INDEX_L1;
594 if (hns_roce_check_bt_null(table->bt_l1, index->l1,
595 chunk_ba_num))
596 index->inited |= HEM_INDEX_L0;
597 }
598 }
599
600 if (table->type < HEM_TYPE_MTT) {
601 if (hop_num == HNS_ROCE_HOP_NUM_0)
602 step_idx = 0;
603 else
604 step_idx = hop_num;
605
606 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx);
607 if (ret)
608 ibdev_warn(ibdev, "failed to clear hop%u HEM, ret = %d.\n",
609 hop_num, ret);
610
611 if (index->inited & HEM_INDEX_L1) {
612 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 1);
613 if (ret)
614 ibdev_warn(ibdev, "failed to clear HEM step 1, ret = %d.\n",
615 ret);
616 }
617
618 if (index->inited & HEM_INDEX_L0) {
619 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0);
620 if (ret)
621 ibdev_warn(ibdev, "failed to clear HEM step 0, ret = %d.\n",
622 ret);
623 }
624 }
625}
626
627static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
628 struct hns_roce_hem_table *table,
629 unsigned long obj,
630 int check_refcount)
631{
632 struct ib_device *ibdev = &hr_dev->ib_dev;
633 struct hns_roce_hem_index index = {};
634 struct hns_roce_hem_mhop mhop = {};
635 int ret;
636
637 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
638 if (ret) {
639 ibdev_err(ibdev, "calc hem config failed!\n");
640 return;
641 }
642
643 if (!check_refcount)
644 mutex_lock(&table->mutex);
645 else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount,
646 &table->mutex))
647 return;
648
649 clear_mhop_hem(hr_dev, table, obj, &mhop, &index);
650 free_mhop_hem(hr_dev, table, &mhop, &index);
651
652 mutex_unlock(&table->mutex);
653}
654
655void hns_roce_table_put(struct hns_roce_dev *hr_dev,
656 struct hns_roce_hem_table *table, unsigned long obj)
657{
658 struct device *dev = hr_dev->dev;
659 unsigned long i;
660 int ret;
661
662 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
663 hns_roce_table_mhop_put(hr_dev, table, obj, 1);
664 return;
665 }
666
667 i = obj / (table->table_chunk_size / table->obj_size);
668
669 if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount,
670 &table->mutex))
671 return;
672
673 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT);
674 if (ret)
675 dev_warn(dev, "failed to clear HEM base address, ret = %d.\n",
676 ret);
677
678 hns_roce_free_hem(hr_dev, table->hem[i]);
679 table->hem[i] = NULL;
680
681 mutex_unlock(&table->mutex);
682}
683
684void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
685 struct hns_roce_hem_table *table,
686 unsigned long obj, dma_addr_t *dma_handle)
687{
688 struct hns_roce_hem_mhop mhop;
689 struct hns_roce_hem *hem;
690 unsigned long mhop_obj = obj;
691 unsigned long obj_per_chunk;
692 unsigned long idx_offset;
693 int offset, dma_offset;
694 void *addr = NULL;
695 u32 hem_idx = 0;
696 int i, j;
697
698 mutex_lock(&table->mutex);
699
700 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {
701 obj_per_chunk = table->table_chunk_size / table->obj_size;
702 hem = table->hem[obj / obj_per_chunk];
703 idx_offset = obj % obj_per_chunk;
704 dma_offset = offset = idx_offset * table->obj_size;
705 } else {
706 u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */
707
708 if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop))
709 goto out;
710 /* mtt mhop */
711 i = mhop.l0_idx;
712 j = mhop.l1_idx;
713 if (mhop.hop_num == 2)
714 hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j;
715 else if (mhop.hop_num == 1 ||
716 mhop.hop_num == HNS_ROCE_HOP_NUM_0)
717 hem_idx = i;
718
719 hem = table->hem[hem_idx];
720 dma_offset = offset = obj * seg_size % mhop.bt_chunk_size;
721 if (mhop.hop_num == 2)
722 dma_offset = offset = 0;
723 }
724
725 if (!hem)
726 goto out;
727
728 *dma_handle = hem->dma + dma_offset;
729 addr = hem->buf + offset;
730
731out:
732 mutex_unlock(&table->mutex);
733 return addr;
734}
735
736int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
737 struct hns_roce_hem_table *table, u32 type,
738 unsigned long obj_size, unsigned long nobj)
739{
740 unsigned long obj_per_chunk;
741 unsigned long num_hem;
742
743 if (!hns_roce_check_whether_mhop(hr_dev, type)) {
744 table->table_chunk_size = hr_dev->caps.chunk_sz;
745 obj_per_chunk = table->table_chunk_size / obj_size;
746 num_hem = DIV_ROUND_UP(nobj, obj_per_chunk);
747
748 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
749 if (!table->hem)
750 return -ENOMEM;
751 } else {
752 struct hns_roce_hem_mhop mhop = {};
753 unsigned long buf_chunk_size;
754 unsigned long bt_chunk_size;
755 unsigned long bt_chunk_num;
756 unsigned long num_bt_l0;
757 u32 hop_num;
758
759 if (get_hem_table_config(hr_dev, &mhop, type))
760 return -EINVAL;
761
762 buf_chunk_size = mhop.buf_chunk_size;
763 bt_chunk_size = mhop.bt_chunk_size;
764 num_bt_l0 = mhop.ba_l0_num;
765 hop_num = mhop.hop_num;
766
767 obj_per_chunk = buf_chunk_size / obj_size;
768 num_hem = DIV_ROUND_UP(nobj, obj_per_chunk);
769 bt_chunk_num = bt_chunk_size / BA_BYTE_LEN;
770
771 if (type >= HEM_TYPE_MTT)
772 num_bt_l0 = bt_chunk_num;
773
774 table->hem = kcalloc(num_hem, sizeof(*table->hem),
775 GFP_KERNEL);
776 if (!table->hem)
777 goto err_kcalloc_hem_buf;
778
779 if (check_whether_bt_num_3(type, hop_num)) {
780 unsigned long num_bt_l1;
781
782 num_bt_l1 = DIV_ROUND_UP(num_hem, bt_chunk_num);
783 table->bt_l1 = kcalloc(num_bt_l1,
784 sizeof(*table->bt_l1),
785 GFP_KERNEL);
786 if (!table->bt_l1)
787 goto err_kcalloc_bt_l1;
788
789 table->bt_l1_dma_addr = kcalloc(num_bt_l1,
790 sizeof(*table->bt_l1_dma_addr),
791 GFP_KERNEL);
792
793 if (!table->bt_l1_dma_addr)
794 goto err_kcalloc_l1_dma;
795 }
796
797 if (check_whether_bt_num_2(type, hop_num) ||
798 check_whether_bt_num_3(type, hop_num)) {
799 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),
800 GFP_KERNEL);
801 if (!table->bt_l0)
802 goto err_kcalloc_bt_l0;
803
804 table->bt_l0_dma_addr = kcalloc(num_bt_l0,
805 sizeof(*table->bt_l0_dma_addr),
806 GFP_KERNEL);
807 if (!table->bt_l0_dma_addr)
808 goto err_kcalloc_l0_dma;
809 }
810 }
811
812 table->type = type;
813 table->num_hem = num_hem;
814 table->obj_size = obj_size;
815 mutex_init(&table->mutex);
816
817 return 0;
818
819err_kcalloc_l0_dma:
820 kfree(table->bt_l0);
821 table->bt_l0 = NULL;
822
823err_kcalloc_bt_l0:
824 kfree(table->bt_l1_dma_addr);
825 table->bt_l1_dma_addr = NULL;
826
827err_kcalloc_l1_dma:
828 kfree(table->bt_l1);
829 table->bt_l1 = NULL;
830
831err_kcalloc_bt_l1:
832 kfree(table->hem);
833 table->hem = NULL;
834
835err_kcalloc_hem_buf:
836 return -ENOMEM;
837}
838
839static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
840 struct hns_roce_hem_table *table)
841{
842 struct hns_roce_hem_mhop mhop;
843 u32 buf_chunk_size;
844 u64 obj;
845 int i;
846
847 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))
848 return;
849 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
850 mhop.bt_chunk_size;
851
852 for (i = 0; i < table->num_hem; ++i) {
853 obj = i * buf_chunk_size / table->obj_size;
854 if (table->hem[i])
855 hns_roce_table_mhop_put(hr_dev, table, obj, 0);
856 }
857
858 kfree(table->hem);
859 table->hem = NULL;
860 kfree(table->bt_l1);
861 table->bt_l1 = NULL;
862 kfree(table->bt_l1_dma_addr);
863 table->bt_l1_dma_addr = NULL;
864 kfree(table->bt_l0);
865 table->bt_l0 = NULL;
866 kfree(table->bt_l0_dma_addr);
867 table->bt_l0_dma_addr = NULL;
868}
869
870void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
871 struct hns_roce_hem_table *table)
872{
873 struct device *dev = hr_dev->dev;
874 unsigned long i;
875 int obj;
876 int ret;
877
878 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
879 hns_roce_cleanup_mhop_hem_table(hr_dev, table);
880 return;
881 }
882
883 for (i = 0; i < table->num_hem; ++i)
884 if (table->hem[i]) {
885 obj = i * table->table_chunk_size / table->obj_size;
886 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0);
887 if (ret)
888 dev_err(dev, "clear HEM base address failed, ret = %d.\n",
889 ret);
890
891 hns_roce_free_hem(hr_dev, table->hem[i]);
892 }
893
894 kfree(table->hem);
895}
896
897void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
898{
899 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
900 hns_roce_cleanup_hem_table(hr_dev,
901 &hr_dev->srq_table.table);
902 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
903 if (hr_dev->caps.qpc_timer_entry_sz)
904 hns_roce_cleanup_hem_table(hr_dev,
905 &hr_dev->qpc_timer_table);
906 if (hr_dev->caps.cqc_timer_entry_sz)
907 hns_roce_cleanup_hem_table(hr_dev,
908 &hr_dev->cqc_timer_table);
909 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)
910 hns_roce_cleanup_hem_table(hr_dev,
911 &hr_dev->qp_table.sccc_table);
912 if (hr_dev->caps.trrl_entry_sz)
913 hns_roce_cleanup_hem_table(hr_dev,
914 &hr_dev->qp_table.trrl_table);
915
916 if (hr_dev->caps.gmv_entry_sz)
917 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->gmv_table);
918
919 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
920 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
921 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
922}
923
924struct hns_roce_hem_item {
925 struct list_head list; /* link all hems in the same bt level */
926 struct list_head sibling; /* link all hems in last hop for mtt */
927 void *addr;
928 dma_addr_t dma_addr;
929 size_t count; /* max ba numbers */
930 int start; /* start buf offset in this hem */
931 int end; /* end buf offset in this hem */
932};
933
934/* All HEM items are linked in a tree structure */
935struct hns_roce_hem_head {
936 struct list_head branch[HNS_ROCE_MAX_BT_REGION];
937 struct list_head root;
938 struct list_head leaf;
939};
940
941static struct hns_roce_hem_item *
942hem_list_alloc_item(struct hns_roce_dev *hr_dev, int start, int end, int count,
943 bool exist_bt)
944{
945 struct hns_roce_hem_item *hem;
946
947 hem = kzalloc(sizeof(*hem), GFP_KERNEL);
948 if (!hem)
949 return NULL;
950
951 if (exist_bt) {
952 hem->addr = dma_alloc_coherent(hr_dev->dev, count * BA_BYTE_LEN,
953 &hem->dma_addr, GFP_KERNEL);
954 if (!hem->addr) {
955 kfree(hem);
956 return NULL;
957 }
958 }
959
960 hem->count = count;
961 hem->start = start;
962 hem->end = end;
963 INIT_LIST_HEAD(&hem->list);
964 INIT_LIST_HEAD(&hem->sibling);
965
966 return hem;
967}
968
969static void hem_list_free_item(struct hns_roce_dev *hr_dev,
970 struct hns_roce_hem_item *hem, bool exist_bt)
971{
972 if (exist_bt)
973 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN,
974 hem->addr, hem->dma_addr);
975 kfree(hem);
976}
977
978static void hem_list_free_all(struct hns_roce_dev *hr_dev,
979 struct list_head *head, bool exist_bt)
980{
981 struct hns_roce_hem_item *hem, *temp_hem;
982
983 list_for_each_entry_safe(hem, temp_hem, head, list) {
984 list_del(&hem->list);
985 hem_list_free_item(hr_dev, hem, exist_bt);
986 }
987}
988
989static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr,
990 u64 table_addr)
991{
992 *(u64 *)(base_addr) = table_addr;
993}
994
995/* assign L0 table address to hem from root bt */
996static void hem_list_assign_bt(struct hns_roce_dev *hr_dev,
997 struct hns_roce_hem_item *hem, void *cpu_addr,
998 u64 phy_addr)
999{
1000 hem->addr = cpu_addr;
1001 hem->dma_addr = (dma_addr_t)phy_addr;
1002}
1003
1004static inline bool hem_list_page_is_in_range(struct hns_roce_hem_item *hem,
1005 int offset)
1006{
1007 return (hem->start <= offset && offset <= hem->end);
1008}
1009
1010static struct hns_roce_hem_item *hem_list_search_item(struct list_head *ba_list,
1011 int page_offset)
1012{
1013 struct hns_roce_hem_item *hem, *temp_hem;
1014 struct hns_roce_hem_item *found = NULL;
1015
1016 list_for_each_entry_safe(hem, temp_hem, ba_list, list) {
1017 if (hem_list_page_is_in_range(hem, page_offset)) {
1018 found = hem;
1019 break;
1020 }
1021 }
1022
1023 return found;
1024}
1025
1026static bool hem_list_is_bottom_bt(int hopnum, int bt_level)
1027{
1028 /*
1029 * hopnum base address table levels
1030 * 0 L0(buf)
1031 * 1 L0 -> buf
1032 * 2 L0 -> L1 -> buf
1033 * 3 L0 -> L1 -> L2 -> buf
1034 */
1035 return bt_level >= (hopnum ? hopnum - 1 : hopnum);
1036}
1037
1038/*
1039 * calc base address entries num
1040 * @hopnum: num of mutihop addressing
1041 * @bt_level: base address table level
1042 * @unit: ba entries per bt page
1043 */
1044static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit)
1045{
1046 u32 step;
1047 int max;
1048 int i;
1049
1050 if (hopnum <= bt_level)
1051 return 0;
1052 /*
1053 * hopnum bt_level range
1054 * 1 0 unit
1055 * ------------
1056 * 2 0 unit * unit
1057 * 2 1 unit
1058 * ------------
1059 * 3 0 unit * unit * unit
1060 * 3 1 unit * unit
1061 * 3 2 unit
1062 */
1063 step = 1;
1064 max = hopnum - bt_level;
1065 for (i = 0; i < max; i++)
1066 step = step * unit;
1067
1068 return step;
1069}
1070
1071/*
1072 * calc the root ba entries which could cover all regions
1073 * @regions: buf region array
1074 * @region_cnt: array size of @regions
1075 * @unit: ba entries per bt page
1076 */
1077int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,
1078 int region_cnt, int unit)
1079{
1080 struct hns_roce_buf_region *r;
1081 int total = 0;
1082 int step;
1083 int i;
1084
1085 for (i = 0; i < region_cnt; i++) {
1086 r = (struct hns_roce_buf_region *)®ions[i];
1087 if (r->hopnum > 1) {
1088 step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1089 if (step > 0)
1090 total += (r->count + step - 1) / step;
1091 } else {
1092 total += r->count;
1093 }
1094 }
1095
1096 return total;
1097}
1098
1099static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev,
1100 const struct hns_roce_buf_region *r, int unit,
1101 int offset, struct list_head *mid_bt,
1102 struct list_head *btm_bt)
1103{
1104 struct hns_roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL };
1105 struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL];
1106 struct hns_roce_hem_item *cur, *pre;
1107 const int hopnum = r->hopnum;
1108 int start_aligned;
1109 int distance;
1110 int ret = 0;
1111 int max_ofs;
1112 int level;
1113 u32 step;
1114 int end;
1115
1116 if (hopnum <= 1)
1117 return 0;
1118
1119 if (hopnum > HNS_ROCE_MAX_BT_LEVEL) {
1120 dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum);
1121 return -EINVAL;
1122 }
1123
1124 if (offset < r->offset) {
1125 dev_err(hr_dev->dev, "invalid offset %d, min %u!\n",
1126 offset, r->offset);
1127 return -EINVAL;
1128 }
1129
1130 distance = offset - r->offset;
1131 max_ofs = r->offset + r->count - 1;
1132 for (level = 0; level < hopnum; level++)
1133 INIT_LIST_HEAD(&temp_list[level]);
1134
1135 /* config L1 bt to last bt and link them to corresponding parent */
1136 for (level = 1; level < hopnum; level++) {
1137 cur = hem_list_search_item(&mid_bt[level], offset);
1138 if (cur) {
1139 hem_ptrs[level] = cur;
1140 continue;
1141 }
1142
1143 step = hem_list_calc_ba_range(hopnum, level, unit);
1144 if (step < 1) {
1145 ret = -EINVAL;
1146 goto err_exit;
1147 }
1148
1149 start_aligned = (distance / step) * step + r->offset;
1150 end = min_t(int, start_aligned + step - 1, max_ofs);
1151 cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit,
1152 true);
1153 if (!cur) {
1154 ret = -ENOMEM;
1155 goto err_exit;
1156 }
1157 hem_ptrs[level] = cur;
1158 list_add(&cur->list, &temp_list[level]);
1159 if (hem_list_is_bottom_bt(hopnum, level))
1160 list_add(&cur->sibling, &temp_list[0]);
1161
1162 /* link bt to parent bt */
1163 if (level > 1) {
1164 pre = hem_ptrs[level - 1];
1165 step = (cur->start - pre->start) / step * BA_BYTE_LEN;
1166 hem_list_link_bt(hr_dev, pre->addr + step,
1167 cur->dma_addr);
1168 }
1169 }
1170
1171 list_splice(&temp_list[0], btm_bt);
1172 for (level = 1; level < hopnum; level++)
1173 list_splice(&temp_list[level], &mid_bt[level]);
1174
1175 return 0;
1176
1177err_exit:
1178 for (level = 1; level < hopnum; level++)
1179 hem_list_free_all(hr_dev, &temp_list[level], true);
1180
1181 return ret;
1182}
1183
1184static struct hns_roce_hem_item *
1185alloc_root_hem(struct hns_roce_dev *hr_dev, int unit, int *max_ba_num,
1186 const struct hns_roce_buf_region *regions, int region_cnt)
1187{
1188 const struct hns_roce_buf_region *r;
1189 struct hns_roce_hem_item *hem;
1190 int ba_num;
1191 int offset;
1192
1193 ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit);
1194 if (ba_num < 1)
1195 return ERR_PTR(-ENOMEM);
1196
1197 if (ba_num > unit)
1198 return ERR_PTR(-ENOBUFS);
1199
1200 offset = regions[0].offset;
1201 /* indicate to last region */
1202 r = ®ions[region_cnt - 1];
1203 hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1,
1204 ba_num, true);
1205 if (!hem)
1206 return ERR_PTR(-ENOMEM);
1207
1208 *max_ba_num = ba_num;
1209
1210 return hem;
1211}
1212
1213static int alloc_fake_root_bt(struct hns_roce_dev *hr_dev, void *cpu_base,
1214 u64 phy_base, const struct hns_roce_buf_region *r,
1215 struct list_head *branch_head,
1216 struct list_head *leaf_head)
1217{
1218 struct hns_roce_hem_item *hem;
1219
1220 hem = hem_list_alloc_item(hr_dev, r->offset, r->offset + r->count - 1,
1221 r->count, false);
1222 if (!hem)
1223 return -ENOMEM;
1224
1225 hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base);
1226 list_add(&hem->list, branch_head);
1227 list_add(&hem->sibling, leaf_head);
1228
1229 return r->count;
1230}
1231
1232static int setup_middle_bt(struct hns_roce_dev *hr_dev, void *cpu_base,
1233 int unit, const struct hns_roce_buf_region *r,
1234 const struct list_head *branch_head)
1235{
1236 struct hns_roce_hem_item *hem, *temp_hem;
1237 int total = 0;
1238 int offset;
1239 int step;
1240
1241 step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1242 if (step < 1)
1243 return -EINVAL;
1244
1245 /* if exist mid bt, link L1 to L0 */
1246 list_for_each_entry_safe(hem, temp_hem, branch_head, list) {
1247 offset = (hem->start - r->offset) / step * BA_BYTE_LEN;
1248 hem_list_link_bt(hr_dev, cpu_base + offset, hem->dma_addr);
1249 total++;
1250 }
1251
1252 return total;
1253}
1254
1255static int
1256setup_root_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list,
1257 int unit, int max_ba_num, struct hns_roce_hem_head *head,
1258 const struct hns_roce_buf_region *regions, int region_cnt)
1259{
1260 const struct hns_roce_buf_region *r;
1261 struct hns_roce_hem_item *root_hem;
1262 void *cpu_base;
1263 u64 phy_base;
1264 int i, total;
1265 int ret;
1266
1267 root_hem = list_first_entry(&head->root,
1268 struct hns_roce_hem_item, list);
1269 if (!root_hem)
1270 return -ENOMEM;
1271
1272 total = 0;
1273 for (i = 0; i < region_cnt && total < max_ba_num; i++) {
1274 r = ®ions[i];
1275 if (!r->count)
1276 continue;
1277
1278 /* all regions's mid[x][0] shared the root_bt's trunk */
1279 cpu_base = root_hem->addr + total * BA_BYTE_LEN;
1280 phy_base = root_hem->dma_addr + total * BA_BYTE_LEN;
1281
1282 /* if hopnum is 0 or 1, cut a new fake hem from the root bt
1283 * which's address share to all regions.
1284 */
1285 if (hem_list_is_bottom_bt(r->hopnum, 0))
1286 ret = alloc_fake_root_bt(hr_dev, cpu_base, phy_base, r,
1287 &head->branch[i], &head->leaf);
1288 else
1289 ret = setup_middle_bt(hr_dev, cpu_base, unit, r,
1290 &hem_list->mid_bt[i][1]);
1291
1292 if (ret < 0)
1293 return ret;
1294
1295 total += ret;
1296 }
1297
1298 list_splice(&head->leaf, &hem_list->btm_bt);
1299 list_splice(&head->root, &hem_list->root_bt);
1300 for (i = 0; i < region_cnt; i++)
1301 list_splice(&head->branch[i], &hem_list->mid_bt[i][0]);
1302
1303 return 0;
1304}
1305
1306static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
1307 struct hns_roce_hem_list *hem_list, int unit,
1308 const struct hns_roce_buf_region *regions,
1309 int region_cnt)
1310{
1311 struct hns_roce_hem_item *root_hem;
1312 struct hns_roce_hem_head head;
1313 int max_ba_num;
1314 int ret;
1315 int i;
1316
1317 root_hem = hem_list_search_item(&hem_list->root_bt, regions[0].offset);
1318 if (root_hem)
1319 return 0;
1320
1321 max_ba_num = 0;
1322 root_hem = alloc_root_hem(hr_dev, unit, &max_ba_num, regions,
1323 region_cnt);
1324 if (IS_ERR(root_hem))
1325 return PTR_ERR(root_hem);
1326
1327 /* List head for storing all allocated HEM items */
1328 INIT_LIST_HEAD(&head.root);
1329 INIT_LIST_HEAD(&head.leaf);
1330 for (i = 0; i < region_cnt; i++)
1331 INIT_LIST_HEAD(&head.branch[i]);
1332
1333 hem_list->root_ba = root_hem->dma_addr;
1334 list_add(&root_hem->list, &head.root);
1335 ret = setup_root_hem(hr_dev, hem_list, unit, max_ba_num, &head, regions,
1336 region_cnt);
1337 if (ret) {
1338 for (i = 0; i < region_cnt; i++)
1339 hem_list_free_all(hr_dev, &head.branch[i], false);
1340
1341 hem_list_free_all(hr_dev, &head.root, true);
1342 }
1343
1344 return ret;
1345}
1346
1347/* construct the base address table and link them by address hop config */
1348int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
1349 struct hns_roce_hem_list *hem_list,
1350 const struct hns_roce_buf_region *regions,
1351 int region_cnt, unsigned int bt_pg_shift)
1352{
1353 const struct hns_roce_buf_region *r;
1354 int ofs, end;
1355 int unit;
1356 int ret;
1357 int i;
1358
1359 if (region_cnt > HNS_ROCE_MAX_BT_REGION) {
1360 dev_err(hr_dev->dev, "invalid region region_cnt %d!\n",
1361 region_cnt);
1362 return -EINVAL;
1363 }
1364
1365 unit = (1 << bt_pg_shift) / BA_BYTE_LEN;
1366 for (i = 0; i < region_cnt; i++) {
1367 r = ®ions[i];
1368 if (!r->count)
1369 continue;
1370
1371 end = r->offset + r->count;
1372 for (ofs = r->offset; ofs < end; ofs += unit) {
1373 ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs,
1374 hem_list->mid_bt[i],
1375 &hem_list->btm_bt);
1376 if (ret) {
1377 dev_err(hr_dev->dev,
1378 "alloc hem trunk fail ret = %d!\n", ret);
1379 goto err_alloc;
1380 }
1381 }
1382 }
1383
1384 ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions,
1385 region_cnt);
1386 if (ret)
1387 dev_err(hr_dev->dev, "alloc hem root fail ret = %d!\n", ret);
1388 else
1389 return 0;
1390
1391err_alloc:
1392 hns_roce_hem_list_release(hr_dev, hem_list);
1393
1394 return ret;
1395}
1396
1397void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,
1398 struct hns_roce_hem_list *hem_list)
1399{
1400 int i, j;
1401
1402 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1403 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1404 hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j],
1405 j != 0);
1406
1407 hem_list_free_all(hr_dev, &hem_list->root_bt, true);
1408 INIT_LIST_HEAD(&hem_list->btm_bt);
1409 hem_list->root_ba = 0;
1410}
1411
1412void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list)
1413{
1414 int i, j;
1415
1416 INIT_LIST_HEAD(&hem_list->root_bt);
1417 INIT_LIST_HEAD(&hem_list->btm_bt);
1418 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1419 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1420 INIT_LIST_HEAD(&hem_list->mid_bt[i][j]);
1421}
1422
1423void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
1424 struct hns_roce_hem_list *hem_list,
1425 int offset, int *mtt_cnt)
1426{
1427 struct list_head *head = &hem_list->btm_bt;
1428 struct hns_roce_hem_item *hem, *temp_hem;
1429 void *cpu_base = NULL;
1430 int nr = 0;
1431
1432 list_for_each_entry_safe(hem, temp_hem, head, sibling) {
1433 if (hem_list_page_is_in_range(hem, offset)) {
1434 nr = offset - hem->start;
1435 cpu_base = hem->addr + nr * BA_BYTE_LEN;
1436 nr = hem->end + 1 - offset;
1437 break;
1438 }
1439 }
1440
1441 if (mtt_cnt)
1442 *mtt_cnt = nr;
1443
1444 return cpu_base;
1445}