Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0+
2// Copyright 2017 IBM Corp.
3#include <linux/module.h>
4#include <linux/pci.h>
5#include <linux/idr.h>
6#include <asm/pnv-ocxl.h>
7#include "ocxl_internal.h"
8
9/*
10 * Any opencapi device which wants to use this 'generic' driver should
11 * use the 0x062B device ID. Vendors should define the subsystem
12 * vendor/device ID to help differentiate devices.
13 */
14static const struct pci_device_id ocxl_pci_tbl[] = {
15 { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x062B), },
16 { }
17};
18MODULE_DEVICE_TABLE(pci, ocxl_pci_tbl);
19
20
21static struct ocxl_fn *ocxl_fn_get(struct ocxl_fn *fn)
22{
23 return (get_device(&fn->dev) == NULL) ? NULL : fn;
24}
25
26static void ocxl_fn_put(struct ocxl_fn *fn)
27{
28 put_device(&fn->dev);
29}
30
31struct ocxl_afu *ocxl_afu_get(struct ocxl_afu *afu)
32{
33 return (get_device(&afu->dev) == NULL) ? NULL : afu;
34}
35
36void ocxl_afu_put(struct ocxl_afu *afu)
37{
38 put_device(&afu->dev);
39}
40
41static struct ocxl_afu *alloc_afu(struct ocxl_fn *fn)
42{
43 struct ocxl_afu *afu;
44
45 afu = kzalloc(sizeof(struct ocxl_afu), GFP_KERNEL);
46 if (!afu)
47 return NULL;
48
49 mutex_init(&afu->contexts_lock);
50 mutex_init(&afu->afu_control_lock);
51 idr_init(&afu->contexts_idr);
52 afu->fn = fn;
53 ocxl_fn_get(fn);
54 return afu;
55}
56
57static void free_afu(struct ocxl_afu *afu)
58{
59 idr_destroy(&afu->contexts_idr);
60 ocxl_fn_put(afu->fn);
61 kfree(afu);
62}
63
64static void free_afu_dev(struct device *dev)
65{
66 struct ocxl_afu *afu = to_ocxl_afu(dev);
67
68 ocxl_unregister_afu(afu);
69 free_afu(afu);
70}
71
72static int set_afu_device(struct ocxl_afu *afu, const char *location)
73{
74 struct ocxl_fn *fn = afu->fn;
75 int rc;
76
77 afu->dev.parent = &fn->dev;
78 afu->dev.release = free_afu_dev;
79 rc = dev_set_name(&afu->dev, "%s.%s.%hhu", afu->config.name, location,
80 afu->config.idx);
81 return rc;
82}
83
84static int assign_afu_actag(struct ocxl_afu *afu, struct pci_dev *dev)
85{
86 struct ocxl_fn *fn = afu->fn;
87 int actag_count, actag_offset;
88
89 /*
90 * if there were not enough actags for the function, each afu
91 * reduces its count as well
92 */
93 actag_count = afu->config.actag_supported *
94 fn->actag_enabled / fn->actag_supported;
95 actag_offset = ocxl_actag_afu_alloc(fn, actag_count);
96 if (actag_offset < 0) {
97 dev_err(&afu->dev, "Can't allocate %d actags for AFU: %d\n",
98 actag_count, actag_offset);
99 return actag_offset;
100 }
101 afu->actag_base = fn->actag_base + actag_offset;
102 afu->actag_enabled = actag_count;
103
104 ocxl_config_set_afu_actag(dev, afu->config.dvsec_afu_control_pos,
105 afu->actag_base, afu->actag_enabled);
106 dev_dbg(&afu->dev, "actag base=%d enabled=%d\n",
107 afu->actag_base, afu->actag_enabled);
108 return 0;
109}
110
111static void reclaim_afu_actag(struct ocxl_afu *afu)
112{
113 struct ocxl_fn *fn = afu->fn;
114 int start_offset, size;
115
116 start_offset = afu->actag_base - fn->actag_base;
117 size = afu->actag_enabled;
118 ocxl_actag_afu_free(afu->fn, start_offset, size);
119}
120
121static int assign_afu_pasid(struct ocxl_afu *afu, struct pci_dev *dev)
122{
123 struct ocxl_fn *fn = afu->fn;
124 int pasid_count, pasid_offset;
125
126 /*
127 * We only support the case where the function configuration
128 * requested enough PASIDs to cover all AFUs.
129 */
130 pasid_count = 1 << afu->config.pasid_supported_log;
131 pasid_offset = ocxl_pasid_afu_alloc(fn, pasid_count);
132 if (pasid_offset < 0) {
133 dev_err(&afu->dev, "Can't allocate %d PASIDs for AFU: %d\n",
134 pasid_count, pasid_offset);
135 return pasid_offset;
136 }
137 afu->pasid_base = fn->pasid_base + pasid_offset;
138 afu->pasid_count = 0;
139 afu->pasid_max = pasid_count;
140
141 ocxl_config_set_afu_pasid(dev, afu->config.dvsec_afu_control_pos,
142 afu->pasid_base,
143 afu->config.pasid_supported_log);
144 dev_dbg(&afu->dev, "PASID base=%d, enabled=%d\n",
145 afu->pasid_base, pasid_count);
146 return 0;
147}
148
149static void reclaim_afu_pasid(struct ocxl_afu *afu)
150{
151 struct ocxl_fn *fn = afu->fn;
152 int start_offset, size;
153
154 start_offset = afu->pasid_base - fn->pasid_base;
155 size = 1 << afu->config.pasid_supported_log;
156 ocxl_pasid_afu_free(afu->fn, start_offset, size);
157}
158
159static int reserve_fn_bar(struct ocxl_fn *fn, int bar)
160{
161 struct pci_dev *dev = to_pci_dev(fn->dev.parent);
162 int rc, idx;
163
164 if (bar != 0 && bar != 2 && bar != 4)
165 return -EINVAL;
166
167 idx = bar >> 1;
168 if (fn->bar_used[idx]++ == 0) {
169 rc = pci_request_region(dev, bar, "ocxl");
170 if (rc)
171 return rc;
172 }
173 return 0;
174}
175
176static void release_fn_bar(struct ocxl_fn *fn, int bar)
177{
178 struct pci_dev *dev = to_pci_dev(fn->dev.parent);
179 int idx;
180
181 if (bar != 0 && bar != 2 && bar != 4)
182 return;
183
184 idx = bar >> 1;
185 if (--fn->bar_used[idx] == 0)
186 pci_release_region(dev, bar);
187 WARN_ON(fn->bar_used[idx] < 0);
188}
189
190static int map_mmio_areas(struct ocxl_afu *afu, struct pci_dev *dev)
191{
192 int rc;
193
194 rc = reserve_fn_bar(afu->fn, afu->config.global_mmio_bar);
195 if (rc)
196 return rc;
197
198 rc = reserve_fn_bar(afu->fn, afu->config.pp_mmio_bar);
199 if (rc) {
200 release_fn_bar(afu->fn, afu->config.global_mmio_bar);
201 return rc;
202 }
203
204 afu->global_mmio_start =
205 pci_resource_start(dev, afu->config.global_mmio_bar) +
206 afu->config.global_mmio_offset;
207 afu->pp_mmio_start =
208 pci_resource_start(dev, afu->config.pp_mmio_bar) +
209 afu->config.pp_mmio_offset;
210
211 afu->global_mmio_ptr = ioremap(afu->global_mmio_start,
212 afu->config.global_mmio_size);
213 if (!afu->global_mmio_ptr) {
214 release_fn_bar(afu->fn, afu->config.pp_mmio_bar);
215 release_fn_bar(afu->fn, afu->config.global_mmio_bar);
216 dev_err(&dev->dev, "Error mapping global mmio area\n");
217 return -ENOMEM;
218 }
219
220 /*
221 * Leave an empty page between the per-process mmio area and
222 * the AFU interrupt mappings
223 */
224 afu->irq_base_offset = afu->config.pp_mmio_stride + PAGE_SIZE;
225 return 0;
226}
227
228static void unmap_mmio_areas(struct ocxl_afu *afu)
229{
230 if (afu->global_mmio_ptr) {
231 iounmap(afu->global_mmio_ptr);
232 afu->global_mmio_ptr = NULL;
233 }
234 afu->global_mmio_start = 0;
235 afu->pp_mmio_start = 0;
236 release_fn_bar(afu->fn, afu->config.pp_mmio_bar);
237 release_fn_bar(afu->fn, afu->config.global_mmio_bar);
238}
239
240static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev)
241{
242 int rc;
243
244 rc = ocxl_config_read_afu(dev, &afu->fn->config, &afu->config, afu_idx);
245 if (rc)
246 return rc;
247
248 rc = set_afu_device(afu, dev_name(&dev->dev));
249 if (rc)
250 return rc;
251
252 rc = assign_afu_actag(afu, dev);
253 if (rc)
254 return rc;
255
256 rc = assign_afu_pasid(afu, dev);
257 if (rc) {
258 reclaim_afu_actag(afu);
259 return rc;
260 }
261
262 rc = map_mmio_areas(afu, dev);
263 if (rc) {
264 reclaim_afu_pasid(afu);
265 reclaim_afu_actag(afu);
266 return rc;
267 }
268 return 0;
269}
270
271static void deconfigure_afu(struct ocxl_afu *afu)
272{
273 unmap_mmio_areas(afu);
274 reclaim_afu_pasid(afu);
275 reclaim_afu_actag(afu);
276}
277
278static int activate_afu(struct pci_dev *dev, struct ocxl_afu *afu)
279{
280 int rc;
281
282 ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 1);
283 /*
284 * Char device creation is the last step, as processes can
285 * call our driver immediately, so all our inits must be finished.
286 */
287 rc = ocxl_create_cdev(afu);
288 if (rc)
289 return rc;
290 return 0;
291}
292
293static void deactivate_afu(struct ocxl_afu *afu)
294{
295 struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent);
296
297 ocxl_destroy_cdev(afu);
298 ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 0);
299}
300
301static int init_afu(struct pci_dev *dev, struct ocxl_fn *fn, u8 afu_idx)
302{
303 int rc;
304 struct ocxl_afu *afu;
305
306 afu = alloc_afu(fn);
307 if (!afu)
308 return -ENOMEM;
309
310 rc = configure_afu(afu, afu_idx, dev);
311 if (rc) {
312 free_afu(afu);
313 return rc;
314 }
315
316 rc = ocxl_register_afu(afu);
317 if (rc)
318 goto err;
319
320 rc = ocxl_sysfs_add_afu(afu);
321 if (rc)
322 goto err;
323
324 rc = activate_afu(dev, afu);
325 if (rc)
326 goto err_sys;
327
328 list_add_tail(&afu->list, &fn->afu_list);
329 return 0;
330
331err_sys:
332 ocxl_sysfs_remove_afu(afu);
333err:
334 deconfigure_afu(afu);
335 device_unregister(&afu->dev);
336 return rc;
337}
338
339static void remove_afu(struct ocxl_afu *afu)
340{
341 list_del(&afu->list);
342 ocxl_context_detach_all(afu);
343 deactivate_afu(afu);
344 ocxl_sysfs_remove_afu(afu);
345 deconfigure_afu(afu);
346 device_unregister(&afu->dev);
347}
348
349static struct ocxl_fn *alloc_function(struct pci_dev *dev)
350{
351 struct ocxl_fn *fn;
352
353 fn = kzalloc(sizeof(struct ocxl_fn), GFP_KERNEL);
354 if (!fn)
355 return NULL;
356
357 INIT_LIST_HEAD(&fn->afu_list);
358 INIT_LIST_HEAD(&fn->pasid_list);
359 INIT_LIST_HEAD(&fn->actag_list);
360 return fn;
361}
362
363static void free_function(struct ocxl_fn *fn)
364{
365 WARN_ON(!list_empty(&fn->afu_list));
366 WARN_ON(!list_empty(&fn->pasid_list));
367 kfree(fn);
368}
369
370static void free_function_dev(struct device *dev)
371{
372 struct ocxl_fn *fn = to_ocxl_function(dev);
373
374 free_function(fn);
375}
376
377static int set_function_device(struct ocxl_fn *fn, struct pci_dev *dev)
378{
379 int rc;
380
381 fn->dev.parent = &dev->dev;
382 fn->dev.release = free_function_dev;
383 rc = dev_set_name(&fn->dev, "ocxlfn.%s", dev_name(&dev->dev));
384 if (rc)
385 return rc;
386 pci_set_drvdata(dev, fn);
387 return 0;
388}
389
390static int assign_function_actag(struct ocxl_fn *fn)
391{
392 struct pci_dev *dev = to_pci_dev(fn->dev.parent);
393 u16 base, enabled, supported;
394 int rc;
395
396 rc = ocxl_config_get_actag_info(dev, &base, &enabled, &supported);
397 if (rc)
398 return rc;
399
400 fn->actag_base = base;
401 fn->actag_enabled = enabled;
402 fn->actag_supported = supported;
403
404 ocxl_config_set_actag(dev, fn->config.dvsec_function_pos,
405 fn->actag_base, fn->actag_enabled);
406 dev_dbg(&fn->dev, "actag range starting at %d, enabled %d\n",
407 fn->actag_base, fn->actag_enabled);
408 return 0;
409}
410
411static int set_function_pasid(struct ocxl_fn *fn)
412{
413 struct pci_dev *dev = to_pci_dev(fn->dev.parent);
414 int rc, desired_count, max_count;
415
416 /* A function may not require any PASID */
417 if (fn->config.max_pasid_log < 0)
418 return 0;
419
420 rc = ocxl_config_get_pasid_info(dev, &max_count);
421 if (rc)
422 return rc;
423
424 desired_count = 1 << fn->config.max_pasid_log;
425
426 if (desired_count > max_count) {
427 dev_err(&fn->dev,
428 "Function requires more PASIDs than is available (%d vs. %d)\n",
429 desired_count, max_count);
430 return -ENOSPC;
431 }
432
433 fn->pasid_base = 0;
434 return 0;
435}
436
437static int configure_function(struct ocxl_fn *fn, struct pci_dev *dev)
438{
439 int rc;
440
441 rc = pci_enable_device(dev);
442 if (rc) {
443 dev_err(&dev->dev, "pci_enable_device failed: %d\n", rc);
444 return rc;
445 }
446
447 /*
448 * Once it has been confirmed to work on our hardware, we
449 * should reset the function, to force the adapter to restart
450 * from scratch.
451 * A function reset would also reset all its AFUs.
452 *
453 * Some hints for implementation:
454 *
455 * - there's not status bit to know when the reset is done. We
456 * should try reading the config space to know when it's
457 * done.
458 * - probably something like:
459 * Reset
460 * wait 100ms
461 * issue config read
462 * allow device up to 1 sec to return success on config
463 * read before declaring it broken
464 *
465 * Some shared logic on the card (CFG, TLX) won't be reset, so
466 * there's no guarantee that it will be enough.
467 */
468 rc = ocxl_config_read_function(dev, &fn->config);
469 if (rc)
470 return rc;
471
472 rc = set_function_device(fn, dev);
473 if (rc)
474 return rc;
475
476 rc = assign_function_actag(fn);
477 if (rc)
478 return rc;
479
480 rc = set_function_pasid(fn);
481 if (rc)
482 return rc;
483
484 rc = ocxl_link_setup(dev, 0, &fn->link);
485 if (rc)
486 return rc;
487
488 rc = ocxl_config_set_TL(dev, fn->config.dvsec_tl_pos);
489 if (rc) {
490 ocxl_link_release(dev, fn->link);
491 return rc;
492 }
493 return 0;
494}
495
496static void deconfigure_function(struct ocxl_fn *fn)
497{
498 struct pci_dev *dev = to_pci_dev(fn->dev.parent);
499
500 ocxl_link_release(dev, fn->link);
501 pci_disable_device(dev);
502}
503
504static struct ocxl_fn *init_function(struct pci_dev *dev)
505{
506 struct ocxl_fn *fn;
507 int rc;
508
509 fn = alloc_function(dev);
510 if (!fn)
511 return ERR_PTR(-ENOMEM);
512
513 rc = configure_function(fn, dev);
514 if (rc) {
515 free_function(fn);
516 return ERR_PTR(rc);
517 }
518
519 rc = device_register(&fn->dev);
520 if (rc) {
521 deconfigure_function(fn);
522 put_device(&fn->dev);
523 return ERR_PTR(rc);
524 }
525 return fn;
526}
527
528static void remove_function(struct ocxl_fn *fn)
529{
530 deconfigure_function(fn);
531 device_unregister(&fn->dev);
532}
533
534static int ocxl_probe(struct pci_dev *dev, const struct pci_device_id *id)
535{
536 int rc, afu_count = 0;
537 u8 afu;
538 struct ocxl_fn *fn;
539
540 if (!radix_enabled()) {
541 dev_err(&dev->dev, "Unsupported memory model (hash)\n");
542 return -ENODEV;
543 }
544
545 fn = init_function(dev);
546 if (IS_ERR(fn)) {
547 dev_err(&dev->dev, "function init failed: %li\n",
548 PTR_ERR(fn));
549 return PTR_ERR(fn);
550 }
551
552 for (afu = 0; afu <= fn->config.max_afu_index; afu++) {
553 rc = ocxl_config_check_afu_index(dev, &fn->config, afu);
554 if (rc > 0) {
555 rc = init_afu(dev, fn, afu);
556 if (rc) {
557 dev_err(&dev->dev,
558 "Can't initialize AFU index %d\n", afu);
559 continue;
560 }
561 afu_count++;
562 }
563 }
564 dev_info(&dev->dev, "%d AFU(s) configured\n", afu_count);
565 return 0;
566}
567
568static void ocxl_remove(struct pci_dev *dev)
569{
570 struct ocxl_afu *afu, *tmp;
571 struct ocxl_fn *fn = pci_get_drvdata(dev);
572
573 list_for_each_entry_safe(afu, tmp, &fn->afu_list, list) {
574 remove_afu(afu);
575 }
576 remove_function(fn);
577}
578
579struct pci_driver ocxl_pci_driver = {
580 .name = "ocxl",
581 .id_table = ocxl_pci_tbl,
582 .probe = ocxl_probe,
583 .remove = ocxl_remove,
584 .shutdown = ocxl_remove,
585};