Loading...
Note: File does not exist in v5.9.
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3#include <linux/platform_device.h>
4#include <linux/memregion.h>
5#include <linux/workqueue.h>
6#include <linux/debugfs.h>
7#include <linux/device.h>
8#include <linux/module.h>
9#include <linux/pci.h>
10#include <linux/slab.h>
11#include <linux/idr.h>
12#include <linux/node.h>
13#include <cxlmem.h>
14#include <cxlpci.h>
15#include <cxl.h>
16#include "core.h"
17
18/**
19 * DOC: cxl core
20 *
21 * The CXL core provides a set of interfaces that can be consumed by CXL aware
22 * drivers. The interfaces allow for creation, modification, and destruction of
23 * regions, memory devices, ports, and decoders. CXL aware drivers must register
24 * with the CXL core via these interfaces in order to be able to participate in
25 * cross-device interleave coordination. The CXL core also establishes and
26 * maintains the bridge to the nvdimm subsystem.
27 *
28 * CXL core introduces sysfs hierarchy to control the devices that are
29 * instantiated by the core.
30 */
31
32/*
33 * All changes to the interleave configuration occur with this lock held
34 * for write.
35 */
36DECLARE_RWSEM(cxl_region_rwsem);
37
38static DEFINE_IDA(cxl_port_ida);
39static DEFINE_XARRAY(cxl_root_buses);
40
41int cxl_num_decoders_committed(struct cxl_port *port)
42{
43 lockdep_assert_held(&cxl_region_rwsem);
44
45 return port->commit_end + 1;
46}
47
48static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
49 char *buf)
50{
51 return sysfs_emit(buf, "%s\n", dev->type->name);
52}
53static DEVICE_ATTR_RO(devtype);
54
55static int cxl_device_id(const struct device *dev)
56{
57 if (dev->type == &cxl_nvdimm_bridge_type)
58 return CXL_DEVICE_NVDIMM_BRIDGE;
59 if (dev->type == &cxl_nvdimm_type)
60 return CXL_DEVICE_NVDIMM;
61 if (dev->type == CXL_PMEM_REGION_TYPE())
62 return CXL_DEVICE_PMEM_REGION;
63 if (dev->type == CXL_DAX_REGION_TYPE())
64 return CXL_DEVICE_DAX_REGION;
65 if (is_cxl_port(dev)) {
66 if (is_cxl_root(to_cxl_port(dev)))
67 return CXL_DEVICE_ROOT;
68 return CXL_DEVICE_PORT;
69 }
70 if (is_cxl_memdev(dev))
71 return CXL_DEVICE_MEMORY_EXPANDER;
72 if (dev->type == CXL_REGION_TYPE())
73 return CXL_DEVICE_REGION;
74 if (dev->type == &cxl_pmu_type)
75 return CXL_DEVICE_PMU;
76 return 0;
77}
78
79static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
80 char *buf)
81{
82 return sysfs_emit(buf, CXL_MODALIAS_FMT "\n", cxl_device_id(dev));
83}
84static DEVICE_ATTR_RO(modalias);
85
86static struct attribute *cxl_base_attributes[] = {
87 &dev_attr_devtype.attr,
88 &dev_attr_modalias.attr,
89 NULL,
90};
91
92struct attribute_group cxl_base_attribute_group = {
93 .attrs = cxl_base_attributes,
94};
95
96static ssize_t start_show(struct device *dev, struct device_attribute *attr,
97 char *buf)
98{
99 struct cxl_decoder *cxld = to_cxl_decoder(dev);
100
101 return sysfs_emit(buf, "%#llx\n", cxld->hpa_range.start);
102}
103static DEVICE_ATTR_ADMIN_RO(start);
104
105static ssize_t size_show(struct device *dev, struct device_attribute *attr,
106 char *buf)
107{
108 struct cxl_decoder *cxld = to_cxl_decoder(dev);
109
110 return sysfs_emit(buf, "%#llx\n", range_len(&cxld->hpa_range));
111}
112static DEVICE_ATTR_RO(size);
113
114#define CXL_DECODER_FLAG_ATTR(name, flag) \
115static ssize_t name##_show(struct device *dev, \
116 struct device_attribute *attr, char *buf) \
117{ \
118 struct cxl_decoder *cxld = to_cxl_decoder(dev); \
119 \
120 return sysfs_emit(buf, "%s\n", \
121 (cxld->flags & (flag)) ? "1" : "0"); \
122} \
123static DEVICE_ATTR_RO(name)
124
125CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM);
126CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM);
127CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2);
128CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3);
129CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK);
130
131static ssize_t target_type_show(struct device *dev,
132 struct device_attribute *attr, char *buf)
133{
134 struct cxl_decoder *cxld = to_cxl_decoder(dev);
135
136 switch (cxld->target_type) {
137 case CXL_DECODER_DEVMEM:
138 return sysfs_emit(buf, "accelerator\n");
139 case CXL_DECODER_HOSTONLYMEM:
140 return sysfs_emit(buf, "expander\n");
141 }
142 return -ENXIO;
143}
144static DEVICE_ATTR_RO(target_type);
145
146static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf)
147{
148 struct cxl_decoder *cxld = &cxlsd->cxld;
149 ssize_t offset = 0;
150 int i, rc = 0;
151
152 for (i = 0; i < cxld->interleave_ways; i++) {
153 struct cxl_dport *dport = cxlsd->target[i];
154 struct cxl_dport *next = NULL;
155
156 if (!dport)
157 break;
158
159 if (i + 1 < cxld->interleave_ways)
160 next = cxlsd->target[i + 1];
161 rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
162 next ? "," : "");
163 if (rc < 0)
164 return rc;
165 offset += rc;
166 }
167
168 return offset;
169}
170
171static ssize_t target_list_show(struct device *dev,
172 struct device_attribute *attr, char *buf)
173{
174 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
175 ssize_t offset;
176 int rc;
177
178 guard(rwsem_read)(&cxl_region_rwsem);
179 rc = emit_target_list(cxlsd, buf);
180 if (rc < 0)
181 return rc;
182 offset = rc;
183
184 rc = sysfs_emit_at(buf, offset, "\n");
185 if (rc < 0)
186 return rc;
187
188 return offset + rc;
189}
190static DEVICE_ATTR_RO(target_list);
191
192static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
193 char *buf)
194{
195 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
196
197 return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxled->mode));
198}
199
200static ssize_t mode_store(struct device *dev, struct device_attribute *attr,
201 const char *buf, size_t len)
202{
203 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
204 enum cxl_decoder_mode mode;
205 ssize_t rc;
206
207 if (sysfs_streq(buf, "pmem"))
208 mode = CXL_DECODER_PMEM;
209 else if (sysfs_streq(buf, "ram"))
210 mode = CXL_DECODER_RAM;
211 else
212 return -EINVAL;
213
214 rc = cxl_dpa_set_mode(cxled, mode);
215 if (rc)
216 return rc;
217
218 return len;
219}
220static DEVICE_ATTR_RW(mode);
221
222static ssize_t dpa_resource_show(struct device *dev, struct device_attribute *attr,
223 char *buf)
224{
225 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
226
227 guard(rwsem_read)(&cxl_dpa_rwsem);
228 return sysfs_emit(buf, "%#llx\n", (u64)cxl_dpa_resource_start(cxled));
229}
230static DEVICE_ATTR_RO(dpa_resource);
231
232static ssize_t dpa_size_show(struct device *dev, struct device_attribute *attr,
233 char *buf)
234{
235 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
236 resource_size_t size = cxl_dpa_size(cxled);
237
238 return sysfs_emit(buf, "%pa\n", &size);
239}
240
241static ssize_t dpa_size_store(struct device *dev, struct device_attribute *attr,
242 const char *buf, size_t len)
243{
244 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
245 unsigned long long size;
246 ssize_t rc;
247
248 rc = kstrtoull(buf, 0, &size);
249 if (rc)
250 return rc;
251
252 if (!IS_ALIGNED(size, SZ_256M))
253 return -EINVAL;
254
255 rc = cxl_dpa_free(cxled);
256 if (rc)
257 return rc;
258
259 if (size == 0)
260 return len;
261
262 rc = cxl_dpa_alloc(cxled, size);
263 if (rc)
264 return rc;
265
266 return len;
267}
268static DEVICE_ATTR_RW(dpa_size);
269
270static ssize_t interleave_granularity_show(struct device *dev,
271 struct device_attribute *attr,
272 char *buf)
273{
274 struct cxl_decoder *cxld = to_cxl_decoder(dev);
275
276 return sysfs_emit(buf, "%d\n", cxld->interleave_granularity);
277}
278
279static DEVICE_ATTR_RO(interleave_granularity);
280
281static ssize_t interleave_ways_show(struct device *dev,
282 struct device_attribute *attr, char *buf)
283{
284 struct cxl_decoder *cxld = to_cxl_decoder(dev);
285
286 return sysfs_emit(buf, "%d\n", cxld->interleave_ways);
287}
288
289static DEVICE_ATTR_RO(interleave_ways);
290
291static ssize_t qos_class_show(struct device *dev,
292 struct device_attribute *attr, char *buf)
293{
294 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
295
296 return sysfs_emit(buf, "%d\n", cxlrd->qos_class);
297}
298static DEVICE_ATTR_RO(qos_class);
299
300static struct attribute *cxl_decoder_base_attrs[] = {
301 &dev_attr_start.attr,
302 &dev_attr_size.attr,
303 &dev_attr_locked.attr,
304 &dev_attr_interleave_granularity.attr,
305 &dev_attr_interleave_ways.attr,
306 NULL,
307};
308
309static struct attribute_group cxl_decoder_base_attribute_group = {
310 .attrs = cxl_decoder_base_attrs,
311};
312
313static struct attribute *cxl_decoder_root_attrs[] = {
314 &dev_attr_cap_pmem.attr,
315 &dev_attr_cap_ram.attr,
316 &dev_attr_cap_type2.attr,
317 &dev_attr_cap_type3.attr,
318 &dev_attr_target_list.attr,
319 &dev_attr_qos_class.attr,
320 SET_CXL_REGION_ATTR(create_pmem_region)
321 SET_CXL_REGION_ATTR(create_ram_region)
322 SET_CXL_REGION_ATTR(delete_region)
323 NULL,
324};
325
326static bool can_create_pmem(struct cxl_root_decoder *cxlrd)
327{
328 unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_PMEM;
329
330 return (cxlrd->cxlsd.cxld.flags & flags) == flags;
331}
332
333static bool can_create_ram(struct cxl_root_decoder *cxlrd)
334{
335 unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_RAM;
336
337 return (cxlrd->cxlsd.cxld.flags & flags) == flags;
338}
339
340static umode_t cxl_root_decoder_visible(struct kobject *kobj, struct attribute *a, int n)
341{
342 struct device *dev = kobj_to_dev(kobj);
343 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
344
345 if (a == CXL_REGION_ATTR(create_pmem_region) && !can_create_pmem(cxlrd))
346 return 0;
347
348 if (a == CXL_REGION_ATTR(create_ram_region) && !can_create_ram(cxlrd))
349 return 0;
350
351 if (a == CXL_REGION_ATTR(delete_region) &&
352 !(can_create_pmem(cxlrd) || can_create_ram(cxlrd)))
353 return 0;
354
355 return a->mode;
356}
357
358static struct attribute_group cxl_decoder_root_attribute_group = {
359 .attrs = cxl_decoder_root_attrs,
360 .is_visible = cxl_root_decoder_visible,
361};
362
363static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
364 &cxl_decoder_root_attribute_group,
365 &cxl_decoder_base_attribute_group,
366 &cxl_base_attribute_group,
367 NULL,
368};
369
370static struct attribute *cxl_decoder_switch_attrs[] = {
371 &dev_attr_target_type.attr,
372 &dev_attr_target_list.attr,
373 SET_CXL_REGION_ATTR(region)
374 NULL,
375};
376
377static struct attribute_group cxl_decoder_switch_attribute_group = {
378 .attrs = cxl_decoder_switch_attrs,
379};
380
381static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = {
382 &cxl_decoder_switch_attribute_group,
383 &cxl_decoder_base_attribute_group,
384 &cxl_base_attribute_group,
385 NULL,
386};
387
388static struct attribute *cxl_decoder_endpoint_attrs[] = {
389 &dev_attr_target_type.attr,
390 &dev_attr_mode.attr,
391 &dev_attr_dpa_size.attr,
392 &dev_attr_dpa_resource.attr,
393 SET_CXL_REGION_ATTR(region)
394 NULL,
395};
396
397static struct attribute_group cxl_decoder_endpoint_attribute_group = {
398 .attrs = cxl_decoder_endpoint_attrs,
399};
400
401static const struct attribute_group *cxl_decoder_endpoint_attribute_groups[] = {
402 &cxl_decoder_base_attribute_group,
403 &cxl_decoder_endpoint_attribute_group,
404 &cxl_base_attribute_group,
405 NULL,
406};
407
408static void __cxl_decoder_release(struct cxl_decoder *cxld)
409{
410 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
411
412 ida_free(&port->decoder_ida, cxld->id);
413 put_device(&port->dev);
414}
415
416static void cxl_endpoint_decoder_release(struct device *dev)
417{
418 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
419
420 __cxl_decoder_release(&cxled->cxld);
421 kfree(cxled);
422}
423
424static void cxl_switch_decoder_release(struct device *dev)
425{
426 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
427
428 __cxl_decoder_release(&cxlsd->cxld);
429 kfree(cxlsd);
430}
431
432struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev)
433{
434 if (dev_WARN_ONCE(dev, !is_root_decoder(dev),
435 "not a cxl_root_decoder device\n"))
436 return NULL;
437 return container_of(dev, struct cxl_root_decoder, cxlsd.cxld.dev);
438}
439EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, CXL);
440
441static void cxl_root_decoder_release(struct device *dev)
442{
443 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
444
445 if (atomic_read(&cxlrd->region_id) >= 0)
446 memregion_free(atomic_read(&cxlrd->region_id));
447 __cxl_decoder_release(&cxlrd->cxlsd.cxld);
448 kfree(cxlrd);
449}
450
451static const struct device_type cxl_decoder_endpoint_type = {
452 .name = "cxl_decoder_endpoint",
453 .release = cxl_endpoint_decoder_release,
454 .groups = cxl_decoder_endpoint_attribute_groups,
455};
456
457static const struct device_type cxl_decoder_switch_type = {
458 .name = "cxl_decoder_switch",
459 .release = cxl_switch_decoder_release,
460 .groups = cxl_decoder_switch_attribute_groups,
461};
462
463static const struct device_type cxl_decoder_root_type = {
464 .name = "cxl_decoder_root",
465 .release = cxl_root_decoder_release,
466 .groups = cxl_decoder_root_attribute_groups,
467};
468
469bool is_endpoint_decoder(struct device *dev)
470{
471 return dev->type == &cxl_decoder_endpoint_type;
472}
473EXPORT_SYMBOL_NS_GPL(is_endpoint_decoder, CXL);
474
475bool is_root_decoder(struct device *dev)
476{
477 return dev->type == &cxl_decoder_root_type;
478}
479EXPORT_SYMBOL_NS_GPL(is_root_decoder, CXL);
480
481bool is_switch_decoder(struct device *dev)
482{
483 return is_root_decoder(dev) || dev->type == &cxl_decoder_switch_type;
484}
485EXPORT_SYMBOL_NS_GPL(is_switch_decoder, CXL);
486
487struct cxl_decoder *to_cxl_decoder(struct device *dev)
488{
489 if (dev_WARN_ONCE(dev,
490 !is_switch_decoder(dev) && !is_endpoint_decoder(dev),
491 "not a cxl_decoder device\n"))
492 return NULL;
493 return container_of(dev, struct cxl_decoder, dev);
494}
495EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, CXL);
496
497struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev)
498{
499 if (dev_WARN_ONCE(dev, !is_endpoint_decoder(dev),
500 "not a cxl_endpoint_decoder device\n"))
501 return NULL;
502 return container_of(dev, struct cxl_endpoint_decoder, cxld.dev);
503}
504EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, CXL);
505
506struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev)
507{
508 if (dev_WARN_ONCE(dev, !is_switch_decoder(dev),
509 "not a cxl_switch_decoder device\n"))
510 return NULL;
511 return container_of(dev, struct cxl_switch_decoder, cxld.dev);
512}
513EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, CXL);
514
515static void cxl_ep_release(struct cxl_ep *ep)
516{
517 put_device(ep->ep);
518 kfree(ep);
519}
520
521static void cxl_ep_remove(struct cxl_port *port, struct cxl_ep *ep)
522{
523 if (!ep)
524 return;
525 xa_erase(&port->endpoints, (unsigned long) ep->ep);
526 cxl_ep_release(ep);
527}
528
529static void cxl_port_release(struct device *dev)
530{
531 struct cxl_port *port = to_cxl_port(dev);
532 unsigned long index;
533 struct cxl_ep *ep;
534
535 xa_for_each(&port->endpoints, index, ep)
536 cxl_ep_remove(port, ep);
537 xa_destroy(&port->endpoints);
538 xa_destroy(&port->dports);
539 xa_destroy(&port->regions);
540 ida_free(&cxl_port_ida, port->id);
541 if (is_cxl_root(port))
542 kfree(to_cxl_root(port));
543 else
544 kfree(port);
545}
546
547static ssize_t decoders_committed_show(struct device *dev,
548 struct device_attribute *attr, char *buf)
549{
550 struct cxl_port *port = to_cxl_port(dev);
551 int rc;
552
553 down_read(&cxl_region_rwsem);
554 rc = sysfs_emit(buf, "%d\n", cxl_num_decoders_committed(port));
555 up_read(&cxl_region_rwsem);
556
557 return rc;
558}
559
560static DEVICE_ATTR_RO(decoders_committed);
561
562static struct attribute *cxl_port_attrs[] = {
563 &dev_attr_decoders_committed.attr,
564 NULL,
565};
566
567static struct attribute_group cxl_port_attribute_group = {
568 .attrs = cxl_port_attrs,
569};
570
571static const struct attribute_group *cxl_port_attribute_groups[] = {
572 &cxl_base_attribute_group,
573 &cxl_port_attribute_group,
574 NULL,
575};
576
577static const struct device_type cxl_port_type = {
578 .name = "cxl_port",
579 .release = cxl_port_release,
580 .groups = cxl_port_attribute_groups,
581};
582
583bool is_cxl_port(const struct device *dev)
584{
585 return dev->type == &cxl_port_type;
586}
587EXPORT_SYMBOL_NS_GPL(is_cxl_port, CXL);
588
589struct cxl_port *to_cxl_port(const struct device *dev)
590{
591 if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
592 "not a cxl_port device\n"))
593 return NULL;
594 return container_of(dev, struct cxl_port, dev);
595}
596EXPORT_SYMBOL_NS_GPL(to_cxl_port, CXL);
597
598static void unregister_port(void *_port)
599{
600 struct cxl_port *port = _port;
601 struct cxl_port *parent;
602 struct device *lock_dev;
603
604 if (is_cxl_root(port))
605 parent = NULL;
606 else
607 parent = to_cxl_port(port->dev.parent);
608
609 /*
610 * CXL root port's and the first level of ports are unregistered
611 * under the platform firmware device lock, all other ports are
612 * unregistered while holding their parent port lock.
613 */
614 if (!parent)
615 lock_dev = port->uport_dev;
616 else if (is_cxl_root(parent))
617 lock_dev = parent->uport_dev;
618 else
619 lock_dev = &parent->dev;
620
621 device_lock_assert(lock_dev);
622 port->dead = true;
623 device_unregister(&port->dev);
624}
625
626static void cxl_unlink_uport(void *_port)
627{
628 struct cxl_port *port = _port;
629
630 sysfs_remove_link(&port->dev.kobj, "uport");
631}
632
633static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
634{
635 int rc;
636
637 rc = sysfs_create_link(&port->dev.kobj, &port->uport_dev->kobj,
638 "uport");
639 if (rc)
640 return rc;
641 return devm_add_action_or_reset(host, cxl_unlink_uport, port);
642}
643
644static void cxl_unlink_parent_dport(void *_port)
645{
646 struct cxl_port *port = _port;
647
648 sysfs_remove_link(&port->dev.kobj, "parent_dport");
649}
650
651static int devm_cxl_link_parent_dport(struct device *host,
652 struct cxl_port *port,
653 struct cxl_dport *parent_dport)
654{
655 int rc;
656
657 if (!parent_dport)
658 return 0;
659
660 rc = sysfs_create_link(&port->dev.kobj, &parent_dport->dport_dev->kobj,
661 "parent_dport");
662 if (rc)
663 return rc;
664 return devm_add_action_or_reset(host, cxl_unlink_parent_dport, port);
665}
666
667static struct lock_class_key cxl_port_key;
668
669static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
670 struct cxl_dport *parent_dport)
671{
672 struct cxl_root *cxl_root __free(kfree) = NULL;
673 struct cxl_port *port, *_port __free(kfree) = NULL;
674 struct device *dev;
675 int rc;
676
677 /* No parent_dport, root cxl_port */
678 if (!parent_dport) {
679 cxl_root = kzalloc(sizeof(*cxl_root), GFP_KERNEL);
680 if (!cxl_root)
681 return ERR_PTR(-ENOMEM);
682 } else {
683 _port = kzalloc(sizeof(*port), GFP_KERNEL);
684 if (!_port)
685 return ERR_PTR(-ENOMEM);
686 }
687
688 rc = ida_alloc(&cxl_port_ida, GFP_KERNEL);
689 if (rc < 0)
690 return ERR_PTR(rc);
691
692 if (cxl_root)
693 port = &no_free_ptr(cxl_root)->port;
694 else
695 port = no_free_ptr(_port);
696
697 port->id = rc;
698 port->uport_dev = uport_dev;
699
700 /*
701 * The top-level cxl_port "cxl_root" does not have a cxl_port as
702 * its parent and it does not have any corresponding component
703 * registers as its decode is described by a fixed platform
704 * description.
705 */
706 dev = &port->dev;
707 if (parent_dport) {
708 struct cxl_port *parent_port = parent_dport->port;
709 struct cxl_port *iter;
710
711 dev->parent = &parent_port->dev;
712 port->depth = parent_port->depth + 1;
713 port->parent_dport = parent_dport;
714
715 /*
716 * walk to the host bridge, or the first ancestor that knows
717 * the host bridge
718 */
719 iter = port;
720 while (!iter->host_bridge &&
721 !is_cxl_root(to_cxl_port(iter->dev.parent)))
722 iter = to_cxl_port(iter->dev.parent);
723 if (iter->host_bridge)
724 port->host_bridge = iter->host_bridge;
725 else if (parent_dport->rch)
726 port->host_bridge = parent_dport->dport_dev;
727 else
728 port->host_bridge = iter->uport_dev;
729 dev_dbg(uport_dev, "host-bridge: %s\n",
730 dev_name(port->host_bridge));
731 } else
732 dev->parent = uport_dev;
733
734 ida_init(&port->decoder_ida);
735 port->hdm_end = -1;
736 port->commit_end = -1;
737 xa_init(&port->dports);
738 xa_init(&port->endpoints);
739 xa_init(&port->regions);
740
741 device_initialize(dev);
742 lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);
743 device_set_pm_not_required(dev);
744 dev->bus = &cxl_bus_type;
745 dev->type = &cxl_port_type;
746
747 return port;
748}
749
750static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map,
751 resource_size_t component_reg_phys)
752{
753 *map = (struct cxl_register_map) {
754 .host = host,
755 .reg_type = CXL_REGLOC_RBI_EMPTY,
756 .resource = component_reg_phys,
757 };
758
759 if (component_reg_phys == CXL_RESOURCE_NONE)
760 return 0;
761
762 map->reg_type = CXL_REGLOC_RBI_COMPONENT;
763 map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
764
765 return cxl_setup_regs(map);
766}
767
768static int cxl_port_setup_regs(struct cxl_port *port,
769 resource_size_t component_reg_phys)
770{
771 if (dev_is_platform(port->uport_dev))
772 return 0;
773 return cxl_setup_comp_regs(&port->dev, &port->reg_map,
774 component_reg_phys);
775}
776
777static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
778 resource_size_t component_reg_phys)
779{
780 int rc;
781
782 if (dev_is_platform(dport->dport_dev))
783 return 0;
784
785 /*
786 * use @dport->dport_dev for the context for error messages during
787 * register probing, and fixup @host after the fact, since @host may be
788 * NULL.
789 */
790 rc = cxl_setup_comp_regs(dport->dport_dev, &dport->reg_map,
791 component_reg_phys);
792 dport->reg_map.host = host;
793 return rc;
794}
795
796static struct cxl_port *__devm_cxl_add_port(struct device *host,
797 struct device *uport_dev,
798 resource_size_t component_reg_phys,
799 struct cxl_dport *parent_dport)
800{
801 struct cxl_port *port;
802 struct device *dev;
803 int rc;
804
805 port = cxl_port_alloc(uport_dev, parent_dport);
806 if (IS_ERR(port))
807 return port;
808
809 dev = &port->dev;
810 if (is_cxl_memdev(uport_dev)) {
811 struct cxl_memdev *cxlmd = to_cxl_memdev(uport_dev);
812 struct cxl_dev_state *cxlds = cxlmd->cxlds;
813
814 rc = dev_set_name(dev, "endpoint%d", port->id);
815 if (rc)
816 goto err;
817
818 /*
819 * The endpoint driver already enumerated the component and RAS
820 * registers. Reuse that enumeration while prepping them to be
821 * mapped by the cxl_port driver.
822 */
823 port->reg_map = cxlds->reg_map;
824 port->reg_map.host = &port->dev;
825 } else if (parent_dport) {
826 rc = dev_set_name(dev, "port%d", port->id);
827 if (rc)
828 goto err;
829
830 rc = cxl_port_setup_regs(port, component_reg_phys);
831 if (rc)
832 goto err;
833 } else
834 rc = dev_set_name(dev, "root%d", port->id);
835 if (rc)
836 goto err;
837
838 rc = device_add(dev);
839 if (rc)
840 goto err;
841
842 rc = devm_add_action_or_reset(host, unregister_port, port);
843 if (rc)
844 return ERR_PTR(rc);
845
846 rc = devm_cxl_link_uport(host, port);
847 if (rc)
848 return ERR_PTR(rc);
849
850 rc = devm_cxl_link_parent_dport(host, port, parent_dport);
851 if (rc)
852 return ERR_PTR(rc);
853
854 if (parent_dport && dev_is_pci(uport_dev))
855 port->pci_latency = cxl_pci_get_latency(to_pci_dev(uport_dev));
856
857 return port;
858
859err:
860 put_device(dev);
861 return ERR_PTR(rc);
862}
863
864/**
865 * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
866 * @host: host device for devm operations
867 * @uport_dev: "physical" device implementing this upstream port
868 * @component_reg_phys: (optional) for configurable cxl_port instances
869 * @parent_dport: next hop up in the CXL memory decode hierarchy
870 */
871struct cxl_port *devm_cxl_add_port(struct device *host,
872 struct device *uport_dev,
873 resource_size_t component_reg_phys,
874 struct cxl_dport *parent_dport)
875{
876 struct cxl_port *port, *parent_port;
877
878 port = __devm_cxl_add_port(host, uport_dev, component_reg_phys,
879 parent_dport);
880
881 parent_port = parent_dport ? parent_dport->port : NULL;
882 if (IS_ERR(port)) {
883 dev_dbg(uport_dev, "Failed to add%s%s%s: %ld\n",
884 parent_port ? " port to " : "",
885 parent_port ? dev_name(&parent_port->dev) : "",
886 parent_port ? "" : " root port",
887 PTR_ERR(port));
888 } else {
889 dev_dbg(uport_dev, "%s added%s%s%s\n",
890 dev_name(&port->dev),
891 parent_port ? " to " : "",
892 parent_port ? dev_name(&parent_port->dev) : "",
893 parent_port ? "" : " (root port)");
894 }
895
896 return port;
897}
898EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, CXL);
899
900struct cxl_root *devm_cxl_add_root(struct device *host,
901 const struct cxl_root_ops *ops)
902{
903 struct cxl_root *cxl_root;
904 struct cxl_port *port;
905
906 port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL);
907 if (IS_ERR(port))
908 return (struct cxl_root *)port;
909
910 cxl_root = to_cxl_root(port);
911 cxl_root->ops = ops;
912 return cxl_root;
913}
914EXPORT_SYMBOL_NS_GPL(devm_cxl_add_root, CXL);
915
916struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port)
917{
918 /* There is no pci_bus associated with a CXL platform-root port */
919 if (is_cxl_root(port))
920 return NULL;
921
922 if (dev_is_pci(port->uport_dev)) {
923 struct pci_dev *pdev = to_pci_dev(port->uport_dev);
924
925 return pdev->subordinate;
926 }
927
928 return xa_load(&cxl_root_buses, (unsigned long)port->uport_dev);
929}
930EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, CXL);
931
932static void unregister_pci_bus(void *uport_dev)
933{
934 xa_erase(&cxl_root_buses, (unsigned long)uport_dev);
935}
936
937int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev,
938 struct pci_bus *bus)
939{
940 int rc;
941
942 if (dev_is_pci(uport_dev))
943 return -EINVAL;
944
945 rc = xa_insert(&cxl_root_buses, (unsigned long)uport_dev, bus,
946 GFP_KERNEL);
947 if (rc)
948 return rc;
949 return devm_add_action_or_reset(host, unregister_pci_bus, uport_dev);
950}
951EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, CXL);
952
953static bool dev_is_cxl_root_child(struct device *dev)
954{
955 struct cxl_port *port, *parent;
956
957 if (!is_cxl_port(dev))
958 return false;
959
960 port = to_cxl_port(dev);
961 if (is_cxl_root(port))
962 return false;
963
964 parent = to_cxl_port(port->dev.parent);
965 if (is_cxl_root(parent))
966 return true;
967
968 return false;
969}
970
971struct cxl_root *find_cxl_root(struct cxl_port *port)
972{
973 struct cxl_port *iter = port;
974
975 while (iter && !is_cxl_root(iter))
976 iter = to_cxl_port(iter->dev.parent);
977
978 if (!iter)
979 return NULL;
980 get_device(&iter->dev);
981 return to_cxl_root(iter);
982}
983EXPORT_SYMBOL_NS_GPL(find_cxl_root, CXL);
984
985void put_cxl_root(struct cxl_root *cxl_root)
986{
987 if (!cxl_root)
988 return;
989
990 put_device(&cxl_root->port.dev);
991}
992EXPORT_SYMBOL_NS_GPL(put_cxl_root, CXL);
993
994static struct cxl_dport *find_dport(struct cxl_port *port, int id)
995{
996 struct cxl_dport *dport;
997 unsigned long index;
998
999 device_lock_assert(&port->dev);
1000 xa_for_each(&port->dports, index, dport)
1001 if (dport->port_id == id)
1002 return dport;
1003 return NULL;
1004}
1005
1006static int add_dport(struct cxl_port *port, struct cxl_dport *dport)
1007{
1008 struct cxl_dport *dup;
1009 int rc;
1010
1011 device_lock_assert(&port->dev);
1012 dup = find_dport(port, dport->port_id);
1013 if (dup) {
1014 dev_err(&port->dev,
1015 "unable to add dport%d-%s non-unique port id (%s)\n",
1016 dport->port_id, dev_name(dport->dport_dev),
1017 dev_name(dup->dport_dev));
1018 return -EBUSY;
1019 }
1020
1021 rc = xa_insert(&port->dports, (unsigned long)dport->dport_dev, dport,
1022 GFP_KERNEL);
1023 if (rc)
1024 return rc;
1025
1026 port->nr_dports++;
1027 return 0;
1028}
1029
1030/*
1031 * Since root-level CXL dports cannot be enumerated by PCI they are not
1032 * enumerated by the common port driver that acquires the port lock over
1033 * dport add/remove. Instead, root dports are manually added by a
1034 * platform driver and cond_cxl_root_lock() is used to take the missing
1035 * port lock in that case.
1036 */
1037static void cond_cxl_root_lock(struct cxl_port *port)
1038{
1039 if (is_cxl_root(port))
1040 device_lock(&port->dev);
1041}
1042
1043static void cond_cxl_root_unlock(struct cxl_port *port)
1044{
1045 if (is_cxl_root(port))
1046 device_unlock(&port->dev);
1047}
1048
1049static void cxl_dport_remove(void *data)
1050{
1051 struct cxl_dport *dport = data;
1052 struct cxl_port *port = dport->port;
1053
1054 xa_erase(&port->dports, (unsigned long) dport->dport_dev);
1055 put_device(dport->dport_dev);
1056}
1057
1058static void cxl_dport_unlink(void *data)
1059{
1060 struct cxl_dport *dport = data;
1061 struct cxl_port *port = dport->port;
1062 char link_name[CXL_TARGET_STRLEN];
1063
1064 sprintf(link_name, "dport%d", dport->port_id);
1065 sysfs_remove_link(&port->dev.kobj, link_name);
1066}
1067
1068static struct cxl_dport *
1069__devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
1070 int port_id, resource_size_t component_reg_phys,
1071 resource_size_t rcrb)
1072{
1073 char link_name[CXL_TARGET_STRLEN];
1074 struct cxl_dport *dport;
1075 struct device *host;
1076 int rc;
1077
1078 if (is_cxl_root(port))
1079 host = port->uport_dev;
1080 else
1081 host = &port->dev;
1082
1083 if (!host->driver) {
1084 dev_WARN_ONCE(&port->dev, 1, "dport:%s bad devm context\n",
1085 dev_name(dport_dev));
1086 return ERR_PTR(-ENXIO);
1087 }
1088
1089 if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >=
1090 CXL_TARGET_STRLEN)
1091 return ERR_PTR(-EINVAL);
1092
1093 dport = devm_kzalloc(host, sizeof(*dport), GFP_KERNEL);
1094 if (!dport)
1095 return ERR_PTR(-ENOMEM);
1096
1097 dport->dport_dev = dport_dev;
1098 dport->port_id = port_id;
1099 dport->port = port;
1100
1101 if (rcrb == CXL_RESOURCE_NONE) {
1102 rc = cxl_dport_setup_regs(&port->dev, dport,
1103 component_reg_phys);
1104 if (rc)
1105 return ERR_PTR(rc);
1106 } else {
1107 dport->rcrb.base = rcrb;
1108 component_reg_phys = __rcrb_to_component(dport_dev, &dport->rcrb,
1109 CXL_RCRB_DOWNSTREAM);
1110 if (component_reg_phys == CXL_RESOURCE_NONE) {
1111 dev_warn(dport_dev, "Invalid Component Registers in RCRB");
1112 return ERR_PTR(-ENXIO);
1113 }
1114
1115 /*
1116 * RCH @dport is not ready to map until associated with its
1117 * memdev
1118 */
1119 rc = cxl_dport_setup_regs(NULL, dport, component_reg_phys);
1120 if (rc)
1121 return ERR_PTR(rc);
1122
1123 dport->rch = true;
1124 }
1125
1126 if (component_reg_phys != CXL_RESOURCE_NONE)
1127 dev_dbg(dport_dev, "Component Registers found for dport: %pa\n",
1128 &component_reg_phys);
1129
1130 cond_cxl_root_lock(port);
1131 rc = add_dport(port, dport);
1132 cond_cxl_root_unlock(port);
1133 if (rc)
1134 return ERR_PTR(rc);
1135
1136 get_device(dport_dev);
1137 rc = devm_add_action_or_reset(host, cxl_dport_remove, dport);
1138 if (rc)
1139 return ERR_PTR(rc);
1140
1141 rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name);
1142 if (rc)
1143 return ERR_PTR(rc);
1144
1145 rc = devm_add_action_or_reset(host, cxl_dport_unlink, dport);
1146 if (rc)
1147 return ERR_PTR(rc);
1148
1149 if (dev_is_pci(dport_dev))
1150 dport->link_latency = cxl_pci_get_latency(to_pci_dev(dport_dev));
1151
1152 return dport;
1153}
1154
1155/**
1156 * devm_cxl_add_dport - append VH downstream port data to a cxl_port
1157 * @port: the cxl_port that references this dport
1158 * @dport_dev: firmware or PCI device representing the dport
1159 * @port_id: identifier for this dport in a decoder's target list
1160 * @component_reg_phys: optional location of CXL component registers
1161 *
1162 * Note that dports are appended to the devm release action's of the
1163 * either the port's host (for root ports), or the port itself (for
1164 * switch ports)
1165 */
1166struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
1167 struct device *dport_dev, int port_id,
1168 resource_size_t component_reg_phys)
1169{
1170 struct cxl_dport *dport;
1171
1172 dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1173 component_reg_phys, CXL_RESOURCE_NONE);
1174 if (IS_ERR(dport)) {
1175 dev_dbg(dport_dev, "failed to add dport to %s: %ld\n",
1176 dev_name(&port->dev), PTR_ERR(dport));
1177 } else {
1178 dev_dbg(dport_dev, "dport added to %s\n",
1179 dev_name(&port->dev));
1180 }
1181
1182 return dport;
1183}
1184EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL);
1185
1186/**
1187 * devm_cxl_add_rch_dport - append RCH downstream port data to a cxl_port
1188 * @port: the cxl_port that references this dport
1189 * @dport_dev: firmware or PCI device representing the dport
1190 * @port_id: identifier for this dport in a decoder's target list
1191 * @rcrb: mandatory location of a Root Complex Register Block
1192 *
1193 * See CXL 3.0 9.11.8 CXL Devices Attached to an RCH
1194 */
1195struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
1196 struct device *dport_dev, int port_id,
1197 resource_size_t rcrb)
1198{
1199 struct cxl_dport *dport;
1200
1201 if (rcrb == CXL_RESOURCE_NONE) {
1202 dev_dbg(&port->dev, "failed to add RCH dport, missing RCRB\n");
1203 return ERR_PTR(-EINVAL);
1204 }
1205
1206 dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1207 CXL_RESOURCE_NONE, rcrb);
1208 if (IS_ERR(dport)) {
1209 dev_dbg(dport_dev, "failed to add RCH dport to %s: %ld\n",
1210 dev_name(&port->dev), PTR_ERR(dport));
1211 } else {
1212 dev_dbg(dport_dev, "RCH dport added to %s\n",
1213 dev_name(&port->dev));
1214 }
1215
1216 return dport;
1217}
1218EXPORT_SYMBOL_NS_GPL(devm_cxl_add_rch_dport, CXL);
1219
1220static int add_ep(struct cxl_ep *new)
1221{
1222 struct cxl_port *port = new->dport->port;
1223 int rc;
1224
1225 device_lock(&port->dev);
1226 if (port->dead) {
1227 device_unlock(&port->dev);
1228 return -ENXIO;
1229 }
1230 rc = xa_insert(&port->endpoints, (unsigned long)new->ep, new,
1231 GFP_KERNEL);
1232 device_unlock(&port->dev);
1233
1234 return rc;
1235}
1236
1237/**
1238 * cxl_add_ep - register an endpoint's interest in a port
1239 * @dport: the dport that routes to @ep_dev
1240 * @ep_dev: device representing the endpoint
1241 *
1242 * Intermediate CXL ports are scanned based on the arrival of endpoints.
1243 * When those endpoints depart the port can be destroyed once all
1244 * endpoints that care about that port have been removed.
1245 */
1246static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev)
1247{
1248 struct cxl_ep *ep;
1249 int rc;
1250
1251 ep = kzalloc(sizeof(*ep), GFP_KERNEL);
1252 if (!ep)
1253 return -ENOMEM;
1254
1255 ep->ep = get_device(ep_dev);
1256 ep->dport = dport;
1257
1258 rc = add_ep(ep);
1259 if (rc)
1260 cxl_ep_release(ep);
1261 return rc;
1262}
1263
1264struct cxl_find_port_ctx {
1265 const struct device *dport_dev;
1266 const struct cxl_port *parent_port;
1267 struct cxl_dport **dport;
1268};
1269
1270static int match_port_by_dport(struct device *dev, const void *data)
1271{
1272 const struct cxl_find_port_ctx *ctx = data;
1273 struct cxl_dport *dport;
1274 struct cxl_port *port;
1275
1276 if (!is_cxl_port(dev))
1277 return 0;
1278 if (ctx->parent_port && dev->parent != &ctx->parent_port->dev)
1279 return 0;
1280
1281 port = to_cxl_port(dev);
1282 dport = cxl_find_dport_by_dev(port, ctx->dport_dev);
1283 if (ctx->dport)
1284 *ctx->dport = dport;
1285 return dport != NULL;
1286}
1287
1288static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx)
1289{
1290 struct device *dev;
1291
1292 if (!ctx->dport_dev)
1293 return NULL;
1294
1295 dev = bus_find_device(&cxl_bus_type, NULL, ctx, match_port_by_dport);
1296 if (dev)
1297 return to_cxl_port(dev);
1298 return NULL;
1299}
1300
1301static struct cxl_port *find_cxl_port(struct device *dport_dev,
1302 struct cxl_dport **dport)
1303{
1304 struct cxl_find_port_ctx ctx = {
1305 .dport_dev = dport_dev,
1306 .dport = dport,
1307 };
1308 struct cxl_port *port;
1309
1310 port = __find_cxl_port(&ctx);
1311 return port;
1312}
1313
1314static struct cxl_port *find_cxl_port_at(struct cxl_port *parent_port,
1315 struct device *dport_dev,
1316 struct cxl_dport **dport)
1317{
1318 struct cxl_find_port_ctx ctx = {
1319 .dport_dev = dport_dev,
1320 .parent_port = parent_port,
1321 .dport = dport,
1322 };
1323 struct cxl_port *port;
1324
1325 port = __find_cxl_port(&ctx);
1326 return port;
1327}
1328
1329/*
1330 * All users of grandparent() are using it to walk PCIe-like switch port
1331 * hierarchy. A PCIe switch is comprised of a bridge device representing the
1332 * upstream switch port and N bridges representing downstream switch ports. When
1333 * bridges stack the grand-parent of a downstream switch port is another
1334 * downstream switch port in the immediate ancestor switch.
1335 */
1336static struct device *grandparent(struct device *dev)
1337{
1338 if (dev && dev->parent)
1339 return dev->parent->parent;
1340 return NULL;
1341}
1342
1343static struct device *endpoint_host(struct cxl_port *endpoint)
1344{
1345 struct cxl_port *port = to_cxl_port(endpoint->dev.parent);
1346
1347 if (is_cxl_root(port))
1348 return port->uport_dev;
1349 return &port->dev;
1350}
1351
1352static void delete_endpoint(void *data)
1353{
1354 struct cxl_memdev *cxlmd = data;
1355 struct cxl_port *endpoint = cxlmd->endpoint;
1356 struct device *host = endpoint_host(endpoint);
1357
1358 device_lock(host);
1359 if (host->driver && !endpoint->dead) {
1360 devm_release_action(host, cxl_unlink_parent_dport, endpoint);
1361 devm_release_action(host, cxl_unlink_uport, endpoint);
1362 devm_release_action(host, unregister_port, endpoint);
1363 }
1364 cxlmd->endpoint = NULL;
1365 device_unlock(host);
1366 put_device(&endpoint->dev);
1367 put_device(host);
1368}
1369
1370int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
1371{
1372 struct device *host = endpoint_host(endpoint);
1373 struct device *dev = &cxlmd->dev;
1374
1375 get_device(host);
1376 get_device(&endpoint->dev);
1377 cxlmd->endpoint = endpoint;
1378 cxlmd->depth = endpoint->depth;
1379 return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
1380}
1381EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, CXL);
1382
1383/*
1384 * The natural end of life of a non-root 'cxl_port' is when its parent port goes
1385 * through a ->remove() event ("top-down" unregistration). The unnatural trigger
1386 * for a port to be unregistered is when all memdevs beneath that port have gone
1387 * through ->remove(). This "bottom-up" removal selectively removes individual
1388 * child ports manually. This depends on devm_cxl_add_port() to not change is
1389 * devm action registration order, and for dports to have already been
1390 * destroyed by reap_dports().
1391 */
1392static void delete_switch_port(struct cxl_port *port)
1393{
1394 devm_release_action(port->dev.parent, cxl_unlink_parent_dport, port);
1395 devm_release_action(port->dev.parent, cxl_unlink_uport, port);
1396 devm_release_action(port->dev.parent, unregister_port, port);
1397}
1398
1399static void reap_dports(struct cxl_port *port)
1400{
1401 struct cxl_dport *dport;
1402 unsigned long index;
1403
1404 device_lock_assert(&port->dev);
1405
1406 xa_for_each(&port->dports, index, dport) {
1407 devm_release_action(&port->dev, cxl_dport_unlink, dport);
1408 devm_release_action(&port->dev, cxl_dport_remove, dport);
1409 devm_kfree(&port->dev, dport);
1410 }
1411}
1412
1413struct detach_ctx {
1414 struct cxl_memdev *cxlmd;
1415 int depth;
1416};
1417
1418static int port_has_memdev(struct device *dev, const void *data)
1419{
1420 const struct detach_ctx *ctx = data;
1421 struct cxl_port *port;
1422
1423 if (!is_cxl_port(dev))
1424 return 0;
1425
1426 port = to_cxl_port(dev);
1427 if (port->depth != ctx->depth)
1428 return 0;
1429
1430 return !!cxl_ep_load(port, ctx->cxlmd);
1431}
1432
1433static void cxl_detach_ep(void *data)
1434{
1435 struct cxl_memdev *cxlmd = data;
1436
1437 for (int i = cxlmd->depth - 1; i >= 1; i--) {
1438 struct cxl_port *port, *parent_port;
1439 struct detach_ctx ctx = {
1440 .cxlmd = cxlmd,
1441 .depth = i,
1442 };
1443 struct device *dev;
1444 struct cxl_ep *ep;
1445 bool died = false;
1446
1447 dev = bus_find_device(&cxl_bus_type, NULL, &ctx,
1448 port_has_memdev);
1449 if (!dev)
1450 continue;
1451 port = to_cxl_port(dev);
1452
1453 parent_port = to_cxl_port(port->dev.parent);
1454 device_lock(&parent_port->dev);
1455 device_lock(&port->dev);
1456 ep = cxl_ep_load(port, cxlmd);
1457 dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",
1458 ep ? dev_name(ep->ep) : "", dev_name(&port->dev));
1459 cxl_ep_remove(port, ep);
1460 if (ep && !port->dead && xa_empty(&port->endpoints) &&
1461 !is_cxl_root(parent_port) && parent_port->dev.driver) {
1462 /*
1463 * This was the last ep attached to a dynamically
1464 * enumerated port. Block new cxl_add_ep() and garbage
1465 * collect the port.
1466 */
1467 died = true;
1468 port->dead = true;
1469 reap_dports(port);
1470 }
1471 device_unlock(&port->dev);
1472
1473 if (died) {
1474 dev_dbg(&cxlmd->dev, "delete %s\n",
1475 dev_name(&port->dev));
1476 delete_switch_port(port);
1477 }
1478 put_device(&port->dev);
1479 device_unlock(&parent_port->dev);
1480 }
1481}
1482
1483static resource_size_t find_component_registers(struct device *dev)
1484{
1485 struct cxl_register_map map;
1486 struct pci_dev *pdev;
1487
1488 /*
1489 * Theoretically, CXL component registers can be hosted on a
1490 * non-PCI device, in practice, only cxl_test hits this case.
1491 */
1492 if (!dev_is_pci(dev))
1493 return CXL_RESOURCE_NONE;
1494
1495 pdev = to_pci_dev(dev);
1496
1497 cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
1498 return map.resource;
1499}
1500
1501static int add_port_attach_ep(struct cxl_memdev *cxlmd,
1502 struct device *uport_dev,
1503 struct device *dport_dev)
1504{
1505 struct device *dparent = grandparent(dport_dev);
1506 struct cxl_port *port, *parent_port = NULL;
1507 struct cxl_dport *dport, *parent_dport;
1508 resource_size_t component_reg_phys;
1509 int rc;
1510
1511 if (!dparent) {
1512 /*
1513 * The iteration reached the topology root without finding the
1514 * CXL-root 'cxl_port' on a previous iteration, fail for now to
1515 * be re-probed after platform driver attaches.
1516 */
1517 dev_dbg(&cxlmd->dev, "%s is a root dport\n",
1518 dev_name(dport_dev));
1519 return -ENXIO;
1520 }
1521
1522 parent_port = find_cxl_port(dparent, &parent_dport);
1523 if (!parent_port) {
1524 /* iterate to create this parent_port */
1525 return -EAGAIN;
1526 }
1527
1528 device_lock(&parent_port->dev);
1529 if (!parent_port->dev.driver) {
1530 dev_warn(&cxlmd->dev,
1531 "port %s:%s disabled, failed to enumerate CXL.mem\n",
1532 dev_name(&parent_port->dev), dev_name(uport_dev));
1533 port = ERR_PTR(-ENXIO);
1534 goto out;
1535 }
1536
1537 port = find_cxl_port_at(parent_port, dport_dev, &dport);
1538 if (!port) {
1539 component_reg_phys = find_component_registers(uport_dev);
1540 port = devm_cxl_add_port(&parent_port->dev, uport_dev,
1541 component_reg_phys, parent_dport);
1542 /* retry find to pick up the new dport information */
1543 if (!IS_ERR(port))
1544 port = find_cxl_port_at(parent_port, dport_dev, &dport);
1545 }
1546out:
1547 device_unlock(&parent_port->dev);
1548
1549 if (IS_ERR(port))
1550 rc = PTR_ERR(port);
1551 else {
1552 dev_dbg(&cxlmd->dev, "add to new port %s:%s\n",
1553 dev_name(&port->dev), dev_name(port->uport_dev));
1554 rc = cxl_add_ep(dport, &cxlmd->dev);
1555 if (rc == -EBUSY) {
1556 /*
1557 * "can't" happen, but this error code means
1558 * something to the caller, so translate it.
1559 */
1560 rc = -ENXIO;
1561 }
1562 put_device(&port->dev);
1563 }
1564
1565 put_device(&parent_port->dev);
1566 return rc;
1567}
1568
1569int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
1570{
1571 struct device *dev = &cxlmd->dev;
1572 struct device *iter;
1573 int rc;
1574
1575 /*
1576 * Skip intermediate port enumeration in the RCH case, there
1577 * are no ports in between a host bridge and an endpoint.
1578 */
1579 if (cxlmd->cxlds->rcd)
1580 return 0;
1581
1582 rc = devm_add_action_or_reset(&cxlmd->dev, cxl_detach_ep, cxlmd);
1583 if (rc)
1584 return rc;
1585
1586 /*
1587 * Scan for and add all cxl_ports in this device's ancestry.
1588 * Repeat until no more ports are added. Abort if a port add
1589 * attempt fails.
1590 */
1591retry:
1592 for (iter = dev; iter; iter = grandparent(iter)) {
1593 struct device *dport_dev = grandparent(iter);
1594 struct device *uport_dev;
1595 struct cxl_dport *dport;
1596 struct cxl_port *port;
1597
1598 /*
1599 * The terminal "grandparent" in PCI is NULL and @platform_bus
1600 * for platform devices
1601 */
1602 if (!dport_dev || dport_dev == &platform_bus)
1603 return 0;
1604
1605 uport_dev = dport_dev->parent;
1606 if (!uport_dev) {
1607 dev_warn(dev, "at %s no parent for dport: %s\n",
1608 dev_name(iter), dev_name(dport_dev));
1609 return -ENXIO;
1610 }
1611
1612 dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",
1613 dev_name(iter), dev_name(dport_dev),
1614 dev_name(uport_dev));
1615 port = find_cxl_port(dport_dev, &dport);
1616 if (port) {
1617 dev_dbg(&cxlmd->dev,
1618 "found already registered port %s:%s\n",
1619 dev_name(&port->dev),
1620 dev_name(port->uport_dev));
1621 rc = cxl_add_ep(dport, &cxlmd->dev);
1622
1623 /*
1624 * If the endpoint already exists in the port's list,
1625 * that's ok, it was added on a previous pass.
1626 * Otherwise, retry in add_port_attach_ep() after taking
1627 * the parent_port lock as the current port may be being
1628 * reaped.
1629 */
1630 if (rc && rc != -EBUSY) {
1631 put_device(&port->dev);
1632 return rc;
1633 }
1634
1635 /* Any more ports to add between this one and the root? */
1636 if (!dev_is_cxl_root_child(&port->dev)) {
1637 put_device(&port->dev);
1638 continue;
1639 }
1640
1641 put_device(&port->dev);
1642 return 0;
1643 }
1644
1645 rc = add_port_attach_ep(cxlmd, uport_dev, dport_dev);
1646 /* port missing, try to add parent */
1647 if (rc == -EAGAIN)
1648 continue;
1649 /* failed to add ep or port */
1650 if (rc)
1651 return rc;
1652 /* port added, new descendants possible, start over */
1653 goto retry;
1654 }
1655
1656 return 0;
1657}
1658EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, CXL);
1659
1660struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
1661 struct cxl_dport **dport)
1662{
1663 return find_cxl_port(pdev->dev.parent, dport);
1664}
1665EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, CXL);
1666
1667struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
1668 struct cxl_dport **dport)
1669{
1670 return find_cxl_port(grandparent(&cxlmd->dev), dport);
1671}
1672EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, CXL);
1673
1674static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
1675 struct cxl_port *port, int *target_map)
1676{
1677 int i;
1678
1679 if (!target_map)
1680 return 0;
1681
1682 device_lock_assert(&port->dev);
1683
1684 if (xa_empty(&port->dports))
1685 return -EINVAL;
1686
1687 guard(rwsem_write)(&cxl_region_rwsem);
1688 for (i = 0; i < cxlsd->cxld.interleave_ways; i++) {
1689 struct cxl_dport *dport = find_dport(port, target_map[i]);
1690
1691 if (!dport)
1692 return -ENXIO;
1693 cxlsd->target[i] = dport;
1694 }
1695
1696 return 0;
1697}
1698
1699struct cxl_dport *cxl_hb_modulo(struct cxl_root_decoder *cxlrd, int pos)
1700{
1701 struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd;
1702 struct cxl_decoder *cxld = &cxlsd->cxld;
1703 int iw;
1704
1705 iw = cxld->interleave_ways;
1706 if (dev_WARN_ONCE(&cxld->dev, iw != cxlsd->nr_targets,
1707 "misconfigured root decoder\n"))
1708 return NULL;
1709
1710 return cxlrd->cxlsd.target[pos % iw];
1711}
1712EXPORT_SYMBOL_NS_GPL(cxl_hb_modulo, CXL);
1713
1714static struct lock_class_key cxl_decoder_key;
1715
1716/**
1717 * cxl_decoder_init - Common decoder setup / initialization
1718 * @port: owning port of this decoder
1719 * @cxld: common decoder properties to initialize
1720 *
1721 * A port may contain one or more decoders. Each of those decoders
1722 * enable some address space for CXL.mem utilization. A decoder is
1723 * expected to be configured by the caller before registering via
1724 * cxl_decoder_add()
1725 */
1726static int cxl_decoder_init(struct cxl_port *port, struct cxl_decoder *cxld)
1727{
1728 struct device *dev;
1729 int rc;
1730
1731 rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
1732 if (rc < 0)
1733 return rc;
1734
1735 /* need parent to stick around to release the id */
1736 get_device(&port->dev);
1737 cxld->id = rc;
1738
1739 dev = &cxld->dev;
1740 device_initialize(dev);
1741 lockdep_set_class(&dev->mutex, &cxl_decoder_key);
1742 device_set_pm_not_required(dev);
1743 dev->parent = &port->dev;
1744 dev->bus = &cxl_bus_type;
1745
1746 /* Pre initialize an "empty" decoder */
1747 cxld->interleave_ways = 1;
1748 cxld->interleave_granularity = PAGE_SIZE;
1749 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1750 cxld->hpa_range = (struct range) {
1751 .start = 0,
1752 .end = -1,
1753 };
1754
1755 return 0;
1756}
1757
1758static int cxl_switch_decoder_init(struct cxl_port *port,
1759 struct cxl_switch_decoder *cxlsd,
1760 int nr_targets)
1761{
1762 if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
1763 return -EINVAL;
1764
1765 cxlsd->nr_targets = nr_targets;
1766 return cxl_decoder_init(port, &cxlsd->cxld);
1767}
1768
1769/**
1770 * cxl_root_decoder_alloc - Allocate a root level decoder
1771 * @port: owning CXL root of this decoder
1772 * @nr_targets: static number of downstream targets
1773 * @calc_hb: which host bridge covers the n'th position by granularity
1774 *
1775 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1776 * 'CXL root' decoder is one that decodes from a top-level / static platform
1777 * firmware description of CXL resources into a CXL standard decode
1778 * topology.
1779 */
1780struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
1781 unsigned int nr_targets,
1782 cxl_calc_hb_fn calc_hb)
1783{
1784 struct cxl_root_decoder *cxlrd;
1785 struct cxl_switch_decoder *cxlsd;
1786 struct cxl_decoder *cxld;
1787 int rc;
1788
1789 if (!is_cxl_root(port))
1790 return ERR_PTR(-EINVAL);
1791
1792 cxlrd = kzalloc(struct_size(cxlrd, cxlsd.target, nr_targets),
1793 GFP_KERNEL);
1794 if (!cxlrd)
1795 return ERR_PTR(-ENOMEM);
1796
1797 cxlsd = &cxlrd->cxlsd;
1798 rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1799 if (rc) {
1800 kfree(cxlrd);
1801 return ERR_PTR(rc);
1802 }
1803
1804 cxlrd->calc_hb = calc_hb;
1805 mutex_init(&cxlrd->range_lock);
1806
1807 cxld = &cxlsd->cxld;
1808 cxld->dev.type = &cxl_decoder_root_type;
1809 /*
1810 * cxl_root_decoder_release() special cases negative ids to
1811 * detect memregion_alloc() failures.
1812 */
1813 atomic_set(&cxlrd->region_id, -1);
1814 rc = memregion_alloc(GFP_KERNEL);
1815 if (rc < 0) {
1816 put_device(&cxld->dev);
1817 return ERR_PTR(rc);
1818 }
1819
1820 atomic_set(&cxlrd->region_id, rc);
1821 cxlrd->qos_class = CXL_QOS_CLASS_INVALID;
1822 return cxlrd;
1823}
1824EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL);
1825
1826/**
1827 * cxl_switch_decoder_alloc - Allocate a switch level decoder
1828 * @port: owning CXL switch port of this decoder
1829 * @nr_targets: max number of dynamically addressable downstream targets
1830 *
1831 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1832 * 'switch' decoder is any decoder that can be enumerated by PCIe
1833 * topology and the HDM Decoder Capability. This includes the decoders
1834 * that sit between Switch Upstream Ports / Switch Downstream Ports and
1835 * Host Bridges / Root Ports.
1836 */
1837struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
1838 unsigned int nr_targets)
1839{
1840 struct cxl_switch_decoder *cxlsd;
1841 struct cxl_decoder *cxld;
1842 int rc;
1843
1844 if (is_cxl_root(port) || is_cxl_endpoint(port))
1845 return ERR_PTR(-EINVAL);
1846
1847 cxlsd = kzalloc(struct_size(cxlsd, target, nr_targets), GFP_KERNEL);
1848 if (!cxlsd)
1849 return ERR_PTR(-ENOMEM);
1850
1851 rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1852 if (rc) {
1853 kfree(cxlsd);
1854 return ERR_PTR(rc);
1855 }
1856
1857 cxld = &cxlsd->cxld;
1858 cxld->dev.type = &cxl_decoder_switch_type;
1859 return cxlsd;
1860}
1861EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, CXL);
1862
1863/**
1864 * cxl_endpoint_decoder_alloc - Allocate an endpoint decoder
1865 * @port: owning port of this decoder
1866 *
1867 * Return: A new cxl decoder to be registered by cxl_decoder_add()
1868 */
1869struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port)
1870{
1871 struct cxl_endpoint_decoder *cxled;
1872 struct cxl_decoder *cxld;
1873 int rc;
1874
1875 if (!is_cxl_endpoint(port))
1876 return ERR_PTR(-EINVAL);
1877
1878 cxled = kzalloc(sizeof(*cxled), GFP_KERNEL);
1879 if (!cxled)
1880 return ERR_PTR(-ENOMEM);
1881
1882 cxled->pos = -1;
1883 cxld = &cxled->cxld;
1884 rc = cxl_decoder_init(port, cxld);
1885 if (rc) {
1886 kfree(cxled);
1887 return ERR_PTR(rc);
1888 }
1889
1890 cxld->dev.type = &cxl_decoder_endpoint_type;
1891 return cxled;
1892}
1893EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, CXL);
1894
1895/**
1896 * cxl_decoder_add_locked - Add a decoder with targets
1897 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1898 * @target_map: A list of downstream ports that this decoder can direct memory
1899 * traffic to. These numbers should correspond with the port number
1900 * in the PCIe Link Capabilities structure.
1901 *
1902 * Certain types of decoders may not have any targets. The main example of this
1903 * is an endpoint device. A more awkward example is a hostbridge whose root
1904 * ports get hot added (technically possible, though unlikely).
1905 *
1906 * This is the locked variant of cxl_decoder_add().
1907 *
1908 * Context: Process context. Expects the device lock of the port that owns the
1909 * @cxld to be held.
1910 *
1911 * Return: Negative error code if the decoder wasn't properly configured; else
1912 * returns 0.
1913 */
1914int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map)
1915{
1916 struct cxl_port *port;
1917 struct device *dev;
1918 int rc;
1919
1920 if (WARN_ON_ONCE(!cxld))
1921 return -EINVAL;
1922
1923 if (WARN_ON_ONCE(IS_ERR(cxld)))
1924 return PTR_ERR(cxld);
1925
1926 if (cxld->interleave_ways < 1)
1927 return -EINVAL;
1928
1929 dev = &cxld->dev;
1930
1931 port = to_cxl_port(cxld->dev.parent);
1932 if (!is_endpoint_decoder(dev)) {
1933 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
1934
1935 rc = decoder_populate_targets(cxlsd, port, target_map);
1936 if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) {
1937 dev_err(&port->dev,
1938 "Failed to populate active decoder targets\n");
1939 return rc;
1940 }
1941 }
1942
1943 rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);
1944 if (rc)
1945 return rc;
1946
1947 return device_add(dev);
1948}
1949EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, CXL);
1950
1951/**
1952 * cxl_decoder_add - Add a decoder with targets
1953 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1954 * @target_map: A list of downstream ports that this decoder can direct memory
1955 * traffic to. These numbers should correspond with the port number
1956 * in the PCIe Link Capabilities structure.
1957 *
1958 * This is the unlocked variant of cxl_decoder_add_locked().
1959 * See cxl_decoder_add_locked().
1960 *
1961 * Context: Process context. Takes and releases the device lock of the port that
1962 * owns the @cxld.
1963 */
1964int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map)
1965{
1966 struct cxl_port *port;
1967 int rc;
1968
1969 if (WARN_ON_ONCE(!cxld))
1970 return -EINVAL;
1971
1972 if (WARN_ON_ONCE(IS_ERR(cxld)))
1973 return PTR_ERR(cxld);
1974
1975 port = to_cxl_port(cxld->dev.parent);
1976
1977 device_lock(&port->dev);
1978 rc = cxl_decoder_add_locked(cxld, target_map);
1979 device_unlock(&port->dev);
1980
1981 return rc;
1982}
1983EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, CXL);
1984
1985static void cxld_unregister(void *dev)
1986{
1987 struct cxl_endpoint_decoder *cxled;
1988
1989 if (is_endpoint_decoder(dev)) {
1990 cxled = to_cxl_endpoint_decoder(dev);
1991 cxl_decoder_kill_region(cxled);
1992 }
1993
1994 device_unregister(dev);
1995}
1996
1997int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
1998{
1999 return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev);
2000}
2001EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, CXL);
2002
2003/**
2004 * __cxl_driver_register - register a driver for the cxl bus
2005 * @cxl_drv: cxl driver structure to attach
2006 * @owner: owning module/driver
2007 * @modname: KBUILD_MODNAME for parent driver
2008 */
2009int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
2010 const char *modname)
2011{
2012 if (!cxl_drv->probe) {
2013 pr_debug("%s ->probe() must be specified\n", modname);
2014 return -EINVAL;
2015 }
2016
2017 if (!cxl_drv->name) {
2018 pr_debug("%s ->name must be specified\n", modname);
2019 return -EINVAL;
2020 }
2021
2022 if (!cxl_drv->id) {
2023 pr_debug("%s ->id must be specified\n", modname);
2024 return -EINVAL;
2025 }
2026
2027 cxl_drv->drv.bus = &cxl_bus_type;
2028 cxl_drv->drv.owner = owner;
2029 cxl_drv->drv.mod_name = modname;
2030 cxl_drv->drv.name = cxl_drv->name;
2031
2032 return driver_register(&cxl_drv->drv);
2033}
2034EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, CXL);
2035
2036void cxl_driver_unregister(struct cxl_driver *cxl_drv)
2037{
2038 driver_unregister(&cxl_drv->drv);
2039}
2040EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, CXL);
2041
2042static int cxl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
2043{
2044 return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,
2045 cxl_device_id(dev));
2046}
2047
2048static int cxl_bus_match(struct device *dev, struct device_driver *drv)
2049{
2050 return cxl_device_id(dev) == to_cxl_drv(drv)->id;
2051}
2052
2053static int cxl_bus_probe(struct device *dev)
2054{
2055 int rc;
2056
2057 rc = to_cxl_drv(dev->driver)->probe(dev);
2058 dev_dbg(dev, "probe: %d\n", rc);
2059 return rc;
2060}
2061
2062static void cxl_bus_remove(struct device *dev)
2063{
2064 struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
2065
2066 if (cxl_drv->remove)
2067 cxl_drv->remove(dev);
2068}
2069
2070static struct workqueue_struct *cxl_bus_wq;
2071
2072static void cxl_bus_rescan_queue(struct work_struct *w)
2073{
2074 int rc = bus_rescan_devices(&cxl_bus_type);
2075
2076 pr_debug("CXL bus rescan result: %d\n", rc);
2077}
2078
2079void cxl_bus_rescan(void)
2080{
2081 static DECLARE_WORK(rescan_work, cxl_bus_rescan_queue);
2082
2083 queue_work(cxl_bus_wq, &rescan_work);
2084}
2085EXPORT_SYMBOL_NS_GPL(cxl_bus_rescan, CXL);
2086
2087void cxl_bus_drain(void)
2088{
2089 drain_workqueue(cxl_bus_wq);
2090}
2091EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, CXL);
2092
2093bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
2094{
2095 return queue_work(cxl_bus_wq, &cxlmd->detach_work);
2096}
2097EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL);
2098
2099static void combine_coordinates(struct access_coordinate *c1,
2100 struct access_coordinate *c2)
2101{
2102 if (c2->write_bandwidth)
2103 c1->write_bandwidth = min(c1->write_bandwidth,
2104 c2->write_bandwidth);
2105 c1->write_latency += c2->write_latency;
2106
2107 if (c2->read_bandwidth)
2108 c1->read_bandwidth = min(c1->read_bandwidth,
2109 c2->read_bandwidth);
2110 c1->read_latency += c2->read_latency;
2111}
2112
2113/**
2114 * cxl_endpoint_get_perf_coordinates - Retrieve performance numbers stored in dports
2115 * of CXL path
2116 * @port: endpoint cxl_port
2117 * @coord: output performance data
2118 *
2119 * Return: errno on failure, 0 on success.
2120 */
2121int cxl_endpoint_get_perf_coordinates(struct cxl_port *port,
2122 struct access_coordinate *coord)
2123{
2124 struct access_coordinate c = {
2125 .read_bandwidth = UINT_MAX,
2126 .write_bandwidth = UINT_MAX,
2127 };
2128 struct cxl_port *iter = port;
2129 struct cxl_dport *dport;
2130 struct pci_dev *pdev;
2131 unsigned int bw;
2132
2133 if (!is_cxl_endpoint(port))
2134 return -EINVAL;
2135
2136 dport = iter->parent_dport;
2137
2138 /*
2139 * Exit the loop when the parent port of the current port is cxl root.
2140 * The iterative loop starts at the endpoint and gathers the
2141 * latency of the CXL link from the current iter to the next downstream
2142 * port each iteration. If the parent is cxl root then there is
2143 * nothing to gather.
2144 */
2145 while (iter && !is_cxl_root(to_cxl_port(iter->dev.parent))) {
2146 combine_coordinates(&c, &dport->sw_coord);
2147 c.write_latency += dport->link_latency;
2148 c.read_latency += dport->link_latency;
2149
2150 iter = to_cxl_port(iter->dev.parent);
2151 dport = iter->parent_dport;
2152 }
2153
2154 /* Augment with the generic port (host bridge) perf data */
2155 combine_coordinates(&c, &dport->hb_coord);
2156
2157 /* Get the calculated PCI paths bandwidth */
2158 pdev = to_pci_dev(port->uport_dev->parent);
2159 bw = pcie_bandwidth_available(pdev, NULL, NULL, NULL);
2160 if (bw == 0)
2161 return -ENXIO;
2162 bw /= BITS_PER_BYTE;
2163
2164 c.write_bandwidth = min(c.write_bandwidth, bw);
2165 c.read_bandwidth = min(c.read_bandwidth, bw);
2166
2167 *coord = c;
2168
2169 return 0;
2170}
2171EXPORT_SYMBOL_NS_GPL(cxl_endpoint_get_perf_coordinates, CXL);
2172
2173/* for user tooling to ensure port disable work has completed */
2174static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count)
2175{
2176 if (sysfs_streq(buf, "1")) {
2177 flush_workqueue(cxl_bus_wq);
2178 return count;
2179 }
2180
2181 return -EINVAL;
2182}
2183
2184static BUS_ATTR_WO(flush);
2185
2186static struct attribute *cxl_bus_attributes[] = {
2187 &bus_attr_flush.attr,
2188 NULL,
2189};
2190
2191static struct attribute_group cxl_bus_attribute_group = {
2192 .attrs = cxl_bus_attributes,
2193};
2194
2195static const struct attribute_group *cxl_bus_attribute_groups[] = {
2196 &cxl_bus_attribute_group,
2197 NULL,
2198};
2199
2200struct bus_type cxl_bus_type = {
2201 .name = "cxl",
2202 .uevent = cxl_bus_uevent,
2203 .match = cxl_bus_match,
2204 .probe = cxl_bus_probe,
2205 .remove = cxl_bus_remove,
2206 .bus_groups = cxl_bus_attribute_groups,
2207};
2208EXPORT_SYMBOL_NS_GPL(cxl_bus_type, CXL);
2209
2210static struct dentry *cxl_debugfs;
2211
2212struct dentry *cxl_debugfs_create_dir(const char *dir)
2213{
2214 return debugfs_create_dir(dir, cxl_debugfs);
2215}
2216EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, CXL);
2217
2218static __init int cxl_core_init(void)
2219{
2220 int rc;
2221
2222 cxl_debugfs = debugfs_create_dir("cxl", NULL);
2223
2224 cxl_mbox_init();
2225
2226 rc = cxl_memdev_init();
2227 if (rc)
2228 return rc;
2229
2230 cxl_bus_wq = alloc_ordered_workqueue("cxl_port", 0);
2231 if (!cxl_bus_wq) {
2232 rc = -ENOMEM;
2233 goto err_wq;
2234 }
2235
2236 rc = bus_register(&cxl_bus_type);
2237 if (rc)
2238 goto err_bus;
2239
2240 rc = cxl_region_init();
2241 if (rc)
2242 goto err_region;
2243
2244 return 0;
2245
2246err_region:
2247 bus_unregister(&cxl_bus_type);
2248err_bus:
2249 destroy_workqueue(cxl_bus_wq);
2250err_wq:
2251 cxl_memdev_exit();
2252 return rc;
2253}
2254
2255static void cxl_core_exit(void)
2256{
2257 cxl_region_exit();
2258 bus_unregister(&cxl_bus_type);
2259 destroy_workqueue(cxl_bus_wq);
2260 cxl_memdev_exit();
2261 debugfs_remove_recursive(cxl_debugfs);
2262}
2263
2264subsys_initcall(cxl_core_init);
2265module_exit(cxl_core_exit);
2266MODULE_LICENSE("GPL v2");
2267MODULE_IMPORT_NS(CXL);