Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3#include <linux/platform_device.h>
4#include <linux/memregion.h>
5#include <linux/workqueue.h>
6#include <linux/debugfs.h>
7#include <linux/device.h>
8#include <linux/module.h>
9#include <linux/pci.h>
10#include <linux/slab.h>
11#include <linux/idr.h>
12#include <linux/node.h>
13#include <cxl/einj.h>
14#include <cxlmem.h>
15#include <cxlpci.h>
16#include <cxl.h>
17#include "core.h"
18
19/**
20 * DOC: cxl core
21 *
22 * The CXL core provides a set of interfaces that can be consumed by CXL aware
23 * drivers. The interfaces allow for creation, modification, and destruction of
24 * regions, memory devices, ports, and decoders. CXL aware drivers must register
25 * with the CXL core via these interfaces in order to be able to participate in
26 * cross-device interleave coordination. The CXL core also establishes and
27 * maintains the bridge to the nvdimm subsystem.
28 *
29 * CXL core introduces sysfs hierarchy to control the devices that are
30 * instantiated by the core.
31 */
32
33/*
34 * All changes to the interleave configuration occur with this lock held
35 * for write.
36 */
37DECLARE_RWSEM(cxl_region_rwsem);
38
39static DEFINE_IDA(cxl_port_ida);
40static DEFINE_XARRAY(cxl_root_buses);
41
42int cxl_num_decoders_committed(struct cxl_port *port)
43{
44 lockdep_assert_held(&cxl_region_rwsem);
45
46 return port->commit_end + 1;
47}
48
49static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
50 char *buf)
51{
52 return sysfs_emit(buf, "%s\n", dev->type->name);
53}
54static DEVICE_ATTR_RO(devtype);
55
56static int cxl_device_id(const struct device *dev)
57{
58 if (dev->type == &cxl_nvdimm_bridge_type)
59 return CXL_DEVICE_NVDIMM_BRIDGE;
60 if (dev->type == &cxl_nvdimm_type)
61 return CXL_DEVICE_NVDIMM;
62 if (dev->type == CXL_PMEM_REGION_TYPE())
63 return CXL_DEVICE_PMEM_REGION;
64 if (dev->type == CXL_DAX_REGION_TYPE())
65 return CXL_DEVICE_DAX_REGION;
66 if (is_cxl_port(dev)) {
67 if (is_cxl_root(to_cxl_port(dev)))
68 return CXL_DEVICE_ROOT;
69 return CXL_DEVICE_PORT;
70 }
71 if (is_cxl_memdev(dev))
72 return CXL_DEVICE_MEMORY_EXPANDER;
73 if (dev->type == CXL_REGION_TYPE())
74 return CXL_DEVICE_REGION;
75 if (dev->type == &cxl_pmu_type)
76 return CXL_DEVICE_PMU;
77 return 0;
78}
79
80static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
81 char *buf)
82{
83 return sysfs_emit(buf, CXL_MODALIAS_FMT "\n", cxl_device_id(dev));
84}
85static DEVICE_ATTR_RO(modalias);
86
87static struct attribute *cxl_base_attributes[] = {
88 &dev_attr_devtype.attr,
89 &dev_attr_modalias.attr,
90 NULL,
91};
92
93struct attribute_group cxl_base_attribute_group = {
94 .attrs = cxl_base_attributes,
95};
96
97static ssize_t start_show(struct device *dev, struct device_attribute *attr,
98 char *buf)
99{
100 struct cxl_decoder *cxld = to_cxl_decoder(dev);
101
102 return sysfs_emit(buf, "%#llx\n", cxld->hpa_range.start);
103}
104static DEVICE_ATTR_ADMIN_RO(start);
105
106static ssize_t size_show(struct device *dev, struct device_attribute *attr,
107 char *buf)
108{
109 struct cxl_decoder *cxld = to_cxl_decoder(dev);
110
111 return sysfs_emit(buf, "%#llx\n", range_len(&cxld->hpa_range));
112}
113static DEVICE_ATTR_RO(size);
114
115#define CXL_DECODER_FLAG_ATTR(name, flag) \
116static ssize_t name##_show(struct device *dev, \
117 struct device_attribute *attr, char *buf) \
118{ \
119 struct cxl_decoder *cxld = to_cxl_decoder(dev); \
120 \
121 return sysfs_emit(buf, "%s\n", \
122 (cxld->flags & (flag)) ? "1" : "0"); \
123} \
124static DEVICE_ATTR_RO(name)
125
126CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM);
127CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM);
128CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2);
129CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3);
130CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK);
131
132static ssize_t target_type_show(struct device *dev,
133 struct device_attribute *attr, char *buf)
134{
135 struct cxl_decoder *cxld = to_cxl_decoder(dev);
136
137 switch (cxld->target_type) {
138 case CXL_DECODER_DEVMEM:
139 return sysfs_emit(buf, "accelerator\n");
140 case CXL_DECODER_HOSTONLYMEM:
141 return sysfs_emit(buf, "expander\n");
142 }
143 return -ENXIO;
144}
145static DEVICE_ATTR_RO(target_type);
146
147static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf)
148{
149 struct cxl_decoder *cxld = &cxlsd->cxld;
150 ssize_t offset = 0;
151 int i, rc = 0;
152
153 for (i = 0; i < cxld->interleave_ways; i++) {
154 struct cxl_dport *dport = cxlsd->target[i];
155 struct cxl_dport *next = NULL;
156
157 if (!dport)
158 break;
159
160 if (i + 1 < cxld->interleave_ways)
161 next = cxlsd->target[i + 1];
162 rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
163 next ? "," : "");
164 if (rc < 0)
165 return rc;
166 offset += rc;
167 }
168
169 return offset;
170}
171
172static ssize_t target_list_show(struct device *dev,
173 struct device_attribute *attr, char *buf)
174{
175 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
176 ssize_t offset;
177 int rc;
178
179 guard(rwsem_read)(&cxl_region_rwsem);
180 rc = emit_target_list(cxlsd, buf);
181 if (rc < 0)
182 return rc;
183 offset = rc;
184
185 rc = sysfs_emit_at(buf, offset, "\n");
186 if (rc < 0)
187 return rc;
188
189 return offset + rc;
190}
191static DEVICE_ATTR_RO(target_list);
192
193static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
194 char *buf)
195{
196 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
197
198 return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxled->mode));
199}
200
201static ssize_t mode_store(struct device *dev, struct device_attribute *attr,
202 const char *buf, size_t len)
203{
204 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
205 enum cxl_decoder_mode mode;
206 ssize_t rc;
207
208 if (sysfs_streq(buf, "pmem"))
209 mode = CXL_DECODER_PMEM;
210 else if (sysfs_streq(buf, "ram"))
211 mode = CXL_DECODER_RAM;
212 else
213 return -EINVAL;
214
215 rc = cxl_dpa_set_mode(cxled, mode);
216 if (rc)
217 return rc;
218
219 return len;
220}
221static DEVICE_ATTR_RW(mode);
222
223static ssize_t dpa_resource_show(struct device *dev, struct device_attribute *attr,
224 char *buf)
225{
226 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
227
228 guard(rwsem_read)(&cxl_dpa_rwsem);
229 return sysfs_emit(buf, "%#llx\n", (u64)cxl_dpa_resource_start(cxled));
230}
231static DEVICE_ATTR_RO(dpa_resource);
232
233static ssize_t dpa_size_show(struct device *dev, struct device_attribute *attr,
234 char *buf)
235{
236 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
237 resource_size_t size = cxl_dpa_size(cxled);
238
239 return sysfs_emit(buf, "%pa\n", &size);
240}
241
242static ssize_t dpa_size_store(struct device *dev, struct device_attribute *attr,
243 const char *buf, size_t len)
244{
245 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
246 unsigned long long size;
247 ssize_t rc;
248
249 rc = kstrtoull(buf, 0, &size);
250 if (rc)
251 return rc;
252
253 if (!IS_ALIGNED(size, SZ_256M))
254 return -EINVAL;
255
256 rc = cxl_dpa_free(cxled);
257 if (rc)
258 return rc;
259
260 if (size == 0)
261 return len;
262
263 rc = cxl_dpa_alloc(cxled, size);
264 if (rc)
265 return rc;
266
267 return len;
268}
269static DEVICE_ATTR_RW(dpa_size);
270
271static ssize_t interleave_granularity_show(struct device *dev,
272 struct device_attribute *attr,
273 char *buf)
274{
275 struct cxl_decoder *cxld = to_cxl_decoder(dev);
276
277 return sysfs_emit(buf, "%d\n", cxld->interleave_granularity);
278}
279
280static DEVICE_ATTR_RO(interleave_granularity);
281
282static ssize_t interleave_ways_show(struct device *dev,
283 struct device_attribute *attr, char *buf)
284{
285 struct cxl_decoder *cxld = to_cxl_decoder(dev);
286
287 return sysfs_emit(buf, "%d\n", cxld->interleave_ways);
288}
289
290static DEVICE_ATTR_RO(interleave_ways);
291
292static ssize_t qos_class_show(struct device *dev,
293 struct device_attribute *attr, char *buf)
294{
295 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
296
297 return sysfs_emit(buf, "%d\n", cxlrd->qos_class);
298}
299static DEVICE_ATTR_RO(qos_class);
300
301static struct attribute *cxl_decoder_base_attrs[] = {
302 &dev_attr_start.attr,
303 &dev_attr_size.attr,
304 &dev_attr_locked.attr,
305 &dev_attr_interleave_granularity.attr,
306 &dev_attr_interleave_ways.attr,
307 NULL,
308};
309
310static struct attribute_group cxl_decoder_base_attribute_group = {
311 .attrs = cxl_decoder_base_attrs,
312};
313
314static struct attribute *cxl_decoder_root_attrs[] = {
315 &dev_attr_cap_pmem.attr,
316 &dev_attr_cap_ram.attr,
317 &dev_attr_cap_type2.attr,
318 &dev_attr_cap_type3.attr,
319 &dev_attr_target_list.attr,
320 &dev_attr_qos_class.attr,
321 SET_CXL_REGION_ATTR(create_pmem_region)
322 SET_CXL_REGION_ATTR(create_ram_region)
323 SET_CXL_REGION_ATTR(delete_region)
324 NULL,
325};
326
327static bool can_create_pmem(struct cxl_root_decoder *cxlrd)
328{
329 unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_PMEM;
330
331 return (cxlrd->cxlsd.cxld.flags & flags) == flags;
332}
333
334static bool can_create_ram(struct cxl_root_decoder *cxlrd)
335{
336 unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_RAM;
337
338 return (cxlrd->cxlsd.cxld.flags & flags) == flags;
339}
340
341static umode_t cxl_root_decoder_visible(struct kobject *kobj, struct attribute *a, int n)
342{
343 struct device *dev = kobj_to_dev(kobj);
344 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
345
346 if (a == CXL_REGION_ATTR(create_pmem_region) && !can_create_pmem(cxlrd))
347 return 0;
348
349 if (a == CXL_REGION_ATTR(create_ram_region) && !can_create_ram(cxlrd))
350 return 0;
351
352 if (a == CXL_REGION_ATTR(delete_region) &&
353 !(can_create_pmem(cxlrd) || can_create_ram(cxlrd)))
354 return 0;
355
356 return a->mode;
357}
358
359static struct attribute_group cxl_decoder_root_attribute_group = {
360 .attrs = cxl_decoder_root_attrs,
361 .is_visible = cxl_root_decoder_visible,
362};
363
364static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
365 &cxl_decoder_root_attribute_group,
366 &cxl_decoder_base_attribute_group,
367 &cxl_base_attribute_group,
368 NULL,
369};
370
371static struct attribute *cxl_decoder_switch_attrs[] = {
372 &dev_attr_target_type.attr,
373 &dev_attr_target_list.attr,
374 SET_CXL_REGION_ATTR(region)
375 NULL,
376};
377
378static struct attribute_group cxl_decoder_switch_attribute_group = {
379 .attrs = cxl_decoder_switch_attrs,
380};
381
382static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = {
383 &cxl_decoder_switch_attribute_group,
384 &cxl_decoder_base_attribute_group,
385 &cxl_base_attribute_group,
386 NULL,
387};
388
389static struct attribute *cxl_decoder_endpoint_attrs[] = {
390 &dev_attr_target_type.attr,
391 &dev_attr_mode.attr,
392 &dev_attr_dpa_size.attr,
393 &dev_attr_dpa_resource.attr,
394 SET_CXL_REGION_ATTR(region)
395 NULL,
396};
397
398static struct attribute_group cxl_decoder_endpoint_attribute_group = {
399 .attrs = cxl_decoder_endpoint_attrs,
400};
401
402static const struct attribute_group *cxl_decoder_endpoint_attribute_groups[] = {
403 &cxl_decoder_base_attribute_group,
404 &cxl_decoder_endpoint_attribute_group,
405 &cxl_base_attribute_group,
406 NULL,
407};
408
409static void __cxl_decoder_release(struct cxl_decoder *cxld)
410{
411 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
412
413 ida_free(&port->decoder_ida, cxld->id);
414 put_device(&port->dev);
415}
416
417static void cxl_endpoint_decoder_release(struct device *dev)
418{
419 struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
420
421 __cxl_decoder_release(&cxled->cxld);
422 kfree(cxled);
423}
424
425static void cxl_switch_decoder_release(struct device *dev)
426{
427 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
428
429 __cxl_decoder_release(&cxlsd->cxld);
430 kfree(cxlsd);
431}
432
433struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev)
434{
435 if (dev_WARN_ONCE(dev, !is_root_decoder(dev),
436 "not a cxl_root_decoder device\n"))
437 return NULL;
438 return container_of(dev, struct cxl_root_decoder, cxlsd.cxld.dev);
439}
440EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, "CXL");
441
442static void cxl_root_decoder_release(struct device *dev)
443{
444 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
445
446 if (atomic_read(&cxlrd->region_id) >= 0)
447 memregion_free(atomic_read(&cxlrd->region_id));
448 __cxl_decoder_release(&cxlrd->cxlsd.cxld);
449 kfree(cxlrd);
450}
451
452static const struct device_type cxl_decoder_endpoint_type = {
453 .name = "cxl_decoder_endpoint",
454 .release = cxl_endpoint_decoder_release,
455 .groups = cxl_decoder_endpoint_attribute_groups,
456};
457
458static const struct device_type cxl_decoder_switch_type = {
459 .name = "cxl_decoder_switch",
460 .release = cxl_switch_decoder_release,
461 .groups = cxl_decoder_switch_attribute_groups,
462};
463
464static const struct device_type cxl_decoder_root_type = {
465 .name = "cxl_decoder_root",
466 .release = cxl_root_decoder_release,
467 .groups = cxl_decoder_root_attribute_groups,
468};
469
470bool is_endpoint_decoder(struct device *dev)
471{
472 return dev->type == &cxl_decoder_endpoint_type;
473}
474EXPORT_SYMBOL_NS_GPL(is_endpoint_decoder, "CXL");
475
476bool is_root_decoder(struct device *dev)
477{
478 return dev->type == &cxl_decoder_root_type;
479}
480EXPORT_SYMBOL_NS_GPL(is_root_decoder, "CXL");
481
482bool is_switch_decoder(struct device *dev)
483{
484 return is_root_decoder(dev) || dev->type == &cxl_decoder_switch_type;
485}
486EXPORT_SYMBOL_NS_GPL(is_switch_decoder, "CXL");
487
488struct cxl_decoder *to_cxl_decoder(struct device *dev)
489{
490 if (dev_WARN_ONCE(dev,
491 !is_switch_decoder(dev) && !is_endpoint_decoder(dev),
492 "not a cxl_decoder device\n"))
493 return NULL;
494 return container_of(dev, struct cxl_decoder, dev);
495}
496EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, "CXL");
497
498struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev)
499{
500 if (dev_WARN_ONCE(dev, !is_endpoint_decoder(dev),
501 "not a cxl_endpoint_decoder device\n"))
502 return NULL;
503 return container_of(dev, struct cxl_endpoint_decoder, cxld.dev);
504}
505EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, "CXL");
506
507struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev)
508{
509 if (dev_WARN_ONCE(dev, !is_switch_decoder(dev),
510 "not a cxl_switch_decoder device\n"))
511 return NULL;
512 return container_of(dev, struct cxl_switch_decoder, cxld.dev);
513}
514EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, "CXL");
515
516static void cxl_ep_release(struct cxl_ep *ep)
517{
518 put_device(ep->ep);
519 kfree(ep);
520}
521
522static void cxl_ep_remove(struct cxl_port *port, struct cxl_ep *ep)
523{
524 if (!ep)
525 return;
526 xa_erase(&port->endpoints, (unsigned long) ep->ep);
527 cxl_ep_release(ep);
528}
529
530static void cxl_port_release(struct device *dev)
531{
532 struct cxl_port *port = to_cxl_port(dev);
533 unsigned long index;
534 struct cxl_ep *ep;
535
536 xa_for_each(&port->endpoints, index, ep)
537 cxl_ep_remove(port, ep);
538 xa_destroy(&port->endpoints);
539 xa_destroy(&port->dports);
540 xa_destroy(&port->regions);
541 ida_free(&cxl_port_ida, port->id);
542 if (is_cxl_root(port))
543 kfree(to_cxl_root(port));
544 else
545 kfree(port);
546}
547
548static ssize_t decoders_committed_show(struct device *dev,
549 struct device_attribute *attr, char *buf)
550{
551 struct cxl_port *port = to_cxl_port(dev);
552 int rc;
553
554 down_read(&cxl_region_rwsem);
555 rc = sysfs_emit(buf, "%d\n", cxl_num_decoders_committed(port));
556 up_read(&cxl_region_rwsem);
557
558 return rc;
559}
560
561static DEVICE_ATTR_RO(decoders_committed);
562
563static struct attribute *cxl_port_attrs[] = {
564 &dev_attr_decoders_committed.attr,
565 NULL,
566};
567
568static struct attribute_group cxl_port_attribute_group = {
569 .attrs = cxl_port_attrs,
570};
571
572static const struct attribute_group *cxl_port_attribute_groups[] = {
573 &cxl_base_attribute_group,
574 &cxl_port_attribute_group,
575 NULL,
576};
577
578static const struct device_type cxl_port_type = {
579 .name = "cxl_port",
580 .release = cxl_port_release,
581 .groups = cxl_port_attribute_groups,
582};
583
584bool is_cxl_port(const struct device *dev)
585{
586 return dev->type == &cxl_port_type;
587}
588EXPORT_SYMBOL_NS_GPL(is_cxl_port, "CXL");
589
590struct cxl_port *to_cxl_port(const struct device *dev)
591{
592 if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
593 "not a cxl_port device\n"))
594 return NULL;
595 return container_of(dev, struct cxl_port, dev);
596}
597EXPORT_SYMBOL_NS_GPL(to_cxl_port, "CXL");
598
599static void unregister_port(void *_port)
600{
601 struct cxl_port *port = _port;
602 struct cxl_port *parent;
603 struct device *lock_dev;
604
605 if (is_cxl_root(port))
606 parent = NULL;
607 else
608 parent = to_cxl_port(port->dev.parent);
609
610 /*
611 * CXL root port's and the first level of ports are unregistered
612 * under the platform firmware device lock, all other ports are
613 * unregistered while holding their parent port lock.
614 */
615 if (!parent)
616 lock_dev = port->uport_dev;
617 else if (is_cxl_root(parent))
618 lock_dev = parent->uport_dev;
619 else
620 lock_dev = &parent->dev;
621
622 device_lock_assert(lock_dev);
623 port->dead = true;
624 device_unregister(&port->dev);
625}
626
627static void cxl_unlink_uport(void *_port)
628{
629 struct cxl_port *port = _port;
630
631 sysfs_remove_link(&port->dev.kobj, "uport");
632}
633
634static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
635{
636 int rc;
637
638 rc = sysfs_create_link(&port->dev.kobj, &port->uport_dev->kobj,
639 "uport");
640 if (rc)
641 return rc;
642 return devm_add_action_or_reset(host, cxl_unlink_uport, port);
643}
644
645static void cxl_unlink_parent_dport(void *_port)
646{
647 struct cxl_port *port = _port;
648
649 sysfs_remove_link(&port->dev.kobj, "parent_dport");
650}
651
652static int devm_cxl_link_parent_dport(struct device *host,
653 struct cxl_port *port,
654 struct cxl_dport *parent_dport)
655{
656 int rc;
657
658 if (!parent_dport)
659 return 0;
660
661 rc = sysfs_create_link(&port->dev.kobj, &parent_dport->dport_dev->kobj,
662 "parent_dport");
663 if (rc)
664 return rc;
665 return devm_add_action_or_reset(host, cxl_unlink_parent_dport, port);
666}
667
668static struct lock_class_key cxl_port_key;
669
670static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
671 struct cxl_dport *parent_dport)
672{
673 struct cxl_root *cxl_root __free(kfree) = NULL;
674 struct cxl_port *port, *_port __free(kfree) = NULL;
675 struct device *dev;
676 int rc;
677
678 /* No parent_dport, root cxl_port */
679 if (!parent_dport) {
680 cxl_root = kzalloc(sizeof(*cxl_root), GFP_KERNEL);
681 if (!cxl_root)
682 return ERR_PTR(-ENOMEM);
683 } else {
684 _port = kzalloc(sizeof(*port), GFP_KERNEL);
685 if (!_port)
686 return ERR_PTR(-ENOMEM);
687 }
688
689 rc = ida_alloc(&cxl_port_ida, GFP_KERNEL);
690 if (rc < 0)
691 return ERR_PTR(rc);
692
693 if (cxl_root)
694 port = &no_free_ptr(cxl_root)->port;
695 else
696 port = no_free_ptr(_port);
697
698 port->id = rc;
699 port->uport_dev = uport_dev;
700
701 /*
702 * The top-level cxl_port "cxl_root" does not have a cxl_port as
703 * its parent and it does not have any corresponding component
704 * registers as its decode is described by a fixed platform
705 * description.
706 */
707 dev = &port->dev;
708 if (parent_dport) {
709 struct cxl_port *parent_port = parent_dport->port;
710 struct cxl_port *iter;
711
712 dev->parent = &parent_port->dev;
713 port->depth = parent_port->depth + 1;
714 port->parent_dport = parent_dport;
715
716 /*
717 * walk to the host bridge, or the first ancestor that knows
718 * the host bridge
719 */
720 iter = port;
721 while (!iter->host_bridge &&
722 !is_cxl_root(to_cxl_port(iter->dev.parent)))
723 iter = to_cxl_port(iter->dev.parent);
724 if (iter->host_bridge)
725 port->host_bridge = iter->host_bridge;
726 else if (parent_dport->rch)
727 port->host_bridge = parent_dport->dport_dev;
728 else
729 port->host_bridge = iter->uport_dev;
730 dev_dbg(uport_dev, "host-bridge: %s\n",
731 dev_name(port->host_bridge));
732 } else
733 dev->parent = uport_dev;
734
735 ida_init(&port->decoder_ida);
736 port->hdm_end = -1;
737 port->commit_end = -1;
738 xa_init(&port->dports);
739 xa_init(&port->endpoints);
740 xa_init(&port->regions);
741
742 device_initialize(dev);
743 lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);
744 device_set_pm_not_required(dev);
745 dev->bus = &cxl_bus_type;
746 dev->type = &cxl_port_type;
747
748 return port;
749}
750
751static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map,
752 resource_size_t component_reg_phys)
753{
754 *map = (struct cxl_register_map) {
755 .host = host,
756 .reg_type = CXL_REGLOC_RBI_EMPTY,
757 .resource = component_reg_phys,
758 };
759
760 if (component_reg_phys == CXL_RESOURCE_NONE)
761 return 0;
762
763 map->reg_type = CXL_REGLOC_RBI_COMPONENT;
764 map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
765
766 return cxl_setup_regs(map);
767}
768
769static int cxl_port_setup_regs(struct cxl_port *port,
770 resource_size_t component_reg_phys)
771{
772 if (dev_is_platform(port->uport_dev))
773 return 0;
774 return cxl_setup_comp_regs(&port->dev, &port->reg_map,
775 component_reg_phys);
776}
777
778static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
779 resource_size_t component_reg_phys)
780{
781 int rc;
782
783 if (dev_is_platform(dport->dport_dev))
784 return 0;
785
786 /*
787 * use @dport->dport_dev for the context for error messages during
788 * register probing, and fixup @host after the fact, since @host may be
789 * NULL.
790 */
791 rc = cxl_setup_comp_regs(dport->dport_dev, &dport->reg_map,
792 component_reg_phys);
793 dport->reg_map.host = host;
794 return rc;
795}
796
797DEFINE_SHOW_ATTRIBUTE(einj_cxl_available_error_type);
798
799static int cxl_einj_inject(void *data, u64 type)
800{
801 struct cxl_dport *dport = data;
802
803 if (dport->rch)
804 return einj_cxl_inject_rch_error(dport->rcrb.base, type);
805
806 return einj_cxl_inject_error(to_pci_dev(dport->dport_dev), type);
807}
808DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject,
809 "0x%llx\n");
810
811static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport)
812{
813 struct dentry *dir;
814
815 if (!einj_cxl_is_initialized())
816 return;
817
818 /*
819 * dport_dev needs to be a PCIe port for CXL 2.0+ ports because
820 * EINJ expects a dport SBDF to be specified for 2.0 error injection.
821 */
822 if (!dport->rch && !dev_is_pci(dport->dport_dev))
823 return;
824
825 dir = cxl_debugfs_create_dir(dev_name(dport->dport_dev));
826
827 debugfs_create_file("einj_inject", 0200, dir, dport,
828 &cxl_einj_inject_fops);
829}
830
831static int cxl_port_add(struct cxl_port *port,
832 resource_size_t component_reg_phys,
833 struct cxl_dport *parent_dport)
834{
835 struct device *dev __free(put_device) = &port->dev;
836 int rc;
837
838 if (is_cxl_memdev(port->uport_dev)) {
839 struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
840 struct cxl_dev_state *cxlds = cxlmd->cxlds;
841
842 rc = dev_set_name(dev, "endpoint%d", port->id);
843 if (rc)
844 return rc;
845
846 /*
847 * The endpoint driver already enumerated the component and RAS
848 * registers. Reuse that enumeration while prepping them to be
849 * mapped by the cxl_port driver.
850 */
851 port->reg_map = cxlds->reg_map;
852 port->reg_map.host = &port->dev;
853 cxlmd->endpoint = port;
854 } else if (parent_dport) {
855 rc = dev_set_name(dev, "port%d", port->id);
856 if (rc)
857 return rc;
858
859 rc = cxl_port_setup_regs(port, component_reg_phys);
860 if (rc)
861 return rc;
862 } else {
863 rc = dev_set_name(dev, "root%d", port->id);
864 if (rc)
865 return rc;
866 }
867
868 rc = device_add(dev);
869 if (rc)
870 return rc;
871
872 /* Inhibit the cleanup function invoked */
873 dev = NULL;
874 return 0;
875}
876
877static struct cxl_port *__devm_cxl_add_port(struct device *host,
878 struct device *uport_dev,
879 resource_size_t component_reg_phys,
880 struct cxl_dport *parent_dport)
881{
882 struct cxl_port *port;
883 int rc;
884
885 port = cxl_port_alloc(uport_dev, parent_dport);
886 if (IS_ERR(port))
887 return port;
888
889 rc = cxl_port_add(port, component_reg_phys, parent_dport);
890 if (rc)
891 return ERR_PTR(rc);
892
893 rc = devm_add_action_or_reset(host, unregister_port, port);
894 if (rc)
895 return ERR_PTR(rc);
896
897 rc = devm_cxl_link_uport(host, port);
898 if (rc)
899 return ERR_PTR(rc);
900
901 rc = devm_cxl_link_parent_dport(host, port, parent_dport);
902 if (rc)
903 return ERR_PTR(rc);
904
905 if (parent_dport && dev_is_pci(uport_dev))
906 port->pci_latency = cxl_pci_get_latency(to_pci_dev(uport_dev));
907
908 return port;
909}
910
911/**
912 * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
913 * @host: host device for devm operations
914 * @uport_dev: "physical" device implementing this upstream port
915 * @component_reg_phys: (optional) for configurable cxl_port instances
916 * @parent_dport: next hop up in the CXL memory decode hierarchy
917 */
918struct cxl_port *devm_cxl_add_port(struct device *host,
919 struct device *uport_dev,
920 resource_size_t component_reg_phys,
921 struct cxl_dport *parent_dport)
922{
923 struct cxl_port *port, *parent_port;
924
925 port = __devm_cxl_add_port(host, uport_dev, component_reg_phys,
926 parent_dport);
927
928 parent_port = parent_dport ? parent_dport->port : NULL;
929 if (IS_ERR(port)) {
930 dev_dbg(uport_dev, "Failed to add%s%s%s: %ld\n",
931 parent_port ? " port to " : "",
932 parent_port ? dev_name(&parent_port->dev) : "",
933 parent_port ? "" : " root port",
934 PTR_ERR(port));
935 } else {
936 dev_dbg(uport_dev, "%s added%s%s%s\n",
937 dev_name(&port->dev),
938 parent_port ? " to " : "",
939 parent_port ? dev_name(&parent_port->dev) : "",
940 parent_port ? "" : " (root port)");
941 }
942
943 return port;
944}
945EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, "CXL");
946
947struct cxl_root *devm_cxl_add_root(struct device *host,
948 const struct cxl_root_ops *ops)
949{
950 struct cxl_root *cxl_root;
951 struct cxl_port *port;
952
953 port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL);
954 if (IS_ERR(port))
955 return ERR_CAST(port);
956
957 cxl_root = to_cxl_root(port);
958 cxl_root->ops = ops;
959 return cxl_root;
960}
961EXPORT_SYMBOL_NS_GPL(devm_cxl_add_root, "CXL");
962
963struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port)
964{
965 /* There is no pci_bus associated with a CXL platform-root port */
966 if (is_cxl_root(port))
967 return NULL;
968
969 if (dev_is_pci(port->uport_dev)) {
970 struct pci_dev *pdev = to_pci_dev(port->uport_dev);
971
972 return pdev->subordinate;
973 }
974
975 return xa_load(&cxl_root_buses, (unsigned long)port->uport_dev);
976}
977EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, "CXL");
978
979static void unregister_pci_bus(void *uport_dev)
980{
981 xa_erase(&cxl_root_buses, (unsigned long)uport_dev);
982}
983
984int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev,
985 struct pci_bus *bus)
986{
987 int rc;
988
989 if (dev_is_pci(uport_dev))
990 return -EINVAL;
991
992 rc = xa_insert(&cxl_root_buses, (unsigned long)uport_dev, bus,
993 GFP_KERNEL);
994 if (rc)
995 return rc;
996 return devm_add_action_or_reset(host, unregister_pci_bus, uport_dev);
997}
998EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, "CXL");
999
1000static bool dev_is_cxl_root_child(struct device *dev)
1001{
1002 struct cxl_port *port, *parent;
1003
1004 if (!is_cxl_port(dev))
1005 return false;
1006
1007 port = to_cxl_port(dev);
1008 if (is_cxl_root(port))
1009 return false;
1010
1011 parent = to_cxl_port(port->dev.parent);
1012 if (is_cxl_root(parent))
1013 return true;
1014
1015 return false;
1016}
1017
1018struct cxl_root *find_cxl_root(struct cxl_port *port)
1019{
1020 struct cxl_port *iter = port;
1021
1022 while (iter && !is_cxl_root(iter))
1023 iter = to_cxl_port(iter->dev.parent);
1024
1025 if (!iter)
1026 return NULL;
1027 get_device(&iter->dev);
1028 return to_cxl_root(iter);
1029}
1030EXPORT_SYMBOL_NS_GPL(find_cxl_root, "CXL");
1031
1032void put_cxl_root(struct cxl_root *cxl_root)
1033{
1034 if (!cxl_root)
1035 return;
1036
1037 put_device(&cxl_root->port.dev);
1038}
1039EXPORT_SYMBOL_NS_GPL(put_cxl_root, "CXL");
1040
1041static struct cxl_dport *find_dport(struct cxl_port *port, int id)
1042{
1043 struct cxl_dport *dport;
1044 unsigned long index;
1045
1046 device_lock_assert(&port->dev);
1047 xa_for_each(&port->dports, index, dport)
1048 if (dport->port_id == id)
1049 return dport;
1050 return NULL;
1051}
1052
1053static int add_dport(struct cxl_port *port, struct cxl_dport *dport)
1054{
1055 struct cxl_dport *dup;
1056 int rc;
1057
1058 device_lock_assert(&port->dev);
1059 dup = find_dport(port, dport->port_id);
1060 if (dup) {
1061 dev_err(&port->dev,
1062 "unable to add dport%d-%s non-unique port id (%s)\n",
1063 dport->port_id, dev_name(dport->dport_dev),
1064 dev_name(dup->dport_dev));
1065 return -EBUSY;
1066 }
1067
1068 rc = xa_insert(&port->dports, (unsigned long)dport->dport_dev, dport,
1069 GFP_KERNEL);
1070 if (rc)
1071 return rc;
1072
1073 port->nr_dports++;
1074 return 0;
1075}
1076
1077/*
1078 * Since root-level CXL dports cannot be enumerated by PCI they are not
1079 * enumerated by the common port driver that acquires the port lock over
1080 * dport add/remove. Instead, root dports are manually added by a
1081 * platform driver and cond_cxl_root_lock() is used to take the missing
1082 * port lock in that case.
1083 */
1084static void cond_cxl_root_lock(struct cxl_port *port)
1085{
1086 if (is_cxl_root(port))
1087 device_lock(&port->dev);
1088}
1089
1090static void cond_cxl_root_unlock(struct cxl_port *port)
1091{
1092 if (is_cxl_root(port))
1093 device_unlock(&port->dev);
1094}
1095
1096static void cxl_dport_remove(void *data)
1097{
1098 struct cxl_dport *dport = data;
1099 struct cxl_port *port = dport->port;
1100
1101 xa_erase(&port->dports, (unsigned long) dport->dport_dev);
1102 put_device(dport->dport_dev);
1103}
1104
1105static void cxl_dport_unlink(void *data)
1106{
1107 struct cxl_dport *dport = data;
1108 struct cxl_port *port = dport->port;
1109 char link_name[CXL_TARGET_STRLEN];
1110
1111 sprintf(link_name, "dport%d", dport->port_id);
1112 sysfs_remove_link(&port->dev.kobj, link_name);
1113}
1114
1115static struct cxl_dport *
1116__devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
1117 int port_id, resource_size_t component_reg_phys,
1118 resource_size_t rcrb)
1119{
1120 char link_name[CXL_TARGET_STRLEN];
1121 struct cxl_dport *dport;
1122 struct device *host;
1123 int rc;
1124
1125 if (is_cxl_root(port))
1126 host = port->uport_dev;
1127 else
1128 host = &port->dev;
1129
1130 if (!host->driver) {
1131 dev_WARN_ONCE(&port->dev, 1, "dport:%s bad devm context\n",
1132 dev_name(dport_dev));
1133 return ERR_PTR(-ENXIO);
1134 }
1135
1136 if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >=
1137 CXL_TARGET_STRLEN)
1138 return ERR_PTR(-EINVAL);
1139
1140 dport = devm_kzalloc(host, sizeof(*dport), GFP_KERNEL);
1141 if (!dport)
1142 return ERR_PTR(-ENOMEM);
1143
1144 dport->dport_dev = dport_dev;
1145 dport->port_id = port_id;
1146 dport->port = port;
1147
1148 if (rcrb == CXL_RESOURCE_NONE) {
1149 rc = cxl_dport_setup_regs(&port->dev, dport,
1150 component_reg_phys);
1151 if (rc)
1152 return ERR_PTR(rc);
1153 } else {
1154 dport->rcrb.base = rcrb;
1155 component_reg_phys = __rcrb_to_component(dport_dev, &dport->rcrb,
1156 CXL_RCRB_DOWNSTREAM);
1157 if (component_reg_phys == CXL_RESOURCE_NONE) {
1158 dev_warn(dport_dev, "Invalid Component Registers in RCRB");
1159 return ERR_PTR(-ENXIO);
1160 }
1161
1162 /*
1163 * RCH @dport is not ready to map until associated with its
1164 * memdev
1165 */
1166 rc = cxl_dport_setup_regs(NULL, dport, component_reg_phys);
1167 if (rc)
1168 return ERR_PTR(rc);
1169
1170 dport->rch = true;
1171 }
1172
1173 if (component_reg_phys != CXL_RESOURCE_NONE)
1174 dev_dbg(dport_dev, "Component Registers found for dport: %pa\n",
1175 &component_reg_phys);
1176
1177 cond_cxl_root_lock(port);
1178 rc = add_dport(port, dport);
1179 cond_cxl_root_unlock(port);
1180 if (rc)
1181 return ERR_PTR(rc);
1182
1183 get_device(dport_dev);
1184 rc = devm_add_action_or_reset(host, cxl_dport_remove, dport);
1185 if (rc)
1186 return ERR_PTR(rc);
1187
1188 rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name);
1189 if (rc)
1190 return ERR_PTR(rc);
1191
1192 rc = devm_add_action_or_reset(host, cxl_dport_unlink, dport);
1193 if (rc)
1194 return ERR_PTR(rc);
1195
1196 if (dev_is_pci(dport_dev))
1197 dport->link_latency = cxl_pci_get_latency(to_pci_dev(dport_dev));
1198
1199 cxl_debugfs_create_dport_dir(dport);
1200
1201 return dport;
1202}
1203
1204/**
1205 * devm_cxl_add_dport - append VH downstream port data to a cxl_port
1206 * @port: the cxl_port that references this dport
1207 * @dport_dev: firmware or PCI device representing the dport
1208 * @port_id: identifier for this dport in a decoder's target list
1209 * @component_reg_phys: optional location of CXL component registers
1210 *
1211 * Note that dports are appended to the devm release action's of the
1212 * either the port's host (for root ports), or the port itself (for
1213 * switch ports)
1214 */
1215struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
1216 struct device *dport_dev, int port_id,
1217 resource_size_t component_reg_phys)
1218{
1219 struct cxl_dport *dport;
1220
1221 dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1222 component_reg_phys, CXL_RESOURCE_NONE);
1223 if (IS_ERR(dport)) {
1224 dev_dbg(dport_dev, "failed to add dport to %s: %ld\n",
1225 dev_name(&port->dev), PTR_ERR(dport));
1226 } else {
1227 dev_dbg(dport_dev, "dport added to %s\n",
1228 dev_name(&port->dev));
1229 }
1230
1231 return dport;
1232}
1233EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, "CXL");
1234
1235/**
1236 * devm_cxl_add_rch_dport - append RCH downstream port data to a cxl_port
1237 * @port: the cxl_port that references this dport
1238 * @dport_dev: firmware or PCI device representing the dport
1239 * @port_id: identifier for this dport in a decoder's target list
1240 * @rcrb: mandatory location of a Root Complex Register Block
1241 *
1242 * See CXL 3.0 9.11.8 CXL Devices Attached to an RCH
1243 */
1244struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
1245 struct device *dport_dev, int port_id,
1246 resource_size_t rcrb)
1247{
1248 struct cxl_dport *dport;
1249
1250 if (rcrb == CXL_RESOURCE_NONE) {
1251 dev_dbg(&port->dev, "failed to add RCH dport, missing RCRB\n");
1252 return ERR_PTR(-EINVAL);
1253 }
1254
1255 dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1256 CXL_RESOURCE_NONE, rcrb);
1257 if (IS_ERR(dport)) {
1258 dev_dbg(dport_dev, "failed to add RCH dport to %s: %ld\n",
1259 dev_name(&port->dev), PTR_ERR(dport));
1260 } else {
1261 dev_dbg(dport_dev, "RCH dport added to %s\n",
1262 dev_name(&port->dev));
1263 }
1264
1265 return dport;
1266}
1267EXPORT_SYMBOL_NS_GPL(devm_cxl_add_rch_dport, "CXL");
1268
1269static int add_ep(struct cxl_ep *new)
1270{
1271 struct cxl_port *port = new->dport->port;
1272
1273 guard(device)(&port->dev);
1274 if (port->dead)
1275 return -ENXIO;
1276
1277 return xa_insert(&port->endpoints, (unsigned long)new->ep,
1278 new, GFP_KERNEL);
1279}
1280
1281/**
1282 * cxl_add_ep - register an endpoint's interest in a port
1283 * @dport: the dport that routes to @ep_dev
1284 * @ep_dev: device representing the endpoint
1285 *
1286 * Intermediate CXL ports are scanned based on the arrival of endpoints.
1287 * When those endpoints depart the port can be destroyed once all
1288 * endpoints that care about that port have been removed.
1289 */
1290static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev)
1291{
1292 struct cxl_ep *ep;
1293 int rc;
1294
1295 ep = kzalloc(sizeof(*ep), GFP_KERNEL);
1296 if (!ep)
1297 return -ENOMEM;
1298
1299 ep->ep = get_device(ep_dev);
1300 ep->dport = dport;
1301
1302 rc = add_ep(ep);
1303 if (rc)
1304 cxl_ep_release(ep);
1305 return rc;
1306}
1307
1308struct cxl_find_port_ctx {
1309 const struct device *dport_dev;
1310 const struct cxl_port *parent_port;
1311 struct cxl_dport **dport;
1312};
1313
1314static int match_port_by_dport(struct device *dev, const void *data)
1315{
1316 const struct cxl_find_port_ctx *ctx = data;
1317 struct cxl_dport *dport;
1318 struct cxl_port *port;
1319
1320 if (!is_cxl_port(dev))
1321 return 0;
1322 if (ctx->parent_port && dev->parent != &ctx->parent_port->dev)
1323 return 0;
1324
1325 port = to_cxl_port(dev);
1326 dport = cxl_find_dport_by_dev(port, ctx->dport_dev);
1327 if (ctx->dport)
1328 *ctx->dport = dport;
1329 return dport != NULL;
1330}
1331
1332static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx)
1333{
1334 struct device *dev;
1335
1336 if (!ctx->dport_dev)
1337 return NULL;
1338
1339 dev = bus_find_device(&cxl_bus_type, NULL, ctx, match_port_by_dport);
1340 if (dev)
1341 return to_cxl_port(dev);
1342 return NULL;
1343}
1344
1345static struct cxl_port *find_cxl_port(struct device *dport_dev,
1346 struct cxl_dport **dport)
1347{
1348 struct cxl_find_port_ctx ctx = {
1349 .dport_dev = dport_dev,
1350 .dport = dport,
1351 };
1352 struct cxl_port *port;
1353
1354 port = __find_cxl_port(&ctx);
1355 return port;
1356}
1357
1358static struct cxl_port *find_cxl_port_at(struct cxl_port *parent_port,
1359 struct device *dport_dev,
1360 struct cxl_dport **dport)
1361{
1362 struct cxl_find_port_ctx ctx = {
1363 .dport_dev = dport_dev,
1364 .parent_port = parent_port,
1365 .dport = dport,
1366 };
1367 struct cxl_port *port;
1368
1369 port = __find_cxl_port(&ctx);
1370 return port;
1371}
1372
1373/*
1374 * All users of grandparent() are using it to walk PCIe-like switch port
1375 * hierarchy. A PCIe switch is comprised of a bridge device representing the
1376 * upstream switch port and N bridges representing downstream switch ports. When
1377 * bridges stack the grand-parent of a downstream switch port is another
1378 * downstream switch port in the immediate ancestor switch.
1379 */
1380static struct device *grandparent(struct device *dev)
1381{
1382 if (dev && dev->parent)
1383 return dev->parent->parent;
1384 return NULL;
1385}
1386
1387static struct device *endpoint_host(struct cxl_port *endpoint)
1388{
1389 struct cxl_port *port = to_cxl_port(endpoint->dev.parent);
1390
1391 if (is_cxl_root(port))
1392 return port->uport_dev;
1393 return &port->dev;
1394}
1395
1396static void delete_endpoint(void *data)
1397{
1398 struct cxl_memdev *cxlmd = data;
1399 struct cxl_port *endpoint = cxlmd->endpoint;
1400 struct device *host = endpoint_host(endpoint);
1401
1402 scoped_guard(device, host) {
1403 if (host->driver && !endpoint->dead) {
1404 devm_release_action(host, cxl_unlink_parent_dport, endpoint);
1405 devm_release_action(host, cxl_unlink_uport, endpoint);
1406 devm_release_action(host, unregister_port, endpoint);
1407 }
1408 cxlmd->endpoint = NULL;
1409 }
1410 put_device(&endpoint->dev);
1411 put_device(host);
1412}
1413
1414int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
1415{
1416 struct device *host = endpoint_host(endpoint);
1417 struct device *dev = &cxlmd->dev;
1418
1419 get_device(host);
1420 get_device(&endpoint->dev);
1421 cxlmd->depth = endpoint->depth;
1422 return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
1423}
1424EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, "CXL");
1425
1426/*
1427 * The natural end of life of a non-root 'cxl_port' is when its parent port goes
1428 * through a ->remove() event ("top-down" unregistration). The unnatural trigger
1429 * for a port to be unregistered is when all memdevs beneath that port have gone
1430 * through ->remove(). This "bottom-up" removal selectively removes individual
1431 * child ports manually. This depends on devm_cxl_add_port() to not change is
1432 * devm action registration order, and for dports to have already been
1433 * destroyed by reap_dports().
1434 */
1435static void delete_switch_port(struct cxl_port *port)
1436{
1437 devm_release_action(port->dev.parent, cxl_unlink_parent_dport, port);
1438 devm_release_action(port->dev.parent, cxl_unlink_uport, port);
1439 devm_release_action(port->dev.parent, unregister_port, port);
1440}
1441
1442static void reap_dports(struct cxl_port *port)
1443{
1444 struct cxl_dport *dport;
1445 unsigned long index;
1446
1447 device_lock_assert(&port->dev);
1448
1449 xa_for_each(&port->dports, index, dport) {
1450 devm_release_action(&port->dev, cxl_dport_unlink, dport);
1451 devm_release_action(&port->dev, cxl_dport_remove, dport);
1452 devm_kfree(&port->dev, dport);
1453 }
1454}
1455
1456struct detach_ctx {
1457 struct cxl_memdev *cxlmd;
1458 int depth;
1459};
1460
1461static int port_has_memdev(struct device *dev, const void *data)
1462{
1463 const struct detach_ctx *ctx = data;
1464 struct cxl_port *port;
1465
1466 if (!is_cxl_port(dev))
1467 return 0;
1468
1469 port = to_cxl_port(dev);
1470 if (port->depth != ctx->depth)
1471 return 0;
1472
1473 return !!cxl_ep_load(port, ctx->cxlmd);
1474}
1475
1476static void cxl_detach_ep(void *data)
1477{
1478 struct cxl_memdev *cxlmd = data;
1479
1480 for (int i = cxlmd->depth - 1; i >= 1; i--) {
1481 struct cxl_port *port, *parent_port;
1482 struct detach_ctx ctx = {
1483 .cxlmd = cxlmd,
1484 .depth = i,
1485 };
1486 struct cxl_ep *ep;
1487 bool died = false;
1488
1489 struct device *dev __free(put_device) =
1490 bus_find_device(&cxl_bus_type, NULL, &ctx, port_has_memdev);
1491 if (!dev)
1492 continue;
1493 port = to_cxl_port(dev);
1494
1495 parent_port = to_cxl_port(port->dev.parent);
1496 device_lock(&parent_port->dev);
1497 device_lock(&port->dev);
1498 ep = cxl_ep_load(port, cxlmd);
1499 dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",
1500 ep ? dev_name(ep->ep) : "", dev_name(&port->dev));
1501 cxl_ep_remove(port, ep);
1502 if (ep && !port->dead && xa_empty(&port->endpoints) &&
1503 !is_cxl_root(parent_port) && parent_port->dev.driver) {
1504 /*
1505 * This was the last ep attached to a dynamically
1506 * enumerated port. Block new cxl_add_ep() and garbage
1507 * collect the port.
1508 */
1509 died = true;
1510 port->dead = true;
1511 reap_dports(port);
1512 }
1513 device_unlock(&port->dev);
1514
1515 if (died) {
1516 dev_dbg(&cxlmd->dev, "delete %s\n",
1517 dev_name(&port->dev));
1518 delete_switch_port(port);
1519 }
1520 device_unlock(&parent_port->dev);
1521 }
1522}
1523
1524static resource_size_t find_component_registers(struct device *dev)
1525{
1526 struct cxl_register_map map;
1527 struct pci_dev *pdev;
1528
1529 /*
1530 * Theoretically, CXL component registers can be hosted on a
1531 * non-PCI device, in practice, only cxl_test hits this case.
1532 */
1533 if (!dev_is_pci(dev))
1534 return CXL_RESOURCE_NONE;
1535
1536 pdev = to_pci_dev(dev);
1537
1538 cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
1539 return map.resource;
1540}
1541
1542static int add_port_attach_ep(struct cxl_memdev *cxlmd,
1543 struct device *uport_dev,
1544 struct device *dport_dev)
1545{
1546 struct device *dparent = grandparent(dport_dev);
1547 struct cxl_dport *dport, *parent_dport;
1548 resource_size_t component_reg_phys;
1549 int rc;
1550
1551 if (!dparent) {
1552 /*
1553 * The iteration reached the topology root without finding the
1554 * CXL-root 'cxl_port' on a previous iteration, fail for now to
1555 * be re-probed after platform driver attaches.
1556 */
1557 dev_dbg(&cxlmd->dev, "%s is a root dport\n",
1558 dev_name(dport_dev));
1559 return -ENXIO;
1560 }
1561
1562 struct cxl_port *parent_port __free(put_cxl_port) =
1563 find_cxl_port(dparent, &parent_dport);
1564 if (!parent_port) {
1565 /* iterate to create this parent_port */
1566 return -EAGAIN;
1567 }
1568
1569 /*
1570 * Definition with __free() here to keep the sequence of
1571 * dereferencing the device of the port before the parent_port releasing.
1572 */
1573 struct cxl_port *port __free(put_cxl_port) = NULL;
1574 scoped_guard(device, &parent_port->dev) {
1575 if (!parent_port->dev.driver) {
1576 dev_warn(&cxlmd->dev,
1577 "port %s:%s disabled, failed to enumerate CXL.mem\n",
1578 dev_name(&parent_port->dev), dev_name(uport_dev));
1579 return -ENXIO;
1580 }
1581
1582 port = find_cxl_port_at(parent_port, dport_dev, &dport);
1583 if (!port) {
1584 component_reg_phys = find_component_registers(uport_dev);
1585 port = devm_cxl_add_port(&parent_port->dev, uport_dev,
1586 component_reg_phys, parent_dport);
1587 if (IS_ERR(port))
1588 return PTR_ERR(port);
1589
1590 /* retry find to pick up the new dport information */
1591 port = find_cxl_port_at(parent_port, dport_dev, &dport);
1592 if (!port)
1593 return -ENXIO;
1594 }
1595 }
1596
1597 dev_dbg(&cxlmd->dev, "add to new port %s:%s\n",
1598 dev_name(&port->dev), dev_name(port->uport_dev));
1599 rc = cxl_add_ep(dport, &cxlmd->dev);
1600 if (rc == -EBUSY) {
1601 /*
1602 * "can't" happen, but this error code means
1603 * something to the caller, so translate it.
1604 */
1605 rc = -ENXIO;
1606 }
1607
1608 return rc;
1609}
1610
1611int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
1612{
1613 struct device *dev = &cxlmd->dev;
1614 struct device *iter;
1615 int rc;
1616
1617 /*
1618 * Skip intermediate port enumeration in the RCH case, there
1619 * are no ports in between a host bridge and an endpoint.
1620 */
1621 if (cxlmd->cxlds->rcd)
1622 return 0;
1623
1624 rc = devm_add_action_or_reset(&cxlmd->dev, cxl_detach_ep, cxlmd);
1625 if (rc)
1626 return rc;
1627
1628 /*
1629 * Scan for and add all cxl_ports in this device's ancestry.
1630 * Repeat until no more ports are added. Abort if a port add
1631 * attempt fails.
1632 */
1633retry:
1634 for (iter = dev; iter; iter = grandparent(iter)) {
1635 struct device *dport_dev = grandparent(iter);
1636 struct device *uport_dev;
1637 struct cxl_dport *dport;
1638
1639 /*
1640 * The terminal "grandparent" in PCI is NULL and @platform_bus
1641 * for platform devices
1642 */
1643 if (!dport_dev || dport_dev == &platform_bus)
1644 return 0;
1645
1646 uport_dev = dport_dev->parent;
1647 if (!uport_dev) {
1648 dev_warn(dev, "at %s no parent for dport: %s\n",
1649 dev_name(iter), dev_name(dport_dev));
1650 return -ENXIO;
1651 }
1652
1653 dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",
1654 dev_name(iter), dev_name(dport_dev),
1655 dev_name(uport_dev));
1656 struct cxl_port *port __free(put_cxl_port) =
1657 find_cxl_port(dport_dev, &dport);
1658 if (port) {
1659 dev_dbg(&cxlmd->dev,
1660 "found already registered port %s:%s\n",
1661 dev_name(&port->dev),
1662 dev_name(port->uport_dev));
1663 rc = cxl_add_ep(dport, &cxlmd->dev);
1664
1665 /*
1666 * If the endpoint already exists in the port's list,
1667 * that's ok, it was added on a previous pass.
1668 * Otherwise, retry in add_port_attach_ep() after taking
1669 * the parent_port lock as the current port may be being
1670 * reaped.
1671 */
1672 if (rc && rc != -EBUSY)
1673 return rc;
1674
1675 /* Any more ports to add between this one and the root? */
1676 if (!dev_is_cxl_root_child(&port->dev))
1677 continue;
1678
1679 return 0;
1680 }
1681
1682 rc = add_port_attach_ep(cxlmd, uport_dev, dport_dev);
1683 /* port missing, try to add parent */
1684 if (rc == -EAGAIN)
1685 continue;
1686 /* failed to add ep or port */
1687 if (rc)
1688 return rc;
1689 /* port added, new descendants possible, start over */
1690 goto retry;
1691 }
1692
1693 return 0;
1694}
1695EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, "CXL");
1696
1697struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
1698 struct cxl_dport **dport)
1699{
1700 return find_cxl_port(pdev->dev.parent, dport);
1701}
1702EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, "CXL");
1703
1704struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
1705 struct cxl_dport **dport)
1706{
1707 return find_cxl_port(grandparent(&cxlmd->dev), dport);
1708}
1709EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, "CXL");
1710
1711static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
1712 struct cxl_port *port, int *target_map)
1713{
1714 int i;
1715
1716 if (!target_map)
1717 return 0;
1718
1719 device_lock_assert(&port->dev);
1720
1721 if (xa_empty(&port->dports))
1722 return -EINVAL;
1723
1724 guard(rwsem_write)(&cxl_region_rwsem);
1725 for (i = 0; i < cxlsd->cxld.interleave_ways; i++) {
1726 struct cxl_dport *dport = find_dport(port, target_map[i]);
1727
1728 if (!dport)
1729 return -ENXIO;
1730 cxlsd->target[i] = dport;
1731 }
1732
1733 return 0;
1734}
1735
1736static struct lock_class_key cxl_decoder_key;
1737
1738/**
1739 * cxl_decoder_init - Common decoder setup / initialization
1740 * @port: owning port of this decoder
1741 * @cxld: common decoder properties to initialize
1742 *
1743 * A port may contain one or more decoders. Each of those decoders
1744 * enable some address space for CXL.mem utilization. A decoder is
1745 * expected to be configured by the caller before registering via
1746 * cxl_decoder_add()
1747 */
1748static int cxl_decoder_init(struct cxl_port *port, struct cxl_decoder *cxld)
1749{
1750 struct device *dev;
1751 int rc;
1752
1753 rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
1754 if (rc < 0)
1755 return rc;
1756
1757 /* need parent to stick around to release the id */
1758 get_device(&port->dev);
1759 cxld->id = rc;
1760
1761 dev = &cxld->dev;
1762 device_initialize(dev);
1763 lockdep_set_class(&dev->mutex, &cxl_decoder_key);
1764 device_set_pm_not_required(dev);
1765 dev->parent = &port->dev;
1766 dev->bus = &cxl_bus_type;
1767
1768 /* Pre initialize an "empty" decoder */
1769 cxld->interleave_ways = 1;
1770 cxld->interleave_granularity = PAGE_SIZE;
1771 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1772 cxld->hpa_range = (struct range) {
1773 .start = 0,
1774 .end = -1,
1775 };
1776
1777 return 0;
1778}
1779
1780static int cxl_switch_decoder_init(struct cxl_port *port,
1781 struct cxl_switch_decoder *cxlsd,
1782 int nr_targets)
1783{
1784 if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
1785 return -EINVAL;
1786
1787 cxlsd->nr_targets = nr_targets;
1788 return cxl_decoder_init(port, &cxlsd->cxld);
1789}
1790
1791/**
1792 * cxl_root_decoder_alloc - Allocate a root level decoder
1793 * @port: owning CXL root of this decoder
1794 * @nr_targets: static number of downstream targets
1795 *
1796 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1797 * 'CXL root' decoder is one that decodes from a top-level / static platform
1798 * firmware description of CXL resources into a CXL standard decode
1799 * topology.
1800 */
1801struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
1802 unsigned int nr_targets)
1803{
1804 struct cxl_root_decoder *cxlrd;
1805 struct cxl_switch_decoder *cxlsd;
1806 struct cxl_decoder *cxld;
1807 int rc;
1808
1809 if (!is_cxl_root(port))
1810 return ERR_PTR(-EINVAL);
1811
1812 cxlrd = kzalloc(struct_size(cxlrd, cxlsd.target, nr_targets),
1813 GFP_KERNEL);
1814 if (!cxlrd)
1815 return ERR_PTR(-ENOMEM);
1816
1817 cxlsd = &cxlrd->cxlsd;
1818 rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1819 if (rc) {
1820 kfree(cxlrd);
1821 return ERR_PTR(rc);
1822 }
1823
1824 mutex_init(&cxlrd->range_lock);
1825
1826 cxld = &cxlsd->cxld;
1827 cxld->dev.type = &cxl_decoder_root_type;
1828 /*
1829 * cxl_root_decoder_release() special cases negative ids to
1830 * detect memregion_alloc() failures.
1831 */
1832 atomic_set(&cxlrd->region_id, -1);
1833 rc = memregion_alloc(GFP_KERNEL);
1834 if (rc < 0) {
1835 put_device(&cxld->dev);
1836 return ERR_PTR(rc);
1837 }
1838
1839 atomic_set(&cxlrd->region_id, rc);
1840 cxlrd->qos_class = CXL_QOS_CLASS_INVALID;
1841 return cxlrd;
1842}
1843EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, "CXL");
1844
1845/**
1846 * cxl_switch_decoder_alloc - Allocate a switch level decoder
1847 * @port: owning CXL switch port of this decoder
1848 * @nr_targets: max number of dynamically addressable downstream targets
1849 *
1850 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1851 * 'switch' decoder is any decoder that can be enumerated by PCIe
1852 * topology and the HDM Decoder Capability. This includes the decoders
1853 * that sit between Switch Upstream Ports / Switch Downstream Ports and
1854 * Host Bridges / Root Ports.
1855 */
1856struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
1857 unsigned int nr_targets)
1858{
1859 struct cxl_switch_decoder *cxlsd;
1860 struct cxl_decoder *cxld;
1861 int rc;
1862
1863 if (is_cxl_root(port) || is_cxl_endpoint(port))
1864 return ERR_PTR(-EINVAL);
1865
1866 cxlsd = kzalloc(struct_size(cxlsd, target, nr_targets), GFP_KERNEL);
1867 if (!cxlsd)
1868 return ERR_PTR(-ENOMEM);
1869
1870 rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1871 if (rc) {
1872 kfree(cxlsd);
1873 return ERR_PTR(rc);
1874 }
1875
1876 cxld = &cxlsd->cxld;
1877 cxld->dev.type = &cxl_decoder_switch_type;
1878 return cxlsd;
1879}
1880EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, "CXL");
1881
1882/**
1883 * cxl_endpoint_decoder_alloc - Allocate an endpoint decoder
1884 * @port: owning port of this decoder
1885 *
1886 * Return: A new cxl decoder to be registered by cxl_decoder_add()
1887 */
1888struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port)
1889{
1890 struct cxl_endpoint_decoder *cxled;
1891 struct cxl_decoder *cxld;
1892 int rc;
1893
1894 if (!is_cxl_endpoint(port))
1895 return ERR_PTR(-EINVAL);
1896
1897 cxled = kzalloc(sizeof(*cxled), GFP_KERNEL);
1898 if (!cxled)
1899 return ERR_PTR(-ENOMEM);
1900
1901 cxled->pos = -1;
1902 cxld = &cxled->cxld;
1903 rc = cxl_decoder_init(port, cxld);
1904 if (rc) {
1905 kfree(cxled);
1906 return ERR_PTR(rc);
1907 }
1908
1909 cxld->dev.type = &cxl_decoder_endpoint_type;
1910 return cxled;
1911}
1912EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, "CXL");
1913
1914/**
1915 * cxl_decoder_add_locked - Add a decoder with targets
1916 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1917 * @target_map: A list of downstream ports that this decoder can direct memory
1918 * traffic to. These numbers should correspond with the port number
1919 * in the PCIe Link Capabilities structure.
1920 *
1921 * Certain types of decoders may not have any targets. The main example of this
1922 * is an endpoint device. A more awkward example is a hostbridge whose root
1923 * ports get hot added (technically possible, though unlikely).
1924 *
1925 * This is the locked variant of cxl_decoder_add().
1926 *
1927 * Context: Process context. Expects the device lock of the port that owns the
1928 * @cxld to be held.
1929 *
1930 * Return: Negative error code if the decoder wasn't properly configured; else
1931 * returns 0.
1932 */
1933int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map)
1934{
1935 struct cxl_port *port;
1936 struct device *dev;
1937 int rc;
1938
1939 if (WARN_ON_ONCE(!cxld))
1940 return -EINVAL;
1941
1942 if (WARN_ON_ONCE(IS_ERR(cxld)))
1943 return PTR_ERR(cxld);
1944
1945 if (cxld->interleave_ways < 1)
1946 return -EINVAL;
1947
1948 dev = &cxld->dev;
1949
1950 port = to_cxl_port(cxld->dev.parent);
1951 if (!is_endpoint_decoder(dev)) {
1952 struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
1953
1954 rc = decoder_populate_targets(cxlsd, port, target_map);
1955 if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) {
1956 dev_err(&port->dev,
1957 "Failed to populate active decoder targets\n");
1958 return rc;
1959 }
1960 }
1961
1962 rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);
1963 if (rc)
1964 return rc;
1965
1966 return device_add(dev);
1967}
1968EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, "CXL");
1969
1970/**
1971 * cxl_decoder_add - Add a decoder with targets
1972 * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1973 * @target_map: A list of downstream ports that this decoder can direct memory
1974 * traffic to. These numbers should correspond with the port number
1975 * in the PCIe Link Capabilities structure.
1976 *
1977 * This is the unlocked variant of cxl_decoder_add_locked().
1978 * See cxl_decoder_add_locked().
1979 *
1980 * Context: Process context. Takes and releases the device lock of the port that
1981 * owns the @cxld.
1982 */
1983int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map)
1984{
1985 struct cxl_port *port;
1986
1987 if (WARN_ON_ONCE(!cxld))
1988 return -EINVAL;
1989
1990 if (WARN_ON_ONCE(IS_ERR(cxld)))
1991 return PTR_ERR(cxld);
1992
1993 port = to_cxl_port(cxld->dev.parent);
1994
1995 guard(device)(&port->dev);
1996 return cxl_decoder_add_locked(cxld, target_map);
1997}
1998EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, "CXL");
1999
2000static void cxld_unregister(void *dev)
2001{
2002 struct cxl_endpoint_decoder *cxled;
2003
2004 if (is_endpoint_decoder(dev)) {
2005 cxled = to_cxl_endpoint_decoder(dev);
2006 cxl_decoder_kill_region(cxled);
2007 }
2008
2009 device_unregister(dev);
2010}
2011
2012int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
2013{
2014 return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev);
2015}
2016EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, "CXL");
2017
2018/**
2019 * __cxl_driver_register - register a driver for the cxl bus
2020 * @cxl_drv: cxl driver structure to attach
2021 * @owner: owning module/driver
2022 * @modname: KBUILD_MODNAME for parent driver
2023 */
2024int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
2025 const char *modname)
2026{
2027 if (!cxl_drv->probe) {
2028 pr_debug("%s ->probe() must be specified\n", modname);
2029 return -EINVAL;
2030 }
2031
2032 if (!cxl_drv->name) {
2033 pr_debug("%s ->name must be specified\n", modname);
2034 return -EINVAL;
2035 }
2036
2037 if (!cxl_drv->id) {
2038 pr_debug("%s ->id must be specified\n", modname);
2039 return -EINVAL;
2040 }
2041
2042 cxl_drv->drv.bus = &cxl_bus_type;
2043 cxl_drv->drv.owner = owner;
2044 cxl_drv->drv.mod_name = modname;
2045 cxl_drv->drv.name = cxl_drv->name;
2046
2047 return driver_register(&cxl_drv->drv);
2048}
2049EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, "CXL");
2050
2051void cxl_driver_unregister(struct cxl_driver *cxl_drv)
2052{
2053 driver_unregister(&cxl_drv->drv);
2054}
2055EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, "CXL");
2056
2057static int cxl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
2058{
2059 return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,
2060 cxl_device_id(dev));
2061}
2062
2063static int cxl_bus_match(struct device *dev, const struct device_driver *drv)
2064{
2065 return cxl_device_id(dev) == to_cxl_drv(drv)->id;
2066}
2067
2068static int cxl_bus_probe(struct device *dev)
2069{
2070 int rc;
2071
2072 rc = to_cxl_drv(dev->driver)->probe(dev);
2073 dev_dbg(dev, "probe: %d\n", rc);
2074 return rc;
2075}
2076
2077static void cxl_bus_remove(struct device *dev)
2078{
2079 struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
2080
2081 if (cxl_drv->remove)
2082 cxl_drv->remove(dev);
2083}
2084
2085static struct workqueue_struct *cxl_bus_wq;
2086
2087static int cxl_rescan_attach(struct device *dev, void *data)
2088{
2089 int rc = device_attach(dev);
2090
2091 dev_vdbg(dev, "rescan: %s\n", rc ? "attach" : "detached");
2092
2093 return 0;
2094}
2095
2096static void cxl_bus_rescan_queue(struct work_struct *w)
2097{
2098 bus_for_each_dev(&cxl_bus_type, NULL, NULL, cxl_rescan_attach);
2099}
2100
2101void cxl_bus_rescan(void)
2102{
2103 static DECLARE_WORK(rescan_work, cxl_bus_rescan_queue);
2104
2105 queue_work(cxl_bus_wq, &rescan_work);
2106}
2107EXPORT_SYMBOL_NS_GPL(cxl_bus_rescan, "CXL");
2108
2109void cxl_bus_drain(void)
2110{
2111 drain_workqueue(cxl_bus_wq);
2112}
2113EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, "CXL");
2114
2115bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
2116{
2117 return queue_work(cxl_bus_wq, &cxlmd->detach_work);
2118}
2119EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, "CXL");
2120
2121static void add_latency(struct access_coordinate *c, long latency)
2122{
2123 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2124 c[i].write_latency += latency;
2125 c[i].read_latency += latency;
2126 }
2127}
2128
2129static bool coordinates_valid(struct access_coordinate *c)
2130{
2131 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2132 if (c[i].read_bandwidth && c[i].write_bandwidth &&
2133 c[i].read_latency && c[i].write_latency)
2134 continue;
2135 return false;
2136 }
2137
2138 return true;
2139}
2140
2141static void set_min_bandwidth(struct access_coordinate *c, unsigned int bw)
2142{
2143 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2144 c[i].write_bandwidth = min(c[i].write_bandwidth, bw);
2145 c[i].read_bandwidth = min(c[i].read_bandwidth, bw);
2146 }
2147}
2148
2149static void set_access_coordinates(struct access_coordinate *out,
2150 struct access_coordinate *in)
2151{
2152 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++)
2153 out[i] = in[i];
2154}
2155
2156static bool parent_port_is_cxl_root(struct cxl_port *port)
2157{
2158 return is_cxl_root(to_cxl_port(port->dev.parent));
2159}
2160
2161/**
2162 * cxl_endpoint_get_perf_coordinates - Retrieve performance numbers stored in dports
2163 * of CXL path
2164 * @port: endpoint cxl_port
2165 * @coord: output performance data
2166 *
2167 * Return: errno on failure, 0 on success.
2168 */
2169int cxl_endpoint_get_perf_coordinates(struct cxl_port *port,
2170 struct access_coordinate *coord)
2171{
2172 struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
2173 struct access_coordinate c[] = {
2174 {
2175 .read_bandwidth = UINT_MAX,
2176 .write_bandwidth = UINT_MAX,
2177 },
2178 {
2179 .read_bandwidth = UINT_MAX,
2180 .write_bandwidth = UINT_MAX,
2181 },
2182 };
2183 struct cxl_port *iter = port;
2184 struct cxl_dport *dport;
2185 struct pci_dev *pdev;
2186 struct device *dev;
2187 unsigned int bw;
2188 bool is_cxl_root;
2189
2190 if (!is_cxl_endpoint(port))
2191 return -EINVAL;
2192
2193 /*
2194 * Skip calculation for RCD. Expectation is HMAT already covers RCD case
2195 * since RCH does not support hotplug.
2196 */
2197 if (cxlmd->cxlds->rcd)
2198 return 0;
2199
2200 /*
2201 * Exit the loop when the parent port of the current iter port is cxl
2202 * root. The iterative loop starts at the endpoint and gathers the
2203 * latency of the CXL link from the current device/port to the connected
2204 * downstream port each iteration.
2205 */
2206 do {
2207 dport = iter->parent_dport;
2208 iter = to_cxl_port(iter->dev.parent);
2209 is_cxl_root = parent_port_is_cxl_root(iter);
2210
2211 /*
2212 * There's no valid access_coordinate for a root port since RPs do not
2213 * have CDAT and therefore needs to be skipped.
2214 */
2215 if (!is_cxl_root) {
2216 if (!coordinates_valid(dport->coord))
2217 return -EINVAL;
2218 cxl_coordinates_combine(c, c, dport->coord);
2219 }
2220 add_latency(c, dport->link_latency);
2221 } while (!is_cxl_root);
2222
2223 dport = iter->parent_dport;
2224 /* Retrieve HB coords */
2225 if (!coordinates_valid(dport->coord))
2226 return -EINVAL;
2227 cxl_coordinates_combine(c, c, dport->coord);
2228
2229 dev = port->uport_dev->parent;
2230 if (!dev_is_pci(dev))
2231 return -ENODEV;
2232
2233 /* Get the calculated PCI paths bandwidth */
2234 pdev = to_pci_dev(dev);
2235 bw = pcie_bandwidth_available(pdev, NULL, NULL, NULL);
2236 if (bw == 0)
2237 return -ENXIO;
2238 bw /= BITS_PER_BYTE;
2239
2240 set_min_bandwidth(c, bw);
2241 set_access_coordinates(coord, c);
2242
2243 return 0;
2244}
2245EXPORT_SYMBOL_NS_GPL(cxl_endpoint_get_perf_coordinates, "CXL");
2246
2247int cxl_port_get_switch_dport_bandwidth(struct cxl_port *port,
2248 struct access_coordinate *c)
2249{
2250 struct cxl_dport *dport = port->parent_dport;
2251
2252 /* Check this port is connected to a switch DSP and not an RP */
2253 if (parent_port_is_cxl_root(to_cxl_port(port->dev.parent)))
2254 return -ENODEV;
2255
2256 if (!coordinates_valid(dport->coord))
2257 return -EINVAL;
2258
2259 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2260 c[i].read_bandwidth = dport->coord[i].read_bandwidth;
2261 c[i].write_bandwidth = dport->coord[i].write_bandwidth;
2262 }
2263
2264 return 0;
2265}
2266
2267/* for user tooling to ensure port disable work has completed */
2268static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count)
2269{
2270 if (sysfs_streq(buf, "1")) {
2271 flush_workqueue(cxl_bus_wq);
2272 return count;
2273 }
2274
2275 return -EINVAL;
2276}
2277
2278static BUS_ATTR_WO(flush);
2279
2280static struct attribute *cxl_bus_attributes[] = {
2281 &bus_attr_flush.attr,
2282 NULL,
2283};
2284
2285static struct attribute_group cxl_bus_attribute_group = {
2286 .attrs = cxl_bus_attributes,
2287};
2288
2289static const struct attribute_group *cxl_bus_attribute_groups[] = {
2290 &cxl_bus_attribute_group,
2291 NULL,
2292};
2293
2294struct bus_type cxl_bus_type = {
2295 .name = "cxl",
2296 .uevent = cxl_bus_uevent,
2297 .match = cxl_bus_match,
2298 .probe = cxl_bus_probe,
2299 .remove = cxl_bus_remove,
2300 .bus_groups = cxl_bus_attribute_groups,
2301};
2302EXPORT_SYMBOL_NS_GPL(cxl_bus_type, "CXL");
2303
2304static struct dentry *cxl_debugfs;
2305
2306struct dentry *cxl_debugfs_create_dir(const char *dir)
2307{
2308 return debugfs_create_dir(dir, cxl_debugfs);
2309}
2310EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, "CXL");
2311
2312static __init int cxl_core_init(void)
2313{
2314 int rc;
2315
2316 cxl_debugfs = debugfs_create_dir("cxl", NULL);
2317
2318 if (einj_cxl_is_initialized())
2319 debugfs_create_file("einj_types", 0400, cxl_debugfs, NULL,
2320 &einj_cxl_available_error_type_fops);
2321
2322 cxl_mbox_init();
2323
2324 rc = cxl_memdev_init();
2325 if (rc)
2326 return rc;
2327
2328 cxl_bus_wq = alloc_ordered_workqueue("cxl_port", 0);
2329 if (!cxl_bus_wq) {
2330 rc = -ENOMEM;
2331 goto err_wq;
2332 }
2333
2334 rc = bus_register(&cxl_bus_type);
2335 if (rc)
2336 goto err_bus;
2337
2338 rc = cxl_region_init();
2339 if (rc)
2340 goto err_region;
2341
2342 return 0;
2343
2344err_region:
2345 bus_unregister(&cxl_bus_type);
2346err_bus:
2347 destroy_workqueue(cxl_bus_wq);
2348err_wq:
2349 cxl_memdev_exit();
2350 return rc;
2351}
2352
2353static void cxl_core_exit(void)
2354{
2355 cxl_region_exit();
2356 bus_unregister(&cxl_bus_type);
2357 destroy_workqueue(cxl_bus_wq);
2358 cxl_memdev_exit();
2359 debugfs_remove_recursive(cxl_debugfs);
2360}
2361
2362subsys_initcall(cxl_core_init);
2363module_exit(cxl_core_exit);
2364MODULE_DESCRIPTION("CXL: Core Compute Express Link support");
2365MODULE_LICENSE("GPL v2");
2366MODULE_IMPORT_NS("CXL");