Loading...
Note: File does not exist in v5.9.
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3#include <linux/memregion.h>
4#include <linux/genalloc.h>
5#include <linux/device.h>
6#include <linux/module.h>
7#include <linux/slab.h>
8#include <linux/uuid.h>
9#include <linux/idr.h>
10#include <cxlmem.h>
11#include <cxl.h>
12#include "core.h"
13
14/**
15 * DOC: cxl core region
16 *
17 * CXL Regions represent mapped memory capacity in system physical address
18 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
19 * Memory ranges, Regions represent the active mapped capacity by the HDM
20 * Decoder Capability structures throughout the Host Bridges, Switches, and
21 * Endpoints in the topology.
22 *
23 * Region configuration has ordering constraints. UUID may be set at any time
24 * but is only visible for persistent regions.
25 * 1. Interleave granularity
26 * 2. Interleave size
27 * 3. Decoder targets
28 */
29
30/*
31 * All changes to the interleave configuration occur with this lock held
32 * for write.
33 */
34static DECLARE_RWSEM(cxl_region_rwsem);
35
36static struct cxl_region *to_cxl_region(struct device *dev);
37
38static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
39 char *buf)
40{
41 struct cxl_region *cxlr = to_cxl_region(dev);
42 struct cxl_region_params *p = &cxlr->params;
43 ssize_t rc;
44
45 rc = down_read_interruptible(&cxl_region_rwsem);
46 if (rc)
47 return rc;
48 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
49 up_read(&cxl_region_rwsem);
50
51 return rc;
52}
53
54static int is_dup(struct device *match, void *data)
55{
56 struct cxl_region_params *p;
57 struct cxl_region *cxlr;
58 uuid_t *uuid = data;
59
60 if (!is_cxl_region(match))
61 return 0;
62
63 lockdep_assert_held(&cxl_region_rwsem);
64 cxlr = to_cxl_region(match);
65 p = &cxlr->params;
66
67 if (uuid_equal(&p->uuid, uuid)) {
68 dev_dbg(match, "already has uuid: %pUb\n", uuid);
69 return -EBUSY;
70 }
71
72 return 0;
73}
74
75static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
76 const char *buf, size_t len)
77{
78 struct cxl_region *cxlr = to_cxl_region(dev);
79 struct cxl_region_params *p = &cxlr->params;
80 uuid_t temp;
81 ssize_t rc;
82
83 if (len != UUID_STRING_LEN + 1)
84 return -EINVAL;
85
86 rc = uuid_parse(buf, &temp);
87 if (rc)
88 return rc;
89
90 if (uuid_is_null(&temp))
91 return -EINVAL;
92
93 rc = down_write_killable(&cxl_region_rwsem);
94 if (rc)
95 return rc;
96
97 if (uuid_equal(&p->uuid, &temp))
98 goto out;
99
100 rc = -EBUSY;
101 if (p->state >= CXL_CONFIG_ACTIVE)
102 goto out;
103
104 rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
105 if (rc < 0)
106 goto out;
107
108 uuid_copy(&p->uuid, &temp);
109out:
110 up_write(&cxl_region_rwsem);
111
112 if (rc)
113 return rc;
114 return len;
115}
116static DEVICE_ATTR_RW(uuid);
117
118static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
119 struct cxl_region *cxlr)
120{
121 return xa_load(&port->regions, (unsigned long)cxlr);
122}
123
124static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
125{
126 struct cxl_region_params *p = &cxlr->params;
127 int i;
128
129 for (i = count - 1; i >= 0; i--) {
130 struct cxl_endpoint_decoder *cxled = p->targets[i];
131 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
132 struct cxl_port *iter = cxled_to_port(cxled);
133 struct cxl_ep *ep;
134 int rc = 0;
135
136 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
137 iter = to_cxl_port(iter->dev.parent);
138
139 for (ep = cxl_ep_load(iter, cxlmd); iter;
140 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
141 struct cxl_region_ref *cxl_rr;
142 struct cxl_decoder *cxld;
143
144 cxl_rr = cxl_rr_load(iter, cxlr);
145 cxld = cxl_rr->decoder;
146 if (cxld->reset)
147 rc = cxld->reset(cxld);
148 if (rc)
149 return rc;
150 }
151
152 rc = cxled->cxld.reset(&cxled->cxld);
153 if (rc)
154 return rc;
155 }
156
157 return 0;
158}
159
160static int cxl_region_decode_commit(struct cxl_region *cxlr)
161{
162 struct cxl_region_params *p = &cxlr->params;
163 int i, rc = 0;
164
165 for (i = 0; i < p->nr_targets; i++) {
166 struct cxl_endpoint_decoder *cxled = p->targets[i];
167 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
168 struct cxl_region_ref *cxl_rr;
169 struct cxl_decoder *cxld;
170 struct cxl_port *iter;
171 struct cxl_ep *ep;
172
173 /* commit bottom up */
174 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
175 iter = to_cxl_port(iter->dev.parent)) {
176 cxl_rr = cxl_rr_load(iter, cxlr);
177 cxld = cxl_rr->decoder;
178 if (cxld->commit)
179 rc = cxld->commit(cxld);
180 if (rc)
181 break;
182 }
183
184 if (rc) {
185 /* programming @iter failed, teardown */
186 for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
187 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
188 cxl_rr = cxl_rr_load(iter, cxlr);
189 cxld = cxl_rr->decoder;
190 if (cxld->reset)
191 cxld->reset(cxld);
192 }
193
194 cxled->cxld.reset(&cxled->cxld);
195 goto err;
196 }
197 }
198
199 return 0;
200
201err:
202 /* undo the targets that were successfully committed */
203 cxl_region_decode_reset(cxlr, i);
204 return rc;
205}
206
207static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
208 const char *buf, size_t len)
209{
210 struct cxl_region *cxlr = to_cxl_region(dev);
211 struct cxl_region_params *p = &cxlr->params;
212 bool commit;
213 ssize_t rc;
214
215 rc = kstrtobool(buf, &commit);
216 if (rc)
217 return rc;
218
219 rc = down_write_killable(&cxl_region_rwsem);
220 if (rc)
221 return rc;
222
223 /* Already in the requested state? */
224 if (commit && p->state >= CXL_CONFIG_COMMIT)
225 goto out;
226 if (!commit && p->state < CXL_CONFIG_COMMIT)
227 goto out;
228
229 /* Not ready to commit? */
230 if (commit && p->state < CXL_CONFIG_ACTIVE) {
231 rc = -ENXIO;
232 goto out;
233 }
234
235 if (commit)
236 rc = cxl_region_decode_commit(cxlr);
237 else {
238 p->state = CXL_CONFIG_RESET_PENDING;
239 up_write(&cxl_region_rwsem);
240 device_release_driver(&cxlr->dev);
241 down_write(&cxl_region_rwsem);
242
243 /*
244 * The lock was dropped, so need to revalidate that the reset is
245 * still pending.
246 */
247 if (p->state == CXL_CONFIG_RESET_PENDING)
248 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
249 }
250
251 if (rc)
252 goto out;
253
254 if (commit)
255 p->state = CXL_CONFIG_COMMIT;
256 else if (p->state == CXL_CONFIG_RESET_PENDING)
257 p->state = CXL_CONFIG_ACTIVE;
258
259out:
260 up_write(&cxl_region_rwsem);
261
262 if (rc)
263 return rc;
264 return len;
265}
266
267static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
268 char *buf)
269{
270 struct cxl_region *cxlr = to_cxl_region(dev);
271 struct cxl_region_params *p = &cxlr->params;
272 ssize_t rc;
273
274 rc = down_read_interruptible(&cxl_region_rwsem);
275 if (rc)
276 return rc;
277 rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
278 up_read(&cxl_region_rwsem);
279
280 return rc;
281}
282static DEVICE_ATTR_RW(commit);
283
284static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
285 int n)
286{
287 struct device *dev = kobj_to_dev(kobj);
288 struct cxl_region *cxlr = to_cxl_region(dev);
289
290 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
291 return 0;
292 return a->mode;
293}
294
295static ssize_t interleave_ways_show(struct device *dev,
296 struct device_attribute *attr, char *buf)
297{
298 struct cxl_region *cxlr = to_cxl_region(dev);
299 struct cxl_region_params *p = &cxlr->params;
300 ssize_t rc;
301
302 rc = down_read_interruptible(&cxl_region_rwsem);
303 if (rc)
304 return rc;
305 rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
306 up_read(&cxl_region_rwsem);
307
308 return rc;
309}
310
311static const struct attribute_group *get_cxl_region_target_group(void);
312
313static ssize_t interleave_ways_store(struct device *dev,
314 struct device_attribute *attr,
315 const char *buf, size_t len)
316{
317 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
318 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
319 struct cxl_region *cxlr = to_cxl_region(dev);
320 struct cxl_region_params *p = &cxlr->params;
321 unsigned int val, save;
322 int rc;
323 u8 iw;
324
325 rc = kstrtouint(buf, 0, &val);
326 if (rc)
327 return rc;
328
329 rc = ways_to_eiw(val, &iw);
330 if (rc)
331 return rc;
332
333 /*
334 * Even for x3, x9, and x12 interleaves the region interleave must be a
335 * power of 2 multiple of the host bridge interleave.
336 */
337 if (!is_power_of_2(val / cxld->interleave_ways) ||
338 (val % cxld->interleave_ways)) {
339 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
340 return -EINVAL;
341 }
342
343 rc = down_write_killable(&cxl_region_rwsem);
344 if (rc)
345 return rc;
346 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
347 rc = -EBUSY;
348 goto out;
349 }
350
351 save = p->interleave_ways;
352 p->interleave_ways = val;
353 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
354 if (rc)
355 p->interleave_ways = save;
356out:
357 up_write(&cxl_region_rwsem);
358 if (rc)
359 return rc;
360 return len;
361}
362static DEVICE_ATTR_RW(interleave_ways);
363
364static ssize_t interleave_granularity_show(struct device *dev,
365 struct device_attribute *attr,
366 char *buf)
367{
368 struct cxl_region *cxlr = to_cxl_region(dev);
369 struct cxl_region_params *p = &cxlr->params;
370 ssize_t rc;
371
372 rc = down_read_interruptible(&cxl_region_rwsem);
373 if (rc)
374 return rc;
375 rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
376 up_read(&cxl_region_rwsem);
377
378 return rc;
379}
380
381static ssize_t interleave_granularity_store(struct device *dev,
382 struct device_attribute *attr,
383 const char *buf, size_t len)
384{
385 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
386 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
387 struct cxl_region *cxlr = to_cxl_region(dev);
388 struct cxl_region_params *p = &cxlr->params;
389 int rc, val;
390 u16 ig;
391
392 rc = kstrtoint(buf, 0, &val);
393 if (rc)
394 return rc;
395
396 rc = granularity_to_eig(val, &ig);
397 if (rc)
398 return rc;
399
400 /*
401 * When the host-bridge is interleaved, disallow region granularity !=
402 * root granularity. Regions with a granularity less than the root
403 * interleave result in needing multiple endpoints to support a single
404 * slot in the interleave (possible to suport in the future). Regions
405 * with a granularity greater than the root interleave result in invalid
406 * DPA translations (invalid to support).
407 */
408 if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
409 return -EINVAL;
410
411 rc = down_write_killable(&cxl_region_rwsem);
412 if (rc)
413 return rc;
414 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
415 rc = -EBUSY;
416 goto out;
417 }
418
419 p->interleave_granularity = val;
420out:
421 up_write(&cxl_region_rwsem);
422 if (rc)
423 return rc;
424 return len;
425}
426static DEVICE_ATTR_RW(interleave_granularity);
427
428static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
429 char *buf)
430{
431 struct cxl_region *cxlr = to_cxl_region(dev);
432 struct cxl_region_params *p = &cxlr->params;
433 u64 resource = -1ULL;
434 ssize_t rc;
435
436 rc = down_read_interruptible(&cxl_region_rwsem);
437 if (rc)
438 return rc;
439 if (p->res)
440 resource = p->res->start;
441 rc = sysfs_emit(buf, "%#llx\n", resource);
442 up_read(&cxl_region_rwsem);
443
444 return rc;
445}
446static DEVICE_ATTR_RO(resource);
447
448static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
449{
450 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
451 struct cxl_region_params *p = &cxlr->params;
452 struct resource *res;
453 u32 remainder = 0;
454
455 lockdep_assert_held_write(&cxl_region_rwsem);
456
457 /* Nothing to do... */
458 if (p->res && resource_size(p->res) == size)
459 return 0;
460
461 /* To change size the old size must be freed first */
462 if (p->res)
463 return -EBUSY;
464
465 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
466 return -EBUSY;
467
468 /* ways, granularity and uuid (if PMEM) need to be set before HPA */
469 if (!p->interleave_ways || !p->interleave_granularity ||
470 (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
471 return -ENXIO;
472
473 div_u64_rem(size, SZ_256M * p->interleave_ways, &remainder);
474 if (remainder)
475 return -EINVAL;
476
477 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
478 dev_name(&cxlr->dev));
479 if (IS_ERR(res)) {
480 dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n",
481 PTR_ERR(res));
482 return PTR_ERR(res);
483 }
484
485 p->res = res;
486 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
487
488 return 0;
489}
490
491static void cxl_region_iomem_release(struct cxl_region *cxlr)
492{
493 struct cxl_region_params *p = &cxlr->params;
494
495 if (device_is_registered(&cxlr->dev))
496 lockdep_assert_held_write(&cxl_region_rwsem);
497 if (p->res) {
498 remove_resource(p->res);
499 kfree(p->res);
500 p->res = NULL;
501 }
502}
503
504static int free_hpa(struct cxl_region *cxlr)
505{
506 struct cxl_region_params *p = &cxlr->params;
507
508 lockdep_assert_held_write(&cxl_region_rwsem);
509
510 if (!p->res)
511 return 0;
512
513 if (p->state >= CXL_CONFIG_ACTIVE)
514 return -EBUSY;
515
516 cxl_region_iomem_release(cxlr);
517 p->state = CXL_CONFIG_IDLE;
518 return 0;
519}
520
521static ssize_t size_store(struct device *dev, struct device_attribute *attr,
522 const char *buf, size_t len)
523{
524 struct cxl_region *cxlr = to_cxl_region(dev);
525 u64 val;
526 int rc;
527
528 rc = kstrtou64(buf, 0, &val);
529 if (rc)
530 return rc;
531
532 rc = down_write_killable(&cxl_region_rwsem);
533 if (rc)
534 return rc;
535
536 if (val)
537 rc = alloc_hpa(cxlr, val);
538 else
539 rc = free_hpa(cxlr);
540 up_write(&cxl_region_rwsem);
541
542 if (rc)
543 return rc;
544
545 return len;
546}
547
548static ssize_t size_show(struct device *dev, struct device_attribute *attr,
549 char *buf)
550{
551 struct cxl_region *cxlr = to_cxl_region(dev);
552 struct cxl_region_params *p = &cxlr->params;
553 u64 size = 0;
554 ssize_t rc;
555
556 rc = down_read_interruptible(&cxl_region_rwsem);
557 if (rc)
558 return rc;
559 if (p->res)
560 size = resource_size(p->res);
561 rc = sysfs_emit(buf, "%#llx\n", size);
562 up_read(&cxl_region_rwsem);
563
564 return rc;
565}
566static DEVICE_ATTR_RW(size);
567
568static struct attribute *cxl_region_attrs[] = {
569 &dev_attr_uuid.attr,
570 &dev_attr_commit.attr,
571 &dev_attr_interleave_ways.attr,
572 &dev_attr_interleave_granularity.attr,
573 &dev_attr_resource.attr,
574 &dev_attr_size.attr,
575 NULL,
576};
577
578static const struct attribute_group cxl_region_group = {
579 .attrs = cxl_region_attrs,
580 .is_visible = cxl_region_visible,
581};
582
583static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
584{
585 struct cxl_region_params *p = &cxlr->params;
586 struct cxl_endpoint_decoder *cxled;
587 int rc;
588
589 rc = down_read_interruptible(&cxl_region_rwsem);
590 if (rc)
591 return rc;
592
593 if (pos >= p->interleave_ways) {
594 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
595 p->interleave_ways);
596 rc = -ENXIO;
597 goto out;
598 }
599
600 cxled = p->targets[pos];
601 if (!cxled)
602 rc = sysfs_emit(buf, "\n");
603 else
604 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
605out:
606 up_read(&cxl_region_rwsem);
607
608 return rc;
609}
610
611static int match_free_decoder(struct device *dev, void *data)
612{
613 struct cxl_decoder *cxld;
614 int *id = data;
615
616 if (!is_switch_decoder(dev))
617 return 0;
618
619 cxld = to_cxl_decoder(dev);
620
621 /* enforce ordered allocation */
622 if (cxld->id != *id)
623 return 0;
624
625 if (!cxld->region)
626 return 1;
627
628 (*id)++;
629
630 return 0;
631}
632
633static struct cxl_decoder *cxl_region_find_decoder(struct cxl_port *port,
634 struct cxl_region *cxlr)
635{
636 struct device *dev;
637 int id = 0;
638
639 dev = device_find_child(&port->dev, &id, match_free_decoder);
640 if (!dev)
641 return NULL;
642 /*
643 * This decoder is pinned registered as long as the endpoint decoder is
644 * registered, and endpoint decoder unregistration holds the
645 * cxl_region_rwsem over unregister events, so no need to hold on to
646 * this extra reference.
647 */
648 put_device(dev);
649 return to_cxl_decoder(dev);
650}
651
652static struct cxl_region_ref *alloc_region_ref(struct cxl_port *port,
653 struct cxl_region *cxlr)
654{
655 struct cxl_region_params *p = &cxlr->params;
656 struct cxl_region_ref *cxl_rr, *iter;
657 unsigned long index;
658 int rc;
659
660 xa_for_each(&port->regions, index, iter) {
661 struct cxl_region_params *ip = &iter->region->params;
662
663 if (!ip->res)
664 continue;
665
666 if (ip->res->start > p->res->start) {
667 dev_dbg(&cxlr->dev,
668 "%s: HPA order violation %s:%pr vs %pr\n",
669 dev_name(&port->dev),
670 dev_name(&iter->region->dev), ip->res, p->res);
671 return ERR_PTR(-EBUSY);
672 }
673 }
674
675 cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
676 if (!cxl_rr)
677 return ERR_PTR(-ENOMEM);
678 cxl_rr->port = port;
679 cxl_rr->region = cxlr;
680 cxl_rr->nr_targets = 1;
681 xa_init(&cxl_rr->endpoints);
682
683 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
684 if (rc) {
685 dev_dbg(&cxlr->dev,
686 "%s: failed to track region reference: %d\n",
687 dev_name(&port->dev), rc);
688 kfree(cxl_rr);
689 return ERR_PTR(rc);
690 }
691
692 return cxl_rr;
693}
694
695static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
696{
697 struct cxl_region *cxlr = cxl_rr->region;
698 struct cxl_decoder *cxld = cxl_rr->decoder;
699
700 if (!cxld)
701 return;
702
703 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
704 if (cxld->region == cxlr) {
705 cxld->region = NULL;
706 put_device(&cxlr->dev);
707 }
708}
709
710static void free_region_ref(struct cxl_region_ref *cxl_rr)
711{
712 struct cxl_port *port = cxl_rr->port;
713 struct cxl_region *cxlr = cxl_rr->region;
714
715 cxl_rr_free_decoder(cxl_rr);
716 xa_erase(&port->regions, (unsigned long)cxlr);
717 xa_destroy(&cxl_rr->endpoints);
718 kfree(cxl_rr);
719}
720
721static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
722 struct cxl_endpoint_decoder *cxled)
723{
724 int rc;
725 struct cxl_port *port = cxl_rr->port;
726 struct cxl_region *cxlr = cxl_rr->region;
727 struct cxl_decoder *cxld = cxl_rr->decoder;
728 struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
729
730 if (ep) {
731 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
732 GFP_KERNEL);
733 if (rc)
734 return rc;
735 }
736 cxl_rr->nr_eps++;
737
738 if (!cxld->region) {
739 cxld->region = cxlr;
740 get_device(&cxlr->dev);
741 }
742
743 return 0;
744}
745
746static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
747 struct cxl_endpoint_decoder *cxled,
748 struct cxl_region_ref *cxl_rr)
749{
750 struct cxl_decoder *cxld;
751
752 if (port == cxled_to_port(cxled))
753 cxld = &cxled->cxld;
754 else
755 cxld = cxl_region_find_decoder(port, cxlr);
756 if (!cxld) {
757 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
758 dev_name(&port->dev));
759 return -EBUSY;
760 }
761
762 if (cxld->region) {
763 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
764 dev_name(&port->dev), dev_name(&cxld->dev),
765 dev_name(&cxld->region->dev));
766 return -EBUSY;
767 }
768
769 cxl_rr->decoder = cxld;
770 return 0;
771}
772
773/**
774 * cxl_port_attach_region() - track a region's interest in a port by endpoint
775 * @port: port to add a new region reference 'struct cxl_region_ref'
776 * @cxlr: region to attach to @port
777 * @cxled: endpoint decoder used to create or further pin a region reference
778 * @pos: interleave position of @cxled in @cxlr
779 *
780 * The attach event is an opportunity to validate CXL decode setup
781 * constraints and record metadata needed for programming HDM decoders,
782 * in particular decoder target lists.
783 *
784 * The steps are:
785 *
786 * - validate that there are no other regions with a higher HPA already
787 * associated with @port
788 * - establish a region reference if one is not already present
789 *
790 * - additionally allocate a decoder instance that will host @cxlr on
791 * @port
792 *
793 * - pin the region reference by the endpoint
794 * - account for how many entries in @port's target list are needed to
795 * cover all of the added endpoints.
796 */
797static int cxl_port_attach_region(struct cxl_port *port,
798 struct cxl_region *cxlr,
799 struct cxl_endpoint_decoder *cxled, int pos)
800{
801 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
802 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
803 struct cxl_region_ref *cxl_rr;
804 bool nr_targets_inc = false;
805 struct cxl_decoder *cxld;
806 unsigned long index;
807 int rc = -EBUSY;
808
809 lockdep_assert_held_write(&cxl_region_rwsem);
810
811 cxl_rr = cxl_rr_load(port, cxlr);
812 if (cxl_rr) {
813 struct cxl_ep *ep_iter;
814 int found = 0;
815
816 /*
817 * Walk the existing endpoints that have been attached to
818 * @cxlr at @port and see if they share the same 'next' port
819 * in the downstream direction. I.e. endpoints that share common
820 * upstream switch.
821 */
822 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
823 if (ep_iter == ep)
824 continue;
825 if (ep_iter->next == ep->next) {
826 found++;
827 break;
828 }
829 }
830
831 /*
832 * New target port, or @port is an endpoint port that always
833 * accounts its own local decode as a target.
834 */
835 if (!found || !ep->next) {
836 cxl_rr->nr_targets++;
837 nr_targets_inc = true;
838 }
839 } else {
840 cxl_rr = alloc_region_ref(port, cxlr);
841 if (IS_ERR(cxl_rr)) {
842 dev_dbg(&cxlr->dev,
843 "%s: failed to allocate region reference\n",
844 dev_name(&port->dev));
845 return PTR_ERR(cxl_rr);
846 }
847 nr_targets_inc = true;
848
849 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
850 if (rc)
851 goto out_erase;
852 }
853 cxld = cxl_rr->decoder;
854
855 rc = cxl_rr_ep_add(cxl_rr, cxled);
856 if (rc) {
857 dev_dbg(&cxlr->dev,
858 "%s: failed to track endpoint %s:%s reference\n",
859 dev_name(&port->dev), dev_name(&cxlmd->dev),
860 dev_name(&cxld->dev));
861 goto out_erase;
862 }
863
864 dev_dbg(&cxlr->dev,
865 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
866 dev_name(port->uport), dev_name(&port->dev),
867 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
868 dev_name(&cxled->cxld.dev), pos,
869 ep ? ep->next ? dev_name(ep->next->uport) :
870 dev_name(&cxlmd->dev) :
871 "none",
872 cxl_rr->nr_eps, cxl_rr->nr_targets);
873
874 return 0;
875out_erase:
876 if (nr_targets_inc)
877 cxl_rr->nr_targets--;
878 if (cxl_rr->nr_eps == 0)
879 free_region_ref(cxl_rr);
880 return rc;
881}
882
883static void cxl_port_detach_region(struct cxl_port *port,
884 struct cxl_region *cxlr,
885 struct cxl_endpoint_decoder *cxled)
886{
887 struct cxl_region_ref *cxl_rr;
888 struct cxl_ep *ep = NULL;
889
890 lockdep_assert_held_write(&cxl_region_rwsem);
891
892 cxl_rr = cxl_rr_load(port, cxlr);
893 if (!cxl_rr)
894 return;
895
896 /*
897 * Endpoint ports do not carry cxl_ep references, and they
898 * never target more than one endpoint by definition
899 */
900 if (cxl_rr->decoder == &cxled->cxld)
901 cxl_rr->nr_eps--;
902 else
903 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
904 if (ep) {
905 struct cxl_ep *ep_iter;
906 unsigned long index;
907 int found = 0;
908
909 cxl_rr->nr_eps--;
910 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
911 if (ep_iter->next == ep->next) {
912 found++;
913 break;
914 }
915 }
916 if (!found)
917 cxl_rr->nr_targets--;
918 }
919
920 if (cxl_rr->nr_eps == 0)
921 free_region_ref(cxl_rr);
922}
923
924static int check_last_peer(struct cxl_endpoint_decoder *cxled,
925 struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
926 int distance)
927{
928 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
929 struct cxl_region *cxlr = cxl_rr->region;
930 struct cxl_region_params *p = &cxlr->params;
931 struct cxl_endpoint_decoder *cxled_peer;
932 struct cxl_port *port = cxl_rr->port;
933 struct cxl_memdev *cxlmd_peer;
934 struct cxl_ep *ep_peer;
935 int pos = cxled->pos;
936
937 /*
938 * If this position wants to share a dport with the last endpoint mapped
939 * then that endpoint, at index 'position - distance', must also be
940 * mapped by this dport.
941 */
942 if (pos < distance) {
943 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
944 dev_name(port->uport), dev_name(&port->dev),
945 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
946 return -ENXIO;
947 }
948 cxled_peer = p->targets[pos - distance];
949 cxlmd_peer = cxled_to_memdev(cxled_peer);
950 ep_peer = cxl_ep_load(port, cxlmd_peer);
951 if (ep->dport != ep_peer->dport) {
952 dev_dbg(&cxlr->dev,
953 "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
954 dev_name(port->uport), dev_name(&port->dev),
955 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
956 dev_name(&cxlmd_peer->dev),
957 dev_name(&cxled_peer->cxld.dev));
958 return -ENXIO;
959 }
960
961 return 0;
962}
963
964static int cxl_port_setup_targets(struct cxl_port *port,
965 struct cxl_region *cxlr,
966 struct cxl_endpoint_decoder *cxled)
967{
968 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
969 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
970 struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
971 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
972 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
973 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
974 struct cxl_region_params *p = &cxlr->params;
975 struct cxl_decoder *cxld = cxl_rr->decoder;
976 struct cxl_switch_decoder *cxlsd;
977 u16 eig, peig;
978 u8 eiw, peiw;
979
980 /*
981 * While root level decoders support x3, x6, x12, switch level
982 * decoders only support powers of 2 up to x16.
983 */
984 if (!is_power_of_2(cxl_rr->nr_targets)) {
985 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
986 dev_name(port->uport), dev_name(&port->dev),
987 cxl_rr->nr_targets);
988 return -EINVAL;
989 }
990
991 cxlsd = to_cxl_switch_decoder(&cxld->dev);
992 if (cxl_rr->nr_targets_set) {
993 int i, distance;
994
995 /*
996 * Passthrough decoders impose no distance requirements between
997 * peers
998 */
999 if (cxl_rr->nr_targets == 1)
1000 distance = 0;
1001 else
1002 distance = p->nr_targets / cxl_rr->nr_targets;
1003 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1004 if (ep->dport == cxlsd->target[i]) {
1005 rc = check_last_peer(cxled, ep, cxl_rr,
1006 distance);
1007 if (rc)
1008 return rc;
1009 goto out_target_set;
1010 }
1011 goto add_target;
1012 }
1013
1014 if (is_cxl_root(parent_port)) {
1015 parent_ig = cxlrd->cxlsd.cxld.interleave_granularity;
1016 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1017 /*
1018 * For purposes of address bit routing, use power-of-2 math for
1019 * switch ports.
1020 */
1021 if (!is_power_of_2(parent_iw))
1022 parent_iw /= 3;
1023 } else {
1024 struct cxl_region_ref *parent_rr;
1025 struct cxl_decoder *parent_cxld;
1026
1027 parent_rr = cxl_rr_load(parent_port, cxlr);
1028 parent_cxld = parent_rr->decoder;
1029 parent_ig = parent_cxld->interleave_granularity;
1030 parent_iw = parent_cxld->interleave_ways;
1031 }
1032
1033 rc = granularity_to_eig(parent_ig, &peig);
1034 if (rc) {
1035 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1036 dev_name(parent_port->uport),
1037 dev_name(&parent_port->dev), parent_ig);
1038 return rc;
1039 }
1040
1041 rc = ways_to_eiw(parent_iw, &peiw);
1042 if (rc) {
1043 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1044 dev_name(parent_port->uport),
1045 dev_name(&parent_port->dev), parent_iw);
1046 return rc;
1047 }
1048
1049 iw = cxl_rr->nr_targets;
1050 rc = ways_to_eiw(iw, &eiw);
1051 if (rc) {
1052 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1053 dev_name(port->uport), dev_name(&port->dev), iw);
1054 return rc;
1055 }
1056
1057 /*
1058 * If @parent_port is masking address bits, pick the next unused address
1059 * bit to route @port's targets.
1060 */
1061 if (parent_iw > 1 && cxl_rr->nr_targets > 1) {
1062 u32 address_bit = max(peig + peiw, eiw + peig);
1063
1064 eig = address_bit - eiw + 1;
1065 } else {
1066 eiw = peiw;
1067 eig = peig;
1068 }
1069
1070 rc = eig_to_granularity(eig, &ig);
1071 if (rc) {
1072 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1073 dev_name(port->uport), dev_name(&port->dev),
1074 256 << eig);
1075 return rc;
1076 }
1077
1078 cxld->interleave_ways = iw;
1079 cxld->interleave_granularity = ig;
1080 cxld->hpa_range = (struct range) {
1081 .start = p->res->start,
1082 .end = p->res->end,
1083 };
1084 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport),
1085 dev_name(&port->dev), iw, ig);
1086add_target:
1087 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1088 dev_dbg(&cxlr->dev,
1089 "%s:%s: targets full trying to add %s:%s at %d\n",
1090 dev_name(port->uport), dev_name(&port->dev),
1091 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1092 return -ENXIO;
1093 }
1094 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1095 inc = 1;
1096out_target_set:
1097 cxl_rr->nr_targets_set += inc;
1098 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1099 dev_name(port->uport), dev_name(&port->dev),
1100 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport),
1101 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1102
1103 return 0;
1104}
1105
1106static void cxl_port_reset_targets(struct cxl_port *port,
1107 struct cxl_region *cxlr)
1108{
1109 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1110 struct cxl_decoder *cxld;
1111
1112 /*
1113 * After the last endpoint has been detached the entire cxl_rr may now
1114 * be gone.
1115 */
1116 if (!cxl_rr)
1117 return;
1118 cxl_rr->nr_targets_set = 0;
1119
1120 cxld = cxl_rr->decoder;
1121 cxld->hpa_range = (struct range) {
1122 .start = 0,
1123 .end = -1,
1124 };
1125}
1126
1127static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1128{
1129 struct cxl_region_params *p = &cxlr->params;
1130 struct cxl_endpoint_decoder *cxled;
1131 struct cxl_memdev *cxlmd;
1132 struct cxl_port *iter;
1133 struct cxl_ep *ep;
1134 int i;
1135
1136 for (i = 0; i < p->nr_targets; i++) {
1137 cxled = p->targets[i];
1138 cxlmd = cxled_to_memdev(cxled);
1139
1140 iter = cxled_to_port(cxled);
1141 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1142 iter = to_cxl_port(iter->dev.parent);
1143
1144 for (ep = cxl_ep_load(iter, cxlmd); iter;
1145 iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1146 cxl_port_reset_targets(iter, cxlr);
1147 }
1148}
1149
1150static int cxl_region_setup_targets(struct cxl_region *cxlr)
1151{
1152 struct cxl_region_params *p = &cxlr->params;
1153 struct cxl_endpoint_decoder *cxled;
1154 struct cxl_memdev *cxlmd;
1155 struct cxl_port *iter;
1156 struct cxl_ep *ep;
1157 int i, rc;
1158
1159 for (i = 0; i < p->nr_targets; i++) {
1160 cxled = p->targets[i];
1161 cxlmd = cxled_to_memdev(cxled);
1162
1163 iter = cxled_to_port(cxled);
1164 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1165 iter = to_cxl_port(iter->dev.parent);
1166
1167 /*
1168 * Descend the topology tree programming targets while
1169 * looking for conflicts.
1170 */
1171 for (ep = cxl_ep_load(iter, cxlmd); iter;
1172 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1173 rc = cxl_port_setup_targets(iter, cxlr, cxled);
1174 if (rc) {
1175 cxl_region_teardown_targets(cxlr);
1176 return rc;
1177 }
1178 }
1179 }
1180
1181 return 0;
1182}
1183
1184static int cxl_region_attach(struct cxl_region *cxlr,
1185 struct cxl_endpoint_decoder *cxled, int pos)
1186{
1187 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1188 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1189 struct cxl_port *ep_port, *root_port, *iter;
1190 struct cxl_region_params *p = &cxlr->params;
1191 struct cxl_dport *dport;
1192 int i, rc = -ENXIO;
1193
1194 if (cxled->mode == CXL_DECODER_DEAD) {
1195 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1196 return -ENODEV;
1197 }
1198
1199 /* all full of members, or interleave config not established? */
1200 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1201 dev_dbg(&cxlr->dev, "region already active\n");
1202 return -EBUSY;
1203 } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1204 dev_dbg(&cxlr->dev, "interleave config missing\n");
1205 return -ENXIO;
1206 }
1207
1208 if (pos < 0 || pos >= p->interleave_ways) {
1209 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1210 p->interleave_ways);
1211 return -ENXIO;
1212 }
1213
1214 if (p->targets[pos] == cxled)
1215 return 0;
1216
1217 if (p->targets[pos]) {
1218 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1219 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1220
1221 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1222 pos, dev_name(&cxlmd_target->dev),
1223 dev_name(&cxled_target->cxld.dev));
1224 return -EBUSY;
1225 }
1226
1227 for (i = 0; i < p->interleave_ways; i++) {
1228 struct cxl_endpoint_decoder *cxled_target;
1229 struct cxl_memdev *cxlmd_target;
1230
1231 cxled_target = p->targets[i];
1232 if (!cxled_target)
1233 continue;
1234
1235 cxlmd_target = cxled_to_memdev(cxled_target);
1236 if (cxlmd_target == cxlmd) {
1237 dev_dbg(&cxlr->dev,
1238 "%s already specified at position %d via: %s\n",
1239 dev_name(&cxlmd->dev), pos,
1240 dev_name(&cxled_target->cxld.dev));
1241 return -EBUSY;
1242 }
1243 }
1244
1245 ep_port = cxled_to_port(cxled);
1246 root_port = cxlrd_to_port(cxlrd);
1247 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1248 if (!dport) {
1249 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1250 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1251 dev_name(cxlr->dev.parent));
1252 return -ENXIO;
1253 }
1254
1255 if (cxlrd->calc_hb(cxlrd, pos) != dport) {
1256 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1257 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1258 dev_name(&cxlrd->cxlsd.cxld.dev));
1259 return -ENXIO;
1260 }
1261
1262 if (cxled->cxld.target_type != cxlr->type) {
1263 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1264 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1265 cxled->cxld.target_type, cxlr->type);
1266 return -ENXIO;
1267 }
1268
1269 if (!cxled->dpa_res) {
1270 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1271 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1272 return -ENXIO;
1273 }
1274
1275 if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1276 resource_size(p->res)) {
1277 dev_dbg(&cxlr->dev,
1278 "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1279 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1280 (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1281 (u64)resource_size(p->res));
1282 return -EINVAL;
1283 }
1284
1285 for (iter = ep_port; !is_cxl_root(iter);
1286 iter = to_cxl_port(iter->dev.parent)) {
1287 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1288 if (rc)
1289 goto err;
1290 }
1291
1292 p->targets[pos] = cxled;
1293 cxled->pos = pos;
1294 p->nr_targets++;
1295
1296 if (p->nr_targets == p->interleave_ways) {
1297 rc = cxl_region_setup_targets(cxlr);
1298 if (rc)
1299 goto err_decrement;
1300 p->state = CXL_CONFIG_ACTIVE;
1301 }
1302
1303 cxled->cxld.interleave_ways = p->interleave_ways;
1304 cxled->cxld.interleave_granularity = p->interleave_granularity;
1305 cxled->cxld.hpa_range = (struct range) {
1306 .start = p->res->start,
1307 .end = p->res->end,
1308 };
1309
1310 return 0;
1311
1312err_decrement:
1313 p->nr_targets--;
1314err:
1315 for (iter = ep_port; !is_cxl_root(iter);
1316 iter = to_cxl_port(iter->dev.parent))
1317 cxl_port_detach_region(iter, cxlr, cxled);
1318 return rc;
1319}
1320
1321static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
1322{
1323 struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
1324 struct cxl_region *cxlr = cxled->cxld.region;
1325 struct cxl_region_params *p;
1326 int rc = 0;
1327
1328 lockdep_assert_held_write(&cxl_region_rwsem);
1329
1330 if (!cxlr)
1331 return 0;
1332
1333 p = &cxlr->params;
1334 get_device(&cxlr->dev);
1335
1336 if (p->state > CXL_CONFIG_ACTIVE) {
1337 /*
1338 * TODO: tear down all impacted regions if a device is
1339 * removed out of order
1340 */
1341 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
1342 if (rc)
1343 goto out;
1344 p->state = CXL_CONFIG_ACTIVE;
1345 }
1346
1347 for (iter = ep_port; !is_cxl_root(iter);
1348 iter = to_cxl_port(iter->dev.parent))
1349 cxl_port_detach_region(iter, cxlr, cxled);
1350
1351 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
1352 p->targets[cxled->pos] != cxled) {
1353 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1354
1355 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
1356 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1357 cxled->pos);
1358 goto out;
1359 }
1360
1361 if (p->state == CXL_CONFIG_ACTIVE) {
1362 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
1363 cxl_region_teardown_targets(cxlr);
1364 }
1365 p->targets[cxled->pos] = NULL;
1366 p->nr_targets--;
1367 cxled->cxld.hpa_range = (struct range) {
1368 .start = 0,
1369 .end = -1,
1370 };
1371
1372 /* notify the region driver that one of its targets has departed */
1373 up_write(&cxl_region_rwsem);
1374 device_release_driver(&cxlr->dev);
1375 down_write(&cxl_region_rwsem);
1376out:
1377 put_device(&cxlr->dev);
1378 return rc;
1379}
1380
1381void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
1382{
1383 down_write(&cxl_region_rwsem);
1384 cxled->mode = CXL_DECODER_DEAD;
1385 cxl_region_detach(cxled);
1386 up_write(&cxl_region_rwsem);
1387}
1388
1389static int attach_target(struct cxl_region *cxlr, const char *decoder, int pos)
1390{
1391 struct device *dev;
1392 int rc;
1393
1394 dev = bus_find_device_by_name(&cxl_bus_type, NULL, decoder);
1395 if (!dev)
1396 return -ENODEV;
1397
1398 if (!is_endpoint_decoder(dev)) {
1399 put_device(dev);
1400 return -EINVAL;
1401 }
1402
1403 rc = down_write_killable(&cxl_region_rwsem);
1404 if (rc)
1405 goto out;
1406 down_read(&cxl_dpa_rwsem);
1407 rc = cxl_region_attach(cxlr, to_cxl_endpoint_decoder(dev), pos);
1408 if (rc == 0)
1409 set_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags);
1410 up_read(&cxl_dpa_rwsem);
1411 up_write(&cxl_region_rwsem);
1412out:
1413 put_device(dev);
1414 return rc;
1415}
1416
1417static int detach_target(struct cxl_region *cxlr, int pos)
1418{
1419 struct cxl_region_params *p = &cxlr->params;
1420 int rc;
1421
1422 rc = down_write_killable(&cxl_region_rwsem);
1423 if (rc)
1424 return rc;
1425
1426 if (pos >= p->interleave_ways) {
1427 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1428 p->interleave_ways);
1429 rc = -ENXIO;
1430 goto out;
1431 }
1432
1433 if (!p->targets[pos]) {
1434 rc = 0;
1435 goto out;
1436 }
1437
1438 rc = cxl_region_detach(p->targets[pos]);
1439out:
1440 up_write(&cxl_region_rwsem);
1441 return rc;
1442}
1443
1444static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
1445 size_t len)
1446{
1447 int rc;
1448
1449 if (sysfs_streq(buf, "\n"))
1450 rc = detach_target(cxlr, pos);
1451 else
1452 rc = attach_target(cxlr, buf, pos);
1453
1454 if (rc < 0)
1455 return rc;
1456 return len;
1457}
1458
1459#define TARGET_ATTR_RW(n) \
1460static ssize_t target##n##_show( \
1461 struct device *dev, struct device_attribute *attr, char *buf) \
1462{ \
1463 return show_targetN(to_cxl_region(dev), buf, (n)); \
1464} \
1465static ssize_t target##n##_store(struct device *dev, \
1466 struct device_attribute *attr, \
1467 const char *buf, size_t len) \
1468{ \
1469 return store_targetN(to_cxl_region(dev), buf, (n), len); \
1470} \
1471static DEVICE_ATTR_RW(target##n)
1472
1473TARGET_ATTR_RW(0);
1474TARGET_ATTR_RW(1);
1475TARGET_ATTR_RW(2);
1476TARGET_ATTR_RW(3);
1477TARGET_ATTR_RW(4);
1478TARGET_ATTR_RW(5);
1479TARGET_ATTR_RW(6);
1480TARGET_ATTR_RW(7);
1481TARGET_ATTR_RW(8);
1482TARGET_ATTR_RW(9);
1483TARGET_ATTR_RW(10);
1484TARGET_ATTR_RW(11);
1485TARGET_ATTR_RW(12);
1486TARGET_ATTR_RW(13);
1487TARGET_ATTR_RW(14);
1488TARGET_ATTR_RW(15);
1489
1490static struct attribute *target_attrs[] = {
1491 &dev_attr_target0.attr,
1492 &dev_attr_target1.attr,
1493 &dev_attr_target2.attr,
1494 &dev_attr_target3.attr,
1495 &dev_attr_target4.attr,
1496 &dev_attr_target5.attr,
1497 &dev_attr_target6.attr,
1498 &dev_attr_target7.attr,
1499 &dev_attr_target8.attr,
1500 &dev_attr_target9.attr,
1501 &dev_attr_target10.attr,
1502 &dev_attr_target11.attr,
1503 &dev_attr_target12.attr,
1504 &dev_attr_target13.attr,
1505 &dev_attr_target14.attr,
1506 &dev_attr_target15.attr,
1507 NULL,
1508};
1509
1510static umode_t cxl_region_target_visible(struct kobject *kobj,
1511 struct attribute *a, int n)
1512{
1513 struct device *dev = kobj_to_dev(kobj);
1514 struct cxl_region *cxlr = to_cxl_region(dev);
1515 struct cxl_region_params *p = &cxlr->params;
1516
1517 if (n < p->interleave_ways)
1518 return a->mode;
1519 return 0;
1520}
1521
1522static const struct attribute_group cxl_region_target_group = {
1523 .attrs = target_attrs,
1524 .is_visible = cxl_region_target_visible,
1525};
1526
1527static const struct attribute_group *get_cxl_region_target_group(void)
1528{
1529 return &cxl_region_target_group;
1530}
1531
1532static const struct attribute_group *region_groups[] = {
1533 &cxl_base_attribute_group,
1534 &cxl_region_group,
1535 &cxl_region_target_group,
1536 NULL,
1537};
1538
1539static void cxl_region_release(struct device *dev)
1540{
1541 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
1542 struct cxl_region *cxlr = to_cxl_region(dev);
1543 int id = atomic_read(&cxlrd->region_id);
1544
1545 /*
1546 * Try to reuse the recently idled id rather than the cached
1547 * next id to prevent the region id space from increasing
1548 * unnecessarily.
1549 */
1550 if (cxlr->id < id)
1551 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
1552 memregion_free(id);
1553 goto out;
1554 }
1555
1556 memregion_free(cxlr->id);
1557out:
1558 put_device(dev->parent);
1559 kfree(cxlr);
1560}
1561
1562const struct device_type cxl_region_type = {
1563 .name = "cxl_region",
1564 .release = cxl_region_release,
1565 .groups = region_groups
1566};
1567
1568bool is_cxl_region(struct device *dev)
1569{
1570 return dev->type == &cxl_region_type;
1571}
1572EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL);
1573
1574static struct cxl_region *to_cxl_region(struct device *dev)
1575{
1576 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
1577 "not a cxl_region device\n"))
1578 return NULL;
1579
1580 return container_of(dev, struct cxl_region, dev);
1581}
1582
1583static void unregister_region(void *dev)
1584{
1585 struct cxl_region *cxlr = to_cxl_region(dev);
1586 struct cxl_region_params *p = &cxlr->params;
1587 int i;
1588
1589 device_del(dev);
1590
1591 /*
1592 * Now that region sysfs is shutdown, the parameter block is now
1593 * read-only, so no need to hold the region rwsem to access the
1594 * region parameters.
1595 */
1596 for (i = 0; i < p->interleave_ways; i++)
1597 detach_target(cxlr, i);
1598
1599 cxl_region_iomem_release(cxlr);
1600 put_device(dev);
1601}
1602
1603static struct lock_class_key cxl_region_key;
1604
1605static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
1606{
1607 struct cxl_region *cxlr;
1608 struct device *dev;
1609
1610 cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
1611 if (!cxlr) {
1612 memregion_free(id);
1613 return ERR_PTR(-ENOMEM);
1614 }
1615
1616 dev = &cxlr->dev;
1617 device_initialize(dev);
1618 lockdep_set_class(&dev->mutex, &cxl_region_key);
1619 dev->parent = &cxlrd->cxlsd.cxld.dev;
1620 /*
1621 * Keep root decoder pinned through cxl_region_release to fixup
1622 * region id allocations
1623 */
1624 get_device(dev->parent);
1625 device_set_pm_not_required(dev);
1626 dev->bus = &cxl_bus_type;
1627 dev->type = &cxl_region_type;
1628 cxlr->id = id;
1629
1630 return cxlr;
1631}
1632
1633/**
1634 * devm_cxl_add_region - Adds a region to a decoder
1635 * @cxlrd: root decoder
1636 * @id: memregion id to create, or memregion_free() on failure
1637 * @mode: mode for the endpoint decoders of this region
1638 * @type: select whether this is an expander or accelerator (type-2 or type-3)
1639 *
1640 * This is the second step of region initialization. Regions exist within an
1641 * address space which is mapped by a @cxlrd.
1642 *
1643 * Return: 0 if the region was added to the @cxlrd, else returns negative error
1644 * code. The region will be named "regionZ" where Z is the unique region number.
1645 */
1646static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
1647 int id,
1648 enum cxl_decoder_mode mode,
1649 enum cxl_decoder_type type)
1650{
1651 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
1652 struct cxl_region *cxlr;
1653 struct device *dev;
1654 int rc;
1655
1656 cxlr = cxl_region_alloc(cxlrd, id);
1657 if (IS_ERR(cxlr))
1658 return cxlr;
1659 cxlr->mode = mode;
1660 cxlr->type = type;
1661
1662 dev = &cxlr->dev;
1663 rc = dev_set_name(dev, "region%d", id);
1664 if (rc)
1665 goto err;
1666
1667 rc = device_add(dev);
1668 if (rc)
1669 goto err;
1670
1671 rc = devm_add_action_or_reset(port->uport, unregister_region, cxlr);
1672 if (rc)
1673 return ERR_PTR(rc);
1674
1675 dev_dbg(port->uport, "%s: created %s\n",
1676 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
1677 return cxlr;
1678
1679err:
1680 put_device(dev);
1681 return ERR_PTR(rc);
1682}
1683
1684static ssize_t create_pmem_region_show(struct device *dev,
1685 struct device_attribute *attr, char *buf)
1686{
1687 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
1688
1689 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
1690}
1691
1692static ssize_t create_pmem_region_store(struct device *dev,
1693 struct device_attribute *attr,
1694 const char *buf, size_t len)
1695{
1696 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
1697 struct cxl_region *cxlr;
1698 int id, rc;
1699
1700 rc = sscanf(buf, "region%d\n", &id);
1701 if (rc != 1)
1702 return -EINVAL;
1703
1704 rc = memregion_alloc(GFP_KERNEL);
1705 if (rc < 0)
1706 return rc;
1707
1708 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
1709 memregion_free(rc);
1710 return -EBUSY;
1711 }
1712
1713 cxlr = devm_cxl_add_region(cxlrd, id, CXL_DECODER_PMEM,
1714 CXL_DECODER_EXPANDER);
1715 if (IS_ERR(cxlr))
1716 return PTR_ERR(cxlr);
1717
1718 return len;
1719}
1720DEVICE_ATTR_RW(create_pmem_region);
1721
1722static ssize_t region_show(struct device *dev, struct device_attribute *attr,
1723 char *buf)
1724{
1725 struct cxl_decoder *cxld = to_cxl_decoder(dev);
1726 ssize_t rc;
1727
1728 rc = down_read_interruptible(&cxl_region_rwsem);
1729 if (rc)
1730 return rc;
1731
1732 if (cxld->region)
1733 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
1734 else
1735 rc = sysfs_emit(buf, "\n");
1736 up_read(&cxl_region_rwsem);
1737
1738 return rc;
1739}
1740DEVICE_ATTR_RO(region);
1741
1742static struct cxl_region *
1743cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
1744{
1745 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
1746 struct device *region_dev;
1747
1748 region_dev = device_find_child_by_name(&cxld->dev, name);
1749 if (!region_dev)
1750 return ERR_PTR(-ENODEV);
1751
1752 return to_cxl_region(region_dev);
1753}
1754
1755static ssize_t delete_region_store(struct device *dev,
1756 struct device_attribute *attr,
1757 const char *buf, size_t len)
1758{
1759 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
1760 struct cxl_port *port = to_cxl_port(dev->parent);
1761 struct cxl_region *cxlr;
1762
1763 cxlr = cxl_find_region_by_name(cxlrd, buf);
1764 if (IS_ERR(cxlr))
1765 return PTR_ERR(cxlr);
1766
1767 devm_release_action(port->uport, unregister_region, cxlr);
1768 put_device(&cxlr->dev);
1769
1770 return len;
1771}
1772DEVICE_ATTR_WO(delete_region);
1773
1774static void cxl_pmem_region_release(struct device *dev)
1775{
1776 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
1777 int i;
1778
1779 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
1780 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
1781
1782 put_device(&cxlmd->dev);
1783 }
1784
1785 kfree(cxlr_pmem);
1786}
1787
1788static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
1789 &cxl_base_attribute_group,
1790 NULL,
1791};
1792
1793const struct device_type cxl_pmem_region_type = {
1794 .name = "cxl_pmem_region",
1795 .release = cxl_pmem_region_release,
1796 .groups = cxl_pmem_region_attribute_groups,
1797};
1798
1799bool is_cxl_pmem_region(struct device *dev)
1800{
1801 return dev->type == &cxl_pmem_region_type;
1802}
1803EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL);
1804
1805struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
1806{
1807 if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
1808 "not a cxl_pmem_region device\n"))
1809 return NULL;
1810 return container_of(dev, struct cxl_pmem_region, dev);
1811}
1812EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL);
1813
1814static struct lock_class_key cxl_pmem_region_key;
1815
1816static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr)
1817{
1818 struct cxl_region_params *p = &cxlr->params;
1819 struct cxl_nvdimm_bridge *cxl_nvb;
1820 struct cxl_pmem_region *cxlr_pmem;
1821 struct device *dev;
1822 int i;
1823
1824 down_read(&cxl_region_rwsem);
1825 if (p->state != CXL_CONFIG_COMMIT) {
1826 cxlr_pmem = ERR_PTR(-ENXIO);
1827 goto out;
1828 }
1829
1830 cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets),
1831 GFP_KERNEL);
1832 if (!cxlr_pmem) {
1833 cxlr_pmem = ERR_PTR(-ENOMEM);
1834 goto out;
1835 }
1836
1837 cxlr_pmem->hpa_range.start = p->res->start;
1838 cxlr_pmem->hpa_range.end = p->res->end;
1839
1840 /* Snapshot the region configuration underneath the cxl_region_rwsem */
1841 cxlr_pmem->nr_mappings = p->nr_targets;
1842 for (i = 0; i < p->nr_targets; i++) {
1843 struct cxl_endpoint_decoder *cxled = p->targets[i];
1844 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1845 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
1846
1847 /*
1848 * Regions never span CXL root devices, so by definition the
1849 * bridge for one device is the same for all.
1850 */
1851 if (i == 0) {
1852 cxl_nvb = cxl_find_nvdimm_bridge(&cxlmd->dev);
1853 if (!cxl_nvb) {
1854 cxlr_pmem = ERR_PTR(-ENODEV);
1855 goto out;
1856 }
1857 cxlr->cxl_nvb = cxl_nvb;
1858 }
1859 m->cxlmd = cxlmd;
1860 get_device(&cxlmd->dev);
1861 m->start = cxled->dpa_res->start;
1862 m->size = resource_size(cxled->dpa_res);
1863 m->position = i;
1864 }
1865
1866 dev = &cxlr_pmem->dev;
1867 cxlr_pmem->cxlr = cxlr;
1868 cxlr->cxlr_pmem = cxlr_pmem;
1869 device_initialize(dev);
1870 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
1871 device_set_pm_not_required(dev);
1872 dev->parent = &cxlr->dev;
1873 dev->bus = &cxl_bus_type;
1874 dev->type = &cxl_pmem_region_type;
1875out:
1876 up_read(&cxl_region_rwsem);
1877
1878 return cxlr_pmem;
1879}
1880
1881static void cxlr_pmem_unregister(void *_cxlr_pmem)
1882{
1883 struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
1884 struct cxl_region *cxlr = cxlr_pmem->cxlr;
1885 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
1886
1887 /*
1888 * Either the bridge is in ->remove() context under the device_lock(),
1889 * or cxlr_release_nvdimm() is cancelling the bridge's release action
1890 * for @cxlr_pmem and doing it itself (while manually holding the bridge
1891 * lock).
1892 */
1893 device_lock_assert(&cxl_nvb->dev);
1894 cxlr->cxlr_pmem = NULL;
1895 cxlr_pmem->cxlr = NULL;
1896 device_unregister(&cxlr_pmem->dev);
1897}
1898
1899static void cxlr_release_nvdimm(void *_cxlr)
1900{
1901 struct cxl_region *cxlr = _cxlr;
1902 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
1903
1904 device_lock(&cxl_nvb->dev);
1905 if (cxlr->cxlr_pmem)
1906 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
1907 cxlr->cxlr_pmem);
1908 device_unlock(&cxl_nvb->dev);
1909 cxlr->cxl_nvb = NULL;
1910 put_device(&cxl_nvb->dev);
1911}
1912
1913/**
1914 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
1915 * @cxlr: parent CXL region for this pmem region bridge device
1916 *
1917 * Return: 0 on success negative error code on failure.
1918 */
1919static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
1920{
1921 struct cxl_pmem_region *cxlr_pmem;
1922 struct cxl_nvdimm_bridge *cxl_nvb;
1923 struct device *dev;
1924 int rc;
1925
1926 cxlr_pmem = cxl_pmem_region_alloc(cxlr);
1927 if (IS_ERR(cxlr_pmem))
1928 return PTR_ERR(cxlr_pmem);
1929 cxl_nvb = cxlr->cxl_nvb;
1930
1931 dev = &cxlr_pmem->dev;
1932 rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
1933 if (rc)
1934 goto err;
1935
1936 rc = device_add(dev);
1937 if (rc)
1938 goto err;
1939
1940 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
1941 dev_name(dev));
1942
1943 device_lock(&cxl_nvb->dev);
1944 if (cxl_nvb->dev.driver)
1945 rc = devm_add_action_or_reset(&cxl_nvb->dev,
1946 cxlr_pmem_unregister, cxlr_pmem);
1947 else
1948 rc = -ENXIO;
1949 device_unlock(&cxl_nvb->dev);
1950
1951 if (rc)
1952 goto err_bridge;
1953
1954 /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
1955 return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
1956
1957err:
1958 put_device(dev);
1959err_bridge:
1960 put_device(&cxl_nvb->dev);
1961 cxlr->cxl_nvb = NULL;
1962 return rc;
1963}
1964
1965static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
1966{
1967 if (!test_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags))
1968 return 0;
1969
1970 if (!cpu_cache_has_invalidate_memregion()) {
1971 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
1972 dev_warn(
1973 &cxlr->dev,
1974 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
1975 clear_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags);
1976 return 0;
1977 } else {
1978 dev_err(&cxlr->dev,
1979 "Failed to synchronize CPU cache state\n");
1980 return -ENXIO;
1981 }
1982 }
1983
1984 cpu_cache_invalidate_memregion(IORES_DESC_CXL);
1985 clear_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags);
1986 return 0;
1987}
1988
1989static int cxl_region_probe(struct device *dev)
1990{
1991 struct cxl_region *cxlr = to_cxl_region(dev);
1992 struct cxl_region_params *p = &cxlr->params;
1993 int rc;
1994
1995 rc = down_read_interruptible(&cxl_region_rwsem);
1996 if (rc) {
1997 dev_dbg(&cxlr->dev, "probe interrupted\n");
1998 return rc;
1999 }
2000
2001 if (p->state < CXL_CONFIG_COMMIT) {
2002 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
2003 rc = -ENXIO;
2004 goto out;
2005 }
2006
2007 rc = cxl_region_invalidate_memregion(cxlr);
2008
2009 /*
2010 * From this point on any path that changes the region's state away from
2011 * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
2012 */
2013out:
2014 up_read(&cxl_region_rwsem);
2015
2016 if (rc)
2017 return rc;
2018
2019 switch (cxlr->mode) {
2020 case CXL_DECODER_PMEM:
2021 return devm_cxl_add_pmem_region(cxlr);
2022 default:
2023 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
2024 cxlr->mode);
2025 return -ENXIO;
2026 }
2027}
2028
2029static struct cxl_driver cxl_region_driver = {
2030 .name = "cxl_region",
2031 .probe = cxl_region_probe,
2032 .id = CXL_DEVICE_REGION,
2033};
2034
2035int cxl_region_init(void)
2036{
2037 return cxl_driver_register(&cxl_region_driver);
2038}
2039
2040void cxl_region_exit(void)
2041{
2042 cxl_driver_unregister(&cxl_region_driver);
2043}
2044
2045MODULE_IMPORT_NS(CXL);
2046MODULE_IMPORT_NS(DEVMEM);
2047MODULE_ALIAS_CXL(CXL_DEVICE_REGION);