Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3#include <linux/memregion.h>
4#include <linux/genalloc.h>
5#include <linux/device.h>
6#include <linux/module.h>
7#include <linux/memory.h>
8#include <linux/slab.h>
9#include <linux/uuid.h>
10#include <linux/sort.h>
11#include <linux/idr.h>
12#include <linux/memory-tiers.h>
13#include <cxlmem.h>
14#include <cxl.h>
15#include "core.h"
16
17/**
18 * DOC: cxl core region
19 *
20 * CXL Regions represent mapped memory capacity in system physical address
21 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
22 * Memory ranges, Regions represent the active mapped capacity by the HDM
23 * Decoder Capability structures throughout the Host Bridges, Switches, and
24 * Endpoints in the topology.
25 *
26 * Region configuration has ordering constraints. UUID may be set at any time
27 * but is only visible for persistent regions.
28 * 1. Interleave granularity
29 * 2. Interleave size
30 * 3. Decoder targets
31 */
32
33static struct cxl_region *to_cxl_region(struct device *dev);
34
35#define __ACCESS_ATTR_RO(_level, _name) { \
36 .attr = { .name = __stringify(_name), .mode = 0444 }, \
37 .show = _name##_access##_level##_show, \
38}
39
40#define ACCESS_DEVICE_ATTR_RO(level, name) \
41 struct device_attribute dev_attr_access##level##_##name = __ACCESS_ATTR_RO(level, name)
42
43#define ACCESS_ATTR_RO(level, attrib) \
44static ssize_t attrib##_access##level##_show(struct device *dev, \
45 struct device_attribute *attr, \
46 char *buf) \
47{ \
48 struct cxl_region *cxlr = to_cxl_region(dev); \
49 \
50 if (cxlr->coord[level].attrib == 0) \
51 return -ENOENT; \
52 \
53 return sysfs_emit(buf, "%u\n", cxlr->coord[level].attrib); \
54} \
55static ACCESS_DEVICE_ATTR_RO(level, attrib)
56
57ACCESS_ATTR_RO(0, read_bandwidth);
58ACCESS_ATTR_RO(0, read_latency);
59ACCESS_ATTR_RO(0, write_bandwidth);
60ACCESS_ATTR_RO(0, write_latency);
61
62#define ACCESS_ATTR_DECLARE(level, attrib) \
63 (&dev_attr_access##level##_##attrib.attr)
64
65static struct attribute *access0_coordinate_attrs[] = {
66 ACCESS_ATTR_DECLARE(0, read_bandwidth),
67 ACCESS_ATTR_DECLARE(0, write_bandwidth),
68 ACCESS_ATTR_DECLARE(0, read_latency),
69 ACCESS_ATTR_DECLARE(0, write_latency),
70 NULL
71};
72
73ACCESS_ATTR_RO(1, read_bandwidth);
74ACCESS_ATTR_RO(1, read_latency);
75ACCESS_ATTR_RO(1, write_bandwidth);
76ACCESS_ATTR_RO(1, write_latency);
77
78static struct attribute *access1_coordinate_attrs[] = {
79 ACCESS_ATTR_DECLARE(1, read_bandwidth),
80 ACCESS_ATTR_DECLARE(1, write_bandwidth),
81 ACCESS_ATTR_DECLARE(1, read_latency),
82 ACCESS_ATTR_DECLARE(1, write_latency),
83 NULL
84};
85
86#define ACCESS_VISIBLE(level) \
87static umode_t cxl_region_access##level##_coordinate_visible( \
88 struct kobject *kobj, struct attribute *a, int n) \
89{ \
90 struct device *dev = kobj_to_dev(kobj); \
91 struct cxl_region *cxlr = to_cxl_region(dev); \
92 \
93 if (a == &dev_attr_access##level##_read_latency.attr && \
94 cxlr->coord[level].read_latency == 0) \
95 return 0; \
96 \
97 if (a == &dev_attr_access##level##_write_latency.attr && \
98 cxlr->coord[level].write_latency == 0) \
99 return 0; \
100 \
101 if (a == &dev_attr_access##level##_read_bandwidth.attr && \
102 cxlr->coord[level].read_bandwidth == 0) \
103 return 0; \
104 \
105 if (a == &dev_attr_access##level##_write_bandwidth.attr && \
106 cxlr->coord[level].write_bandwidth == 0) \
107 return 0; \
108 \
109 return a->mode; \
110}
111
112ACCESS_VISIBLE(0);
113ACCESS_VISIBLE(1);
114
115static const struct attribute_group cxl_region_access0_coordinate_group = {
116 .name = "access0",
117 .attrs = access0_coordinate_attrs,
118 .is_visible = cxl_region_access0_coordinate_visible,
119};
120
121static const struct attribute_group *get_cxl_region_access0_group(void)
122{
123 return &cxl_region_access0_coordinate_group;
124}
125
126static const struct attribute_group cxl_region_access1_coordinate_group = {
127 .name = "access1",
128 .attrs = access1_coordinate_attrs,
129 .is_visible = cxl_region_access1_coordinate_visible,
130};
131
132static const struct attribute_group *get_cxl_region_access1_group(void)
133{
134 return &cxl_region_access1_coordinate_group;
135}
136
137static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
138 char *buf)
139{
140 struct cxl_region *cxlr = to_cxl_region(dev);
141 struct cxl_region_params *p = &cxlr->params;
142 ssize_t rc;
143
144 rc = down_read_interruptible(&cxl_region_rwsem);
145 if (rc)
146 return rc;
147 if (cxlr->mode != CXL_DECODER_PMEM)
148 rc = sysfs_emit(buf, "\n");
149 else
150 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
151 up_read(&cxl_region_rwsem);
152
153 return rc;
154}
155
156static int is_dup(struct device *match, void *data)
157{
158 struct cxl_region_params *p;
159 struct cxl_region *cxlr;
160 uuid_t *uuid = data;
161
162 if (!is_cxl_region(match))
163 return 0;
164
165 lockdep_assert_held(&cxl_region_rwsem);
166 cxlr = to_cxl_region(match);
167 p = &cxlr->params;
168
169 if (uuid_equal(&p->uuid, uuid)) {
170 dev_dbg(match, "already has uuid: %pUb\n", uuid);
171 return -EBUSY;
172 }
173
174 return 0;
175}
176
177static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
178 const char *buf, size_t len)
179{
180 struct cxl_region *cxlr = to_cxl_region(dev);
181 struct cxl_region_params *p = &cxlr->params;
182 uuid_t temp;
183 ssize_t rc;
184
185 if (len != UUID_STRING_LEN + 1)
186 return -EINVAL;
187
188 rc = uuid_parse(buf, &temp);
189 if (rc)
190 return rc;
191
192 if (uuid_is_null(&temp))
193 return -EINVAL;
194
195 rc = down_write_killable(&cxl_region_rwsem);
196 if (rc)
197 return rc;
198
199 if (uuid_equal(&p->uuid, &temp))
200 goto out;
201
202 rc = -EBUSY;
203 if (p->state >= CXL_CONFIG_ACTIVE)
204 goto out;
205
206 rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
207 if (rc < 0)
208 goto out;
209
210 uuid_copy(&p->uuid, &temp);
211out:
212 up_write(&cxl_region_rwsem);
213
214 if (rc)
215 return rc;
216 return len;
217}
218static DEVICE_ATTR_RW(uuid);
219
220static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
221 struct cxl_region *cxlr)
222{
223 return xa_load(&port->regions, (unsigned long)cxlr);
224}
225
226static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
227{
228 if (!cpu_cache_has_invalidate_memregion()) {
229 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
230 dev_info_once(
231 &cxlr->dev,
232 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
233 return 0;
234 } else {
235 dev_WARN(&cxlr->dev,
236 "Failed to synchronize CPU cache state\n");
237 return -ENXIO;
238 }
239 }
240
241 cpu_cache_invalidate_memregion(IORES_DESC_CXL);
242 return 0;
243}
244
245static void cxl_region_decode_reset(struct cxl_region *cxlr, int count)
246{
247 struct cxl_region_params *p = &cxlr->params;
248 int i;
249
250 /*
251 * Before region teardown attempt to flush, evict any data cached for
252 * this region, or scream loudly about missing arch / platform support
253 * for CXL teardown.
254 */
255 cxl_region_invalidate_memregion(cxlr);
256
257 for (i = count - 1; i >= 0; i--) {
258 struct cxl_endpoint_decoder *cxled = p->targets[i];
259 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
260 struct cxl_port *iter = cxled_to_port(cxled);
261 struct cxl_dev_state *cxlds = cxlmd->cxlds;
262 struct cxl_ep *ep;
263
264 if (cxlds->rcd)
265 goto endpoint_reset;
266
267 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
268 iter = to_cxl_port(iter->dev.parent);
269
270 for (ep = cxl_ep_load(iter, cxlmd); iter;
271 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
272 struct cxl_region_ref *cxl_rr;
273 struct cxl_decoder *cxld;
274
275 cxl_rr = cxl_rr_load(iter, cxlr);
276 cxld = cxl_rr->decoder;
277 if (cxld->reset)
278 cxld->reset(cxld);
279 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
280 }
281
282endpoint_reset:
283 cxled->cxld.reset(&cxled->cxld);
284 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
285 }
286
287 /* all decoders associated with this region have been torn down */
288 clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
289}
290
291static int commit_decoder(struct cxl_decoder *cxld)
292{
293 struct cxl_switch_decoder *cxlsd = NULL;
294
295 if (cxld->commit)
296 return cxld->commit(cxld);
297
298 if (is_switch_decoder(&cxld->dev))
299 cxlsd = to_cxl_switch_decoder(&cxld->dev);
300
301 if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
302 "->commit() is required\n"))
303 return -ENXIO;
304 return 0;
305}
306
307static int cxl_region_decode_commit(struct cxl_region *cxlr)
308{
309 struct cxl_region_params *p = &cxlr->params;
310 int i, rc = 0;
311
312 for (i = 0; i < p->nr_targets; i++) {
313 struct cxl_endpoint_decoder *cxled = p->targets[i];
314 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
315 struct cxl_region_ref *cxl_rr;
316 struct cxl_decoder *cxld;
317 struct cxl_port *iter;
318 struct cxl_ep *ep;
319
320 /* commit bottom up */
321 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
322 iter = to_cxl_port(iter->dev.parent)) {
323 cxl_rr = cxl_rr_load(iter, cxlr);
324 cxld = cxl_rr->decoder;
325 rc = commit_decoder(cxld);
326 if (rc)
327 break;
328 }
329
330 if (rc) {
331 /* programming @iter failed, teardown */
332 for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
333 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
334 cxl_rr = cxl_rr_load(iter, cxlr);
335 cxld = cxl_rr->decoder;
336 if (cxld->reset)
337 cxld->reset(cxld);
338 }
339
340 cxled->cxld.reset(&cxled->cxld);
341 goto err;
342 }
343 }
344
345 return 0;
346
347err:
348 /* undo the targets that were successfully committed */
349 cxl_region_decode_reset(cxlr, i);
350 return rc;
351}
352
353static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
354 const char *buf, size_t len)
355{
356 struct cxl_region *cxlr = to_cxl_region(dev);
357 struct cxl_region_params *p = &cxlr->params;
358 bool commit;
359 ssize_t rc;
360
361 rc = kstrtobool(buf, &commit);
362 if (rc)
363 return rc;
364
365 rc = down_write_killable(&cxl_region_rwsem);
366 if (rc)
367 return rc;
368
369 /* Already in the requested state? */
370 if (commit && p->state >= CXL_CONFIG_COMMIT)
371 goto out;
372 if (!commit && p->state < CXL_CONFIG_COMMIT)
373 goto out;
374
375 /* Not ready to commit? */
376 if (commit && p->state < CXL_CONFIG_ACTIVE) {
377 rc = -ENXIO;
378 goto out;
379 }
380
381 /*
382 * Invalidate caches before region setup to drop any speculative
383 * consumption of this address space
384 */
385 rc = cxl_region_invalidate_memregion(cxlr);
386 if (rc)
387 goto out;
388
389 if (commit) {
390 rc = cxl_region_decode_commit(cxlr);
391 if (rc == 0)
392 p->state = CXL_CONFIG_COMMIT;
393 } else {
394 p->state = CXL_CONFIG_RESET_PENDING;
395 up_write(&cxl_region_rwsem);
396 device_release_driver(&cxlr->dev);
397 down_write(&cxl_region_rwsem);
398
399 /*
400 * The lock was dropped, so need to revalidate that the reset is
401 * still pending.
402 */
403 if (p->state == CXL_CONFIG_RESET_PENDING) {
404 cxl_region_decode_reset(cxlr, p->interleave_ways);
405 p->state = CXL_CONFIG_ACTIVE;
406 }
407 }
408
409out:
410 up_write(&cxl_region_rwsem);
411
412 if (rc)
413 return rc;
414 return len;
415}
416
417static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
418 char *buf)
419{
420 struct cxl_region *cxlr = to_cxl_region(dev);
421 struct cxl_region_params *p = &cxlr->params;
422 ssize_t rc;
423
424 rc = down_read_interruptible(&cxl_region_rwsem);
425 if (rc)
426 return rc;
427 rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
428 up_read(&cxl_region_rwsem);
429
430 return rc;
431}
432static DEVICE_ATTR_RW(commit);
433
434static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
435 int n)
436{
437 struct device *dev = kobj_to_dev(kobj);
438 struct cxl_region *cxlr = to_cxl_region(dev);
439
440 /*
441 * Support tooling that expects to find a 'uuid' attribute for all
442 * regions regardless of mode.
443 */
444 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
445 return 0444;
446 return a->mode;
447}
448
449static ssize_t interleave_ways_show(struct device *dev,
450 struct device_attribute *attr, char *buf)
451{
452 struct cxl_region *cxlr = to_cxl_region(dev);
453 struct cxl_region_params *p = &cxlr->params;
454 ssize_t rc;
455
456 rc = down_read_interruptible(&cxl_region_rwsem);
457 if (rc)
458 return rc;
459 rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
460 up_read(&cxl_region_rwsem);
461
462 return rc;
463}
464
465static const struct attribute_group *get_cxl_region_target_group(void);
466
467static ssize_t interleave_ways_store(struct device *dev,
468 struct device_attribute *attr,
469 const char *buf, size_t len)
470{
471 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
472 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
473 struct cxl_region *cxlr = to_cxl_region(dev);
474 struct cxl_region_params *p = &cxlr->params;
475 unsigned int val, save;
476 int rc;
477 u8 iw;
478
479 rc = kstrtouint(buf, 0, &val);
480 if (rc)
481 return rc;
482
483 rc = ways_to_eiw(val, &iw);
484 if (rc)
485 return rc;
486
487 /*
488 * Even for x3, x6, and x12 interleaves the region interleave must be a
489 * power of 2 multiple of the host bridge interleave.
490 */
491 if (!is_power_of_2(val / cxld->interleave_ways) ||
492 (val % cxld->interleave_ways)) {
493 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
494 return -EINVAL;
495 }
496
497 rc = down_write_killable(&cxl_region_rwsem);
498 if (rc)
499 return rc;
500 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
501 rc = -EBUSY;
502 goto out;
503 }
504
505 save = p->interleave_ways;
506 p->interleave_ways = val;
507 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
508 if (rc)
509 p->interleave_ways = save;
510out:
511 up_write(&cxl_region_rwsem);
512 if (rc)
513 return rc;
514 return len;
515}
516static DEVICE_ATTR_RW(interleave_ways);
517
518static ssize_t interleave_granularity_show(struct device *dev,
519 struct device_attribute *attr,
520 char *buf)
521{
522 struct cxl_region *cxlr = to_cxl_region(dev);
523 struct cxl_region_params *p = &cxlr->params;
524 ssize_t rc;
525
526 rc = down_read_interruptible(&cxl_region_rwsem);
527 if (rc)
528 return rc;
529 rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
530 up_read(&cxl_region_rwsem);
531
532 return rc;
533}
534
535static ssize_t interleave_granularity_store(struct device *dev,
536 struct device_attribute *attr,
537 const char *buf, size_t len)
538{
539 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
540 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
541 struct cxl_region *cxlr = to_cxl_region(dev);
542 struct cxl_region_params *p = &cxlr->params;
543 int rc, val;
544 u16 ig;
545
546 rc = kstrtoint(buf, 0, &val);
547 if (rc)
548 return rc;
549
550 rc = granularity_to_eig(val, &ig);
551 if (rc)
552 return rc;
553
554 /*
555 * When the host-bridge is interleaved, disallow region granularity !=
556 * root granularity. Regions with a granularity less than the root
557 * interleave result in needing multiple endpoints to support a single
558 * slot in the interleave (possible to support in the future). Regions
559 * with a granularity greater than the root interleave result in invalid
560 * DPA translations (invalid to support).
561 */
562 if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
563 return -EINVAL;
564
565 rc = down_write_killable(&cxl_region_rwsem);
566 if (rc)
567 return rc;
568 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
569 rc = -EBUSY;
570 goto out;
571 }
572
573 p->interleave_granularity = val;
574out:
575 up_write(&cxl_region_rwsem);
576 if (rc)
577 return rc;
578 return len;
579}
580static DEVICE_ATTR_RW(interleave_granularity);
581
582static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
583 char *buf)
584{
585 struct cxl_region *cxlr = to_cxl_region(dev);
586 struct cxl_region_params *p = &cxlr->params;
587 u64 resource = -1ULL;
588 ssize_t rc;
589
590 rc = down_read_interruptible(&cxl_region_rwsem);
591 if (rc)
592 return rc;
593 if (p->res)
594 resource = p->res->start;
595 rc = sysfs_emit(buf, "%#llx\n", resource);
596 up_read(&cxl_region_rwsem);
597
598 return rc;
599}
600static DEVICE_ATTR_RO(resource);
601
602static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
603 char *buf)
604{
605 struct cxl_region *cxlr = to_cxl_region(dev);
606
607 return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
608}
609static DEVICE_ATTR_RO(mode);
610
611static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
612{
613 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
614 struct cxl_region_params *p = &cxlr->params;
615 struct resource *res;
616 u64 remainder = 0;
617
618 lockdep_assert_held_write(&cxl_region_rwsem);
619
620 /* Nothing to do... */
621 if (p->res && resource_size(p->res) == size)
622 return 0;
623
624 /* To change size the old size must be freed first */
625 if (p->res)
626 return -EBUSY;
627
628 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
629 return -EBUSY;
630
631 /* ways, granularity and uuid (if PMEM) need to be set before HPA */
632 if (!p->interleave_ways || !p->interleave_granularity ||
633 (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
634 return -ENXIO;
635
636 div64_u64_rem(size, (u64)SZ_256M * p->interleave_ways, &remainder);
637 if (remainder)
638 return -EINVAL;
639
640 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
641 dev_name(&cxlr->dev));
642 if (IS_ERR(res)) {
643 dev_dbg(&cxlr->dev,
644 "HPA allocation error (%ld) for size:%pap in %s %pr\n",
645 PTR_ERR(res), &size, cxlrd->res->name, cxlrd->res);
646 return PTR_ERR(res);
647 }
648
649 p->res = res;
650 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
651
652 return 0;
653}
654
655static void cxl_region_iomem_release(struct cxl_region *cxlr)
656{
657 struct cxl_region_params *p = &cxlr->params;
658
659 if (device_is_registered(&cxlr->dev))
660 lockdep_assert_held_write(&cxl_region_rwsem);
661 if (p->res) {
662 /*
663 * Autodiscovered regions may not have been able to insert their
664 * resource.
665 */
666 if (p->res->parent)
667 remove_resource(p->res);
668 kfree(p->res);
669 p->res = NULL;
670 }
671}
672
673static int free_hpa(struct cxl_region *cxlr)
674{
675 struct cxl_region_params *p = &cxlr->params;
676
677 lockdep_assert_held_write(&cxl_region_rwsem);
678
679 if (!p->res)
680 return 0;
681
682 if (p->state >= CXL_CONFIG_ACTIVE)
683 return -EBUSY;
684
685 cxl_region_iomem_release(cxlr);
686 p->state = CXL_CONFIG_IDLE;
687 return 0;
688}
689
690static ssize_t size_store(struct device *dev, struct device_attribute *attr,
691 const char *buf, size_t len)
692{
693 struct cxl_region *cxlr = to_cxl_region(dev);
694 u64 val;
695 int rc;
696
697 rc = kstrtou64(buf, 0, &val);
698 if (rc)
699 return rc;
700
701 rc = down_write_killable(&cxl_region_rwsem);
702 if (rc)
703 return rc;
704
705 if (val)
706 rc = alloc_hpa(cxlr, val);
707 else
708 rc = free_hpa(cxlr);
709 up_write(&cxl_region_rwsem);
710
711 if (rc)
712 return rc;
713
714 return len;
715}
716
717static ssize_t size_show(struct device *dev, struct device_attribute *attr,
718 char *buf)
719{
720 struct cxl_region *cxlr = to_cxl_region(dev);
721 struct cxl_region_params *p = &cxlr->params;
722 u64 size = 0;
723 ssize_t rc;
724
725 rc = down_read_interruptible(&cxl_region_rwsem);
726 if (rc)
727 return rc;
728 if (p->res)
729 size = resource_size(p->res);
730 rc = sysfs_emit(buf, "%#llx\n", size);
731 up_read(&cxl_region_rwsem);
732
733 return rc;
734}
735static DEVICE_ATTR_RW(size);
736
737static struct attribute *cxl_region_attrs[] = {
738 &dev_attr_uuid.attr,
739 &dev_attr_commit.attr,
740 &dev_attr_interleave_ways.attr,
741 &dev_attr_interleave_granularity.attr,
742 &dev_attr_resource.attr,
743 &dev_attr_size.attr,
744 &dev_attr_mode.attr,
745 NULL,
746};
747
748static const struct attribute_group cxl_region_group = {
749 .attrs = cxl_region_attrs,
750 .is_visible = cxl_region_visible,
751};
752
753static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
754{
755 struct cxl_region_params *p = &cxlr->params;
756 struct cxl_endpoint_decoder *cxled;
757 int rc;
758
759 rc = down_read_interruptible(&cxl_region_rwsem);
760 if (rc)
761 return rc;
762
763 if (pos >= p->interleave_ways) {
764 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
765 p->interleave_ways);
766 rc = -ENXIO;
767 goto out;
768 }
769
770 cxled = p->targets[pos];
771 if (!cxled)
772 rc = sysfs_emit(buf, "\n");
773 else
774 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
775out:
776 up_read(&cxl_region_rwsem);
777
778 return rc;
779}
780
781static int check_commit_order(struct device *dev, const void *data)
782{
783 struct cxl_decoder *cxld = to_cxl_decoder(dev);
784
785 /*
786 * if port->commit_end is not the only free decoder, then out of
787 * order shutdown has occurred, block further allocations until
788 * that is resolved
789 */
790 if (((cxld->flags & CXL_DECODER_F_ENABLE) == 0))
791 return -EBUSY;
792 return 0;
793}
794
795static int match_free_decoder(struct device *dev, void *data)
796{
797 struct cxl_port *port = to_cxl_port(dev->parent);
798 struct cxl_decoder *cxld;
799 int rc;
800
801 if (!is_switch_decoder(dev))
802 return 0;
803
804 cxld = to_cxl_decoder(dev);
805
806 if (cxld->id != port->commit_end + 1)
807 return 0;
808
809 if (cxld->region) {
810 dev_dbg(dev->parent,
811 "next decoder to commit (%s) is already reserved (%s)\n",
812 dev_name(dev), dev_name(&cxld->region->dev));
813 return 0;
814 }
815
816 rc = device_for_each_child_reverse_from(dev->parent, dev, NULL,
817 check_commit_order);
818 if (rc) {
819 dev_dbg(dev->parent,
820 "unable to allocate %s due to out of order shutdown\n",
821 dev_name(dev));
822 return 0;
823 }
824 return 1;
825}
826
827static int match_auto_decoder(struct device *dev, void *data)
828{
829 struct cxl_region_params *p = data;
830 struct cxl_decoder *cxld;
831 struct range *r;
832
833 if (!is_switch_decoder(dev))
834 return 0;
835
836 cxld = to_cxl_decoder(dev);
837 r = &cxld->hpa_range;
838
839 if (p->res && p->res->start == r->start && p->res->end == r->end)
840 return 1;
841
842 return 0;
843}
844
845static struct cxl_decoder *
846cxl_region_find_decoder(struct cxl_port *port,
847 struct cxl_endpoint_decoder *cxled,
848 struct cxl_region *cxlr)
849{
850 struct device *dev;
851
852 if (port == cxled_to_port(cxled))
853 return &cxled->cxld;
854
855 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
856 dev = device_find_child(&port->dev, &cxlr->params,
857 match_auto_decoder);
858 else
859 dev = device_find_child(&port->dev, NULL, match_free_decoder);
860 if (!dev)
861 return NULL;
862 /*
863 * This decoder is pinned registered as long as the endpoint decoder is
864 * registered, and endpoint decoder unregistration holds the
865 * cxl_region_rwsem over unregister events, so no need to hold on to
866 * this extra reference.
867 */
868 put_device(dev);
869 return to_cxl_decoder(dev);
870}
871
872static bool auto_order_ok(struct cxl_port *port, struct cxl_region *cxlr_iter,
873 struct cxl_decoder *cxld)
874{
875 struct cxl_region_ref *rr = cxl_rr_load(port, cxlr_iter);
876 struct cxl_decoder *cxld_iter = rr->decoder;
877
878 /*
879 * Allow the out of order assembly of auto-discovered regions.
880 * Per CXL Spec 3.1 8.2.4.20.12 software must commit decoders
881 * in HPA order. Confirm that the decoder with the lesser HPA
882 * starting address has the lesser id.
883 */
884 dev_dbg(&cxld->dev, "check for HPA violation %s:%d < %s:%d\n",
885 dev_name(&cxld->dev), cxld->id,
886 dev_name(&cxld_iter->dev), cxld_iter->id);
887
888 if (cxld_iter->id > cxld->id)
889 return true;
890
891 return false;
892}
893
894static struct cxl_region_ref *
895alloc_region_ref(struct cxl_port *port, struct cxl_region *cxlr,
896 struct cxl_endpoint_decoder *cxled)
897{
898 struct cxl_region_params *p = &cxlr->params;
899 struct cxl_region_ref *cxl_rr, *iter;
900 unsigned long index;
901 int rc;
902
903 xa_for_each(&port->regions, index, iter) {
904 struct cxl_region_params *ip = &iter->region->params;
905
906 if (!ip->res || ip->res->start < p->res->start)
907 continue;
908
909 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
910 struct cxl_decoder *cxld;
911
912 cxld = cxl_region_find_decoder(port, cxled, cxlr);
913 if (auto_order_ok(port, iter->region, cxld))
914 continue;
915 }
916 dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n",
917 dev_name(&port->dev),
918 dev_name(&iter->region->dev), ip->res, p->res);
919
920 return ERR_PTR(-EBUSY);
921 }
922
923 cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
924 if (!cxl_rr)
925 return ERR_PTR(-ENOMEM);
926 cxl_rr->port = port;
927 cxl_rr->region = cxlr;
928 cxl_rr->nr_targets = 1;
929 xa_init(&cxl_rr->endpoints);
930
931 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
932 if (rc) {
933 dev_dbg(&cxlr->dev,
934 "%s: failed to track region reference: %d\n",
935 dev_name(&port->dev), rc);
936 kfree(cxl_rr);
937 return ERR_PTR(rc);
938 }
939
940 return cxl_rr;
941}
942
943static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
944{
945 struct cxl_region *cxlr = cxl_rr->region;
946 struct cxl_decoder *cxld = cxl_rr->decoder;
947
948 if (!cxld)
949 return;
950
951 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
952 if (cxld->region == cxlr) {
953 cxld->region = NULL;
954 put_device(&cxlr->dev);
955 }
956}
957
958static void free_region_ref(struct cxl_region_ref *cxl_rr)
959{
960 struct cxl_port *port = cxl_rr->port;
961 struct cxl_region *cxlr = cxl_rr->region;
962
963 cxl_rr_free_decoder(cxl_rr);
964 xa_erase(&port->regions, (unsigned long)cxlr);
965 xa_destroy(&cxl_rr->endpoints);
966 kfree(cxl_rr);
967}
968
969static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
970 struct cxl_endpoint_decoder *cxled)
971{
972 int rc;
973 struct cxl_port *port = cxl_rr->port;
974 struct cxl_region *cxlr = cxl_rr->region;
975 struct cxl_decoder *cxld = cxl_rr->decoder;
976 struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
977
978 if (ep) {
979 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
980 GFP_KERNEL);
981 if (rc)
982 return rc;
983 }
984 cxl_rr->nr_eps++;
985
986 if (!cxld->region) {
987 cxld->region = cxlr;
988 get_device(&cxlr->dev);
989 }
990
991 return 0;
992}
993
994static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
995 struct cxl_endpoint_decoder *cxled,
996 struct cxl_region_ref *cxl_rr)
997{
998 struct cxl_decoder *cxld;
999
1000 cxld = cxl_region_find_decoder(port, cxled, cxlr);
1001 if (!cxld) {
1002 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
1003 dev_name(&port->dev));
1004 return -EBUSY;
1005 }
1006
1007 if (cxld->region) {
1008 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
1009 dev_name(&port->dev), dev_name(&cxld->dev),
1010 dev_name(&cxld->region->dev));
1011 return -EBUSY;
1012 }
1013
1014 /*
1015 * Endpoints should already match the region type, but backstop that
1016 * assumption with an assertion. Switch-decoders change mapping-type
1017 * based on what is mapped when they are assigned to a region.
1018 */
1019 dev_WARN_ONCE(&cxlr->dev,
1020 port == cxled_to_port(cxled) &&
1021 cxld->target_type != cxlr->type,
1022 "%s:%s mismatch decoder type %d -> %d\n",
1023 dev_name(&cxled_to_memdev(cxled)->dev),
1024 dev_name(&cxld->dev), cxld->target_type, cxlr->type);
1025 cxld->target_type = cxlr->type;
1026 cxl_rr->decoder = cxld;
1027 return 0;
1028}
1029
1030/**
1031 * cxl_port_attach_region() - track a region's interest in a port by endpoint
1032 * @port: port to add a new region reference 'struct cxl_region_ref'
1033 * @cxlr: region to attach to @port
1034 * @cxled: endpoint decoder used to create or further pin a region reference
1035 * @pos: interleave position of @cxled in @cxlr
1036 *
1037 * The attach event is an opportunity to validate CXL decode setup
1038 * constraints and record metadata needed for programming HDM decoders,
1039 * in particular decoder target lists.
1040 *
1041 * The steps are:
1042 *
1043 * - validate that there are no other regions with a higher HPA already
1044 * associated with @port
1045 * - establish a region reference if one is not already present
1046 *
1047 * - additionally allocate a decoder instance that will host @cxlr on
1048 * @port
1049 *
1050 * - pin the region reference by the endpoint
1051 * - account for how many entries in @port's target list are needed to
1052 * cover all of the added endpoints.
1053 */
1054static int cxl_port_attach_region(struct cxl_port *port,
1055 struct cxl_region *cxlr,
1056 struct cxl_endpoint_decoder *cxled, int pos)
1057{
1058 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1059 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1060 struct cxl_region_ref *cxl_rr;
1061 bool nr_targets_inc = false;
1062 struct cxl_decoder *cxld;
1063 unsigned long index;
1064 int rc = -EBUSY;
1065
1066 lockdep_assert_held_write(&cxl_region_rwsem);
1067
1068 cxl_rr = cxl_rr_load(port, cxlr);
1069 if (cxl_rr) {
1070 struct cxl_ep *ep_iter;
1071 int found = 0;
1072
1073 /*
1074 * Walk the existing endpoints that have been attached to
1075 * @cxlr at @port and see if they share the same 'next' port
1076 * in the downstream direction. I.e. endpoints that share common
1077 * upstream switch.
1078 */
1079 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1080 if (ep_iter == ep)
1081 continue;
1082 if (ep_iter->next == ep->next) {
1083 found++;
1084 break;
1085 }
1086 }
1087
1088 /*
1089 * New target port, or @port is an endpoint port that always
1090 * accounts its own local decode as a target.
1091 */
1092 if (!found || !ep->next) {
1093 cxl_rr->nr_targets++;
1094 nr_targets_inc = true;
1095 }
1096 } else {
1097 cxl_rr = alloc_region_ref(port, cxlr, cxled);
1098 if (IS_ERR(cxl_rr)) {
1099 dev_dbg(&cxlr->dev,
1100 "%s: failed to allocate region reference\n",
1101 dev_name(&port->dev));
1102 return PTR_ERR(cxl_rr);
1103 }
1104 nr_targets_inc = true;
1105
1106 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
1107 if (rc)
1108 goto out_erase;
1109 }
1110 cxld = cxl_rr->decoder;
1111
1112 /*
1113 * the number of targets should not exceed the target_count
1114 * of the decoder
1115 */
1116 if (is_switch_decoder(&cxld->dev)) {
1117 struct cxl_switch_decoder *cxlsd;
1118
1119 cxlsd = to_cxl_switch_decoder(&cxld->dev);
1120 if (cxl_rr->nr_targets > cxlsd->nr_targets) {
1121 dev_dbg(&cxlr->dev,
1122 "%s:%s %s add: %s:%s @ %d overflows targets: %d\n",
1123 dev_name(port->uport_dev), dev_name(&port->dev),
1124 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1125 dev_name(&cxled->cxld.dev), pos,
1126 cxlsd->nr_targets);
1127 rc = -ENXIO;
1128 goto out_erase;
1129 }
1130 }
1131
1132 rc = cxl_rr_ep_add(cxl_rr, cxled);
1133 if (rc) {
1134 dev_dbg(&cxlr->dev,
1135 "%s: failed to track endpoint %s:%s reference\n",
1136 dev_name(&port->dev), dev_name(&cxlmd->dev),
1137 dev_name(&cxld->dev));
1138 goto out_erase;
1139 }
1140
1141 dev_dbg(&cxlr->dev,
1142 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
1143 dev_name(port->uport_dev), dev_name(&port->dev),
1144 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1145 dev_name(&cxled->cxld.dev), pos,
1146 ep ? ep->next ? dev_name(ep->next->uport_dev) :
1147 dev_name(&cxlmd->dev) :
1148 "none",
1149 cxl_rr->nr_eps, cxl_rr->nr_targets);
1150
1151 return 0;
1152out_erase:
1153 if (nr_targets_inc)
1154 cxl_rr->nr_targets--;
1155 if (cxl_rr->nr_eps == 0)
1156 free_region_ref(cxl_rr);
1157 return rc;
1158}
1159
1160static void cxl_port_detach_region(struct cxl_port *port,
1161 struct cxl_region *cxlr,
1162 struct cxl_endpoint_decoder *cxled)
1163{
1164 struct cxl_region_ref *cxl_rr;
1165 struct cxl_ep *ep = NULL;
1166
1167 lockdep_assert_held_write(&cxl_region_rwsem);
1168
1169 cxl_rr = cxl_rr_load(port, cxlr);
1170 if (!cxl_rr)
1171 return;
1172
1173 /*
1174 * Endpoint ports do not carry cxl_ep references, and they
1175 * never target more than one endpoint by definition
1176 */
1177 if (cxl_rr->decoder == &cxled->cxld)
1178 cxl_rr->nr_eps--;
1179 else
1180 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
1181 if (ep) {
1182 struct cxl_ep *ep_iter;
1183 unsigned long index;
1184 int found = 0;
1185
1186 cxl_rr->nr_eps--;
1187 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1188 if (ep_iter->next == ep->next) {
1189 found++;
1190 break;
1191 }
1192 }
1193 if (!found)
1194 cxl_rr->nr_targets--;
1195 }
1196
1197 if (cxl_rr->nr_eps == 0)
1198 free_region_ref(cxl_rr);
1199}
1200
1201static int check_last_peer(struct cxl_endpoint_decoder *cxled,
1202 struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
1203 int distance)
1204{
1205 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1206 struct cxl_region *cxlr = cxl_rr->region;
1207 struct cxl_region_params *p = &cxlr->params;
1208 struct cxl_endpoint_decoder *cxled_peer;
1209 struct cxl_port *port = cxl_rr->port;
1210 struct cxl_memdev *cxlmd_peer;
1211 struct cxl_ep *ep_peer;
1212 int pos = cxled->pos;
1213
1214 /*
1215 * If this position wants to share a dport with the last endpoint mapped
1216 * then that endpoint, at index 'position - distance', must also be
1217 * mapped by this dport.
1218 */
1219 if (pos < distance) {
1220 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
1221 dev_name(port->uport_dev), dev_name(&port->dev),
1222 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1223 return -ENXIO;
1224 }
1225 cxled_peer = p->targets[pos - distance];
1226 cxlmd_peer = cxled_to_memdev(cxled_peer);
1227 ep_peer = cxl_ep_load(port, cxlmd_peer);
1228 if (ep->dport != ep_peer->dport) {
1229 dev_dbg(&cxlr->dev,
1230 "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
1231 dev_name(port->uport_dev), dev_name(&port->dev),
1232 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
1233 dev_name(&cxlmd_peer->dev),
1234 dev_name(&cxled_peer->cxld.dev));
1235 return -ENXIO;
1236 }
1237
1238 return 0;
1239}
1240
1241static int check_interleave_cap(struct cxl_decoder *cxld, int iw, int ig)
1242{
1243 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
1244 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
1245 unsigned int interleave_mask;
1246 u8 eiw;
1247 u16 eig;
1248 int high_pos, low_pos;
1249
1250 if (!test_bit(iw, &cxlhdm->iw_cap_mask))
1251 return -ENXIO;
1252 /*
1253 * Per CXL specification r3.1(8.2.4.20.13 Decoder Protection),
1254 * if eiw < 8:
1255 * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + 8 + eiw]
1256 * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0]
1257 *
1258 * when the eiw is 0, all the bits of HPAOFFSET[51: 0] are used, the
1259 * interleave bits are none.
1260 *
1261 * if eiw >= 8:
1262 * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + eiw] / 3
1263 * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0]
1264 *
1265 * when the eiw is 8, all the bits of HPAOFFSET[51: 0] are used, the
1266 * interleave bits are none.
1267 */
1268 ways_to_eiw(iw, &eiw);
1269 if (eiw == 0 || eiw == 8)
1270 return 0;
1271
1272 granularity_to_eig(ig, &eig);
1273 if (eiw > 8)
1274 high_pos = eiw + eig - 1;
1275 else
1276 high_pos = eiw + eig + 7;
1277 low_pos = eig + 8;
1278 interleave_mask = GENMASK(high_pos, low_pos);
1279 if (interleave_mask & ~cxlhdm->interleave_mask)
1280 return -ENXIO;
1281
1282 return 0;
1283}
1284
1285static int cxl_port_setup_targets(struct cxl_port *port,
1286 struct cxl_region *cxlr,
1287 struct cxl_endpoint_decoder *cxled)
1288{
1289 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1290 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1291 struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1292 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1293 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1294 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1295 struct cxl_region_params *p = &cxlr->params;
1296 struct cxl_decoder *cxld = cxl_rr->decoder;
1297 struct cxl_switch_decoder *cxlsd;
1298 struct cxl_port *iter = port;
1299 u16 eig, peig;
1300 u8 eiw, peiw;
1301
1302 /*
1303 * While root level decoders support x3, x6, x12, switch level
1304 * decoders only support powers of 2 up to x16.
1305 */
1306 if (!is_power_of_2(cxl_rr->nr_targets)) {
1307 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1308 dev_name(port->uport_dev), dev_name(&port->dev),
1309 cxl_rr->nr_targets);
1310 return -EINVAL;
1311 }
1312
1313 cxlsd = to_cxl_switch_decoder(&cxld->dev);
1314 if (cxl_rr->nr_targets_set) {
1315 int i, distance = 1;
1316 struct cxl_region_ref *cxl_rr_iter;
1317
1318 /*
1319 * The "distance" between peer downstream ports represents which
1320 * endpoint positions in the region interleave a given port can
1321 * host.
1322 *
1323 * For example, at the root of a hierarchy the distance is
1324 * always 1 as every index targets a different host-bridge. At
1325 * each subsequent switch level those ports map every Nth region
1326 * position where N is the width of the switch == distance.
1327 */
1328 do {
1329 cxl_rr_iter = cxl_rr_load(iter, cxlr);
1330 distance *= cxl_rr_iter->nr_targets;
1331 iter = to_cxl_port(iter->dev.parent);
1332 } while (!is_cxl_root(iter));
1333 distance *= cxlrd->cxlsd.cxld.interleave_ways;
1334
1335 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1336 if (ep->dport == cxlsd->target[i]) {
1337 rc = check_last_peer(cxled, ep, cxl_rr,
1338 distance);
1339 if (rc)
1340 return rc;
1341 goto out_target_set;
1342 }
1343 goto add_target;
1344 }
1345
1346 if (is_cxl_root(parent_port)) {
1347 /*
1348 * Root decoder IG is always set to value in CFMWS which
1349 * may be different than this region's IG. We can use the
1350 * region's IG here since interleave_granularity_store()
1351 * does not allow interleaved host-bridges with
1352 * root IG != region IG.
1353 */
1354 parent_ig = p->interleave_granularity;
1355 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1356 /*
1357 * For purposes of address bit routing, use power-of-2 math for
1358 * switch ports.
1359 */
1360 if (!is_power_of_2(parent_iw))
1361 parent_iw /= 3;
1362 } else {
1363 struct cxl_region_ref *parent_rr;
1364 struct cxl_decoder *parent_cxld;
1365
1366 parent_rr = cxl_rr_load(parent_port, cxlr);
1367 parent_cxld = parent_rr->decoder;
1368 parent_ig = parent_cxld->interleave_granularity;
1369 parent_iw = parent_cxld->interleave_ways;
1370 }
1371
1372 rc = granularity_to_eig(parent_ig, &peig);
1373 if (rc) {
1374 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1375 dev_name(parent_port->uport_dev),
1376 dev_name(&parent_port->dev), parent_ig);
1377 return rc;
1378 }
1379
1380 rc = ways_to_eiw(parent_iw, &peiw);
1381 if (rc) {
1382 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1383 dev_name(parent_port->uport_dev),
1384 dev_name(&parent_port->dev), parent_iw);
1385 return rc;
1386 }
1387
1388 iw = cxl_rr->nr_targets;
1389 rc = ways_to_eiw(iw, &eiw);
1390 if (rc) {
1391 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1392 dev_name(port->uport_dev), dev_name(&port->dev), iw);
1393 return rc;
1394 }
1395
1396 /*
1397 * Interleave granularity is a multiple of @parent_port granularity.
1398 * Multiplier is the parent port interleave ways.
1399 */
1400 rc = granularity_to_eig(parent_ig * parent_iw, &eig);
1401 if (rc) {
1402 dev_dbg(&cxlr->dev,
1403 "%s: invalid granularity calculation (%d * %d)\n",
1404 dev_name(&parent_port->dev), parent_ig, parent_iw);
1405 return rc;
1406 }
1407
1408 rc = eig_to_granularity(eig, &ig);
1409 if (rc) {
1410 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1411 dev_name(port->uport_dev), dev_name(&port->dev),
1412 256 << eig);
1413 return rc;
1414 }
1415
1416 if (iw > 8 || iw > cxlsd->nr_targets) {
1417 dev_dbg(&cxlr->dev,
1418 "%s:%s:%s: ways: %d overflows targets: %d\n",
1419 dev_name(port->uport_dev), dev_name(&port->dev),
1420 dev_name(&cxld->dev), iw, cxlsd->nr_targets);
1421 return -ENXIO;
1422 }
1423
1424 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1425 if (cxld->interleave_ways != iw ||
1426 cxld->interleave_granularity != ig ||
1427 cxld->hpa_range.start != p->res->start ||
1428 cxld->hpa_range.end != p->res->end ||
1429 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1430 dev_err(&cxlr->dev,
1431 "%s:%s %s expected iw: %d ig: %d %pr\n",
1432 dev_name(port->uport_dev), dev_name(&port->dev),
1433 __func__, iw, ig, p->res);
1434 dev_err(&cxlr->dev,
1435 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1436 dev_name(port->uport_dev), dev_name(&port->dev),
1437 __func__, cxld->interleave_ways,
1438 cxld->interleave_granularity,
1439 (cxld->flags & CXL_DECODER_F_ENABLE) ?
1440 "enabled" :
1441 "disabled",
1442 cxld->hpa_range.start, cxld->hpa_range.end);
1443 return -ENXIO;
1444 }
1445 } else {
1446 rc = check_interleave_cap(cxld, iw, ig);
1447 if (rc) {
1448 dev_dbg(&cxlr->dev,
1449 "%s:%s iw: %d ig: %d is not supported\n",
1450 dev_name(port->uport_dev),
1451 dev_name(&port->dev), iw, ig);
1452 return rc;
1453 }
1454
1455 cxld->interleave_ways = iw;
1456 cxld->interleave_granularity = ig;
1457 cxld->hpa_range = (struct range) {
1458 .start = p->res->start,
1459 .end = p->res->end,
1460 };
1461 }
1462 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
1463 dev_name(&port->dev), iw, ig);
1464add_target:
1465 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1466 dev_dbg(&cxlr->dev,
1467 "%s:%s: targets full trying to add %s:%s at %d\n",
1468 dev_name(port->uport_dev), dev_name(&port->dev),
1469 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1470 return -ENXIO;
1471 }
1472 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1473 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1474 dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1475 dev_name(port->uport_dev), dev_name(&port->dev),
1476 dev_name(&cxlsd->cxld.dev),
1477 dev_name(ep->dport->dport_dev),
1478 cxl_rr->nr_targets_set);
1479 return -ENXIO;
1480 }
1481 } else
1482 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1483 inc = 1;
1484out_target_set:
1485 cxl_rr->nr_targets_set += inc;
1486 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1487 dev_name(port->uport_dev), dev_name(&port->dev),
1488 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
1489 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1490
1491 return 0;
1492}
1493
1494static void cxl_port_reset_targets(struct cxl_port *port,
1495 struct cxl_region *cxlr)
1496{
1497 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1498 struct cxl_decoder *cxld;
1499
1500 /*
1501 * After the last endpoint has been detached the entire cxl_rr may now
1502 * be gone.
1503 */
1504 if (!cxl_rr)
1505 return;
1506 cxl_rr->nr_targets_set = 0;
1507
1508 cxld = cxl_rr->decoder;
1509 cxld->hpa_range = (struct range) {
1510 .start = 0,
1511 .end = -1,
1512 };
1513}
1514
1515static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1516{
1517 struct cxl_region_params *p = &cxlr->params;
1518 struct cxl_endpoint_decoder *cxled;
1519 struct cxl_dev_state *cxlds;
1520 struct cxl_memdev *cxlmd;
1521 struct cxl_port *iter;
1522 struct cxl_ep *ep;
1523 int i;
1524
1525 /*
1526 * In the auto-discovery case skip automatic teardown since the
1527 * address space is already active
1528 */
1529 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1530 return;
1531
1532 for (i = 0; i < p->nr_targets; i++) {
1533 cxled = p->targets[i];
1534 cxlmd = cxled_to_memdev(cxled);
1535 cxlds = cxlmd->cxlds;
1536
1537 if (cxlds->rcd)
1538 continue;
1539
1540 iter = cxled_to_port(cxled);
1541 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1542 iter = to_cxl_port(iter->dev.parent);
1543
1544 for (ep = cxl_ep_load(iter, cxlmd); iter;
1545 iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1546 cxl_port_reset_targets(iter, cxlr);
1547 }
1548}
1549
1550static int cxl_region_setup_targets(struct cxl_region *cxlr)
1551{
1552 struct cxl_region_params *p = &cxlr->params;
1553 struct cxl_endpoint_decoder *cxled;
1554 struct cxl_dev_state *cxlds;
1555 int i, rc, rch = 0, vh = 0;
1556 struct cxl_memdev *cxlmd;
1557 struct cxl_port *iter;
1558 struct cxl_ep *ep;
1559
1560 for (i = 0; i < p->nr_targets; i++) {
1561 cxled = p->targets[i];
1562 cxlmd = cxled_to_memdev(cxled);
1563 cxlds = cxlmd->cxlds;
1564
1565 /* validate that all targets agree on topology */
1566 if (!cxlds->rcd) {
1567 vh++;
1568 } else {
1569 rch++;
1570 continue;
1571 }
1572
1573 iter = cxled_to_port(cxled);
1574 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1575 iter = to_cxl_port(iter->dev.parent);
1576
1577 /*
1578 * Descend the topology tree programming / validating
1579 * targets while looking for conflicts.
1580 */
1581 for (ep = cxl_ep_load(iter, cxlmd); iter;
1582 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1583 rc = cxl_port_setup_targets(iter, cxlr, cxled);
1584 if (rc) {
1585 cxl_region_teardown_targets(cxlr);
1586 return rc;
1587 }
1588 }
1589 }
1590
1591 if (rch && vh) {
1592 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1593 cxl_region_teardown_targets(cxlr);
1594 return -ENXIO;
1595 }
1596
1597 return 0;
1598}
1599
1600static int cxl_region_validate_position(struct cxl_region *cxlr,
1601 struct cxl_endpoint_decoder *cxled,
1602 int pos)
1603{
1604 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1605 struct cxl_region_params *p = &cxlr->params;
1606 int i;
1607
1608 if (pos < 0 || pos >= p->interleave_ways) {
1609 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1610 p->interleave_ways);
1611 return -ENXIO;
1612 }
1613
1614 if (p->targets[pos] == cxled)
1615 return 0;
1616
1617 if (p->targets[pos]) {
1618 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1619 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1620
1621 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1622 pos, dev_name(&cxlmd_target->dev),
1623 dev_name(&cxled_target->cxld.dev));
1624 return -EBUSY;
1625 }
1626
1627 for (i = 0; i < p->interleave_ways; i++) {
1628 struct cxl_endpoint_decoder *cxled_target;
1629 struct cxl_memdev *cxlmd_target;
1630
1631 cxled_target = p->targets[i];
1632 if (!cxled_target)
1633 continue;
1634
1635 cxlmd_target = cxled_to_memdev(cxled_target);
1636 if (cxlmd_target == cxlmd) {
1637 dev_dbg(&cxlr->dev,
1638 "%s already specified at position %d via: %s\n",
1639 dev_name(&cxlmd->dev), pos,
1640 dev_name(&cxled_target->cxld.dev));
1641 return -EBUSY;
1642 }
1643 }
1644
1645 return 0;
1646}
1647
1648static int cxl_region_attach_position(struct cxl_region *cxlr,
1649 struct cxl_root_decoder *cxlrd,
1650 struct cxl_endpoint_decoder *cxled,
1651 const struct cxl_dport *dport, int pos)
1652{
1653 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1654 struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd;
1655 struct cxl_decoder *cxld = &cxlsd->cxld;
1656 int iw = cxld->interleave_ways;
1657 struct cxl_port *iter;
1658 int rc;
1659
1660 if (dport != cxlrd->cxlsd.target[pos % iw]) {
1661 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1662 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1663 dev_name(&cxlrd->cxlsd.cxld.dev));
1664 return -ENXIO;
1665 }
1666
1667 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1668 iter = to_cxl_port(iter->dev.parent)) {
1669 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1670 if (rc)
1671 goto err;
1672 }
1673
1674 return 0;
1675
1676err:
1677 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1678 iter = to_cxl_port(iter->dev.parent))
1679 cxl_port_detach_region(iter, cxlr, cxled);
1680 return rc;
1681}
1682
1683static int cxl_region_attach_auto(struct cxl_region *cxlr,
1684 struct cxl_endpoint_decoder *cxled, int pos)
1685{
1686 struct cxl_region_params *p = &cxlr->params;
1687
1688 if (cxled->state != CXL_DECODER_STATE_AUTO) {
1689 dev_err(&cxlr->dev,
1690 "%s: unable to add decoder to autodetected region\n",
1691 dev_name(&cxled->cxld.dev));
1692 return -EINVAL;
1693 }
1694
1695 if (pos >= 0) {
1696 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1697 dev_name(&cxled->cxld.dev), pos);
1698 return -EINVAL;
1699 }
1700
1701 if (p->nr_targets >= p->interleave_ways) {
1702 dev_err(&cxlr->dev, "%s: no more target slots available\n",
1703 dev_name(&cxled->cxld.dev));
1704 return -ENXIO;
1705 }
1706
1707 /*
1708 * Temporarily record the endpoint decoder into the target array. Yes,
1709 * this means that userspace can view devices in the wrong position
1710 * before the region activates, and must be careful to understand when
1711 * it might be racing region autodiscovery.
1712 */
1713 pos = p->nr_targets;
1714 p->targets[pos] = cxled;
1715 cxled->pos = pos;
1716 p->nr_targets++;
1717
1718 return 0;
1719}
1720
1721static int cmp_interleave_pos(const void *a, const void *b)
1722{
1723 struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1724 struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1725
1726 return cxled_a->pos - cxled_b->pos;
1727}
1728
1729static struct cxl_port *next_port(struct cxl_port *port)
1730{
1731 if (!port->parent_dport)
1732 return NULL;
1733 return port->parent_dport->port;
1734}
1735
1736static int match_switch_decoder_by_range(struct device *dev, void *data)
1737{
1738 struct cxl_switch_decoder *cxlsd;
1739 struct range *r1, *r2 = data;
1740
1741 if (!is_switch_decoder(dev))
1742 return 0;
1743
1744 cxlsd = to_cxl_switch_decoder(dev);
1745 r1 = &cxlsd->cxld.hpa_range;
1746
1747 if (is_root_decoder(dev))
1748 return range_contains(r1, r2);
1749 return (r1->start == r2->start && r1->end == r2->end);
1750}
1751
1752static int find_pos_and_ways(struct cxl_port *port, struct range *range,
1753 int *pos, int *ways)
1754{
1755 struct cxl_switch_decoder *cxlsd;
1756 struct cxl_port *parent;
1757 struct device *dev;
1758 int rc = -ENXIO;
1759
1760 parent = next_port(port);
1761 if (!parent)
1762 return rc;
1763
1764 dev = device_find_child(&parent->dev, range,
1765 match_switch_decoder_by_range);
1766 if (!dev) {
1767 dev_err(port->uport_dev,
1768 "failed to find decoder mapping %#llx-%#llx\n",
1769 range->start, range->end);
1770 return rc;
1771 }
1772 cxlsd = to_cxl_switch_decoder(dev);
1773 *ways = cxlsd->cxld.interleave_ways;
1774
1775 for (int i = 0; i < *ways; i++) {
1776 if (cxlsd->target[i] == port->parent_dport) {
1777 *pos = i;
1778 rc = 0;
1779 break;
1780 }
1781 }
1782 put_device(dev);
1783
1784 return rc;
1785}
1786
1787/**
1788 * cxl_calc_interleave_pos() - calculate an endpoint position in a region
1789 * @cxled: endpoint decoder member of given region
1790 *
1791 * The endpoint position is calculated by traversing the topology from
1792 * the endpoint to the root decoder and iteratively applying this
1793 * calculation:
1794 *
1795 * position = position * parent_ways + parent_pos;
1796 *
1797 * ...where @position is inferred from switch and root decoder target lists.
1798 *
1799 * Return: position >= 0 on success
1800 * -ENXIO on failure
1801 */
1802static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled)
1803{
1804 struct cxl_port *iter, *port = cxled_to_port(cxled);
1805 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1806 struct range *range = &cxled->cxld.hpa_range;
1807 int parent_ways = 0, parent_pos = 0, pos = 0;
1808 int rc;
1809
1810 /*
1811 * Example: the expected interleave order of the 4-way region shown
1812 * below is: mem0, mem2, mem1, mem3
1813 *
1814 * root_port
1815 * / \
1816 * host_bridge_0 host_bridge_1
1817 * | | | |
1818 * mem0 mem1 mem2 mem3
1819 *
1820 * In the example the calculator will iterate twice. The first iteration
1821 * uses the mem position in the host-bridge and the ways of the host-
1822 * bridge to generate the first, or local, position. The second
1823 * iteration uses the host-bridge position in the root_port and the ways
1824 * of the root_port to refine the position.
1825 *
1826 * A trace of the calculation per endpoint looks like this:
1827 * mem0: pos = 0 * 2 + 0 mem2: pos = 0 * 2 + 0
1828 * pos = 0 * 2 + 0 pos = 0 * 2 + 1
1829 * pos: 0 pos: 1
1830 *
1831 * mem1: pos = 0 * 2 + 1 mem3: pos = 0 * 2 + 1
1832 * pos = 1 * 2 + 0 pos = 1 * 2 + 1
1833 * pos: 2 pos = 3
1834 *
1835 * Note that while this example is simple, the method applies to more
1836 * complex topologies, including those with switches.
1837 */
1838
1839 /* Iterate from endpoint to root_port refining the position */
1840 for (iter = port; iter; iter = next_port(iter)) {
1841 if (is_cxl_root(iter))
1842 break;
1843
1844 rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways);
1845 if (rc)
1846 return rc;
1847
1848 pos = pos * parent_ways + parent_pos;
1849 }
1850
1851 dev_dbg(&cxlmd->dev,
1852 "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n",
1853 dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent),
1854 dev_name(&port->dev), range->start, range->end, pos);
1855
1856 return pos;
1857}
1858
1859static int cxl_region_sort_targets(struct cxl_region *cxlr)
1860{
1861 struct cxl_region_params *p = &cxlr->params;
1862 int i, rc = 0;
1863
1864 for (i = 0; i < p->nr_targets; i++) {
1865 struct cxl_endpoint_decoder *cxled = p->targets[i];
1866
1867 cxled->pos = cxl_calc_interleave_pos(cxled);
1868 /*
1869 * Record that sorting failed, but still continue to calc
1870 * cxled->pos so that follow-on code paths can reliably
1871 * do p->targets[cxled->pos] to self-reference their entry.
1872 */
1873 if (cxled->pos < 0)
1874 rc = -ENXIO;
1875 }
1876 /* Keep the cxlr target list in interleave position order */
1877 sort(p->targets, p->nr_targets, sizeof(p->targets[0]),
1878 cmp_interleave_pos, NULL);
1879
1880 dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1881 return rc;
1882}
1883
1884static int cxl_region_attach(struct cxl_region *cxlr,
1885 struct cxl_endpoint_decoder *cxled, int pos)
1886{
1887 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1888 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1889 struct cxl_region_params *p = &cxlr->params;
1890 struct cxl_port *ep_port, *root_port;
1891 struct cxl_dport *dport;
1892 int rc = -ENXIO;
1893
1894 rc = check_interleave_cap(&cxled->cxld, p->interleave_ways,
1895 p->interleave_granularity);
1896 if (rc) {
1897 dev_dbg(&cxlr->dev, "%s iw: %d ig: %d is not supported\n",
1898 dev_name(&cxled->cxld.dev), p->interleave_ways,
1899 p->interleave_granularity);
1900 return rc;
1901 }
1902
1903 if (cxled->mode != cxlr->mode) {
1904 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
1905 dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
1906 return -EINVAL;
1907 }
1908
1909 if (cxled->mode == CXL_DECODER_DEAD) {
1910 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1911 return -ENODEV;
1912 }
1913
1914 /* all full of members, or interleave config not established? */
1915 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1916 dev_dbg(&cxlr->dev, "region already active\n");
1917 return -EBUSY;
1918 } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1919 dev_dbg(&cxlr->dev, "interleave config missing\n");
1920 return -ENXIO;
1921 }
1922
1923 if (p->nr_targets >= p->interleave_ways) {
1924 dev_dbg(&cxlr->dev, "region already has %d endpoints\n",
1925 p->nr_targets);
1926 return -EINVAL;
1927 }
1928
1929 ep_port = cxled_to_port(cxled);
1930 root_port = cxlrd_to_port(cxlrd);
1931 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1932 if (!dport) {
1933 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1934 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1935 dev_name(cxlr->dev.parent));
1936 return -ENXIO;
1937 }
1938
1939 if (cxled->cxld.target_type != cxlr->type) {
1940 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1941 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1942 cxled->cxld.target_type, cxlr->type);
1943 return -ENXIO;
1944 }
1945
1946 if (!cxled->dpa_res) {
1947 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1948 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1949 return -ENXIO;
1950 }
1951
1952 if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1953 resource_size(p->res)) {
1954 dev_dbg(&cxlr->dev,
1955 "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1956 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1957 (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1958 (u64)resource_size(p->res));
1959 return -EINVAL;
1960 }
1961
1962 cxl_region_perf_data_calculate(cxlr, cxled);
1963
1964 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1965 int i;
1966
1967 rc = cxl_region_attach_auto(cxlr, cxled, pos);
1968 if (rc)
1969 return rc;
1970
1971 /* await more targets to arrive... */
1972 if (p->nr_targets < p->interleave_ways)
1973 return 0;
1974
1975 /*
1976 * All targets are here, which implies all PCI enumeration that
1977 * affects this region has been completed. Walk the topology to
1978 * sort the devices into their relative region decode position.
1979 */
1980 rc = cxl_region_sort_targets(cxlr);
1981 if (rc)
1982 return rc;
1983
1984 for (i = 0; i < p->nr_targets; i++) {
1985 cxled = p->targets[i];
1986 ep_port = cxled_to_port(cxled);
1987 dport = cxl_find_dport_by_dev(root_port,
1988 ep_port->host_bridge);
1989 rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1990 dport, i);
1991 if (rc)
1992 return rc;
1993 }
1994
1995 rc = cxl_region_setup_targets(cxlr);
1996 if (rc)
1997 return rc;
1998
1999 /*
2000 * If target setup succeeds in the autodiscovery case
2001 * then the region is already committed.
2002 */
2003 p->state = CXL_CONFIG_COMMIT;
2004 cxl_region_shared_upstream_bandwidth_update(cxlr);
2005
2006 return 0;
2007 }
2008
2009 rc = cxl_region_validate_position(cxlr, cxled, pos);
2010 if (rc)
2011 return rc;
2012
2013 rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
2014 if (rc)
2015 return rc;
2016
2017 p->targets[pos] = cxled;
2018 cxled->pos = pos;
2019 p->nr_targets++;
2020
2021 if (p->nr_targets == p->interleave_ways) {
2022 rc = cxl_region_setup_targets(cxlr);
2023 if (rc)
2024 return rc;
2025 p->state = CXL_CONFIG_ACTIVE;
2026 cxl_region_shared_upstream_bandwidth_update(cxlr);
2027 }
2028
2029 cxled->cxld.interleave_ways = p->interleave_ways;
2030 cxled->cxld.interleave_granularity = p->interleave_granularity;
2031 cxled->cxld.hpa_range = (struct range) {
2032 .start = p->res->start,
2033 .end = p->res->end,
2034 };
2035
2036 if (p->nr_targets != p->interleave_ways)
2037 return 0;
2038
2039 /*
2040 * Test the auto-discovery position calculator function
2041 * against this successfully created user-defined region.
2042 * A fail message here means that this interleave config
2043 * will fail when presented as CXL_REGION_F_AUTO.
2044 */
2045 for (int i = 0; i < p->nr_targets; i++) {
2046 struct cxl_endpoint_decoder *cxled = p->targets[i];
2047 int test_pos;
2048
2049 test_pos = cxl_calc_interleave_pos(cxled);
2050 dev_dbg(&cxled->cxld.dev,
2051 "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n",
2052 (test_pos == cxled->pos) ? "success" : "fail",
2053 test_pos, cxled->pos);
2054 }
2055
2056 return 0;
2057}
2058
2059static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
2060{
2061 struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
2062 struct cxl_region *cxlr = cxled->cxld.region;
2063 struct cxl_region_params *p;
2064 int rc = 0;
2065
2066 lockdep_assert_held_write(&cxl_region_rwsem);
2067
2068 if (!cxlr)
2069 return 0;
2070
2071 p = &cxlr->params;
2072 get_device(&cxlr->dev);
2073
2074 if (p->state > CXL_CONFIG_ACTIVE) {
2075 cxl_region_decode_reset(cxlr, p->interleave_ways);
2076 p->state = CXL_CONFIG_ACTIVE;
2077 }
2078
2079 for (iter = ep_port; !is_cxl_root(iter);
2080 iter = to_cxl_port(iter->dev.parent))
2081 cxl_port_detach_region(iter, cxlr, cxled);
2082
2083 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
2084 p->targets[cxled->pos] != cxled) {
2085 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2086
2087 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
2088 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2089 cxled->pos);
2090 goto out;
2091 }
2092
2093 if (p->state == CXL_CONFIG_ACTIVE) {
2094 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2095 cxl_region_teardown_targets(cxlr);
2096 }
2097 p->targets[cxled->pos] = NULL;
2098 p->nr_targets--;
2099 cxled->cxld.hpa_range = (struct range) {
2100 .start = 0,
2101 .end = -1,
2102 };
2103
2104 /* notify the region driver that one of its targets has departed */
2105 up_write(&cxl_region_rwsem);
2106 device_release_driver(&cxlr->dev);
2107 down_write(&cxl_region_rwsem);
2108out:
2109 put_device(&cxlr->dev);
2110 return rc;
2111}
2112
2113void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
2114{
2115 down_write(&cxl_region_rwsem);
2116 cxled->mode = CXL_DECODER_DEAD;
2117 cxl_region_detach(cxled);
2118 up_write(&cxl_region_rwsem);
2119}
2120
2121static int attach_target(struct cxl_region *cxlr,
2122 struct cxl_endpoint_decoder *cxled, int pos,
2123 unsigned int state)
2124{
2125 int rc = 0;
2126
2127 if (state == TASK_INTERRUPTIBLE)
2128 rc = down_write_killable(&cxl_region_rwsem);
2129 else
2130 down_write(&cxl_region_rwsem);
2131 if (rc)
2132 return rc;
2133
2134 down_read(&cxl_dpa_rwsem);
2135 rc = cxl_region_attach(cxlr, cxled, pos);
2136 up_read(&cxl_dpa_rwsem);
2137 up_write(&cxl_region_rwsem);
2138 return rc;
2139}
2140
2141static int detach_target(struct cxl_region *cxlr, int pos)
2142{
2143 struct cxl_region_params *p = &cxlr->params;
2144 int rc;
2145
2146 rc = down_write_killable(&cxl_region_rwsem);
2147 if (rc)
2148 return rc;
2149
2150 if (pos >= p->interleave_ways) {
2151 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
2152 p->interleave_ways);
2153 rc = -ENXIO;
2154 goto out;
2155 }
2156
2157 if (!p->targets[pos]) {
2158 rc = 0;
2159 goto out;
2160 }
2161
2162 rc = cxl_region_detach(p->targets[pos]);
2163out:
2164 up_write(&cxl_region_rwsem);
2165 return rc;
2166}
2167
2168static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
2169 size_t len)
2170{
2171 int rc;
2172
2173 if (sysfs_streq(buf, "\n"))
2174 rc = detach_target(cxlr, pos);
2175 else {
2176 struct device *dev;
2177
2178 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
2179 if (!dev)
2180 return -ENODEV;
2181
2182 if (!is_endpoint_decoder(dev)) {
2183 rc = -EINVAL;
2184 goto out;
2185 }
2186
2187 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
2188 TASK_INTERRUPTIBLE);
2189out:
2190 put_device(dev);
2191 }
2192
2193 if (rc < 0)
2194 return rc;
2195 return len;
2196}
2197
2198#define TARGET_ATTR_RW(n) \
2199static ssize_t target##n##_show( \
2200 struct device *dev, struct device_attribute *attr, char *buf) \
2201{ \
2202 return show_targetN(to_cxl_region(dev), buf, (n)); \
2203} \
2204static ssize_t target##n##_store(struct device *dev, \
2205 struct device_attribute *attr, \
2206 const char *buf, size_t len) \
2207{ \
2208 return store_targetN(to_cxl_region(dev), buf, (n), len); \
2209} \
2210static DEVICE_ATTR_RW(target##n)
2211
2212TARGET_ATTR_RW(0);
2213TARGET_ATTR_RW(1);
2214TARGET_ATTR_RW(2);
2215TARGET_ATTR_RW(3);
2216TARGET_ATTR_RW(4);
2217TARGET_ATTR_RW(5);
2218TARGET_ATTR_RW(6);
2219TARGET_ATTR_RW(7);
2220TARGET_ATTR_RW(8);
2221TARGET_ATTR_RW(9);
2222TARGET_ATTR_RW(10);
2223TARGET_ATTR_RW(11);
2224TARGET_ATTR_RW(12);
2225TARGET_ATTR_RW(13);
2226TARGET_ATTR_RW(14);
2227TARGET_ATTR_RW(15);
2228
2229static struct attribute *target_attrs[] = {
2230 &dev_attr_target0.attr,
2231 &dev_attr_target1.attr,
2232 &dev_attr_target2.attr,
2233 &dev_attr_target3.attr,
2234 &dev_attr_target4.attr,
2235 &dev_attr_target5.attr,
2236 &dev_attr_target6.attr,
2237 &dev_attr_target7.attr,
2238 &dev_attr_target8.attr,
2239 &dev_attr_target9.attr,
2240 &dev_attr_target10.attr,
2241 &dev_attr_target11.attr,
2242 &dev_attr_target12.attr,
2243 &dev_attr_target13.attr,
2244 &dev_attr_target14.attr,
2245 &dev_attr_target15.attr,
2246 NULL,
2247};
2248
2249static umode_t cxl_region_target_visible(struct kobject *kobj,
2250 struct attribute *a, int n)
2251{
2252 struct device *dev = kobj_to_dev(kobj);
2253 struct cxl_region *cxlr = to_cxl_region(dev);
2254 struct cxl_region_params *p = &cxlr->params;
2255
2256 if (n < p->interleave_ways)
2257 return a->mode;
2258 return 0;
2259}
2260
2261static const struct attribute_group cxl_region_target_group = {
2262 .attrs = target_attrs,
2263 .is_visible = cxl_region_target_visible,
2264};
2265
2266static const struct attribute_group *get_cxl_region_target_group(void)
2267{
2268 return &cxl_region_target_group;
2269}
2270
2271static const struct attribute_group *region_groups[] = {
2272 &cxl_base_attribute_group,
2273 &cxl_region_group,
2274 &cxl_region_target_group,
2275 &cxl_region_access0_coordinate_group,
2276 &cxl_region_access1_coordinate_group,
2277 NULL,
2278};
2279
2280static void cxl_region_release(struct device *dev)
2281{
2282 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
2283 struct cxl_region *cxlr = to_cxl_region(dev);
2284 int id = atomic_read(&cxlrd->region_id);
2285
2286 /*
2287 * Try to reuse the recently idled id rather than the cached
2288 * next id to prevent the region id space from increasing
2289 * unnecessarily.
2290 */
2291 if (cxlr->id < id)
2292 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
2293 memregion_free(id);
2294 goto out;
2295 }
2296
2297 memregion_free(cxlr->id);
2298out:
2299 put_device(dev->parent);
2300 kfree(cxlr);
2301}
2302
2303const struct device_type cxl_region_type = {
2304 .name = "cxl_region",
2305 .release = cxl_region_release,
2306 .groups = region_groups
2307};
2308
2309bool is_cxl_region(struct device *dev)
2310{
2311 return dev->type == &cxl_region_type;
2312}
2313EXPORT_SYMBOL_NS_GPL(is_cxl_region, "CXL");
2314
2315static struct cxl_region *to_cxl_region(struct device *dev)
2316{
2317 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2318 "not a cxl_region device\n"))
2319 return NULL;
2320
2321 return container_of(dev, struct cxl_region, dev);
2322}
2323
2324static void unregister_region(void *_cxlr)
2325{
2326 struct cxl_region *cxlr = _cxlr;
2327 struct cxl_region_params *p = &cxlr->params;
2328 int i;
2329
2330 device_del(&cxlr->dev);
2331
2332 /*
2333 * Now that region sysfs is shutdown, the parameter block is now
2334 * read-only, so no need to hold the region rwsem to access the
2335 * region parameters.
2336 */
2337 for (i = 0; i < p->interleave_ways; i++)
2338 detach_target(cxlr, i);
2339
2340 cxl_region_iomem_release(cxlr);
2341 put_device(&cxlr->dev);
2342}
2343
2344static struct lock_class_key cxl_region_key;
2345
2346static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2347{
2348 struct cxl_region *cxlr;
2349 struct device *dev;
2350
2351 cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2352 if (!cxlr) {
2353 memregion_free(id);
2354 return ERR_PTR(-ENOMEM);
2355 }
2356
2357 dev = &cxlr->dev;
2358 device_initialize(dev);
2359 lockdep_set_class(&dev->mutex, &cxl_region_key);
2360 dev->parent = &cxlrd->cxlsd.cxld.dev;
2361 /*
2362 * Keep root decoder pinned through cxl_region_release to fixup
2363 * region id allocations
2364 */
2365 get_device(dev->parent);
2366 device_set_pm_not_required(dev);
2367 dev->bus = &cxl_bus_type;
2368 dev->type = &cxl_region_type;
2369 cxlr->id = id;
2370
2371 return cxlr;
2372}
2373
2374static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid)
2375{
2376 int cset = 0;
2377 int rc;
2378
2379 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2380 if (cxlr->coord[i].read_bandwidth) {
2381 rc = 0;
2382 if (cxl_need_node_perf_attrs_update(nid))
2383 node_set_perf_attrs(nid, &cxlr->coord[i], i);
2384 else
2385 rc = cxl_update_hmat_access_coordinates(nid, cxlr, i);
2386
2387 if (rc == 0)
2388 cset++;
2389 }
2390 }
2391
2392 if (!cset)
2393 return false;
2394
2395 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access0_group());
2396 if (rc)
2397 dev_dbg(&cxlr->dev, "Failed to update access0 group\n");
2398
2399 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access1_group());
2400 if (rc)
2401 dev_dbg(&cxlr->dev, "Failed to update access1 group\n");
2402
2403 return true;
2404}
2405
2406static int cxl_region_perf_attrs_callback(struct notifier_block *nb,
2407 unsigned long action, void *arg)
2408{
2409 struct cxl_region *cxlr = container_of(nb, struct cxl_region,
2410 memory_notifier);
2411 struct memory_notify *mnb = arg;
2412 int nid = mnb->status_change_nid;
2413 int region_nid;
2414
2415 if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
2416 return NOTIFY_DONE;
2417
2418 /*
2419 * No need to hold cxl_region_rwsem; region parameters are stable
2420 * within the cxl_region driver.
2421 */
2422 region_nid = phys_to_target_node(cxlr->params.res->start);
2423 if (nid != region_nid)
2424 return NOTIFY_DONE;
2425
2426 if (!cxl_region_update_coordinates(cxlr, nid))
2427 return NOTIFY_DONE;
2428
2429 return NOTIFY_OK;
2430}
2431
2432static int cxl_region_calculate_adistance(struct notifier_block *nb,
2433 unsigned long nid, void *data)
2434{
2435 struct cxl_region *cxlr = container_of(nb, struct cxl_region,
2436 adist_notifier);
2437 struct access_coordinate *perf;
2438 int *adist = data;
2439 int region_nid;
2440
2441 /*
2442 * No need to hold cxl_region_rwsem; region parameters are stable
2443 * within the cxl_region driver.
2444 */
2445 region_nid = phys_to_target_node(cxlr->params.res->start);
2446 if (nid != region_nid)
2447 return NOTIFY_OK;
2448
2449 perf = &cxlr->coord[ACCESS_COORDINATE_CPU];
2450
2451 if (mt_perf_to_adistance(perf, adist))
2452 return NOTIFY_OK;
2453
2454 return NOTIFY_STOP;
2455}
2456
2457/**
2458 * devm_cxl_add_region - Adds a region to a decoder
2459 * @cxlrd: root decoder
2460 * @id: memregion id to create, or memregion_free() on failure
2461 * @mode: mode for the endpoint decoders of this region
2462 * @type: select whether this is an expander or accelerator (type-2 or type-3)
2463 *
2464 * This is the second step of region initialization. Regions exist within an
2465 * address space which is mapped by a @cxlrd.
2466 *
2467 * Return: 0 if the region was added to the @cxlrd, else returns negative error
2468 * code. The region will be named "regionZ" where Z is the unique region number.
2469 */
2470static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2471 int id,
2472 enum cxl_decoder_mode mode,
2473 enum cxl_decoder_type type)
2474{
2475 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2476 struct cxl_region *cxlr;
2477 struct device *dev;
2478 int rc;
2479
2480 cxlr = cxl_region_alloc(cxlrd, id);
2481 if (IS_ERR(cxlr))
2482 return cxlr;
2483 cxlr->mode = mode;
2484 cxlr->type = type;
2485
2486 dev = &cxlr->dev;
2487 rc = dev_set_name(dev, "region%d", id);
2488 if (rc)
2489 goto err;
2490
2491 rc = device_add(dev);
2492 if (rc)
2493 goto err;
2494
2495 rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2496 if (rc)
2497 return ERR_PTR(rc);
2498
2499 dev_dbg(port->uport_dev, "%s: created %s\n",
2500 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2501 return cxlr;
2502
2503err:
2504 put_device(dev);
2505 return ERR_PTR(rc);
2506}
2507
2508static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2509{
2510 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2511}
2512
2513static ssize_t create_pmem_region_show(struct device *dev,
2514 struct device_attribute *attr, char *buf)
2515{
2516 return __create_region_show(to_cxl_root_decoder(dev), buf);
2517}
2518
2519static ssize_t create_ram_region_show(struct device *dev,
2520 struct device_attribute *attr, char *buf)
2521{
2522 return __create_region_show(to_cxl_root_decoder(dev), buf);
2523}
2524
2525static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2526 enum cxl_decoder_mode mode, int id)
2527{
2528 int rc;
2529
2530 switch (mode) {
2531 case CXL_DECODER_RAM:
2532 case CXL_DECODER_PMEM:
2533 break;
2534 default:
2535 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2536 return ERR_PTR(-EINVAL);
2537 }
2538
2539 rc = memregion_alloc(GFP_KERNEL);
2540 if (rc < 0)
2541 return ERR_PTR(rc);
2542
2543 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2544 memregion_free(rc);
2545 return ERR_PTR(-EBUSY);
2546 }
2547
2548 return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
2549}
2550
2551static ssize_t create_region_store(struct device *dev, const char *buf,
2552 size_t len, enum cxl_decoder_mode mode)
2553{
2554 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2555 struct cxl_region *cxlr;
2556 int rc, id;
2557
2558 rc = sscanf(buf, "region%d\n", &id);
2559 if (rc != 1)
2560 return -EINVAL;
2561
2562 cxlr = __create_region(cxlrd, mode, id);
2563 if (IS_ERR(cxlr))
2564 return PTR_ERR(cxlr);
2565
2566 return len;
2567}
2568
2569static ssize_t create_pmem_region_store(struct device *dev,
2570 struct device_attribute *attr,
2571 const char *buf, size_t len)
2572{
2573 return create_region_store(dev, buf, len, CXL_DECODER_PMEM);
2574}
2575DEVICE_ATTR_RW(create_pmem_region);
2576
2577static ssize_t create_ram_region_store(struct device *dev,
2578 struct device_attribute *attr,
2579 const char *buf, size_t len)
2580{
2581 return create_region_store(dev, buf, len, CXL_DECODER_RAM);
2582}
2583DEVICE_ATTR_RW(create_ram_region);
2584
2585static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2586 char *buf)
2587{
2588 struct cxl_decoder *cxld = to_cxl_decoder(dev);
2589 ssize_t rc;
2590
2591 rc = down_read_interruptible(&cxl_region_rwsem);
2592 if (rc)
2593 return rc;
2594
2595 if (cxld->region)
2596 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2597 else
2598 rc = sysfs_emit(buf, "\n");
2599 up_read(&cxl_region_rwsem);
2600
2601 return rc;
2602}
2603DEVICE_ATTR_RO(region);
2604
2605static struct cxl_region *
2606cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2607{
2608 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2609 struct device *region_dev;
2610
2611 region_dev = device_find_child_by_name(&cxld->dev, name);
2612 if (!region_dev)
2613 return ERR_PTR(-ENODEV);
2614
2615 return to_cxl_region(region_dev);
2616}
2617
2618static ssize_t delete_region_store(struct device *dev,
2619 struct device_attribute *attr,
2620 const char *buf, size_t len)
2621{
2622 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2623 struct cxl_port *port = to_cxl_port(dev->parent);
2624 struct cxl_region *cxlr;
2625
2626 cxlr = cxl_find_region_by_name(cxlrd, buf);
2627 if (IS_ERR(cxlr))
2628 return PTR_ERR(cxlr);
2629
2630 devm_release_action(port->uport_dev, unregister_region, cxlr);
2631 put_device(&cxlr->dev);
2632
2633 return len;
2634}
2635DEVICE_ATTR_WO(delete_region);
2636
2637static void cxl_pmem_region_release(struct device *dev)
2638{
2639 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2640 int i;
2641
2642 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2643 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2644
2645 put_device(&cxlmd->dev);
2646 }
2647
2648 kfree(cxlr_pmem);
2649}
2650
2651static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2652 &cxl_base_attribute_group,
2653 NULL,
2654};
2655
2656const struct device_type cxl_pmem_region_type = {
2657 .name = "cxl_pmem_region",
2658 .release = cxl_pmem_region_release,
2659 .groups = cxl_pmem_region_attribute_groups,
2660};
2661
2662bool is_cxl_pmem_region(struct device *dev)
2663{
2664 return dev->type == &cxl_pmem_region_type;
2665}
2666EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, "CXL");
2667
2668struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2669{
2670 if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2671 "not a cxl_pmem_region device\n"))
2672 return NULL;
2673 return container_of(dev, struct cxl_pmem_region, dev);
2674}
2675EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, "CXL");
2676
2677struct cxl_poison_context {
2678 struct cxl_port *port;
2679 enum cxl_decoder_mode mode;
2680 u64 offset;
2681};
2682
2683static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2684 struct cxl_poison_context *ctx)
2685{
2686 struct cxl_dev_state *cxlds = cxlmd->cxlds;
2687 u64 offset, length;
2688 int rc = 0;
2689
2690 /*
2691 * Collect poison for the remaining unmapped resources
2692 * after poison is collected by committed endpoints.
2693 *
2694 * Knowing that PMEM must always follow RAM, get poison
2695 * for unmapped resources based on the last decoder's mode:
2696 * ram: scan remains of ram range, then any pmem range
2697 * pmem: scan remains of pmem range
2698 */
2699
2700 if (ctx->mode == CXL_DECODER_RAM) {
2701 offset = ctx->offset;
2702 length = resource_size(&cxlds->ram_res) - offset;
2703 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2704 if (rc == -EFAULT)
2705 rc = 0;
2706 if (rc)
2707 return rc;
2708 }
2709 if (ctx->mode == CXL_DECODER_PMEM) {
2710 offset = ctx->offset;
2711 length = resource_size(&cxlds->dpa_res) - offset;
2712 if (!length)
2713 return 0;
2714 } else if (resource_size(&cxlds->pmem_res)) {
2715 offset = cxlds->pmem_res.start;
2716 length = resource_size(&cxlds->pmem_res);
2717 } else {
2718 return 0;
2719 }
2720
2721 return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2722}
2723
2724static int poison_by_decoder(struct device *dev, void *arg)
2725{
2726 struct cxl_poison_context *ctx = arg;
2727 struct cxl_endpoint_decoder *cxled;
2728 struct cxl_memdev *cxlmd;
2729 u64 offset, length;
2730 int rc = 0;
2731
2732 if (!is_endpoint_decoder(dev))
2733 return rc;
2734
2735 cxled = to_cxl_endpoint_decoder(dev);
2736 if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2737 return rc;
2738
2739 /*
2740 * Regions are only created with single mode decoders: pmem or ram.
2741 * Linux does not support mixed mode decoders. This means that
2742 * reading poison per endpoint decoder adheres to the requirement
2743 * that poison reads of pmem and ram must be separated.
2744 * CXL 3.0 Spec 8.2.9.8.4.1
2745 */
2746 if (cxled->mode == CXL_DECODER_MIXED) {
2747 dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2748 return rc;
2749 }
2750
2751 cxlmd = cxled_to_memdev(cxled);
2752 if (cxled->skip) {
2753 offset = cxled->dpa_res->start - cxled->skip;
2754 length = cxled->skip;
2755 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2756 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2757 rc = 0;
2758 if (rc)
2759 return rc;
2760 }
2761
2762 offset = cxled->dpa_res->start;
2763 length = cxled->dpa_res->end - offset + 1;
2764 rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2765 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2766 rc = 0;
2767 if (rc)
2768 return rc;
2769
2770 /* Iterate until commit_end is reached */
2771 if (cxled->cxld.id == ctx->port->commit_end) {
2772 ctx->offset = cxled->dpa_res->end + 1;
2773 ctx->mode = cxled->mode;
2774 return 1;
2775 }
2776
2777 return 0;
2778}
2779
2780int cxl_get_poison_by_endpoint(struct cxl_port *port)
2781{
2782 struct cxl_poison_context ctx;
2783 int rc = 0;
2784
2785 ctx = (struct cxl_poison_context) {
2786 .port = port
2787 };
2788
2789 rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2790 if (rc == 1)
2791 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
2792 &ctx);
2793
2794 return rc;
2795}
2796
2797struct cxl_dpa_to_region_context {
2798 struct cxl_region *cxlr;
2799 u64 dpa;
2800};
2801
2802static int __cxl_dpa_to_region(struct device *dev, void *arg)
2803{
2804 struct cxl_dpa_to_region_context *ctx = arg;
2805 struct cxl_endpoint_decoder *cxled;
2806 struct cxl_region *cxlr;
2807 u64 dpa = ctx->dpa;
2808
2809 if (!is_endpoint_decoder(dev))
2810 return 0;
2811
2812 cxled = to_cxl_endpoint_decoder(dev);
2813 if (!cxled || !cxled->dpa_res || !resource_size(cxled->dpa_res))
2814 return 0;
2815
2816 if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start)
2817 return 0;
2818
2819 /*
2820 * Stop the region search (return 1) when an endpoint mapping is
2821 * found. The region may not be fully constructed so offering
2822 * the cxlr in the context structure is not guaranteed.
2823 */
2824 cxlr = cxled->cxld.region;
2825 if (cxlr)
2826 dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa,
2827 dev_name(&cxlr->dev));
2828 else
2829 dev_dbg(dev, "dpa:0x%llx mapped in endpoint:%s\n", dpa,
2830 dev_name(dev));
2831
2832 ctx->cxlr = cxlr;
2833
2834 return 1;
2835}
2836
2837struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa)
2838{
2839 struct cxl_dpa_to_region_context ctx;
2840 struct cxl_port *port;
2841
2842 ctx = (struct cxl_dpa_to_region_context) {
2843 .dpa = dpa,
2844 };
2845 port = cxlmd->endpoint;
2846 if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port))
2847 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region);
2848
2849 return ctx.cxlr;
2850}
2851
2852static bool cxl_is_hpa_in_chunk(u64 hpa, struct cxl_region *cxlr, int pos)
2853{
2854 struct cxl_region_params *p = &cxlr->params;
2855 int gran = p->interleave_granularity;
2856 int ways = p->interleave_ways;
2857 u64 offset;
2858
2859 /* Is the hpa in an expected chunk for its pos(-ition) */
2860 offset = hpa - p->res->start;
2861 offset = do_div(offset, gran * ways);
2862 if ((offset >= pos * gran) && (offset < (pos + 1) * gran))
2863 return true;
2864
2865 dev_dbg(&cxlr->dev,
2866 "Addr trans fail: hpa 0x%llx not in expected chunk\n", hpa);
2867
2868 return false;
2869}
2870
2871u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd,
2872 u64 dpa)
2873{
2874 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
2875 u64 dpa_offset, hpa_offset, bits_upper, mask_upper, hpa;
2876 struct cxl_region_params *p = &cxlr->params;
2877 struct cxl_endpoint_decoder *cxled = NULL;
2878 u16 eig = 0;
2879 u8 eiw = 0;
2880 int pos;
2881
2882 for (int i = 0; i < p->nr_targets; i++) {
2883 cxled = p->targets[i];
2884 if (cxlmd == cxled_to_memdev(cxled))
2885 break;
2886 }
2887 if (!cxled || cxlmd != cxled_to_memdev(cxled))
2888 return ULLONG_MAX;
2889
2890 pos = cxled->pos;
2891 ways_to_eiw(p->interleave_ways, &eiw);
2892 granularity_to_eig(p->interleave_granularity, &eig);
2893
2894 /*
2895 * The device position in the region interleave set was removed
2896 * from the offset at HPA->DPA translation. To reconstruct the
2897 * HPA, place the 'pos' in the offset.
2898 *
2899 * The placement of 'pos' in the HPA is determined by interleave
2900 * ways and granularity and is defined in the CXL Spec 3.0 Section
2901 * 8.2.4.19.13 Implementation Note: Device Decode Logic
2902 */
2903
2904 /* Remove the dpa base */
2905 dpa_offset = dpa - cxl_dpa_resource_start(cxled);
2906
2907 mask_upper = GENMASK_ULL(51, eig + 8);
2908
2909 if (eiw < 8) {
2910 hpa_offset = (dpa_offset & mask_upper) << eiw;
2911 hpa_offset |= pos << (eig + 8);
2912 } else {
2913 bits_upper = (dpa_offset & mask_upper) >> (eig + 8);
2914 bits_upper = bits_upper * 3;
2915 hpa_offset = ((bits_upper << (eiw - 8)) + pos) << (eig + 8);
2916 }
2917
2918 /* The lower bits remain unchanged */
2919 hpa_offset |= dpa_offset & GENMASK_ULL(eig + 7, 0);
2920
2921 /* Apply the hpa_offset to the region base address */
2922 hpa = hpa_offset + p->res->start;
2923
2924 /* Root decoder translation overrides typical modulo decode */
2925 if (cxlrd->hpa_to_spa)
2926 hpa = cxlrd->hpa_to_spa(cxlrd, hpa);
2927
2928 if (hpa < p->res->start || hpa > p->res->end) {
2929 dev_dbg(&cxlr->dev,
2930 "Addr trans fail: hpa 0x%llx not in region\n", hpa);
2931 return ULLONG_MAX;
2932 }
2933
2934 /* Simple chunk check, by pos & gran, only applies to modulo decodes */
2935 if (!cxlrd->hpa_to_spa && (!cxl_is_hpa_in_chunk(hpa, cxlr, pos)))
2936 return ULLONG_MAX;
2937
2938 return hpa;
2939}
2940
2941static struct lock_class_key cxl_pmem_region_key;
2942
2943static int cxl_pmem_region_alloc(struct cxl_region *cxlr)
2944{
2945 struct cxl_region_params *p = &cxlr->params;
2946 struct cxl_nvdimm_bridge *cxl_nvb;
2947 struct device *dev;
2948 int i;
2949
2950 guard(rwsem_read)(&cxl_region_rwsem);
2951 if (p->state != CXL_CONFIG_COMMIT)
2952 return -ENXIO;
2953
2954 struct cxl_pmem_region *cxlr_pmem __free(kfree) =
2955 kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets), GFP_KERNEL);
2956 if (!cxlr_pmem)
2957 return -ENOMEM;
2958
2959 cxlr_pmem->hpa_range.start = p->res->start;
2960 cxlr_pmem->hpa_range.end = p->res->end;
2961
2962 /* Snapshot the region configuration underneath the cxl_region_rwsem */
2963 cxlr_pmem->nr_mappings = p->nr_targets;
2964 for (i = 0; i < p->nr_targets; i++) {
2965 struct cxl_endpoint_decoder *cxled = p->targets[i];
2966 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2967 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
2968
2969 /*
2970 * Regions never span CXL root devices, so by definition the
2971 * bridge for one device is the same for all.
2972 */
2973 if (i == 0) {
2974 cxl_nvb = cxl_find_nvdimm_bridge(cxlmd->endpoint);
2975 if (!cxl_nvb)
2976 return -ENODEV;
2977 cxlr->cxl_nvb = cxl_nvb;
2978 }
2979 m->cxlmd = cxlmd;
2980 get_device(&cxlmd->dev);
2981 m->start = cxled->dpa_res->start;
2982 m->size = resource_size(cxled->dpa_res);
2983 m->position = i;
2984 }
2985
2986 dev = &cxlr_pmem->dev;
2987 device_initialize(dev);
2988 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
2989 device_set_pm_not_required(dev);
2990 dev->parent = &cxlr->dev;
2991 dev->bus = &cxl_bus_type;
2992 dev->type = &cxl_pmem_region_type;
2993 cxlr_pmem->cxlr = cxlr;
2994 cxlr->cxlr_pmem = no_free_ptr(cxlr_pmem);
2995
2996 return 0;
2997}
2998
2999static void cxl_dax_region_release(struct device *dev)
3000{
3001 struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
3002
3003 kfree(cxlr_dax);
3004}
3005
3006static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
3007 &cxl_base_attribute_group,
3008 NULL,
3009};
3010
3011const struct device_type cxl_dax_region_type = {
3012 .name = "cxl_dax_region",
3013 .release = cxl_dax_region_release,
3014 .groups = cxl_dax_region_attribute_groups,
3015};
3016
3017static bool is_cxl_dax_region(struct device *dev)
3018{
3019 return dev->type == &cxl_dax_region_type;
3020}
3021
3022struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
3023{
3024 if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
3025 "not a cxl_dax_region device\n"))
3026 return NULL;
3027 return container_of(dev, struct cxl_dax_region, dev);
3028}
3029EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, "CXL");
3030
3031static struct lock_class_key cxl_dax_region_key;
3032
3033static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
3034{
3035 struct cxl_region_params *p = &cxlr->params;
3036 struct cxl_dax_region *cxlr_dax;
3037 struct device *dev;
3038
3039 down_read(&cxl_region_rwsem);
3040 if (p->state != CXL_CONFIG_COMMIT) {
3041 cxlr_dax = ERR_PTR(-ENXIO);
3042 goto out;
3043 }
3044
3045 cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
3046 if (!cxlr_dax) {
3047 cxlr_dax = ERR_PTR(-ENOMEM);
3048 goto out;
3049 }
3050
3051 cxlr_dax->hpa_range.start = p->res->start;
3052 cxlr_dax->hpa_range.end = p->res->end;
3053
3054 dev = &cxlr_dax->dev;
3055 cxlr_dax->cxlr = cxlr;
3056 device_initialize(dev);
3057 lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
3058 device_set_pm_not_required(dev);
3059 dev->parent = &cxlr->dev;
3060 dev->bus = &cxl_bus_type;
3061 dev->type = &cxl_dax_region_type;
3062out:
3063 up_read(&cxl_region_rwsem);
3064
3065 return cxlr_dax;
3066}
3067
3068static void cxlr_pmem_unregister(void *_cxlr_pmem)
3069{
3070 struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
3071 struct cxl_region *cxlr = cxlr_pmem->cxlr;
3072 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
3073
3074 /*
3075 * Either the bridge is in ->remove() context under the device_lock(),
3076 * or cxlr_release_nvdimm() is cancelling the bridge's release action
3077 * for @cxlr_pmem and doing it itself (while manually holding the bridge
3078 * lock).
3079 */
3080 device_lock_assert(&cxl_nvb->dev);
3081 cxlr->cxlr_pmem = NULL;
3082 cxlr_pmem->cxlr = NULL;
3083 device_unregister(&cxlr_pmem->dev);
3084}
3085
3086static void cxlr_release_nvdimm(void *_cxlr)
3087{
3088 struct cxl_region *cxlr = _cxlr;
3089 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
3090
3091 scoped_guard(device, &cxl_nvb->dev) {
3092 if (cxlr->cxlr_pmem)
3093 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
3094 cxlr->cxlr_pmem);
3095 }
3096 cxlr->cxl_nvb = NULL;
3097 put_device(&cxl_nvb->dev);
3098}
3099
3100/**
3101 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
3102 * @cxlr: parent CXL region for this pmem region bridge device
3103 *
3104 * Return: 0 on success negative error code on failure.
3105 */
3106static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
3107{
3108 struct cxl_pmem_region *cxlr_pmem;
3109 struct cxl_nvdimm_bridge *cxl_nvb;
3110 struct device *dev;
3111 int rc;
3112
3113 rc = cxl_pmem_region_alloc(cxlr);
3114 if (rc)
3115 return rc;
3116 cxlr_pmem = cxlr->cxlr_pmem;
3117 cxl_nvb = cxlr->cxl_nvb;
3118
3119 dev = &cxlr_pmem->dev;
3120 rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
3121 if (rc)
3122 goto err;
3123
3124 rc = device_add(dev);
3125 if (rc)
3126 goto err;
3127
3128 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
3129 dev_name(dev));
3130
3131 scoped_guard(device, &cxl_nvb->dev) {
3132 if (cxl_nvb->dev.driver)
3133 rc = devm_add_action_or_reset(&cxl_nvb->dev,
3134 cxlr_pmem_unregister,
3135 cxlr_pmem);
3136 else
3137 rc = -ENXIO;
3138 }
3139
3140 if (rc)
3141 goto err_bridge;
3142
3143 /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
3144 return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
3145
3146err:
3147 put_device(dev);
3148err_bridge:
3149 put_device(&cxl_nvb->dev);
3150 cxlr->cxl_nvb = NULL;
3151 return rc;
3152}
3153
3154static void cxlr_dax_unregister(void *_cxlr_dax)
3155{
3156 struct cxl_dax_region *cxlr_dax = _cxlr_dax;
3157
3158 device_unregister(&cxlr_dax->dev);
3159}
3160
3161static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
3162{
3163 struct cxl_dax_region *cxlr_dax;
3164 struct device *dev;
3165 int rc;
3166
3167 cxlr_dax = cxl_dax_region_alloc(cxlr);
3168 if (IS_ERR(cxlr_dax))
3169 return PTR_ERR(cxlr_dax);
3170
3171 dev = &cxlr_dax->dev;
3172 rc = dev_set_name(dev, "dax_region%d", cxlr->id);
3173 if (rc)
3174 goto err;
3175
3176 rc = device_add(dev);
3177 if (rc)
3178 goto err;
3179
3180 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
3181 dev_name(dev));
3182
3183 return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
3184 cxlr_dax);
3185err:
3186 put_device(dev);
3187 return rc;
3188}
3189
3190static int match_root_decoder_by_range(struct device *dev, void *data)
3191{
3192 struct range *r1, *r2 = data;
3193 struct cxl_root_decoder *cxlrd;
3194
3195 if (!is_root_decoder(dev))
3196 return 0;
3197
3198 cxlrd = to_cxl_root_decoder(dev);
3199 r1 = &cxlrd->cxlsd.cxld.hpa_range;
3200 return range_contains(r1, r2);
3201}
3202
3203static int match_region_by_range(struct device *dev, void *data)
3204{
3205 struct cxl_region_params *p;
3206 struct cxl_region *cxlr;
3207 struct range *r = data;
3208 int rc = 0;
3209
3210 if (!is_cxl_region(dev))
3211 return 0;
3212
3213 cxlr = to_cxl_region(dev);
3214 p = &cxlr->params;
3215
3216 down_read(&cxl_region_rwsem);
3217 if (p->res && p->res->start == r->start && p->res->end == r->end)
3218 rc = 1;
3219 up_read(&cxl_region_rwsem);
3220
3221 return rc;
3222}
3223
3224/* Establish an empty region covering the given HPA range */
3225static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
3226 struct cxl_endpoint_decoder *cxled)
3227{
3228 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3229 struct cxl_port *port = cxlrd_to_port(cxlrd);
3230 struct range *hpa = &cxled->cxld.hpa_range;
3231 struct cxl_region_params *p;
3232 struct cxl_region *cxlr;
3233 struct resource *res;
3234 int rc;
3235
3236 do {
3237 cxlr = __create_region(cxlrd, cxled->mode,
3238 atomic_read(&cxlrd->region_id));
3239 } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
3240
3241 if (IS_ERR(cxlr)) {
3242 dev_err(cxlmd->dev.parent,
3243 "%s:%s: %s failed assign region: %ld\n",
3244 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3245 __func__, PTR_ERR(cxlr));
3246 return cxlr;
3247 }
3248
3249 down_write(&cxl_region_rwsem);
3250 p = &cxlr->params;
3251 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
3252 dev_err(cxlmd->dev.parent,
3253 "%s:%s: %s autodiscovery interrupted\n",
3254 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3255 __func__);
3256 rc = -EBUSY;
3257 goto err;
3258 }
3259
3260 set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
3261
3262 res = kmalloc(sizeof(*res), GFP_KERNEL);
3263 if (!res) {
3264 rc = -ENOMEM;
3265 goto err;
3266 }
3267
3268 *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
3269 dev_name(&cxlr->dev));
3270 rc = insert_resource(cxlrd->res, res);
3271 if (rc) {
3272 /*
3273 * Platform-firmware may not have split resources like "System
3274 * RAM" on CXL window boundaries see cxl_region_iomem_release()
3275 */
3276 dev_warn(cxlmd->dev.parent,
3277 "%s:%s: %s %s cannot insert resource\n",
3278 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3279 __func__, dev_name(&cxlr->dev));
3280 }
3281
3282 p->res = res;
3283 p->interleave_ways = cxled->cxld.interleave_ways;
3284 p->interleave_granularity = cxled->cxld.interleave_granularity;
3285 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
3286
3287 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
3288 if (rc)
3289 goto err;
3290
3291 dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
3292 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
3293 dev_name(&cxlr->dev), p->res, p->interleave_ways,
3294 p->interleave_granularity);
3295
3296 /* ...to match put_device() in cxl_add_to_region() */
3297 get_device(&cxlr->dev);
3298 up_write(&cxl_region_rwsem);
3299
3300 return cxlr;
3301
3302err:
3303 up_write(&cxl_region_rwsem);
3304 devm_release_action(port->uport_dev, unregister_region, cxlr);
3305 return ERR_PTR(rc);
3306}
3307
3308int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
3309{
3310 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3311 struct range *hpa = &cxled->cxld.hpa_range;
3312 struct cxl_decoder *cxld = &cxled->cxld;
3313 struct device *cxlrd_dev, *region_dev;
3314 struct cxl_root_decoder *cxlrd;
3315 struct cxl_region_params *p;
3316 struct cxl_region *cxlr;
3317 bool attach = false;
3318 int rc;
3319
3320 cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
3321 match_root_decoder_by_range);
3322 if (!cxlrd_dev) {
3323 dev_err(cxlmd->dev.parent,
3324 "%s:%s no CXL window for range %#llx:%#llx\n",
3325 dev_name(&cxlmd->dev), dev_name(&cxld->dev),
3326 cxld->hpa_range.start, cxld->hpa_range.end);
3327 return -ENXIO;
3328 }
3329
3330 cxlrd = to_cxl_root_decoder(cxlrd_dev);
3331
3332 /*
3333 * Ensure that if multiple threads race to construct_region() for @hpa
3334 * one does the construction and the others add to that.
3335 */
3336 mutex_lock(&cxlrd->range_lock);
3337 region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
3338 match_region_by_range);
3339 if (!region_dev) {
3340 cxlr = construct_region(cxlrd, cxled);
3341 region_dev = &cxlr->dev;
3342 } else
3343 cxlr = to_cxl_region(region_dev);
3344 mutex_unlock(&cxlrd->range_lock);
3345
3346 rc = PTR_ERR_OR_ZERO(cxlr);
3347 if (rc)
3348 goto out;
3349
3350 attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
3351
3352 down_read(&cxl_region_rwsem);
3353 p = &cxlr->params;
3354 attach = p->state == CXL_CONFIG_COMMIT;
3355 up_read(&cxl_region_rwsem);
3356
3357 if (attach) {
3358 /*
3359 * If device_attach() fails the range may still be active via
3360 * the platform-firmware memory map, otherwise the driver for
3361 * regions is local to this file, so driver matching can't fail.
3362 */
3363 if (device_attach(&cxlr->dev) < 0)
3364 dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
3365 p->res);
3366 }
3367
3368 put_device(region_dev);
3369out:
3370 put_device(cxlrd_dev);
3371 return rc;
3372}
3373EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, "CXL");
3374
3375static int is_system_ram(struct resource *res, void *arg)
3376{
3377 struct cxl_region *cxlr = arg;
3378 struct cxl_region_params *p = &cxlr->params;
3379
3380 dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
3381 return 1;
3382}
3383
3384static void shutdown_notifiers(void *_cxlr)
3385{
3386 struct cxl_region *cxlr = _cxlr;
3387
3388 unregister_memory_notifier(&cxlr->memory_notifier);
3389 unregister_mt_adistance_algorithm(&cxlr->adist_notifier);
3390}
3391
3392static int cxl_region_probe(struct device *dev)
3393{
3394 struct cxl_region *cxlr = to_cxl_region(dev);
3395 struct cxl_region_params *p = &cxlr->params;
3396 int rc;
3397
3398 rc = down_read_interruptible(&cxl_region_rwsem);
3399 if (rc) {
3400 dev_dbg(&cxlr->dev, "probe interrupted\n");
3401 return rc;
3402 }
3403
3404 if (p->state < CXL_CONFIG_COMMIT) {
3405 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
3406 rc = -ENXIO;
3407 goto out;
3408 }
3409
3410 if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
3411 dev_err(&cxlr->dev,
3412 "failed to activate, re-commit region and retry\n");
3413 rc = -ENXIO;
3414 goto out;
3415 }
3416
3417 /*
3418 * From this point on any path that changes the region's state away from
3419 * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
3420 */
3421out:
3422 up_read(&cxl_region_rwsem);
3423
3424 if (rc)
3425 return rc;
3426
3427 cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback;
3428 cxlr->memory_notifier.priority = CXL_CALLBACK_PRI;
3429 register_memory_notifier(&cxlr->memory_notifier);
3430
3431 cxlr->adist_notifier.notifier_call = cxl_region_calculate_adistance;
3432 cxlr->adist_notifier.priority = 100;
3433 register_mt_adistance_algorithm(&cxlr->adist_notifier);
3434
3435 rc = devm_add_action_or_reset(&cxlr->dev, shutdown_notifiers, cxlr);
3436 if (rc)
3437 return rc;
3438
3439 switch (cxlr->mode) {
3440 case CXL_DECODER_PMEM:
3441 return devm_cxl_add_pmem_region(cxlr);
3442 case CXL_DECODER_RAM:
3443 /*
3444 * The region can not be manged by CXL if any portion of
3445 * it is already online as 'System RAM'
3446 */
3447 if (walk_iomem_res_desc(IORES_DESC_NONE,
3448 IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
3449 p->res->start, p->res->end, cxlr,
3450 is_system_ram) > 0)
3451 return 0;
3452 return devm_cxl_add_dax_region(cxlr);
3453 default:
3454 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
3455 cxlr->mode);
3456 return -ENXIO;
3457 }
3458}
3459
3460static struct cxl_driver cxl_region_driver = {
3461 .name = "cxl_region",
3462 .probe = cxl_region_probe,
3463 .id = CXL_DEVICE_REGION,
3464};
3465
3466int cxl_region_init(void)
3467{
3468 return cxl_driver_register(&cxl_region_driver);
3469}
3470
3471void cxl_region_exit(void)
3472{
3473 cxl_driver_unregister(&cxl_region_driver);
3474}
3475
3476MODULE_IMPORT_NS("CXL");
3477MODULE_IMPORT_NS("DEVMEM");
3478MODULE_ALIAS_CXL(CXL_DEVICE_REGION);
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3#include <linux/memregion.h>
4#include <linux/genalloc.h>
5#include <linux/device.h>
6#include <linux/module.h>
7#include <linux/memory.h>
8#include <linux/slab.h>
9#include <linux/uuid.h>
10#include <linux/sort.h>
11#include <linux/idr.h>
12#include <cxlmem.h>
13#include <cxl.h>
14#include "core.h"
15
16/**
17 * DOC: cxl core region
18 *
19 * CXL Regions represent mapped memory capacity in system physical address
20 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
21 * Memory ranges, Regions represent the active mapped capacity by the HDM
22 * Decoder Capability structures throughout the Host Bridges, Switches, and
23 * Endpoints in the topology.
24 *
25 * Region configuration has ordering constraints. UUID may be set at any time
26 * but is only visible for persistent regions.
27 * 1. Interleave granularity
28 * 2. Interleave size
29 * 3. Decoder targets
30 */
31
32static struct cxl_region *to_cxl_region(struct device *dev);
33
34#define __ACCESS_ATTR_RO(_level, _name) { \
35 .attr = { .name = __stringify(_name), .mode = 0444 }, \
36 .show = _name##_access##_level##_show, \
37}
38
39#define ACCESS_DEVICE_ATTR_RO(level, name) \
40 struct device_attribute dev_attr_access##level##_##name = __ACCESS_ATTR_RO(level, name)
41
42#define ACCESS_ATTR_RO(level, attrib) \
43static ssize_t attrib##_access##level##_show(struct device *dev, \
44 struct device_attribute *attr, \
45 char *buf) \
46{ \
47 struct cxl_region *cxlr = to_cxl_region(dev); \
48 \
49 if (cxlr->coord[level].attrib == 0) \
50 return -ENOENT; \
51 \
52 return sysfs_emit(buf, "%u\n", cxlr->coord[level].attrib); \
53} \
54static ACCESS_DEVICE_ATTR_RO(level, attrib)
55
56ACCESS_ATTR_RO(0, read_bandwidth);
57ACCESS_ATTR_RO(0, read_latency);
58ACCESS_ATTR_RO(0, write_bandwidth);
59ACCESS_ATTR_RO(0, write_latency);
60
61#define ACCESS_ATTR_DECLARE(level, attrib) \
62 (&dev_attr_access##level##_##attrib.attr)
63
64static struct attribute *access0_coordinate_attrs[] = {
65 ACCESS_ATTR_DECLARE(0, read_bandwidth),
66 ACCESS_ATTR_DECLARE(0, write_bandwidth),
67 ACCESS_ATTR_DECLARE(0, read_latency),
68 ACCESS_ATTR_DECLARE(0, write_latency),
69 NULL
70};
71
72ACCESS_ATTR_RO(1, read_bandwidth);
73ACCESS_ATTR_RO(1, read_latency);
74ACCESS_ATTR_RO(1, write_bandwidth);
75ACCESS_ATTR_RO(1, write_latency);
76
77static struct attribute *access1_coordinate_attrs[] = {
78 ACCESS_ATTR_DECLARE(1, read_bandwidth),
79 ACCESS_ATTR_DECLARE(1, write_bandwidth),
80 ACCESS_ATTR_DECLARE(1, read_latency),
81 ACCESS_ATTR_DECLARE(1, write_latency),
82 NULL
83};
84
85#define ACCESS_VISIBLE(level) \
86static umode_t cxl_region_access##level##_coordinate_visible( \
87 struct kobject *kobj, struct attribute *a, int n) \
88{ \
89 struct device *dev = kobj_to_dev(kobj); \
90 struct cxl_region *cxlr = to_cxl_region(dev); \
91 \
92 if (a == &dev_attr_access##level##_read_latency.attr && \
93 cxlr->coord[level].read_latency == 0) \
94 return 0; \
95 \
96 if (a == &dev_attr_access##level##_write_latency.attr && \
97 cxlr->coord[level].write_latency == 0) \
98 return 0; \
99 \
100 if (a == &dev_attr_access##level##_read_bandwidth.attr && \
101 cxlr->coord[level].read_bandwidth == 0) \
102 return 0; \
103 \
104 if (a == &dev_attr_access##level##_write_bandwidth.attr && \
105 cxlr->coord[level].write_bandwidth == 0) \
106 return 0; \
107 \
108 return a->mode; \
109}
110
111ACCESS_VISIBLE(0);
112ACCESS_VISIBLE(1);
113
114static const struct attribute_group cxl_region_access0_coordinate_group = {
115 .name = "access0",
116 .attrs = access0_coordinate_attrs,
117 .is_visible = cxl_region_access0_coordinate_visible,
118};
119
120static const struct attribute_group *get_cxl_region_access0_group(void)
121{
122 return &cxl_region_access0_coordinate_group;
123}
124
125static const struct attribute_group cxl_region_access1_coordinate_group = {
126 .name = "access1",
127 .attrs = access1_coordinate_attrs,
128 .is_visible = cxl_region_access1_coordinate_visible,
129};
130
131static const struct attribute_group *get_cxl_region_access1_group(void)
132{
133 return &cxl_region_access1_coordinate_group;
134}
135
136static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
137 char *buf)
138{
139 struct cxl_region *cxlr = to_cxl_region(dev);
140 struct cxl_region_params *p = &cxlr->params;
141 ssize_t rc;
142
143 rc = down_read_interruptible(&cxl_region_rwsem);
144 if (rc)
145 return rc;
146 if (cxlr->mode != CXL_DECODER_PMEM)
147 rc = sysfs_emit(buf, "\n");
148 else
149 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
150 up_read(&cxl_region_rwsem);
151
152 return rc;
153}
154
155static int is_dup(struct device *match, void *data)
156{
157 struct cxl_region_params *p;
158 struct cxl_region *cxlr;
159 uuid_t *uuid = data;
160
161 if (!is_cxl_region(match))
162 return 0;
163
164 lockdep_assert_held(&cxl_region_rwsem);
165 cxlr = to_cxl_region(match);
166 p = &cxlr->params;
167
168 if (uuid_equal(&p->uuid, uuid)) {
169 dev_dbg(match, "already has uuid: %pUb\n", uuid);
170 return -EBUSY;
171 }
172
173 return 0;
174}
175
176static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
177 const char *buf, size_t len)
178{
179 struct cxl_region *cxlr = to_cxl_region(dev);
180 struct cxl_region_params *p = &cxlr->params;
181 uuid_t temp;
182 ssize_t rc;
183
184 if (len != UUID_STRING_LEN + 1)
185 return -EINVAL;
186
187 rc = uuid_parse(buf, &temp);
188 if (rc)
189 return rc;
190
191 if (uuid_is_null(&temp))
192 return -EINVAL;
193
194 rc = down_write_killable(&cxl_region_rwsem);
195 if (rc)
196 return rc;
197
198 if (uuid_equal(&p->uuid, &temp))
199 goto out;
200
201 rc = -EBUSY;
202 if (p->state >= CXL_CONFIG_ACTIVE)
203 goto out;
204
205 rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
206 if (rc < 0)
207 goto out;
208
209 uuid_copy(&p->uuid, &temp);
210out:
211 up_write(&cxl_region_rwsem);
212
213 if (rc)
214 return rc;
215 return len;
216}
217static DEVICE_ATTR_RW(uuid);
218
219static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
220 struct cxl_region *cxlr)
221{
222 return xa_load(&port->regions, (unsigned long)cxlr);
223}
224
225static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
226{
227 if (!cpu_cache_has_invalidate_memregion()) {
228 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
229 dev_info_once(
230 &cxlr->dev,
231 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
232 return 0;
233 } else {
234 dev_err(&cxlr->dev,
235 "Failed to synchronize CPU cache state\n");
236 return -ENXIO;
237 }
238 }
239
240 cpu_cache_invalidate_memregion(IORES_DESC_CXL);
241 return 0;
242}
243
244static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
245{
246 struct cxl_region_params *p = &cxlr->params;
247 int i, rc = 0;
248
249 /*
250 * Before region teardown attempt to flush, and if the flush
251 * fails cancel the region teardown for data consistency
252 * concerns
253 */
254 rc = cxl_region_invalidate_memregion(cxlr);
255 if (rc)
256 return rc;
257
258 for (i = count - 1; i >= 0; i--) {
259 struct cxl_endpoint_decoder *cxled = p->targets[i];
260 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
261 struct cxl_port *iter = cxled_to_port(cxled);
262 struct cxl_dev_state *cxlds = cxlmd->cxlds;
263 struct cxl_ep *ep;
264
265 if (cxlds->rcd)
266 goto endpoint_reset;
267
268 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
269 iter = to_cxl_port(iter->dev.parent);
270
271 for (ep = cxl_ep_load(iter, cxlmd); iter;
272 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
273 struct cxl_region_ref *cxl_rr;
274 struct cxl_decoder *cxld;
275
276 cxl_rr = cxl_rr_load(iter, cxlr);
277 cxld = cxl_rr->decoder;
278 if (cxld->reset)
279 rc = cxld->reset(cxld);
280 if (rc)
281 return rc;
282 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
283 }
284
285endpoint_reset:
286 rc = cxled->cxld.reset(&cxled->cxld);
287 if (rc)
288 return rc;
289 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
290 }
291
292 /* all decoders associated with this region have been torn down */
293 clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
294
295 return 0;
296}
297
298static int commit_decoder(struct cxl_decoder *cxld)
299{
300 struct cxl_switch_decoder *cxlsd = NULL;
301
302 if (cxld->commit)
303 return cxld->commit(cxld);
304
305 if (is_switch_decoder(&cxld->dev))
306 cxlsd = to_cxl_switch_decoder(&cxld->dev);
307
308 if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
309 "->commit() is required\n"))
310 return -ENXIO;
311 return 0;
312}
313
314static int cxl_region_decode_commit(struct cxl_region *cxlr)
315{
316 struct cxl_region_params *p = &cxlr->params;
317 int i, rc = 0;
318
319 for (i = 0; i < p->nr_targets; i++) {
320 struct cxl_endpoint_decoder *cxled = p->targets[i];
321 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
322 struct cxl_region_ref *cxl_rr;
323 struct cxl_decoder *cxld;
324 struct cxl_port *iter;
325 struct cxl_ep *ep;
326
327 /* commit bottom up */
328 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
329 iter = to_cxl_port(iter->dev.parent)) {
330 cxl_rr = cxl_rr_load(iter, cxlr);
331 cxld = cxl_rr->decoder;
332 rc = commit_decoder(cxld);
333 if (rc)
334 break;
335 }
336
337 if (rc) {
338 /* programming @iter failed, teardown */
339 for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
340 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
341 cxl_rr = cxl_rr_load(iter, cxlr);
342 cxld = cxl_rr->decoder;
343 if (cxld->reset)
344 cxld->reset(cxld);
345 }
346
347 cxled->cxld.reset(&cxled->cxld);
348 goto err;
349 }
350 }
351
352 return 0;
353
354err:
355 /* undo the targets that were successfully committed */
356 cxl_region_decode_reset(cxlr, i);
357 return rc;
358}
359
360static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
361 const char *buf, size_t len)
362{
363 struct cxl_region *cxlr = to_cxl_region(dev);
364 struct cxl_region_params *p = &cxlr->params;
365 bool commit;
366 ssize_t rc;
367
368 rc = kstrtobool(buf, &commit);
369 if (rc)
370 return rc;
371
372 rc = down_write_killable(&cxl_region_rwsem);
373 if (rc)
374 return rc;
375
376 /* Already in the requested state? */
377 if (commit && p->state >= CXL_CONFIG_COMMIT)
378 goto out;
379 if (!commit && p->state < CXL_CONFIG_COMMIT)
380 goto out;
381
382 /* Not ready to commit? */
383 if (commit && p->state < CXL_CONFIG_ACTIVE) {
384 rc = -ENXIO;
385 goto out;
386 }
387
388 /*
389 * Invalidate caches before region setup to drop any speculative
390 * consumption of this address space
391 */
392 rc = cxl_region_invalidate_memregion(cxlr);
393 if (rc)
394 goto out;
395
396 if (commit) {
397 rc = cxl_region_decode_commit(cxlr);
398 if (rc == 0)
399 p->state = CXL_CONFIG_COMMIT;
400 } else {
401 p->state = CXL_CONFIG_RESET_PENDING;
402 up_write(&cxl_region_rwsem);
403 device_release_driver(&cxlr->dev);
404 down_write(&cxl_region_rwsem);
405
406 /*
407 * The lock was dropped, so need to revalidate that the reset is
408 * still pending.
409 */
410 if (p->state == CXL_CONFIG_RESET_PENDING) {
411 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
412 /*
413 * Revert to committed since there may still be active
414 * decoders associated with this region, or move forward
415 * to active to mark the reset successful
416 */
417 if (rc)
418 p->state = CXL_CONFIG_COMMIT;
419 else
420 p->state = CXL_CONFIG_ACTIVE;
421 }
422 }
423
424out:
425 up_write(&cxl_region_rwsem);
426
427 if (rc)
428 return rc;
429 return len;
430}
431
432static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
433 char *buf)
434{
435 struct cxl_region *cxlr = to_cxl_region(dev);
436 struct cxl_region_params *p = &cxlr->params;
437 ssize_t rc;
438
439 rc = down_read_interruptible(&cxl_region_rwsem);
440 if (rc)
441 return rc;
442 rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
443 up_read(&cxl_region_rwsem);
444
445 return rc;
446}
447static DEVICE_ATTR_RW(commit);
448
449static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
450 int n)
451{
452 struct device *dev = kobj_to_dev(kobj);
453 struct cxl_region *cxlr = to_cxl_region(dev);
454
455 /*
456 * Support tooling that expects to find a 'uuid' attribute for all
457 * regions regardless of mode.
458 */
459 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
460 return 0444;
461 return a->mode;
462}
463
464static ssize_t interleave_ways_show(struct device *dev,
465 struct device_attribute *attr, char *buf)
466{
467 struct cxl_region *cxlr = to_cxl_region(dev);
468 struct cxl_region_params *p = &cxlr->params;
469 ssize_t rc;
470
471 rc = down_read_interruptible(&cxl_region_rwsem);
472 if (rc)
473 return rc;
474 rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
475 up_read(&cxl_region_rwsem);
476
477 return rc;
478}
479
480static const struct attribute_group *get_cxl_region_target_group(void);
481
482static ssize_t interleave_ways_store(struct device *dev,
483 struct device_attribute *attr,
484 const char *buf, size_t len)
485{
486 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
487 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
488 struct cxl_region *cxlr = to_cxl_region(dev);
489 struct cxl_region_params *p = &cxlr->params;
490 unsigned int val, save;
491 int rc;
492 u8 iw;
493
494 rc = kstrtouint(buf, 0, &val);
495 if (rc)
496 return rc;
497
498 rc = ways_to_eiw(val, &iw);
499 if (rc)
500 return rc;
501
502 /*
503 * Even for x3, x6, and x12 interleaves the region interleave must be a
504 * power of 2 multiple of the host bridge interleave.
505 */
506 if (!is_power_of_2(val / cxld->interleave_ways) ||
507 (val % cxld->interleave_ways)) {
508 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
509 return -EINVAL;
510 }
511
512 rc = down_write_killable(&cxl_region_rwsem);
513 if (rc)
514 return rc;
515 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
516 rc = -EBUSY;
517 goto out;
518 }
519
520 save = p->interleave_ways;
521 p->interleave_ways = val;
522 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
523 if (rc)
524 p->interleave_ways = save;
525out:
526 up_write(&cxl_region_rwsem);
527 if (rc)
528 return rc;
529 return len;
530}
531static DEVICE_ATTR_RW(interleave_ways);
532
533static ssize_t interleave_granularity_show(struct device *dev,
534 struct device_attribute *attr,
535 char *buf)
536{
537 struct cxl_region *cxlr = to_cxl_region(dev);
538 struct cxl_region_params *p = &cxlr->params;
539 ssize_t rc;
540
541 rc = down_read_interruptible(&cxl_region_rwsem);
542 if (rc)
543 return rc;
544 rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
545 up_read(&cxl_region_rwsem);
546
547 return rc;
548}
549
550static ssize_t interleave_granularity_store(struct device *dev,
551 struct device_attribute *attr,
552 const char *buf, size_t len)
553{
554 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
555 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
556 struct cxl_region *cxlr = to_cxl_region(dev);
557 struct cxl_region_params *p = &cxlr->params;
558 int rc, val;
559 u16 ig;
560
561 rc = kstrtoint(buf, 0, &val);
562 if (rc)
563 return rc;
564
565 rc = granularity_to_eig(val, &ig);
566 if (rc)
567 return rc;
568
569 /*
570 * When the host-bridge is interleaved, disallow region granularity !=
571 * root granularity. Regions with a granularity less than the root
572 * interleave result in needing multiple endpoints to support a single
573 * slot in the interleave (possible to support in the future). Regions
574 * with a granularity greater than the root interleave result in invalid
575 * DPA translations (invalid to support).
576 */
577 if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
578 return -EINVAL;
579
580 rc = down_write_killable(&cxl_region_rwsem);
581 if (rc)
582 return rc;
583 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
584 rc = -EBUSY;
585 goto out;
586 }
587
588 p->interleave_granularity = val;
589out:
590 up_write(&cxl_region_rwsem);
591 if (rc)
592 return rc;
593 return len;
594}
595static DEVICE_ATTR_RW(interleave_granularity);
596
597static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
598 char *buf)
599{
600 struct cxl_region *cxlr = to_cxl_region(dev);
601 struct cxl_region_params *p = &cxlr->params;
602 u64 resource = -1ULL;
603 ssize_t rc;
604
605 rc = down_read_interruptible(&cxl_region_rwsem);
606 if (rc)
607 return rc;
608 if (p->res)
609 resource = p->res->start;
610 rc = sysfs_emit(buf, "%#llx\n", resource);
611 up_read(&cxl_region_rwsem);
612
613 return rc;
614}
615static DEVICE_ATTR_RO(resource);
616
617static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
618 char *buf)
619{
620 struct cxl_region *cxlr = to_cxl_region(dev);
621
622 return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
623}
624static DEVICE_ATTR_RO(mode);
625
626static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
627{
628 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
629 struct cxl_region_params *p = &cxlr->params;
630 struct resource *res;
631 u64 remainder = 0;
632
633 lockdep_assert_held_write(&cxl_region_rwsem);
634
635 /* Nothing to do... */
636 if (p->res && resource_size(p->res) == size)
637 return 0;
638
639 /* To change size the old size must be freed first */
640 if (p->res)
641 return -EBUSY;
642
643 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
644 return -EBUSY;
645
646 /* ways, granularity and uuid (if PMEM) need to be set before HPA */
647 if (!p->interleave_ways || !p->interleave_granularity ||
648 (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
649 return -ENXIO;
650
651 div64_u64_rem(size, (u64)SZ_256M * p->interleave_ways, &remainder);
652 if (remainder)
653 return -EINVAL;
654
655 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
656 dev_name(&cxlr->dev));
657 if (IS_ERR(res)) {
658 dev_dbg(&cxlr->dev,
659 "HPA allocation error (%ld) for size:%pap in %s %pr\n",
660 PTR_ERR(res), &size, cxlrd->res->name, cxlrd->res);
661 return PTR_ERR(res);
662 }
663
664 p->res = res;
665 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
666
667 return 0;
668}
669
670static void cxl_region_iomem_release(struct cxl_region *cxlr)
671{
672 struct cxl_region_params *p = &cxlr->params;
673
674 if (device_is_registered(&cxlr->dev))
675 lockdep_assert_held_write(&cxl_region_rwsem);
676 if (p->res) {
677 /*
678 * Autodiscovered regions may not have been able to insert their
679 * resource.
680 */
681 if (p->res->parent)
682 remove_resource(p->res);
683 kfree(p->res);
684 p->res = NULL;
685 }
686}
687
688static int free_hpa(struct cxl_region *cxlr)
689{
690 struct cxl_region_params *p = &cxlr->params;
691
692 lockdep_assert_held_write(&cxl_region_rwsem);
693
694 if (!p->res)
695 return 0;
696
697 if (p->state >= CXL_CONFIG_ACTIVE)
698 return -EBUSY;
699
700 cxl_region_iomem_release(cxlr);
701 p->state = CXL_CONFIG_IDLE;
702 return 0;
703}
704
705static ssize_t size_store(struct device *dev, struct device_attribute *attr,
706 const char *buf, size_t len)
707{
708 struct cxl_region *cxlr = to_cxl_region(dev);
709 u64 val;
710 int rc;
711
712 rc = kstrtou64(buf, 0, &val);
713 if (rc)
714 return rc;
715
716 rc = down_write_killable(&cxl_region_rwsem);
717 if (rc)
718 return rc;
719
720 if (val)
721 rc = alloc_hpa(cxlr, val);
722 else
723 rc = free_hpa(cxlr);
724 up_write(&cxl_region_rwsem);
725
726 if (rc)
727 return rc;
728
729 return len;
730}
731
732static ssize_t size_show(struct device *dev, struct device_attribute *attr,
733 char *buf)
734{
735 struct cxl_region *cxlr = to_cxl_region(dev);
736 struct cxl_region_params *p = &cxlr->params;
737 u64 size = 0;
738 ssize_t rc;
739
740 rc = down_read_interruptible(&cxl_region_rwsem);
741 if (rc)
742 return rc;
743 if (p->res)
744 size = resource_size(p->res);
745 rc = sysfs_emit(buf, "%#llx\n", size);
746 up_read(&cxl_region_rwsem);
747
748 return rc;
749}
750static DEVICE_ATTR_RW(size);
751
752static struct attribute *cxl_region_attrs[] = {
753 &dev_attr_uuid.attr,
754 &dev_attr_commit.attr,
755 &dev_attr_interleave_ways.attr,
756 &dev_attr_interleave_granularity.attr,
757 &dev_attr_resource.attr,
758 &dev_attr_size.attr,
759 &dev_attr_mode.attr,
760 NULL,
761};
762
763static const struct attribute_group cxl_region_group = {
764 .attrs = cxl_region_attrs,
765 .is_visible = cxl_region_visible,
766};
767
768static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
769{
770 struct cxl_region_params *p = &cxlr->params;
771 struct cxl_endpoint_decoder *cxled;
772 int rc;
773
774 rc = down_read_interruptible(&cxl_region_rwsem);
775 if (rc)
776 return rc;
777
778 if (pos >= p->interleave_ways) {
779 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
780 p->interleave_ways);
781 rc = -ENXIO;
782 goto out;
783 }
784
785 cxled = p->targets[pos];
786 if (!cxled)
787 rc = sysfs_emit(buf, "\n");
788 else
789 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
790out:
791 up_read(&cxl_region_rwsem);
792
793 return rc;
794}
795
796static int match_free_decoder(struct device *dev, void *data)
797{
798 struct cxl_decoder *cxld;
799 int *id = data;
800
801 if (!is_switch_decoder(dev))
802 return 0;
803
804 cxld = to_cxl_decoder(dev);
805
806 /* enforce ordered allocation */
807 if (cxld->id != *id)
808 return 0;
809
810 if (!cxld->region)
811 return 1;
812
813 (*id)++;
814
815 return 0;
816}
817
818static int match_auto_decoder(struct device *dev, void *data)
819{
820 struct cxl_region_params *p = data;
821 struct cxl_decoder *cxld;
822 struct range *r;
823
824 if (!is_switch_decoder(dev))
825 return 0;
826
827 cxld = to_cxl_decoder(dev);
828 r = &cxld->hpa_range;
829
830 if (p->res && p->res->start == r->start && p->res->end == r->end)
831 return 1;
832
833 return 0;
834}
835
836static struct cxl_decoder *
837cxl_region_find_decoder(struct cxl_port *port,
838 struct cxl_endpoint_decoder *cxled,
839 struct cxl_region *cxlr)
840{
841 struct device *dev;
842 int id = 0;
843
844 if (port == cxled_to_port(cxled))
845 return &cxled->cxld;
846
847 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
848 dev = device_find_child(&port->dev, &cxlr->params,
849 match_auto_decoder);
850 else
851 dev = device_find_child(&port->dev, &id, match_free_decoder);
852 if (!dev)
853 return NULL;
854 /*
855 * This decoder is pinned registered as long as the endpoint decoder is
856 * registered, and endpoint decoder unregistration holds the
857 * cxl_region_rwsem over unregister events, so no need to hold on to
858 * this extra reference.
859 */
860 put_device(dev);
861 return to_cxl_decoder(dev);
862}
863
864static bool auto_order_ok(struct cxl_port *port, struct cxl_region *cxlr_iter,
865 struct cxl_decoder *cxld)
866{
867 struct cxl_region_ref *rr = cxl_rr_load(port, cxlr_iter);
868 struct cxl_decoder *cxld_iter = rr->decoder;
869
870 /*
871 * Allow the out of order assembly of auto-discovered regions.
872 * Per CXL Spec 3.1 8.2.4.20.12 software must commit decoders
873 * in HPA order. Confirm that the decoder with the lesser HPA
874 * starting address has the lesser id.
875 */
876 dev_dbg(&cxld->dev, "check for HPA violation %s:%d < %s:%d\n",
877 dev_name(&cxld->dev), cxld->id,
878 dev_name(&cxld_iter->dev), cxld_iter->id);
879
880 if (cxld_iter->id > cxld->id)
881 return true;
882
883 return false;
884}
885
886static struct cxl_region_ref *
887alloc_region_ref(struct cxl_port *port, struct cxl_region *cxlr,
888 struct cxl_endpoint_decoder *cxled)
889{
890 struct cxl_region_params *p = &cxlr->params;
891 struct cxl_region_ref *cxl_rr, *iter;
892 unsigned long index;
893 int rc;
894
895 xa_for_each(&port->regions, index, iter) {
896 struct cxl_region_params *ip = &iter->region->params;
897
898 if (!ip->res || ip->res->start < p->res->start)
899 continue;
900
901 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
902 struct cxl_decoder *cxld;
903
904 cxld = cxl_region_find_decoder(port, cxled, cxlr);
905 if (auto_order_ok(port, iter->region, cxld))
906 continue;
907 }
908 dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n",
909 dev_name(&port->dev),
910 dev_name(&iter->region->dev), ip->res, p->res);
911
912 return ERR_PTR(-EBUSY);
913 }
914
915 cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
916 if (!cxl_rr)
917 return ERR_PTR(-ENOMEM);
918 cxl_rr->port = port;
919 cxl_rr->region = cxlr;
920 cxl_rr->nr_targets = 1;
921 xa_init(&cxl_rr->endpoints);
922
923 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
924 if (rc) {
925 dev_dbg(&cxlr->dev,
926 "%s: failed to track region reference: %d\n",
927 dev_name(&port->dev), rc);
928 kfree(cxl_rr);
929 return ERR_PTR(rc);
930 }
931
932 return cxl_rr;
933}
934
935static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
936{
937 struct cxl_region *cxlr = cxl_rr->region;
938 struct cxl_decoder *cxld = cxl_rr->decoder;
939
940 if (!cxld)
941 return;
942
943 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
944 if (cxld->region == cxlr) {
945 cxld->region = NULL;
946 put_device(&cxlr->dev);
947 }
948}
949
950static void free_region_ref(struct cxl_region_ref *cxl_rr)
951{
952 struct cxl_port *port = cxl_rr->port;
953 struct cxl_region *cxlr = cxl_rr->region;
954
955 cxl_rr_free_decoder(cxl_rr);
956 xa_erase(&port->regions, (unsigned long)cxlr);
957 xa_destroy(&cxl_rr->endpoints);
958 kfree(cxl_rr);
959}
960
961static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
962 struct cxl_endpoint_decoder *cxled)
963{
964 int rc;
965 struct cxl_port *port = cxl_rr->port;
966 struct cxl_region *cxlr = cxl_rr->region;
967 struct cxl_decoder *cxld = cxl_rr->decoder;
968 struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
969
970 if (ep) {
971 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
972 GFP_KERNEL);
973 if (rc)
974 return rc;
975 }
976 cxl_rr->nr_eps++;
977
978 if (!cxld->region) {
979 cxld->region = cxlr;
980 get_device(&cxlr->dev);
981 }
982
983 return 0;
984}
985
986static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
987 struct cxl_endpoint_decoder *cxled,
988 struct cxl_region_ref *cxl_rr)
989{
990 struct cxl_decoder *cxld;
991
992 cxld = cxl_region_find_decoder(port, cxled, cxlr);
993 if (!cxld) {
994 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
995 dev_name(&port->dev));
996 return -EBUSY;
997 }
998
999 if (cxld->region) {
1000 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
1001 dev_name(&port->dev), dev_name(&cxld->dev),
1002 dev_name(&cxld->region->dev));
1003 return -EBUSY;
1004 }
1005
1006 /*
1007 * Endpoints should already match the region type, but backstop that
1008 * assumption with an assertion. Switch-decoders change mapping-type
1009 * based on what is mapped when they are assigned to a region.
1010 */
1011 dev_WARN_ONCE(&cxlr->dev,
1012 port == cxled_to_port(cxled) &&
1013 cxld->target_type != cxlr->type,
1014 "%s:%s mismatch decoder type %d -> %d\n",
1015 dev_name(&cxled_to_memdev(cxled)->dev),
1016 dev_name(&cxld->dev), cxld->target_type, cxlr->type);
1017 cxld->target_type = cxlr->type;
1018 cxl_rr->decoder = cxld;
1019 return 0;
1020}
1021
1022/**
1023 * cxl_port_attach_region() - track a region's interest in a port by endpoint
1024 * @port: port to add a new region reference 'struct cxl_region_ref'
1025 * @cxlr: region to attach to @port
1026 * @cxled: endpoint decoder used to create or further pin a region reference
1027 * @pos: interleave position of @cxled in @cxlr
1028 *
1029 * The attach event is an opportunity to validate CXL decode setup
1030 * constraints and record metadata needed for programming HDM decoders,
1031 * in particular decoder target lists.
1032 *
1033 * The steps are:
1034 *
1035 * - validate that there are no other regions with a higher HPA already
1036 * associated with @port
1037 * - establish a region reference if one is not already present
1038 *
1039 * - additionally allocate a decoder instance that will host @cxlr on
1040 * @port
1041 *
1042 * - pin the region reference by the endpoint
1043 * - account for how many entries in @port's target list are needed to
1044 * cover all of the added endpoints.
1045 */
1046static int cxl_port_attach_region(struct cxl_port *port,
1047 struct cxl_region *cxlr,
1048 struct cxl_endpoint_decoder *cxled, int pos)
1049{
1050 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1051 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1052 struct cxl_region_ref *cxl_rr;
1053 bool nr_targets_inc = false;
1054 struct cxl_decoder *cxld;
1055 unsigned long index;
1056 int rc = -EBUSY;
1057
1058 lockdep_assert_held_write(&cxl_region_rwsem);
1059
1060 cxl_rr = cxl_rr_load(port, cxlr);
1061 if (cxl_rr) {
1062 struct cxl_ep *ep_iter;
1063 int found = 0;
1064
1065 /*
1066 * Walk the existing endpoints that have been attached to
1067 * @cxlr at @port and see if they share the same 'next' port
1068 * in the downstream direction. I.e. endpoints that share common
1069 * upstream switch.
1070 */
1071 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1072 if (ep_iter == ep)
1073 continue;
1074 if (ep_iter->next == ep->next) {
1075 found++;
1076 break;
1077 }
1078 }
1079
1080 /*
1081 * New target port, or @port is an endpoint port that always
1082 * accounts its own local decode as a target.
1083 */
1084 if (!found || !ep->next) {
1085 cxl_rr->nr_targets++;
1086 nr_targets_inc = true;
1087 }
1088 } else {
1089 cxl_rr = alloc_region_ref(port, cxlr, cxled);
1090 if (IS_ERR(cxl_rr)) {
1091 dev_dbg(&cxlr->dev,
1092 "%s: failed to allocate region reference\n",
1093 dev_name(&port->dev));
1094 return PTR_ERR(cxl_rr);
1095 }
1096 nr_targets_inc = true;
1097
1098 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
1099 if (rc)
1100 goto out_erase;
1101 }
1102 cxld = cxl_rr->decoder;
1103
1104 rc = cxl_rr_ep_add(cxl_rr, cxled);
1105 if (rc) {
1106 dev_dbg(&cxlr->dev,
1107 "%s: failed to track endpoint %s:%s reference\n",
1108 dev_name(&port->dev), dev_name(&cxlmd->dev),
1109 dev_name(&cxld->dev));
1110 goto out_erase;
1111 }
1112
1113 dev_dbg(&cxlr->dev,
1114 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
1115 dev_name(port->uport_dev), dev_name(&port->dev),
1116 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1117 dev_name(&cxled->cxld.dev), pos,
1118 ep ? ep->next ? dev_name(ep->next->uport_dev) :
1119 dev_name(&cxlmd->dev) :
1120 "none",
1121 cxl_rr->nr_eps, cxl_rr->nr_targets);
1122
1123 return 0;
1124out_erase:
1125 if (nr_targets_inc)
1126 cxl_rr->nr_targets--;
1127 if (cxl_rr->nr_eps == 0)
1128 free_region_ref(cxl_rr);
1129 return rc;
1130}
1131
1132static void cxl_port_detach_region(struct cxl_port *port,
1133 struct cxl_region *cxlr,
1134 struct cxl_endpoint_decoder *cxled)
1135{
1136 struct cxl_region_ref *cxl_rr;
1137 struct cxl_ep *ep = NULL;
1138
1139 lockdep_assert_held_write(&cxl_region_rwsem);
1140
1141 cxl_rr = cxl_rr_load(port, cxlr);
1142 if (!cxl_rr)
1143 return;
1144
1145 /*
1146 * Endpoint ports do not carry cxl_ep references, and they
1147 * never target more than one endpoint by definition
1148 */
1149 if (cxl_rr->decoder == &cxled->cxld)
1150 cxl_rr->nr_eps--;
1151 else
1152 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
1153 if (ep) {
1154 struct cxl_ep *ep_iter;
1155 unsigned long index;
1156 int found = 0;
1157
1158 cxl_rr->nr_eps--;
1159 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1160 if (ep_iter->next == ep->next) {
1161 found++;
1162 break;
1163 }
1164 }
1165 if (!found)
1166 cxl_rr->nr_targets--;
1167 }
1168
1169 if (cxl_rr->nr_eps == 0)
1170 free_region_ref(cxl_rr);
1171}
1172
1173static int check_last_peer(struct cxl_endpoint_decoder *cxled,
1174 struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
1175 int distance)
1176{
1177 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1178 struct cxl_region *cxlr = cxl_rr->region;
1179 struct cxl_region_params *p = &cxlr->params;
1180 struct cxl_endpoint_decoder *cxled_peer;
1181 struct cxl_port *port = cxl_rr->port;
1182 struct cxl_memdev *cxlmd_peer;
1183 struct cxl_ep *ep_peer;
1184 int pos = cxled->pos;
1185
1186 /*
1187 * If this position wants to share a dport with the last endpoint mapped
1188 * then that endpoint, at index 'position - distance', must also be
1189 * mapped by this dport.
1190 */
1191 if (pos < distance) {
1192 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
1193 dev_name(port->uport_dev), dev_name(&port->dev),
1194 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1195 return -ENXIO;
1196 }
1197 cxled_peer = p->targets[pos - distance];
1198 cxlmd_peer = cxled_to_memdev(cxled_peer);
1199 ep_peer = cxl_ep_load(port, cxlmd_peer);
1200 if (ep->dport != ep_peer->dport) {
1201 dev_dbg(&cxlr->dev,
1202 "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
1203 dev_name(port->uport_dev), dev_name(&port->dev),
1204 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
1205 dev_name(&cxlmd_peer->dev),
1206 dev_name(&cxled_peer->cxld.dev));
1207 return -ENXIO;
1208 }
1209
1210 return 0;
1211}
1212
1213static int cxl_port_setup_targets(struct cxl_port *port,
1214 struct cxl_region *cxlr,
1215 struct cxl_endpoint_decoder *cxled)
1216{
1217 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1218 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1219 struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1220 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1221 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1222 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1223 struct cxl_region_params *p = &cxlr->params;
1224 struct cxl_decoder *cxld = cxl_rr->decoder;
1225 struct cxl_switch_decoder *cxlsd;
1226 u16 eig, peig;
1227 u8 eiw, peiw;
1228
1229 /*
1230 * While root level decoders support x3, x6, x12, switch level
1231 * decoders only support powers of 2 up to x16.
1232 */
1233 if (!is_power_of_2(cxl_rr->nr_targets)) {
1234 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1235 dev_name(port->uport_dev), dev_name(&port->dev),
1236 cxl_rr->nr_targets);
1237 return -EINVAL;
1238 }
1239
1240 cxlsd = to_cxl_switch_decoder(&cxld->dev);
1241 if (cxl_rr->nr_targets_set) {
1242 int i, distance;
1243
1244 /*
1245 * Passthrough decoders impose no distance requirements between
1246 * peers
1247 */
1248 if (cxl_rr->nr_targets == 1)
1249 distance = 0;
1250 else
1251 distance = p->nr_targets / cxl_rr->nr_targets;
1252 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1253 if (ep->dport == cxlsd->target[i]) {
1254 rc = check_last_peer(cxled, ep, cxl_rr,
1255 distance);
1256 if (rc)
1257 return rc;
1258 goto out_target_set;
1259 }
1260 goto add_target;
1261 }
1262
1263 if (is_cxl_root(parent_port)) {
1264 /*
1265 * Root decoder IG is always set to value in CFMWS which
1266 * may be different than this region's IG. We can use the
1267 * region's IG here since interleave_granularity_store()
1268 * does not allow interleaved host-bridges with
1269 * root IG != region IG.
1270 */
1271 parent_ig = p->interleave_granularity;
1272 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1273 /*
1274 * For purposes of address bit routing, use power-of-2 math for
1275 * switch ports.
1276 */
1277 if (!is_power_of_2(parent_iw))
1278 parent_iw /= 3;
1279 } else {
1280 struct cxl_region_ref *parent_rr;
1281 struct cxl_decoder *parent_cxld;
1282
1283 parent_rr = cxl_rr_load(parent_port, cxlr);
1284 parent_cxld = parent_rr->decoder;
1285 parent_ig = parent_cxld->interleave_granularity;
1286 parent_iw = parent_cxld->interleave_ways;
1287 }
1288
1289 rc = granularity_to_eig(parent_ig, &peig);
1290 if (rc) {
1291 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1292 dev_name(parent_port->uport_dev),
1293 dev_name(&parent_port->dev), parent_ig);
1294 return rc;
1295 }
1296
1297 rc = ways_to_eiw(parent_iw, &peiw);
1298 if (rc) {
1299 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1300 dev_name(parent_port->uport_dev),
1301 dev_name(&parent_port->dev), parent_iw);
1302 return rc;
1303 }
1304
1305 iw = cxl_rr->nr_targets;
1306 rc = ways_to_eiw(iw, &eiw);
1307 if (rc) {
1308 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1309 dev_name(port->uport_dev), dev_name(&port->dev), iw);
1310 return rc;
1311 }
1312
1313 /*
1314 * Interleave granularity is a multiple of @parent_port granularity.
1315 * Multiplier is the parent port interleave ways.
1316 */
1317 rc = granularity_to_eig(parent_ig * parent_iw, &eig);
1318 if (rc) {
1319 dev_dbg(&cxlr->dev,
1320 "%s: invalid granularity calculation (%d * %d)\n",
1321 dev_name(&parent_port->dev), parent_ig, parent_iw);
1322 return rc;
1323 }
1324
1325 rc = eig_to_granularity(eig, &ig);
1326 if (rc) {
1327 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1328 dev_name(port->uport_dev), dev_name(&port->dev),
1329 256 << eig);
1330 return rc;
1331 }
1332
1333 if (iw > 8 || iw > cxlsd->nr_targets) {
1334 dev_dbg(&cxlr->dev,
1335 "%s:%s:%s: ways: %d overflows targets: %d\n",
1336 dev_name(port->uport_dev), dev_name(&port->dev),
1337 dev_name(&cxld->dev), iw, cxlsd->nr_targets);
1338 return -ENXIO;
1339 }
1340
1341 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1342 if (cxld->interleave_ways != iw ||
1343 cxld->interleave_granularity != ig ||
1344 cxld->hpa_range.start != p->res->start ||
1345 cxld->hpa_range.end != p->res->end ||
1346 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1347 dev_err(&cxlr->dev,
1348 "%s:%s %s expected iw: %d ig: %d %pr\n",
1349 dev_name(port->uport_dev), dev_name(&port->dev),
1350 __func__, iw, ig, p->res);
1351 dev_err(&cxlr->dev,
1352 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1353 dev_name(port->uport_dev), dev_name(&port->dev),
1354 __func__, cxld->interleave_ways,
1355 cxld->interleave_granularity,
1356 (cxld->flags & CXL_DECODER_F_ENABLE) ?
1357 "enabled" :
1358 "disabled",
1359 cxld->hpa_range.start, cxld->hpa_range.end);
1360 return -ENXIO;
1361 }
1362 } else {
1363 cxld->interleave_ways = iw;
1364 cxld->interleave_granularity = ig;
1365 cxld->hpa_range = (struct range) {
1366 .start = p->res->start,
1367 .end = p->res->end,
1368 };
1369 }
1370 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
1371 dev_name(&port->dev), iw, ig);
1372add_target:
1373 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1374 dev_dbg(&cxlr->dev,
1375 "%s:%s: targets full trying to add %s:%s at %d\n",
1376 dev_name(port->uport_dev), dev_name(&port->dev),
1377 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1378 return -ENXIO;
1379 }
1380 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1381 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1382 dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1383 dev_name(port->uport_dev), dev_name(&port->dev),
1384 dev_name(&cxlsd->cxld.dev),
1385 dev_name(ep->dport->dport_dev),
1386 cxl_rr->nr_targets_set);
1387 return -ENXIO;
1388 }
1389 } else
1390 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1391 inc = 1;
1392out_target_set:
1393 cxl_rr->nr_targets_set += inc;
1394 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1395 dev_name(port->uport_dev), dev_name(&port->dev),
1396 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
1397 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1398
1399 return 0;
1400}
1401
1402static void cxl_port_reset_targets(struct cxl_port *port,
1403 struct cxl_region *cxlr)
1404{
1405 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1406 struct cxl_decoder *cxld;
1407
1408 /*
1409 * After the last endpoint has been detached the entire cxl_rr may now
1410 * be gone.
1411 */
1412 if (!cxl_rr)
1413 return;
1414 cxl_rr->nr_targets_set = 0;
1415
1416 cxld = cxl_rr->decoder;
1417 cxld->hpa_range = (struct range) {
1418 .start = 0,
1419 .end = -1,
1420 };
1421}
1422
1423static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1424{
1425 struct cxl_region_params *p = &cxlr->params;
1426 struct cxl_endpoint_decoder *cxled;
1427 struct cxl_dev_state *cxlds;
1428 struct cxl_memdev *cxlmd;
1429 struct cxl_port *iter;
1430 struct cxl_ep *ep;
1431 int i;
1432
1433 /*
1434 * In the auto-discovery case skip automatic teardown since the
1435 * address space is already active
1436 */
1437 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1438 return;
1439
1440 for (i = 0; i < p->nr_targets; i++) {
1441 cxled = p->targets[i];
1442 cxlmd = cxled_to_memdev(cxled);
1443 cxlds = cxlmd->cxlds;
1444
1445 if (cxlds->rcd)
1446 continue;
1447
1448 iter = cxled_to_port(cxled);
1449 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1450 iter = to_cxl_port(iter->dev.parent);
1451
1452 for (ep = cxl_ep_load(iter, cxlmd); iter;
1453 iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1454 cxl_port_reset_targets(iter, cxlr);
1455 }
1456}
1457
1458static int cxl_region_setup_targets(struct cxl_region *cxlr)
1459{
1460 struct cxl_region_params *p = &cxlr->params;
1461 struct cxl_endpoint_decoder *cxled;
1462 struct cxl_dev_state *cxlds;
1463 int i, rc, rch = 0, vh = 0;
1464 struct cxl_memdev *cxlmd;
1465 struct cxl_port *iter;
1466 struct cxl_ep *ep;
1467
1468 for (i = 0; i < p->nr_targets; i++) {
1469 cxled = p->targets[i];
1470 cxlmd = cxled_to_memdev(cxled);
1471 cxlds = cxlmd->cxlds;
1472
1473 /* validate that all targets agree on topology */
1474 if (!cxlds->rcd) {
1475 vh++;
1476 } else {
1477 rch++;
1478 continue;
1479 }
1480
1481 iter = cxled_to_port(cxled);
1482 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1483 iter = to_cxl_port(iter->dev.parent);
1484
1485 /*
1486 * Descend the topology tree programming / validating
1487 * targets while looking for conflicts.
1488 */
1489 for (ep = cxl_ep_load(iter, cxlmd); iter;
1490 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1491 rc = cxl_port_setup_targets(iter, cxlr, cxled);
1492 if (rc) {
1493 cxl_region_teardown_targets(cxlr);
1494 return rc;
1495 }
1496 }
1497 }
1498
1499 if (rch && vh) {
1500 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1501 cxl_region_teardown_targets(cxlr);
1502 return -ENXIO;
1503 }
1504
1505 return 0;
1506}
1507
1508static int cxl_region_validate_position(struct cxl_region *cxlr,
1509 struct cxl_endpoint_decoder *cxled,
1510 int pos)
1511{
1512 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1513 struct cxl_region_params *p = &cxlr->params;
1514 int i;
1515
1516 if (pos < 0 || pos >= p->interleave_ways) {
1517 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1518 p->interleave_ways);
1519 return -ENXIO;
1520 }
1521
1522 if (p->targets[pos] == cxled)
1523 return 0;
1524
1525 if (p->targets[pos]) {
1526 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1527 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1528
1529 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1530 pos, dev_name(&cxlmd_target->dev),
1531 dev_name(&cxled_target->cxld.dev));
1532 return -EBUSY;
1533 }
1534
1535 for (i = 0; i < p->interleave_ways; i++) {
1536 struct cxl_endpoint_decoder *cxled_target;
1537 struct cxl_memdev *cxlmd_target;
1538
1539 cxled_target = p->targets[i];
1540 if (!cxled_target)
1541 continue;
1542
1543 cxlmd_target = cxled_to_memdev(cxled_target);
1544 if (cxlmd_target == cxlmd) {
1545 dev_dbg(&cxlr->dev,
1546 "%s already specified at position %d via: %s\n",
1547 dev_name(&cxlmd->dev), pos,
1548 dev_name(&cxled_target->cxld.dev));
1549 return -EBUSY;
1550 }
1551 }
1552
1553 return 0;
1554}
1555
1556static int cxl_region_attach_position(struct cxl_region *cxlr,
1557 struct cxl_root_decoder *cxlrd,
1558 struct cxl_endpoint_decoder *cxled,
1559 const struct cxl_dport *dport, int pos)
1560{
1561 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1562 struct cxl_port *iter;
1563 int rc;
1564
1565 if (cxlrd->calc_hb(cxlrd, pos) != dport) {
1566 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1567 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1568 dev_name(&cxlrd->cxlsd.cxld.dev));
1569 return -ENXIO;
1570 }
1571
1572 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1573 iter = to_cxl_port(iter->dev.parent)) {
1574 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1575 if (rc)
1576 goto err;
1577 }
1578
1579 return 0;
1580
1581err:
1582 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1583 iter = to_cxl_port(iter->dev.parent))
1584 cxl_port_detach_region(iter, cxlr, cxled);
1585 return rc;
1586}
1587
1588static int cxl_region_attach_auto(struct cxl_region *cxlr,
1589 struct cxl_endpoint_decoder *cxled, int pos)
1590{
1591 struct cxl_region_params *p = &cxlr->params;
1592
1593 if (cxled->state != CXL_DECODER_STATE_AUTO) {
1594 dev_err(&cxlr->dev,
1595 "%s: unable to add decoder to autodetected region\n",
1596 dev_name(&cxled->cxld.dev));
1597 return -EINVAL;
1598 }
1599
1600 if (pos >= 0) {
1601 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1602 dev_name(&cxled->cxld.dev), pos);
1603 return -EINVAL;
1604 }
1605
1606 if (p->nr_targets >= p->interleave_ways) {
1607 dev_err(&cxlr->dev, "%s: no more target slots available\n",
1608 dev_name(&cxled->cxld.dev));
1609 return -ENXIO;
1610 }
1611
1612 /*
1613 * Temporarily record the endpoint decoder into the target array. Yes,
1614 * this means that userspace can view devices in the wrong position
1615 * before the region activates, and must be careful to understand when
1616 * it might be racing region autodiscovery.
1617 */
1618 pos = p->nr_targets;
1619 p->targets[pos] = cxled;
1620 cxled->pos = pos;
1621 p->nr_targets++;
1622
1623 return 0;
1624}
1625
1626static int cmp_interleave_pos(const void *a, const void *b)
1627{
1628 struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1629 struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1630
1631 return cxled_a->pos - cxled_b->pos;
1632}
1633
1634static struct cxl_port *next_port(struct cxl_port *port)
1635{
1636 if (!port->parent_dport)
1637 return NULL;
1638 return port->parent_dport->port;
1639}
1640
1641static int match_switch_decoder_by_range(struct device *dev, void *data)
1642{
1643 struct cxl_switch_decoder *cxlsd;
1644 struct range *r1, *r2 = data;
1645
1646 if (!is_switch_decoder(dev))
1647 return 0;
1648
1649 cxlsd = to_cxl_switch_decoder(dev);
1650 r1 = &cxlsd->cxld.hpa_range;
1651
1652 if (is_root_decoder(dev))
1653 return range_contains(r1, r2);
1654 return (r1->start == r2->start && r1->end == r2->end);
1655}
1656
1657static int find_pos_and_ways(struct cxl_port *port, struct range *range,
1658 int *pos, int *ways)
1659{
1660 struct cxl_switch_decoder *cxlsd;
1661 struct cxl_port *parent;
1662 struct device *dev;
1663 int rc = -ENXIO;
1664
1665 parent = next_port(port);
1666 if (!parent)
1667 return rc;
1668
1669 dev = device_find_child(&parent->dev, range,
1670 match_switch_decoder_by_range);
1671 if (!dev) {
1672 dev_err(port->uport_dev,
1673 "failed to find decoder mapping %#llx-%#llx\n",
1674 range->start, range->end);
1675 return rc;
1676 }
1677 cxlsd = to_cxl_switch_decoder(dev);
1678 *ways = cxlsd->cxld.interleave_ways;
1679
1680 for (int i = 0; i < *ways; i++) {
1681 if (cxlsd->target[i] == port->parent_dport) {
1682 *pos = i;
1683 rc = 0;
1684 break;
1685 }
1686 }
1687 put_device(dev);
1688
1689 return rc;
1690}
1691
1692/**
1693 * cxl_calc_interleave_pos() - calculate an endpoint position in a region
1694 * @cxled: endpoint decoder member of given region
1695 *
1696 * The endpoint position is calculated by traversing the topology from
1697 * the endpoint to the root decoder and iteratively applying this
1698 * calculation:
1699 *
1700 * position = position * parent_ways + parent_pos;
1701 *
1702 * ...where @position is inferred from switch and root decoder target lists.
1703 *
1704 * Return: position >= 0 on success
1705 * -ENXIO on failure
1706 */
1707static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled)
1708{
1709 struct cxl_port *iter, *port = cxled_to_port(cxled);
1710 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1711 struct range *range = &cxled->cxld.hpa_range;
1712 int parent_ways = 0, parent_pos = 0, pos = 0;
1713 int rc;
1714
1715 /*
1716 * Example: the expected interleave order of the 4-way region shown
1717 * below is: mem0, mem2, mem1, mem3
1718 *
1719 * root_port
1720 * / \
1721 * host_bridge_0 host_bridge_1
1722 * | | | |
1723 * mem0 mem1 mem2 mem3
1724 *
1725 * In the example the calculator will iterate twice. The first iteration
1726 * uses the mem position in the host-bridge and the ways of the host-
1727 * bridge to generate the first, or local, position. The second
1728 * iteration uses the host-bridge position in the root_port and the ways
1729 * of the root_port to refine the position.
1730 *
1731 * A trace of the calculation per endpoint looks like this:
1732 * mem0: pos = 0 * 2 + 0 mem2: pos = 0 * 2 + 0
1733 * pos = 0 * 2 + 0 pos = 0 * 2 + 1
1734 * pos: 0 pos: 1
1735 *
1736 * mem1: pos = 0 * 2 + 1 mem3: pos = 0 * 2 + 1
1737 * pos = 1 * 2 + 0 pos = 1 * 2 + 1
1738 * pos: 2 pos = 3
1739 *
1740 * Note that while this example is simple, the method applies to more
1741 * complex topologies, including those with switches.
1742 */
1743
1744 /* Iterate from endpoint to root_port refining the position */
1745 for (iter = port; iter; iter = next_port(iter)) {
1746 if (is_cxl_root(iter))
1747 break;
1748
1749 rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways);
1750 if (rc)
1751 return rc;
1752
1753 pos = pos * parent_ways + parent_pos;
1754 }
1755
1756 dev_dbg(&cxlmd->dev,
1757 "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n",
1758 dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent),
1759 dev_name(&port->dev), range->start, range->end, pos);
1760
1761 return pos;
1762}
1763
1764static int cxl_region_sort_targets(struct cxl_region *cxlr)
1765{
1766 struct cxl_region_params *p = &cxlr->params;
1767 int i, rc = 0;
1768
1769 for (i = 0; i < p->nr_targets; i++) {
1770 struct cxl_endpoint_decoder *cxled = p->targets[i];
1771
1772 cxled->pos = cxl_calc_interleave_pos(cxled);
1773 /*
1774 * Record that sorting failed, but still continue to calc
1775 * cxled->pos so that follow-on code paths can reliably
1776 * do p->targets[cxled->pos] to self-reference their entry.
1777 */
1778 if (cxled->pos < 0)
1779 rc = -ENXIO;
1780 }
1781 /* Keep the cxlr target list in interleave position order */
1782 sort(p->targets, p->nr_targets, sizeof(p->targets[0]),
1783 cmp_interleave_pos, NULL);
1784
1785 dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1786 return rc;
1787}
1788
1789static int cxl_region_attach(struct cxl_region *cxlr,
1790 struct cxl_endpoint_decoder *cxled, int pos)
1791{
1792 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1793 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1794 struct cxl_region_params *p = &cxlr->params;
1795 struct cxl_port *ep_port, *root_port;
1796 struct cxl_dport *dport;
1797 int rc = -ENXIO;
1798
1799 if (cxled->mode != cxlr->mode) {
1800 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
1801 dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
1802 return -EINVAL;
1803 }
1804
1805 if (cxled->mode == CXL_DECODER_DEAD) {
1806 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1807 return -ENODEV;
1808 }
1809
1810 /* all full of members, or interleave config not established? */
1811 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1812 dev_dbg(&cxlr->dev, "region already active\n");
1813 return -EBUSY;
1814 } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1815 dev_dbg(&cxlr->dev, "interleave config missing\n");
1816 return -ENXIO;
1817 }
1818
1819 if (p->nr_targets >= p->interleave_ways) {
1820 dev_dbg(&cxlr->dev, "region already has %d endpoints\n",
1821 p->nr_targets);
1822 return -EINVAL;
1823 }
1824
1825 ep_port = cxled_to_port(cxled);
1826 root_port = cxlrd_to_port(cxlrd);
1827 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1828 if (!dport) {
1829 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1830 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1831 dev_name(cxlr->dev.parent));
1832 return -ENXIO;
1833 }
1834
1835 if (cxled->cxld.target_type != cxlr->type) {
1836 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1837 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1838 cxled->cxld.target_type, cxlr->type);
1839 return -ENXIO;
1840 }
1841
1842 if (!cxled->dpa_res) {
1843 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1844 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1845 return -ENXIO;
1846 }
1847
1848 if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1849 resource_size(p->res)) {
1850 dev_dbg(&cxlr->dev,
1851 "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1852 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1853 (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1854 (u64)resource_size(p->res));
1855 return -EINVAL;
1856 }
1857
1858 cxl_region_perf_data_calculate(cxlr, cxled);
1859
1860 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1861 int i;
1862
1863 rc = cxl_region_attach_auto(cxlr, cxled, pos);
1864 if (rc)
1865 return rc;
1866
1867 /* await more targets to arrive... */
1868 if (p->nr_targets < p->interleave_ways)
1869 return 0;
1870
1871 /*
1872 * All targets are here, which implies all PCI enumeration that
1873 * affects this region has been completed. Walk the topology to
1874 * sort the devices into their relative region decode position.
1875 */
1876 rc = cxl_region_sort_targets(cxlr);
1877 if (rc)
1878 return rc;
1879
1880 for (i = 0; i < p->nr_targets; i++) {
1881 cxled = p->targets[i];
1882 ep_port = cxled_to_port(cxled);
1883 dport = cxl_find_dport_by_dev(root_port,
1884 ep_port->host_bridge);
1885 rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1886 dport, i);
1887 if (rc)
1888 return rc;
1889 }
1890
1891 rc = cxl_region_setup_targets(cxlr);
1892 if (rc)
1893 return rc;
1894
1895 /*
1896 * If target setup succeeds in the autodiscovery case
1897 * then the region is already committed.
1898 */
1899 p->state = CXL_CONFIG_COMMIT;
1900
1901 return 0;
1902 }
1903
1904 rc = cxl_region_validate_position(cxlr, cxled, pos);
1905 if (rc)
1906 return rc;
1907
1908 rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
1909 if (rc)
1910 return rc;
1911
1912 p->targets[pos] = cxled;
1913 cxled->pos = pos;
1914 p->nr_targets++;
1915
1916 if (p->nr_targets == p->interleave_ways) {
1917 rc = cxl_region_setup_targets(cxlr);
1918 if (rc)
1919 return rc;
1920 p->state = CXL_CONFIG_ACTIVE;
1921 }
1922
1923 cxled->cxld.interleave_ways = p->interleave_ways;
1924 cxled->cxld.interleave_granularity = p->interleave_granularity;
1925 cxled->cxld.hpa_range = (struct range) {
1926 .start = p->res->start,
1927 .end = p->res->end,
1928 };
1929
1930 if (p->nr_targets != p->interleave_ways)
1931 return 0;
1932
1933 /*
1934 * Test the auto-discovery position calculator function
1935 * against this successfully created user-defined region.
1936 * A fail message here means that this interleave config
1937 * will fail when presented as CXL_REGION_F_AUTO.
1938 */
1939 for (int i = 0; i < p->nr_targets; i++) {
1940 struct cxl_endpoint_decoder *cxled = p->targets[i];
1941 int test_pos;
1942
1943 test_pos = cxl_calc_interleave_pos(cxled);
1944 dev_dbg(&cxled->cxld.dev,
1945 "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n",
1946 (test_pos == cxled->pos) ? "success" : "fail",
1947 test_pos, cxled->pos);
1948 }
1949
1950 return 0;
1951}
1952
1953static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
1954{
1955 struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
1956 struct cxl_region *cxlr = cxled->cxld.region;
1957 struct cxl_region_params *p;
1958 int rc = 0;
1959
1960 lockdep_assert_held_write(&cxl_region_rwsem);
1961
1962 if (!cxlr)
1963 return 0;
1964
1965 p = &cxlr->params;
1966 get_device(&cxlr->dev);
1967
1968 if (p->state > CXL_CONFIG_ACTIVE) {
1969 /*
1970 * TODO: tear down all impacted regions if a device is
1971 * removed out of order
1972 */
1973 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
1974 if (rc)
1975 goto out;
1976 p->state = CXL_CONFIG_ACTIVE;
1977 }
1978
1979 for (iter = ep_port; !is_cxl_root(iter);
1980 iter = to_cxl_port(iter->dev.parent))
1981 cxl_port_detach_region(iter, cxlr, cxled);
1982
1983 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
1984 p->targets[cxled->pos] != cxled) {
1985 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1986
1987 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
1988 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1989 cxled->pos);
1990 goto out;
1991 }
1992
1993 if (p->state == CXL_CONFIG_ACTIVE) {
1994 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
1995 cxl_region_teardown_targets(cxlr);
1996 }
1997 p->targets[cxled->pos] = NULL;
1998 p->nr_targets--;
1999 cxled->cxld.hpa_range = (struct range) {
2000 .start = 0,
2001 .end = -1,
2002 };
2003
2004 /* notify the region driver that one of its targets has departed */
2005 up_write(&cxl_region_rwsem);
2006 device_release_driver(&cxlr->dev);
2007 down_write(&cxl_region_rwsem);
2008out:
2009 put_device(&cxlr->dev);
2010 return rc;
2011}
2012
2013void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
2014{
2015 down_write(&cxl_region_rwsem);
2016 cxled->mode = CXL_DECODER_DEAD;
2017 cxl_region_detach(cxled);
2018 up_write(&cxl_region_rwsem);
2019}
2020
2021static int attach_target(struct cxl_region *cxlr,
2022 struct cxl_endpoint_decoder *cxled, int pos,
2023 unsigned int state)
2024{
2025 int rc = 0;
2026
2027 if (state == TASK_INTERRUPTIBLE)
2028 rc = down_write_killable(&cxl_region_rwsem);
2029 else
2030 down_write(&cxl_region_rwsem);
2031 if (rc)
2032 return rc;
2033
2034 down_read(&cxl_dpa_rwsem);
2035 rc = cxl_region_attach(cxlr, cxled, pos);
2036 up_read(&cxl_dpa_rwsem);
2037 up_write(&cxl_region_rwsem);
2038 return rc;
2039}
2040
2041static int detach_target(struct cxl_region *cxlr, int pos)
2042{
2043 struct cxl_region_params *p = &cxlr->params;
2044 int rc;
2045
2046 rc = down_write_killable(&cxl_region_rwsem);
2047 if (rc)
2048 return rc;
2049
2050 if (pos >= p->interleave_ways) {
2051 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
2052 p->interleave_ways);
2053 rc = -ENXIO;
2054 goto out;
2055 }
2056
2057 if (!p->targets[pos]) {
2058 rc = 0;
2059 goto out;
2060 }
2061
2062 rc = cxl_region_detach(p->targets[pos]);
2063out:
2064 up_write(&cxl_region_rwsem);
2065 return rc;
2066}
2067
2068static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
2069 size_t len)
2070{
2071 int rc;
2072
2073 if (sysfs_streq(buf, "\n"))
2074 rc = detach_target(cxlr, pos);
2075 else {
2076 struct device *dev;
2077
2078 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
2079 if (!dev)
2080 return -ENODEV;
2081
2082 if (!is_endpoint_decoder(dev)) {
2083 rc = -EINVAL;
2084 goto out;
2085 }
2086
2087 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
2088 TASK_INTERRUPTIBLE);
2089out:
2090 put_device(dev);
2091 }
2092
2093 if (rc < 0)
2094 return rc;
2095 return len;
2096}
2097
2098#define TARGET_ATTR_RW(n) \
2099static ssize_t target##n##_show( \
2100 struct device *dev, struct device_attribute *attr, char *buf) \
2101{ \
2102 return show_targetN(to_cxl_region(dev), buf, (n)); \
2103} \
2104static ssize_t target##n##_store(struct device *dev, \
2105 struct device_attribute *attr, \
2106 const char *buf, size_t len) \
2107{ \
2108 return store_targetN(to_cxl_region(dev), buf, (n), len); \
2109} \
2110static DEVICE_ATTR_RW(target##n)
2111
2112TARGET_ATTR_RW(0);
2113TARGET_ATTR_RW(1);
2114TARGET_ATTR_RW(2);
2115TARGET_ATTR_RW(3);
2116TARGET_ATTR_RW(4);
2117TARGET_ATTR_RW(5);
2118TARGET_ATTR_RW(6);
2119TARGET_ATTR_RW(7);
2120TARGET_ATTR_RW(8);
2121TARGET_ATTR_RW(9);
2122TARGET_ATTR_RW(10);
2123TARGET_ATTR_RW(11);
2124TARGET_ATTR_RW(12);
2125TARGET_ATTR_RW(13);
2126TARGET_ATTR_RW(14);
2127TARGET_ATTR_RW(15);
2128
2129static struct attribute *target_attrs[] = {
2130 &dev_attr_target0.attr,
2131 &dev_attr_target1.attr,
2132 &dev_attr_target2.attr,
2133 &dev_attr_target3.attr,
2134 &dev_attr_target4.attr,
2135 &dev_attr_target5.attr,
2136 &dev_attr_target6.attr,
2137 &dev_attr_target7.attr,
2138 &dev_attr_target8.attr,
2139 &dev_attr_target9.attr,
2140 &dev_attr_target10.attr,
2141 &dev_attr_target11.attr,
2142 &dev_attr_target12.attr,
2143 &dev_attr_target13.attr,
2144 &dev_attr_target14.attr,
2145 &dev_attr_target15.attr,
2146 NULL,
2147};
2148
2149static umode_t cxl_region_target_visible(struct kobject *kobj,
2150 struct attribute *a, int n)
2151{
2152 struct device *dev = kobj_to_dev(kobj);
2153 struct cxl_region *cxlr = to_cxl_region(dev);
2154 struct cxl_region_params *p = &cxlr->params;
2155
2156 if (n < p->interleave_ways)
2157 return a->mode;
2158 return 0;
2159}
2160
2161static const struct attribute_group cxl_region_target_group = {
2162 .attrs = target_attrs,
2163 .is_visible = cxl_region_target_visible,
2164};
2165
2166static const struct attribute_group *get_cxl_region_target_group(void)
2167{
2168 return &cxl_region_target_group;
2169}
2170
2171static const struct attribute_group *region_groups[] = {
2172 &cxl_base_attribute_group,
2173 &cxl_region_group,
2174 &cxl_region_target_group,
2175 &cxl_region_access0_coordinate_group,
2176 &cxl_region_access1_coordinate_group,
2177 NULL,
2178};
2179
2180static void cxl_region_release(struct device *dev)
2181{
2182 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
2183 struct cxl_region *cxlr = to_cxl_region(dev);
2184 int id = atomic_read(&cxlrd->region_id);
2185
2186 /*
2187 * Try to reuse the recently idled id rather than the cached
2188 * next id to prevent the region id space from increasing
2189 * unnecessarily.
2190 */
2191 if (cxlr->id < id)
2192 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
2193 memregion_free(id);
2194 goto out;
2195 }
2196
2197 memregion_free(cxlr->id);
2198out:
2199 put_device(dev->parent);
2200 kfree(cxlr);
2201}
2202
2203const struct device_type cxl_region_type = {
2204 .name = "cxl_region",
2205 .release = cxl_region_release,
2206 .groups = region_groups
2207};
2208
2209bool is_cxl_region(struct device *dev)
2210{
2211 return dev->type == &cxl_region_type;
2212}
2213EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL);
2214
2215static struct cxl_region *to_cxl_region(struct device *dev)
2216{
2217 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2218 "not a cxl_region device\n"))
2219 return NULL;
2220
2221 return container_of(dev, struct cxl_region, dev);
2222}
2223
2224static void unregister_region(void *_cxlr)
2225{
2226 struct cxl_region *cxlr = _cxlr;
2227 struct cxl_region_params *p = &cxlr->params;
2228 int i;
2229
2230 unregister_memory_notifier(&cxlr->memory_notifier);
2231 device_del(&cxlr->dev);
2232
2233 /*
2234 * Now that region sysfs is shutdown, the parameter block is now
2235 * read-only, so no need to hold the region rwsem to access the
2236 * region parameters.
2237 */
2238 for (i = 0; i < p->interleave_ways; i++)
2239 detach_target(cxlr, i);
2240
2241 cxl_region_iomem_release(cxlr);
2242 put_device(&cxlr->dev);
2243}
2244
2245static struct lock_class_key cxl_region_key;
2246
2247static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2248{
2249 struct cxl_region *cxlr;
2250 struct device *dev;
2251
2252 cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2253 if (!cxlr) {
2254 memregion_free(id);
2255 return ERR_PTR(-ENOMEM);
2256 }
2257
2258 dev = &cxlr->dev;
2259 device_initialize(dev);
2260 lockdep_set_class(&dev->mutex, &cxl_region_key);
2261 dev->parent = &cxlrd->cxlsd.cxld.dev;
2262 /*
2263 * Keep root decoder pinned through cxl_region_release to fixup
2264 * region id allocations
2265 */
2266 get_device(dev->parent);
2267 device_set_pm_not_required(dev);
2268 dev->bus = &cxl_bus_type;
2269 dev->type = &cxl_region_type;
2270 cxlr->id = id;
2271
2272 return cxlr;
2273}
2274
2275static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid)
2276{
2277 int cset = 0;
2278 int rc;
2279
2280 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2281 if (cxlr->coord[i].read_bandwidth) {
2282 rc = 0;
2283 if (cxl_need_node_perf_attrs_update(nid))
2284 node_set_perf_attrs(nid, &cxlr->coord[i], i);
2285 else
2286 rc = cxl_update_hmat_access_coordinates(nid, cxlr, i);
2287
2288 if (rc == 0)
2289 cset++;
2290 }
2291 }
2292
2293 if (!cset)
2294 return false;
2295
2296 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access0_group());
2297 if (rc)
2298 dev_dbg(&cxlr->dev, "Failed to update access0 group\n");
2299
2300 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access1_group());
2301 if (rc)
2302 dev_dbg(&cxlr->dev, "Failed to update access1 group\n");
2303
2304 return true;
2305}
2306
2307static int cxl_region_perf_attrs_callback(struct notifier_block *nb,
2308 unsigned long action, void *arg)
2309{
2310 struct cxl_region *cxlr = container_of(nb, struct cxl_region,
2311 memory_notifier);
2312 struct cxl_region_params *p = &cxlr->params;
2313 struct cxl_endpoint_decoder *cxled = p->targets[0];
2314 struct cxl_decoder *cxld = &cxled->cxld;
2315 struct memory_notify *mnb = arg;
2316 int nid = mnb->status_change_nid;
2317 int region_nid;
2318
2319 if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
2320 return NOTIFY_DONE;
2321
2322 region_nid = phys_to_target_node(cxld->hpa_range.start);
2323 if (nid != region_nid)
2324 return NOTIFY_DONE;
2325
2326 if (!cxl_region_update_coordinates(cxlr, nid))
2327 return NOTIFY_DONE;
2328
2329 return NOTIFY_OK;
2330}
2331
2332/**
2333 * devm_cxl_add_region - Adds a region to a decoder
2334 * @cxlrd: root decoder
2335 * @id: memregion id to create, or memregion_free() on failure
2336 * @mode: mode for the endpoint decoders of this region
2337 * @type: select whether this is an expander or accelerator (type-2 or type-3)
2338 *
2339 * This is the second step of region initialization. Regions exist within an
2340 * address space which is mapped by a @cxlrd.
2341 *
2342 * Return: 0 if the region was added to the @cxlrd, else returns negative error
2343 * code. The region will be named "regionZ" where Z is the unique region number.
2344 */
2345static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2346 int id,
2347 enum cxl_decoder_mode mode,
2348 enum cxl_decoder_type type)
2349{
2350 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2351 struct cxl_region *cxlr;
2352 struct device *dev;
2353 int rc;
2354
2355 switch (mode) {
2356 case CXL_DECODER_RAM:
2357 case CXL_DECODER_PMEM:
2358 break;
2359 default:
2360 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2361 return ERR_PTR(-EINVAL);
2362 }
2363
2364 cxlr = cxl_region_alloc(cxlrd, id);
2365 if (IS_ERR(cxlr))
2366 return cxlr;
2367 cxlr->mode = mode;
2368 cxlr->type = type;
2369
2370 dev = &cxlr->dev;
2371 rc = dev_set_name(dev, "region%d", id);
2372 if (rc)
2373 goto err;
2374
2375 rc = device_add(dev);
2376 if (rc)
2377 goto err;
2378
2379 cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback;
2380 cxlr->memory_notifier.priority = CXL_CALLBACK_PRI;
2381 register_memory_notifier(&cxlr->memory_notifier);
2382
2383 rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2384 if (rc)
2385 return ERR_PTR(rc);
2386
2387 dev_dbg(port->uport_dev, "%s: created %s\n",
2388 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2389 return cxlr;
2390
2391err:
2392 put_device(dev);
2393 return ERR_PTR(rc);
2394}
2395
2396static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2397{
2398 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2399}
2400
2401static ssize_t create_pmem_region_show(struct device *dev,
2402 struct device_attribute *attr, char *buf)
2403{
2404 return __create_region_show(to_cxl_root_decoder(dev), buf);
2405}
2406
2407static ssize_t create_ram_region_show(struct device *dev,
2408 struct device_attribute *attr, char *buf)
2409{
2410 return __create_region_show(to_cxl_root_decoder(dev), buf);
2411}
2412
2413static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2414 enum cxl_decoder_mode mode, int id)
2415{
2416 int rc;
2417
2418 rc = memregion_alloc(GFP_KERNEL);
2419 if (rc < 0)
2420 return ERR_PTR(rc);
2421
2422 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2423 memregion_free(rc);
2424 return ERR_PTR(-EBUSY);
2425 }
2426
2427 return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
2428}
2429
2430static ssize_t create_pmem_region_store(struct device *dev,
2431 struct device_attribute *attr,
2432 const char *buf, size_t len)
2433{
2434 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2435 struct cxl_region *cxlr;
2436 int rc, id;
2437
2438 rc = sscanf(buf, "region%d\n", &id);
2439 if (rc != 1)
2440 return -EINVAL;
2441
2442 cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id);
2443 if (IS_ERR(cxlr))
2444 return PTR_ERR(cxlr);
2445
2446 return len;
2447}
2448DEVICE_ATTR_RW(create_pmem_region);
2449
2450static ssize_t create_ram_region_store(struct device *dev,
2451 struct device_attribute *attr,
2452 const char *buf, size_t len)
2453{
2454 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2455 struct cxl_region *cxlr;
2456 int rc, id;
2457
2458 rc = sscanf(buf, "region%d\n", &id);
2459 if (rc != 1)
2460 return -EINVAL;
2461
2462 cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id);
2463 if (IS_ERR(cxlr))
2464 return PTR_ERR(cxlr);
2465
2466 return len;
2467}
2468DEVICE_ATTR_RW(create_ram_region);
2469
2470static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2471 char *buf)
2472{
2473 struct cxl_decoder *cxld = to_cxl_decoder(dev);
2474 ssize_t rc;
2475
2476 rc = down_read_interruptible(&cxl_region_rwsem);
2477 if (rc)
2478 return rc;
2479
2480 if (cxld->region)
2481 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2482 else
2483 rc = sysfs_emit(buf, "\n");
2484 up_read(&cxl_region_rwsem);
2485
2486 return rc;
2487}
2488DEVICE_ATTR_RO(region);
2489
2490static struct cxl_region *
2491cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2492{
2493 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2494 struct device *region_dev;
2495
2496 region_dev = device_find_child_by_name(&cxld->dev, name);
2497 if (!region_dev)
2498 return ERR_PTR(-ENODEV);
2499
2500 return to_cxl_region(region_dev);
2501}
2502
2503static ssize_t delete_region_store(struct device *dev,
2504 struct device_attribute *attr,
2505 const char *buf, size_t len)
2506{
2507 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2508 struct cxl_port *port = to_cxl_port(dev->parent);
2509 struct cxl_region *cxlr;
2510
2511 cxlr = cxl_find_region_by_name(cxlrd, buf);
2512 if (IS_ERR(cxlr))
2513 return PTR_ERR(cxlr);
2514
2515 devm_release_action(port->uport_dev, unregister_region, cxlr);
2516 put_device(&cxlr->dev);
2517
2518 return len;
2519}
2520DEVICE_ATTR_WO(delete_region);
2521
2522static void cxl_pmem_region_release(struct device *dev)
2523{
2524 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2525 int i;
2526
2527 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2528 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2529
2530 put_device(&cxlmd->dev);
2531 }
2532
2533 kfree(cxlr_pmem);
2534}
2535
2536static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2537 &cxl_base_attribute_group,
2538 NULL,
2539};
2540
2541const struct device_type cxl_pmem_region_type = {
2542 .name = "cxl_pmem_region",
2543 .release = cxl_pmem_region_release,
2544 .groups = cxl_pmem_region_attribute_groups,
2545};
2546
2547bool is_cxl_pmem_region(struct device *dev)
2548{
2549 return dev->type == &cxl_pmem_region_type;
2550}
2551EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL);
2552
2553struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2554{
2555 if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2556 "not a cxl_pmem_region device\n"))
2557 return NULL;
2558 return container_of(dev, struct cxl_pmem_region, dev);
2559}
2560EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL);
2561
2562struct cxl_poison_context {
2563 struct cxl_port *port;
2564 enum cxl_decoder_mode mode;
2565 u64 offset;
2566};
2567
2568static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2569 struct cxl_poison_context *ctx)
2570{
2571 struct cxl_dev_state *cxlds = cxlmd->cxlds;
2572 u64 offset, length;
2573 int rc = 0;
2574
2575 /*
2576 * Collect poison for the remaining unmapped resources
2577 * after poison is collected by committed endpoints.
2578 *
2579 * Knowing that PMEM must always follow RAM, get poison
2580 * for unmapped resources based on the last decoder's mode:
2581 * ram: scan remains of ram range, then any pmem range
2582 * pmem: scan remains of pmem range
2583 */
2584
2585 if (ctx->mode == CXL_DECODER_RAM) {
2586 offset = ctx->offset;
2587 length = resource_size(&cxlds->ram_res) - offset;
2588 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2589 if (rc == -EFAULT)
2590 rc = 0;
2591 if (rc)
2592 return rc;
2593 }
2594 if (ctx->mode == CXL_DECODER_PMEM) {
2595 offset = ctx->offset;
2596 length = resource_size(&cxlds->dpa_res) - offset;
2597 if (!length)
2598 return 0;
2599 } else if (resource_size(&cxlds->pmem_res)) {
2600 offset = cxlds->pmem_res.start;
2601 length = resource_size(&cxlds->pmem_res);
2602 } else {
2603 return 0;
2604 }
2605
2606 return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2607}
2608
2609static int poison_by_decoder(struct device *dev, void *arg)
2610{
2611 struct cxl_poison_context *ctx = arg;
2612 struct cxl_endpoint_decoder *cxled;
2613 struct cxl_memdev *cxlmd;
2614 u64 offset, length;
2615 int rc = 0;
2616
2617 if (!is_endpoint_decoder(dev))
2618 return rc;
2619
2620 cxled = to_cxl_endpoint_decoder(dev);
2621 if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2622 return rc;
2623
2624 /*
2625 * Regions are only created with single mode decoders: pmem or ram.
2626 * Linux does not support mixed mode decoders. This means that
2627 * reading poison per endpoint decoder adheres to the requirement
2628 * that poison reads of pmem and ram must be separated.
2629 * CXL 3.0 Spec 8.2.9.8.4.1
2630 */
2631 if (cxled->mode == CXL_DECODER_MIXED) {
2632 dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2633 return rc;
2634 }
2635
2636 cxlmd = cxled_to_memdev(cxled);
2637 if (cxled->skip) {
2638 offset = cxled->dpa_res->start - cxled->skip;
2639 length = cxled->skip;
2640 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2641 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2642 rc = 0;
2643 if (rc)
2644 return rc;
2645 }
2646
2647 offset = cxled->dpa_res->start;
2648 length = cxled->dpa_res->end - offset + 1;
2649 rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2650 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2651 rc = 0;
2652 if (rc)
2653 return rc;
2654
2655 /* Iterate until commit_end is reached */
2656 if (cxled->cxld.id == ctx->port->commit_end) {
2657 ctx->offset = cxled->dpa_res->end + 1;
2658 ctx->mode = cxled->mode;
2659 return 1;
2660 }
2661
2662 return 0;
2663}
2664
2665int cxl_get_poison_by_endpoint(struct cxl_port *port)
2666{
2667 struct cxl_poison_context ctx;
2668 int rc = 0;
2669
2670 ctx = (struct cxl_poison_context) {
2671 .port = port
2672 };
2673
2674 rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2675 if (rc == 1)
2676 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
2677 &ctx);
2678
2679 return rc;
2680}
2681
2682static struct lock_class_key cxl_pmem_region_key;
2683
2684static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr)
2685{
2686 struct cxl_region_params *p = &cxlr->params;
2687 struct cxl_nvdimm_bridge *cxl_nvb;
2688 struct cxl_pmem_region *cxlr_pmem;
2689 struct device *dev;
2690 int i;
2691
2692 down_read(&cxl_region_rwsem);
2693 if (p->state != CXL_CONFIG_COMMIT) {
2694 cxlr_pmem = ERR_PTR(-ENXIO);
2695 goto out;
2696 }
2697
2698 cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets),
2699 GFP_KERNEL);
2700 if (!cxlr_pmem) {
2701 cxlr_pmem = ERR_PTR(-ENOMEM);
2702 goto out;
2703 }
2704
2705 cxlr_pmem->hpa_range.start = p->res->start;
2706 cxlr_pmem->hpa_range.end = p->res->end;
2707
2708 /* Snapshot the region configuration underneath the cxl_region_rwsem */
2709 cxlr_pmem->nr_mappings = p->nr_targets;
2710 for (i = 0; i < p->nr_targets; i++) {
2711 struct cxl_endpoint_decoder *cxled = p->targets[i];
2712 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2713 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
2714
2715 /*
2716 * Regions never span CXL root devices, so by definition the
2717 * bridge for one device is the same for all.
2718 */
2719 if (i == 0) {
2720 cxl_nvb = cxl_find_nvdimm_bridge(cxlmd);
2721 if (!cxl_nvb) {
2722 kfree(cxlr_pmem);
2723 cxlr_pmem = ERR_PTR(-ENODEV);
2724 goto out;
2725 }
2726 cxlr->cxl_nvb = cxl_nvb;
2727 }
2728 m->cxlmd = cxlmd;
2729 get_device(&cxlmd->dev);
2730 m->start = cxled->dpa_res->start;
2731 m->size = resource_size(cxled->dpa_res);
2732 m->position = i;
2733 }
2734
2735 dev = &cxlr_pmem->dev;
2736 cxlr_pmem->cxlr = cxlr;
2737 cxlr->cxlr_pmem = cxlr_pmem;
2738 device_initialize(dev);
2739 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
2740 device_set_pm_not_required(dev);
2741 dev->parent = &cxlr->dev;
2742 dev->bus = &cxl_bus_type;
2743 dev->type = &cxl_pmem_region_type;
2744out:
2745 up_read(&cxl_region_rwsem);
2746
2747 return cxlr_pmem;
2748}
2749
2750static void cxl_dax_region_release(struct device *dev)
2751{
2752 struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
2753
2754 kfree(cxlr_dax);
2755}
2756
2757static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
2758 &cxl_base_attribute_group,
2759 NULL,
2760};
2761
2762const struct device_type cxl_dax_region_type = {
2763 .name = "cxl_dax_region",
2764 .release = cxl_dax_region_release,
2765 .groups = cxl_dax_region_attribute_groups,
2766};
2767
2768static bool is_cxl_dax_region(struct device *dev)
2769{
2770 return dev->type == &cxl_dax_region_type;
2771}
2772
2773struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
2774{
2775 if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
2776 "not a cxl_dax_region device\n"))
2777 return NULL;
2778 return container_of(dev, struct cxl_dax_region, dev);
2779}
2780EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, CXL);
2781
2782static struct lock_class_key cxl_dax_region_key;
2783
2784static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
2785{
2786 struct cxl_region_params *p = &cxlr->params;
2787 struct cxl_dax_region *cxlr_dax;
2788 struct device *dev;
2789
2790 down_read(&cxl_region_rwsem);
2791 if (p->state != CXL_CONFIG_COMMIT) {
2792 cxlr_dax = ERR_PTR(-ENXIO);
2793 goto out;
2794 }
2795
2796 cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
2797 if (!cxlr_dax) {
2798 cxlr_dax = ERR_PTR(-ENOMEM);
2799 goto out;
2800 }
2801
2802 cxlr_dax->hpa_range.start = p->res->start;
2803 cxlr_dax->hpa_range.end = p->res->end;
2804
2805 dev = &cxlr_dax->dev;
2806 cxlr_dax->cxlr = cxlr;
2807 device_initialize(dev);
2808 lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
2809 device_set_pm_not_required(dev);
2810 dev->parent = &cxlr->dev;
2811 dev->bus = &cxl_bus_type;
2812 dev->type = &cxl_dax_region_type;
2813out:
2814 up_read(&cxl_region_rwsem);
2815
2816 return cxlr_dax;
2817}
2818
2819static void cxlr_pmem_unregister(void *_cxlr_pmem)
2820{
2821 struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
2822 struct cxl_region *cxlr = cxlr_pmem->cxlr;
2823 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2824
2825 /*
2826 * Either the bridge is in ->remove() context under the device_lock(),
2827 * or cxlr_release_nvdimm() is cancelling the bridge's release action
2828 * for @cxlr_pmem and doing it itself (while manually holding the bridge
2829 * lock).
2830 */
2831 device_lock_assert(&cxl_nvb->dev);
2832 cxlr->cxlr_pmem = NULL;
2833 cxlr_pmem->cxlr = NULL;
2834 device_unregister(&cxlr_pmem->dev);
2835}
2836
2837static void cxlr_release_nvdimm(void *_cxlr)
2838{
2839 struct cxl_region *cxlr = _cxlr;
2840 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2841
2842 device_lock(&cxl_nvb->dev);
2843 if (cxlr->cxlr_pmem)
2844 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
2845 cxlr->cxlr_pmem);
2846 device_unlock(&cxl_nvb->dev);
2847 cxlr->cxl_nvb = NULL;
2848 put_device(&cxl_nvb->dev);
2849}
2850
2851/**
2852 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
2853 * @cxlr: parent CXL region for this pmem region bridge device
2854 *
2855 * Return: 0 on success negative error code on failure.
2856 */
2857static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
2858{
2859 struct cxl_pmem_region *cxlr_pmem;
2860 struct cxl_nvdimm_bridge *cxl_nvb;
2861 struct device *dev;
2862 int rc;
2863
2864 cxlr_pmem = cxl_pmem_region_alloc(cxlr);
2865 if (IS_ERR(cxlr_pmem))
2866 return PTR_ERR(cxlr_pmem);
2867 cxl_nvb = cxlr->cxl_nvb;
2868
2869 dev = &cxlr_pmem->dev;
2870 rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
2871 if (rc)
2872 goto err;
2873
2874 rc = device_add(dev);
2875 if (rc)
2876 goto err;
2877
2878 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2879 dev_name(dev));
2880
2881 device_lock(&cxl_nvb->dev);
2882 if (cxl_nvb->dev.driver)
2883 rc = devm_add_action_or_reset(&cxl_nvb->dev,
2884 cxlr_pmem_unregister, cxlr_pmem);
2885 else
2886 rc = -ENXIO;
2887 device_unlock(&cxl_nvb->dev);
2888
2889 if (rc)
2890 goto err_bridge;
2891
2892 /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
2893 return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
2894
2895err:
2896 put_device(dev);
2897err_bridge:
2898 put_device(&cxl_nvb->dev);
2899 cxlr->cxl_nvb = NULL;
2900 return rc;
2901}
2902
2903static void cxlr_dax_unregister(void *_cxlr_dax)
2904{
2905 struct cxl_dax_region *cxlr_dax = _cxlr_dax;
2906
2907 device_unregister(&cxlr_dax->dev);
2908}
2909
2910static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
2911{
2912 struct cxl_dax_region *cxlr_dax;
2913 struct device *dev;
2914 int rc;
2915
2916 cxlr_dax = cxl_dax_region_alloc(cxlr);
2917 if (IS_ERR(cxlr_dax))
2918 return PTR_ERR(cxlr_dax);
2919
2920 dev = &cxlr_dax->dev;
2921 rc = dev_set_name(dev, "dax_region%d", cxlr->id);
2922 if (rc)
2923 goto err;
2924
2925 rc = device_add(dev);
2926 if (rc)
2927 goto err;
2928
2929 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2930 dev_name(dev));
2931
2932 return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
2933 cxlr_dax);
2934err:
2935 put_device(dev);
2936 return rc;
2937}
2938
2939static int match_root_decoder_by_range(struct device *dev, void *data)
2940{
2941 struct range *r1, *r2 = data;
2942 struct cxl_root_decoder *cxlrd;
2943
2944 if (!is_root_decoder(dev))
2945 return 0;
2946
2947 cxlrd = to_cxl_root_decoder(dev);
2948 r1 = &cxlrd->cxlsd.cxld.hpa_range;
2949 return range_contains(r1, r2);
2950}
2951
2952static int match_region_by_range(struct device *dev, void *data)
2953{
2954 struct cxl_region_params *p;
2955 struct cxl_region *cxlr;
2956 struct range *r = data;
2957 int rc = 0;
2958
2959 if (!is_cxl_region(dev))
2960 return 0;
2961
2962 cxlr = to_cxl_region(dev);
2963 p = &cxlr->params;
2964
2965 down_read(&cxl_region_rwsem);
2966 if (p->res && p->res->start == r->start && p->res->end == r->end)
2967 rc = 1;
2968 up_read(&cxl_region_rwsem);
2969
2970 return rc;
2971}
2972
2973/* Establish an empty region covering the given HPA range */
2974static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
2975 struct cxl_endpoint_decoder *cxled)
2976{
2977 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2978 struct cxl_port *port = cxlrd_to_port(cxlrd);
2979 struct range *hpa = &cxled->cxld.hpa_range;
2980 struct cxl_region_params *p;
2981 struct cxl_region *cxlr;
2982 struct resource *res;
2983 int rc;
2984
2985 do {
2986 cxlr = __create_region(cxlrd, cxled->mode,
2987 atomic_read(&cxlrd->region_id));
2988 } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
2989
2990 if (IS_ERR(cxlr)) {
2991 dev_err(cxlmd->dev.parent,
2992 "%s:%s: %s failed assign region: %ld\n",
2993 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2994 __func__, PTR_ERR(cxlr));
2995 return cxlr;
2996 }
2997
2998 down_write(&cxl_region_rwsem);
2999 p = &cxlr->params;
3000 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
3001 dev_err(cxlmd->dev.parent,
3002 "%s:%s: %s autodiscovery interrupted\n",
3003 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3004 __func__);
3005 rc = -EBUSY;
3006 goto err;
3007 }
3008
3009 set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
3010
3011 res = kmalloc(sizeof(*res), GFP_KERNEL);
3012 if (!res) {
3013 rc = -ENOMEM;
3014 goto err;
3015 }
3016
3017 *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
3018 dev_name(&cxlr->dev));
3019 rc = insert_resource(cxlrd->res, res);
3020 if (rc) {
3021 /*
3022 * Platform-firmware may not have split resources like "System
3023 * RAM" on CXL window boundaries see cxl_region_iomem_release()
3024 */
3025 dev_warn(cxlmd->dev.parent,
3026 "%s:%s: %s %s cannot insert resource\n",
3027 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3028 __func__, dev_name(&cxlr->dev));
3029 }
3030
3031 p->res = res;
3032 p->interleave_ways = cxled->cxld.interleave_ways;
3033 p->interleave_granularity = cxled->cxld.interleave_granularity;
3034 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
3035
3036 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
3037 if (rc)
3038 goto err;
3039
3040 dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
3041 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
3042 dev_name(&cxlr->dev), p->res, p->interleave_ways,
3043 p->interleave_granularity);
3044
3045 /* ...to match put_device() in cxl_add_to_region() */
3046 get_device(&cxlr->dev);
3047 up_write(&cxl_region_rwsem);
3048
3049 return cxlr;
3050
3051err:
3052 up_write(&cxl_region_rwsem);
3053 devm_release_action(port->uport_dev, unregister_region, cxlr);
3054 return ERR_PTR(rc);
3055}
3056
3057int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
3058{
3059 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3060 struct range *hpa = &cxled->cxld.hpa_range;
3061 struct cxl_decoder *cxld = &cxled->cxld;
3062 struct device *cxlrd_dev, *region_dev;
3063 struct cxl_root_decoder *cxlrd;
3064 struct cxl_region_params *p;
3065 struct cxl_region *cxlr;
3066 bool attach = false;
3067 int rc;
3068
3069 cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
3070 match_root_decoder_by_range);
3071 if (!cxlrd_dev) {
3072 dev_err(cxlmd->dev.parent,
3073 "%s:%s no CXL window for range %#llx:%#llx\n",
3074 dev_name(&cxlmd->dev), dev_name(&cxld->dev),
3075 cxld->hpa_range.start, cxld->hpa_range.end);
3076 return -ENXIO;
3077 }
3078
3079 cxlrd = to_cxl_root_decoder(cxlrd_dev);
3080
3081 /*
3082 * Ensure that if multiple threads race to construct_region() for @hpa
3083 * one does the construction and the others add to that.
3084 */
3085 mutex_lock(&cxlrd->range_lock);
3086 region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
3087 match_region_by_range);
3088 if (!region_dev) {
3089 cxlr = construct_region(cxlrd, cxled);
3090 region_dev = &cxlr->dev;
3091 } else
3092 cxlr = to_cxl_region(region_dev);
3093 mutex_unlock(&cxlrd->range_lock);
3094
3095 rc = PTR_ERR_OR_ZERO(cxlr);
3096 if (rc)
3097 goto out;
3098
3099 attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
3100
3101 down_read(&cxl_region_rwsem);
3102 p = &cxlr->params;
3103 attach = p->state == CXL_CONFIG_COMMIT;
3104 up_read(&cxl_region_rwsem);
3105
3106 if (attach) {
3107 /*
3108 * If device_attach() fails the range may still be active via
3109 * the platform-firmware memory map, otherwise the driver for
3110 * regions is local to this file, so driver matching can't fail.
3111 */
3112 if (device_attach(&cxlr->dev) < 0)
3113 dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
3114 p->res);
3115 }
3116
3117 put_device(region_dev);
3118out:
3119 put_device(cxlrd_dev);
3120 return rc;
3121}
3122EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL);
3123
3124static int is_system_ram(struct resource *res, void *arg)
3125{
3126 struct cxl_region *cxlr = arg;
3127 struct cxl_region_params *p = &cxlr->params;
3128
3129 dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
3130 return 1;
3131}
3132
3133static int cxl_region_probe(struct device *dev)
3134{
3135 struct cxl_region *cxlr = to_cxl_region(dev);
3136 struct cxl_region_params *p = &cxlr->params;
3137 int rc;
3138
3139 rc = down_read_interruptible(&cxl_region_rwsem);
3140 if (rc) {
3141 dev_dbg(&cxlr->dev, "probe interrupted\n");
3142 return rc;
3143 }
3144
3145 if (p->state < CXL_CONFIG_COMMIT) {
3146 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
3147 rc = -ENXIO;
3148 goto out;
3149 }
3150
3151 if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
3152 dev_err(&cxlr->dev,
3153 "failed to activate, re-commit region and retry\n");
3154 rc = -ENXIO;
3155 goto out;
3156 }
3157
3158 /*
3159 * From this point on any path that changes the region's state away from
3160 * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
3161 */
3162out:
3163 up_read(&cxl_region_rwsem);
3164
3165 if (rc)
3166 return rc;
3167
3168 switch (cxlr->mode) {
3169 case CXL_DECODER_PMEM:
3170 return devm_cxl_add_pmem_region(cxlr);
3171 case CXL_DECODER_RAM:
3172 /*
3173 * The region can not be manged by CXL if any portion of
3174 * it is already online as 'System RAM'
3175 */
3176 if (walk_iomem_res_desc(IORES_DESC_NONE,
3177 IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
3178 p->res->start, p->res->end, cxlr,
3179 is_system_ram) > 0)
3180 return 0;
3181 return devm_cxl_add_dax_region(cxlr);
3182 default:
3183 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
3184 cxlr->mode);
3185 return -ENXIO;
3186 }
3187}
3188
3189static struct cxl_driver cxl_region_driver = {
3190 .name = "cxl_region",
3191 .probe = cxl_region_probe,
3192 .id = CXL_DEVICE_REGION,
3193};
3194
3195int cxl_region_init(void)
3196{
3197 return cxl_driver_register(&cxl_region_driver);
3198}
3199
3200void cxl_region_exit(void)
3201{
3202 cxl_driver_unregister(&cxl_region_driver);
3203}
3204
3205MODULE_IMPORT_NS(CXL);
3206MODULE_IMPORT_NS(DEVMEM);
3207MODULE_ALIAS_CXL(CXL_DEVICE_REGION);