Loading...
Note: File does not exist in v5.9.
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3#include <linux/memregion.h>
4#include <linux/genalloc.h>
5#include <linux/device.h>
6#include <linux/module.h>
7#include <linux/memory.h>
8#include <linux/slab.h>
9#include <linux/uuid.h>
10#include <linux/sort.h>
11#include <linux/idr.h>
12#include <linux/memory-tiers.h>
13#include <cxlmem.h>
14#include <cxl.h>
15#include "core.h"
16
17/**
18 * DOC: cxl core region
19 *
20 * CXL Regions represent mapped memory capacity in system physical address
21 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
22 * Memory ranges, Regions represent the active mapped capacity by the HDM
23 * Decoder Capability structures throughout the Host Bridges, Switches, and
24 * Endpoints in the topology.
25 *
26 * Region configuration has ordering constraints. UUID may be set at any time
27 * but is only visible for persistent regions.
28 * 1. Interleave granularity
29 * 2. Interleave size
30 * 3. Decoder targets
31 */
32
33static struct cxl_region *to_cxl_region(struct device *dev);
34
35#define __ACCESS_ATTR_RO(_level, _name) { \
36 .attr = { .name = __stringify(_name), .mode = 0444 }, \
37 .show = _name##_access##_level##_show, \
38}
39
40#define ACCESS_DEVICE_ATTR_RO(level, name) \
41 struct device_attribute dev_attr_access##level##_##name = __ACCESS_ATTR_RO(level, name)
42
43#define ACCESS_ATTR_RO(level, attrib) \
44static ssize_t attrib##_access##level##_show(struct device *dev, \
45 struct device_attribute *attr, \
46 char *buf) \
47{ \
48 struct cxl_region *cxlr = to_cxl_region(dev); \
49 \
50 if (cxlr->coord[level].attrib == 0) \
51 return -ENOENT; \
52 \
53 return sysfs_emit(buf, "%u\n", cxlr->coord[level].attrib); \
54} \
55static ACCESS_DEVICE_ATTR_RO(level, attrib)
56
57ACCESS_ATTR_RO(0, read_bandwidth);
58ACCESS_ATTR_RO(0, read_latency);
59ACCESS_ATTR_RO(0, write_bandwidth);
60ACCESS_ATTR_RO(0, write_latency);
61
62#define ACCESS_ATTR_DECLARE(level, attrib) \
63 (&dev_attr_access##level##_##attrib.attr)
64
65static struct attribute *access0_coordinate_attrs[] = {
66 ACCESS_ATTR_DECLARE(0, read_bandwidth),
67 ACCESS_ATTR_DECLARE(0, write_bandwidth),
68 ACCESS_ATTR_DECLARE(0, read_latency),
69 ACCESS_ATTR_DECLARE(0, write_latency),
70 NULL
71};
72
73ACCESS_ATTR_RO(1, read_bandwidth);
74ACCESS_ATTR_RO(1, read_latency);
75ACCESS_ATTR_RO(1, write_bandwidth);
76ACCESS_ATTR_RO(1, write_latency);
77
78static struct attribute *access1_coordinate_attrs[] = {
79 ACCESS_ATTR_DECLARE(1, read_bandwidth),
80 ACCESS_ATTR_DECLARE(1, write_bandwidth),
81 ACCESS_ATTR_DECLARE(1, read_latency),
82 ACCESS_ATTR_DECLARE(1, write_latency),
83 NULL
84};
85
86#define ACCESS_VISIBLE(level) \
87static umode_t cxl_region_access##level##_coordinate_visible( \
88 struct kobject *kobj, struct attribute *a, int n) \
89{ \
90 struct device *dev = kobj_to_dev(kobj); \
91 struct cxl_region *cxlr = to_cxl_region(dev); \
92 \
93 if (a == &dev_attr_access##level##_read_latency.attr && \
94 cxlr->coord[level].read_latency == 0) \
95 return 0; \
96 \
97 if (a == &dev_attr_access##level##_write_latency.attr && \
98 cxlr->coord[level].write_latency == 0) \
99 return 0; \
100 \
101 if (a == &dev_attr_access##level##_read_bandwidth.attr && \
102 cxlr->coord[level].read_bandwidth == 0) \
103 return 0; \
104 \
105 if (a == &dev_attr_access##level##_write_bandwidth.attr && \
106 cxlr->coord[level].write_bandwidth == 0) \
107 return 0; \
108 \
109 return a->mode; \
110}
111
112ACCESS_VISIBLE(0);
113ACCESS_VISIBLE(1);
114
115static const struct attribute_group cxl_region_access0_coordinate_group = {
116 .name = "access0",
117 .attrs = access0_coordinate_attrs,
118 .is_visible = cxl_region_access0_coordinate_visible,
119};
120
121static const struct attribute_group *get_cxl_region_access0_group(void)
122{
123 return &cxl_region_access0_coordinate_group;
124}
125
126static const struct attribute_group cxl_region_access1_coordinate_group = {
127 .name = "access1",
128 .attrs = access1_coordinate_attrs,
129 .is_visible = cxl_region_access1_coordinate_visible,
130};
131
132static const struct attribute_group *get_cxl_region_access1_group(void)
133{
134 return &cxl_region_access1_coordinate_group;
135}
136
137static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
138 char *buf)
139{
140 struct cxl_region *cxlr = to_cxl_region(dev);
141 struct cxl_region_params *p = &cxlr->params;
142 ssize_t rc;
143
144 rc = down_read_interruptible(&cxl_region_rwsem);
145 if (rc)
146 return rc;
147 if (cxlr->mode != CXL_DECODER_PMEM)
148 rc = sysfs_emit(buf, "\n");
149 else
150 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
151 up_read(&cxl_region_rwsem);
152
153 return rc;
154}
155
156static int is_dup(struct device *match, void *data)
157{
158 struct cxl_region_params *p;
159 struct cxl_region *cxlr;
160 uuid_t *uuid = data;
161
162 if (!is_cxl_region(match))
163 return 0;
164
165 lockdep_assert_held(&cxl_region_rwsem);
166 cxlr = to_cxl_region(match);
167 p = &cxlr->params;
168
169 if (uuid_equal(&p->uuid, uuid)) {
170 dev_dbg(match, "already has uuid: %pUb\n", uuid);
171 return -EBUSY;
172 }
173
174 return 0;
175}
176
177static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
178 const char *buf, size_t len)
179{
180 struct cxl_region *cxlr = to_cxl_region(dev);
181 struct cxl_region_params *p = &cxlr->params;
182 uuid_t temp;
183 ssize_t rc;
184
185 if (len != UUID_STRING_LEN + 1)
186 return -EINVAL;
187
188 rc = uuid_parse(buf, &temp);
189 if (rc)
190 return rc;
191
192 if (uuid_is_null(&temp))
193 return -EINVAL;
194
195 rc = down_write_killable(&cxl_region_rwsem);
196 if (rc)
197 return rc;
198
199 if (uuid_equal(&p->uuid, &temp))
200 goto out;
201
202 rc = -EBUSY;
203 if (p->state >= CXL_CONFIG_ACTIVE)
204 goto out;
205
206 rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
207 if (rc < 0)
208 goto out;
209
210 uuid_copy(&p->uuid, &temp);
211out:
212 up_write(&cxl_region_rwsem);
213
214 if (rc)
215 return rc;
216 return len;
217}
218static DEVICE_ATTR_RW(uuid);
219
220static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
221 struct cxl_region *cxlr)
222{
223 return xa_load(&port->regions, (unsigned long)cxlr);
224}
225
226static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
227{
228 if (!cpu_cache_has_invalidate_memregion()) {
229 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
230 dev_info_once(
231 &cxlr->dev,
232 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
233 return 0;
234 } else {
235 dev_WARN(&cxlr->dev,
236 "Failed to synchronize CPU cache state\n");
237 return -ENXIO;
238 }
239 }
240
241 cpu_cache_invalidate_memregion(IORES_DESC_CXL);
242 return 0;
243}
244
245static void cxl_region_decode_reset(struct cxl_region *cxlr, int count)
246{
247 struct cxl_region_params *p = &cxlr->params;
248 int i;
249
250 /*
251 * Before region teardown attempt to flush, evict any data cached for
252 * this region, or scream loudly about missing arch / platform support
253 * for CXL teardown.
254 */
255 cxl_region_invalidate_memregion(cxlr);
256
257 for (i = count - 1; i >= 0; i--) {
258 struct cxl_endpoint_decoder *cxled = p->targets[i];
259 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
260 struct cxl_port *iter = cxled_to_port(cxled);
261 struct cxl_dev_state *cxlds = cxlmd->cxlds;
262 struct cxl_ep *ep;
263
264 if (cxlds->rcd)
265 goto endpoint_reset;
266
267 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
268 iter = to_cxl_port(iter->dev.parent);
269
270 for (ep = cxl_ep_load(iter, cxlmd); iter;
271 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
272 struct cxl_region_ref *cxl_rr;
273 struct cxl_decoder *cxld;
274
275 cxl_rr = cxl_rr_load(iter, cxlr);
276 cxld = cxl_rr->decoder;
277 if (cxld->reset)
278 cxld->reset(cxld);
279 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
280 }
281
282endpoint_reset:
283 cxled->cxld.reset(&cxled->cxld);
284 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
285 }
286
287 /* all decoders associated with this region have been torn down */
288 clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
289}
290
291static int commit_decoder(struct cxl_decoder *cxld)
292{
293 struct cxl_switch_decoder *cxlsd = NULL;
294
295 if (cxld->commit)
296 return cxld->commit(cxld);
297
298 if (is_switch_decoder(&cxld->dev))
299 cxlsd = to_cxl_switch_decoder(&cxld->dev);
300
301 if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
302 "->commit() is required\n"))
303 return -ENXIO;
304 return 0;
305}
306
307static int cxl_region_decode_commit(struct cxl_region *cxlr)
308{
309 struct cxl_region_params *p = &cxlr->params;
310 int i, rc = 0;
311
312 for (i = 0; i < p->nr_targets; i++) {
313 struct cxl_endpoint_decoder *cxled = p->targets[i];
314 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
315 struct cxl_region_ref *cxl_rr;
316 struct cxl_decoder *cxld;
317 struct cxl_port *iter;
318 struct cxl_ep *ep;
319
320 /* commit bottom up */
321 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
322 iter = to_cxl_port(iter->dev.parent)) {
323 cxl_rr = cxl_rr_load(iter, cxlr);
324 cxld = cxl_rr->decoder;
325 rc = commit_decoder(cxld);
326 if (rc)
327 break;
328 }
329
330 if (rc) {
331 /* programming @iter failed, teardown */
332 for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
333 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
334 cxl_rr = cxl_rr_load(iter, cxlr);
335 cxld = cxl_rr->decoder;
336 if (cxld->reset)
337 cxld->reset(cxld);
338 }
339
340 cxled->cxld.reset(&cxled->cxld);
341 goto err;
342 }
343 }
344
345 return 0;
346
347err:
348 /* undo the targets that were successfully committed */
349 cxl_region_decode_reset(cxlr, i);
350 return rc;
351}
352
353static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
354 const char *buf, size_t len)
355{
356 struct cxl_region *cxlr = to_cxl_region(dev);
357 struct cxl_region_params *p = &cxlr->params;
358 bool commit;
359 ssize_t rc;
360
361 rc = kstrtobool(buf, &commit);
362 if (rc)
363 return rc;
364
365 rc = down_write_killable(&cxl_region_rwsem);
366 if (rc)
367 return rc;
368
369 /* Already in the requested state? */
370 if (commit && p->state >= CXL_CONFIG_COMMIT)
371 goto out;
372 if (!commit && p->state < CXL_CONFIG_COMMIT)
373 goto out;
374
375 /* Not ready to commit? */
376 if (commit && p->state < CXL_CONFIG_ACTIVE) {
377 rc = -ENXIO;
378 goto out;
379 }
380
381 /*
382 * Invalidate caches before region setup to drop any speculative
383 * consumption of this address space
384 */
385 rc = cxl_region_invalidate_memregion(cxlr);
386 if (rc)
387 goto out;
388
389 if (commit) {
390 rc = cxl_region_decode_commit(cxlr);
391 if (rc == 0)
392 p->state = CXL_CONFIG_COMMIT;
393 } else {
394 p->state = CXL_CONFIG_RESET_PENDING;
395 up_write(&cxl_region_rwsem);
396 device_release_driver(&cxlr->dev);
397 down_write(&cxl_region_rwsem);
398
399 /*
400 * The lock was dropped, so need to revalidate that the reset is
401 * still pending.
402 */
403 if (p->state == CXL_CONFIG_RESET_PENDING) {
404 cxl_region_decode_reset(cxlr, p->interleave_ways);
405 p->state = CXL_CONFIG_ACTIVE;
406 }
407 }
408
409out:
410 up_write(&cxl_region_rwsem);
411
412 if (rc)
413 return rc;
414 return len;
415}
416
417static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
418 char *buf)
419{
420 struct cxl_region *cxlr = to_cxl_region(dev);
421 struct cxl_region_params *p = &cxlr->params;
422 ssize_t rc;
423
424 rc = down_read_interruptible(&cxl_region_rwsem);
425 if (rc)
426 return rc;
427 rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
428 up_read(&cxl_region_rwsem);
429
430 return rc;
431}
432static DEVICE_ATTR_RW(commit);
433
434static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
435 int n)
436{
437 struct device *dev = kobj_to_dev(kobj);
438 struct cxl_region *cxlr = to_cxl_region(dev);
439
440 /*
441 * Support tooling that expects to find a 'uuid' attribute for all
442 * regions regardless of mode.
443 */
444 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
445 return 0444;
446 return a->mode;
447}
448
449static ssize_t interleave_ways_show(struct device *dev,
450 struct device_attribute *attr, char *buf)
451{
452 struct cxl_region *cxlr = to_cxl_region(dev);
453 struct cxl_region_params *p = &cxlr->params;
454 ssize_t rc;
455
456 rc = down_read_interruptible(&cxl_region_rwsem);
457 if (rc)
458 return rc;
459 rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
460 up_read(&cxl_region_rwsem);
461
462 return rc;
463}
464
465static const struct attribute_group *get_cxl_region_target_group(void);
466
467static ssize_t interleave_ways_store(struct device *dev,
468 struct device_attribute *attr,
469 const char *buf, size_t len)
470{
471 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
472 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
473 struct cxl_region *cxlr = to_cxl_region(dev);
474 struct cxl_region_params *p = &cxlr->params;
475 unsigned int val, save;
476 int rc;
477 u8 iw;
478
479 rc = kstrtouint(buf, 0, &val);
480 if (rc)
481 return rc;
482
483 rc = ways_to_eiw(val, &iw);
484 if (rc)
485 return rc;
486
487 /*
488 * Even for x3, x6, and x12 interleaves the region interleave must be a
489 * power of 2 multiple of the host bridge interleave.
490 */
491 if (!is_power_of_2(val / cxld->interleave_ways) ||
492 (val % cxld->interleave_ways)) {
493 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
494 return -EINVAL;
495 }
496
497 rc = down_write_killable(&cxl_region_rwsem);
498 if (rc)
499 return rc;
500 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
501 rc = -EBUSY;
502 goto out;
503 }
504
505 save = p->interleave_ways;
506 p->interleave_ways = val;
507 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
508 if (rc)
509 p->interleave_ways = save;
510out:
511 up_write(&cxl_region_rwsem);
512 if (rc)
513 return rc;
514 return len;
515}
516static DEVICE_ATTR_RW(interleave_ways);
517
518static ssize_t interleave_granularity_show(struct device *dev,
519 struct device_attribute *attr,
520 char *buf)
521{
522 struct cxl_region *cxlr = to_cxl_region(dev);
523 struct cxl_region_params *p = &cxlr->params;
524 ssize_t rc;
525
526 rc = down_read_interruptible(&cxl_region_rwsem);
527 if (rc)
528 return rc;
529 rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
530 up_read(&cxl_region_rwsem);
531
532 return rc;
533}
534
535static ssize_t interleave_granularity_store(struct device *dev,
536 struct device_attribute *attr,
537 const char *buf, size_t len)
538{
539 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
540 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
541 struct cxl_region *cxlr = to_cxl_region(dev);
542 struct cxl_region_params *p = &cxlr->params;
543 int rc, val;
544 u16 ig;
545
546 rc = kstrtoint(buf, 0, &val);
547 if (rc)
548 return rc;
549
550 rc = granularity_to_eig(val, &ig);
551 if (rc)
552 return rc;
553
554 /*
555 * When the host-bridge is interleaved, disallow region granularity !=
556 * root granularity. Regions with a granularity less than the root
557 * interleave result in needing multiple endpoints to support a single
558 * slot in the interleave (possible to support in the future). Regions
559 * with a granularity greater than the root interleave result in invalid
560 * DPA translations (invalid to support).
561 */
562 if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
563 return -EINVAL;
564
565 rc = down_write_killable(&cxl_region_rwsem);
566 if (rc)
567 return rc;
568 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
569 rc = -EBUSY;
570 goto out;
571 }
572
573 p->interleave_granularity = val;
574out:
575 up_write(&cxl_region_rwsem);
576 if (rc)
577 return rc;
578 return len;
579}
580static DEVICE_ATTR_RW(interleave_granularity);
581
582static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
583 char *buf)
584{
585 struct cxl_region *cxlr = to_cxl_region(dev);
586 struct cxl_region_params *p = &cxlr->params;
587 u64 resource = -1ULL;
588 ssize_t rc;
589
590 rc = down_read_interruptible(&cxl_region_rwsem);
591 if (rc)
592 return rc;
593 if (p->res)
594 resource = p->res->start;
595 rc = sysfs_emit(buf, "%#llx\n", resource);
596 up_read(&cxl_region_rwsem);
597
598 return rc;
599}
600static DEVICE_ATTR_RO(resource);
601
602static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
603 char *buf)
604{
605 struct cxl_region *cxlr = to_cxl_region(dev);
606
607 return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
608}
609static DEVICE_ATTR_RO(mode);
610
611static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
612{
613 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
614 struct cxl_region_params *p = &cxlr->params;
615 struct resource *res;
616 u64 remainder = 0;
617
618 lockdep_assert_held_write(&cxl_region_rwsem);
619
620 /* Nothing to do... */
621 if (p->res && resource_size(p->res) == size)
622 return 0;
623
624 /* To change size the old size must be freed first */
625 if (p->res)
626 return -EBUSY;
627
628 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
629 return -EBUSY;
630
631 /* ways, granularity and uuid (if PMEM) need to be set before HPA */
632 if (!p->interleave_ways || !p->interleave_granularity ||
633 (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
634 return -ENXIO;
635
636 div64_u64_rem(size, (u64)SZ_256M * p->interleave_ways, &remainder);
637 if (remainder)
638 return -EINVAL;
639
640 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
641 dev_name(&cxlr->dev));
642 if (IS_ERR(res)) {
643 dev_dbg(&cxlr->dev,
644 "HPA allocation error (%ld) for size:%pap in %s %pr\n",
645 PTR_ERR(res), &size, cxlrd->res->name, cxlrd->res);
646 return PTR_ERR(res);
647 }
648
649 p->res = res;
650 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
651
652 return 0;
653}
654
655static void cxl_region_iomem_release(struct cxl_region *cxlr)
656{
657 struct cxl_region_params *p = &cxlr->params;
658
659 if (device_is_registered(&cxlr->dev))
660 lockdep_assert_held_write(&cxl_region_rwsem);
661 if (p->res) {
662 /*
663 * Autodiscovered regions may not have been able to insert their
664 * resource.
665 */
666 if (p->res->parent)
667 remove_resource(p->res);
668 kfree(p->res);
669 p->res = NULL;
670 }
671}
672
673static int free_hpa(struct cxl_region *cxlr)
674{
675 struct cxl_region_params *p = &cxlr->params;
676
677 lockdep_assert_held_write(&cxl_region_rwsem);
678
679 if (!p->res)
680 return 0;
681
682 if (p->state >= CXL_CONFIG_ACTIVE)
683 return -EBUSY;
684
685 cxl_region_iomem_release(cxlr);
686 p->state = CXL_CONFIG_IDLE;
687 return 0;
688}
689
690static ssize_t size_store(struct device *dev, struct device_attribute *attr,
691 const char *buf, size_t len)
692{
693 struct cxl_region *cxlr = to_cxl_region(dev);
694 u64 val;
695 int rc;
696
697 rc = kstrtou64(buf, 0, &val);
698 if (rc)
699 return rc;
700
701 rc = down_write_killable(&cxl_region_rwsem);
702 if (rc)
703 return rc;
704
705 if (val)
706 rc = alloc_hpa(cxlr, val);
707 else
708 rc = free_hpa(cxlr);
709 up_write(&cxl_region_rwsem);
710
711 if (rc)
712 return rc;
713
714 return len;
715}
716
717static ssize_t size_show(struct device *dev, struct device_attribute *attr,
718 char *buf)
719{
720 struct cxl_region *cxlr = to_cxl_region(dev);
721 struct cxl_region_params *p = &cxlr->params;
722 u64 size = 0;
723 ssize_t rc;
724
725 rc = down_read_interruptible(&cxl_region_rwsem);
726 if (rc)
727 return rc;
728 if (p->res)
729 size = resource_size(p->res);
730 rc = sysfs_emit(buf, "%#llx\n", size);
731 up_read(&cxl_region_rwsem);
732
733 return rc;
734}
735static DEVICE_ATTR_RW(size);
736
737static struct attribute *cxl_region_attrs[] = {
738 &dev_attr_uuid.attr,
739 &dev_attr_commit.attr,
740 &dev_attr_interleave_ways.attr,
741 &dev_attr_interleave_granularity.attr,
742 &dev_attr_resource.attr,
743 &dev_attr_size.attr,
744 &dev_attr_mode.attr,
745 NULL,
746};
747
748static const struct attribute_group cxl_region_group = {
749 .attrs = cxl_region_attrs,
750 .is_visible = cxl_region_visible,
751};
752
753static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
754{
755 struct cxl_region_params *p = &cxlr->params;
756 struct cxl_endpoint_decoder *cxled;
757 int rc;
758
759 rc = down_read_interruptible(&cxl_region_rwsem);
760 if (rc)
761 return rc;
762
763 if (pos >= p->interleave_ways) {
764 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
765 p->interleave_ways);
766 rc = -ENXIO;
767 goto out;
768 }
769
770 cxled = p->targets[pos];
771 if (!cxled)
772 rc = sysfs_emit(buf, "\n");
773 else
774 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
775out:
776 up_read(&cxl_region_rwsem);
777
778 return rc;
779}
780
781static int check_commit_order(struct device *dev, const void *data)
782{
783 struct cxl_decoder *cxld = to_cxl_decoder(dev);
784
785 /*
786 * if port->commit_end is not the only free decoder, then out of
787 * order shutdown has occurred, block further allocations until
788 * that is resolved
789 */
790 if (((cxld->flags & CXL_DECODER_F_ENABLE) == 0))
791 return -EBUSY;
792 return 0;
793}
794
795static int match_free_decoder(struct device *dev, void *data)
796{
797 struct cxl_port *port = to_cxl_port(dev->parent);
798 struct cxl_decoder *cxld;
799 int rc;
800
801 if (!is_switch_decoder(dev))
802 return 0;
803
804 cxld = to_cxl_decoder(dev);
805
806 if (cxld->id != port->commit_end + 1)
807 return 0;
808
809 if (cxld->region) {
810 dev_dbg(dev->parent,
811 "next decoder to commit (%s) is already reserved (%s)\n",
812 dev_name(dev), dev_name(&cxld->region->dev));
813 return 0;
814 }
815
816 rc = device_for_each_child_reverse_from(dev->parent, dev, NULL,
817 check_commit_order);
818 if (rc) {
819 dev_dbg(dev->parent,
820 "unable to allocate %s due to out of order shutdown\n",
821 dev_name(dev));
822 return 0;
823 }
824 return 1;
825}
826
827static int match_auto_decoder(struct device *dev, void *data)
828{
829 struct cxl_region_params *p = data;
830 struct cxl_decoder *cxld;
831 struct range *r;
832
833 if (!is_switch_decoder(dev))
834 return 0;
835
836 cxld = to_cxl_decoder(dev);
837 r = &cxld->hpa_range;
838
839 if (p->res && p->res->start == r->start && p->res->end == r->end)
840 return 1;
841
842 return 0;
843}
844
845static struct cxl_decoder *
846cxl_region_find_decoder(struct cxl_port *port,
847 struct cxl_endpoint_decoder *cxled,
848 struct cxl_region *cxlr)
849{
850 struct device *dev;
851
852 if (port == cxled_to_port(cxled))
853 return &cxled->cxld;
854
855 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
856 dev = device_find_child(&port->dev, &cxlr->params,
857 match_auto_decoder);
858 else
859 dev = device_find_child(&port->dev, NULL, match_free_decoder);
860 if (!dev)
861 return NULL;
862 /*
863 * This decoder is pinned registered as long as the endpoint decoder is
864 * registered, and endpoint decoder unregistration holds the
865 * cxl_region_rwsem over unregister events, so no need to hold on to
866 * this extra reference.
867 */
868 put_device(dev);
869 return to_cxl_decoder(dev);
870}
871
872static bool auto_order_ok(struct cxl_port *port, struct cxl_region *cxlr_iter,
873 struct cxl_decoder *cxld)
874{
875 struct cxl_region_ref *rr = cxl_rr_load(port, cxlr_iter);
876 struct cxl_decoder *cxld_iter = rr->decoder;
877
878 /*
879 * Allow the out of order assembly of auto-discovered regions.
880 * Per CXL Spec 3.1 8.2.4.20.12 software must commit decoders
881 * in HPA order. Confirm that the decoder with the lesser HPA
882 * starting address has the lesser id.
883 */
884 dev_dbg(&cxld->dev, "check for HPA violation %s:%d < %s:%d\n",
885 dev_name(&cxld->dev), cxld->id,
886 dev_name(&cxld_iter->dev), cxld_iter->id);
887
888 if (cxld_iter->id > cxld->id)
889 return true;
890
891 return false;
892}
893
894static struct cxl_region_ref *
895alloc_region_ref(struct cxl_port *port, struct cxl_region *cxlr,
896 struct cxl_endpoint_decoder *cxled)
897{
898 struct cxl_region_params *p = &cxlr->params;
899 struct cxl_region_ref *cxl_rr, *iter;
900 unsigned long index;
901 int rc;
902
903 xa_for_each(&port->regions, index, iter) {
904 struct cxl_region_params *ip = &iter->region->params;
905
906 if (!ip->res || ip->res->start < p->res->start)
907 continue;
908
909 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
910 struct cxl_decoder *cxld;
911
912 cxld = cxl_region_find_decoder(port, cxled, cxlr);
913 if (auto_order_ok(port, iter->region, cxld))
914 continue;
915 }
916 dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n",
917 dev_name(&port->dev),
918 dev_name(&iter->region->dev), ip->res, p->res);
919
920 return ERR_PTR(-EBUSY);
921 }
922
923 cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
924 if (!cxl_rr)
925 return ERR_PTR(-ENOMEM);
926 cxl_rr->port = port;
927 cxl_rr->region = cxlr;
928 cxl_rr->nr_targets = 1;
929 xa_init(&cxl_rr->endpoints);
930
931 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
932 if (rc) {
933 dev_dbg(&cxlr->dev,
934 "%s: failed to track region reference: %d\n",
935 dev_name(&port->dev), rc);
936 kfree(cxl_rr);
937 return ERR_PTR(rc);
938 }
939
940 return cxl_rr;
941}
942
943static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
944{
945 struct cxl_region *cxlr = cxl_rr->region;
946 struct cxl_decoder *cxld = cxl_rr->decoder;
947
948 if (!cxld)
949 return;
950
951 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
952 if (cxld->region == cxlr) {
953 cxld->region = NULL;
954 put_device(&cxlr->dev);
955 }
956}
957
958static void free_region_ref(struct cxl_region_ref *cxl_rr)
959{
960 struct cxl_port *port = cxl_rr->port;
961 struct cxl_region *cxlr = cxl_rr->region;
962
963 cxl_rr_free_decoder(cxl_rr);
964 xa_erase(&port->regions, (unsigned long)cxlr);
965 xa_destroy(&cxl_rr->endpoints);
966 kfree(cxl_rr);
967}
968
969static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
970 struct cxl_endpoint_decoder *cxled)
971{
972 int rc;
973 struct cxl_port *port = cxl_rr->port;
974 struct cxl_region *cxlr = cxl_rr->region;
975 struct cxl_decoder *cxld = cxl_rr->decoder;
976 struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
977
978 if (ep) {
979 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
980 GFP_KERNEL);
981 if (rc)
982 return rc;
983 }
984 cxl_rr->nr_eps++;
985
986 if (!cxld->region) {
987 cxld->region = cxlr;
988 get_device(&cxlr->dev);
989 }
990
991 return 0;
992}
993
994static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
995 struct cxl_endpoint_decoder *cxled,
996 struct cxl_region_ref *cxl_rr)
997{
998 struct cxl_decoder *cxld;
999
1000 cxld = cxl_region_find_decoder(port, cxled, cxlr);
1001 if (!cxld) {
1002 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
1003 dev_name(&port->dev));
1004 return -EBUSY;
1005 }
1006
1007 if (cxld->region) {
1008 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
1009 dev_name(&port->dev), dev_name(&cxld->dev),
1010 dev_name(&cxld->region->dev));
1011 return -EBUSY;
1012 }
1013
1014 /*
1015 * Endpoints should already match the region type, but backstop that
1016 * assumption with an assertion. Switch-decoders change mapping-type
1017 * based on what is mapped when they are assigned to a region.
1018 */
1019 dev_WARN_ONCE(&cxlr->dev,
1020 port == cxled_to_port(cxled) &&
1021 cxld->target_type != cxlr->type,
1022 "%s:%s mismatch decoder type %d -> %d\n",
1023 dev_name(&cxled_to_memdev(cxled)->dev),
1024 dev_name(&cxld->dev), cxld->target_type, cxlr->type);
1025 cxld->target_type = cxlr->type;
1026 cxl_rr->decoder = cxld;
1027 return 0;
1028}
1029
1030/**
1031 * cxl_port_attach_region() - track a region's interest in a port by endpoint
1032 * @port: port to add a new region reference 'struct cxl_region_ref'
1033 * @cxlr: region to attach to @port
1034 * @cxled: endpoint decoder used to create or further pin a region reference
1035 * @pos: interleave position of @cxled in @cxlr
1036 *
1037 * The attach event is an opportunity to validate CXL decode setup
1038 * constraints and record metadata needed for programming HDM decoders,
1039 * in particular decoder target lists.
1040 *
1041 * The steps are:
1042 *
1043 * - validate that there are no other regions with a higher HPA already
1044 * associated with @port
1045 * - establish a region reference if one is not already present
1046 *
1047 * - additionally allocate a decoder instance that will host @cxlr on
1048 * @port
1049 *
1050 * - pin the region reference by the endpoint
1051 * - account for how many entries in @port's target list are needed to
1052 * cover all of the added endpoints.
1053 */
1054static int cxl_port_attach_region(struct cxl_port *port,
1055 struct cxl_region *cxlr,
1056 struct cxl_endpoint_decoder *cxled, int pos)
1057{
1058 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1059 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1060 struct cxl_region_ref *cxl_rr;
1061 bool nr_targets_inc = false;
1062 struct cxl_decoder *cxld;
1063 unsigned long index;
1064 int rc = -EBUSY;
1065
1066 lockdep_assert_held_write(&cxl_region_rwsem);
1067
1068 cxl_rr = cxl_rr_load(port, cxlr);
1069 if (cxl_rr) {
1070 struct cxl_ep *ep_iter;
1071 int found = 0;
1072
1073 /*
1074 * Walk the existing endpoints that have been attached to
1075 * @cxlr at @port and see if they share the same 'next' port
1076 * in the downstream direction. I.e. endpoints that share common
1077 * upstream switch.
1078 */
1079 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1080 if (ep_iter == ep)
1081 continue;
1082 if (ep_iter->next == ep->next) {
1083 found++;
1084 break;
1085 }
1086 }
1087
1088 /*
1089 * New target port, or @port is an endpoint port that always
1090 * accounts its own local decode as a target.
1091 */
1092 if (!found || !ep->next) {
1093 cxl_rr->nr_targets++;
1094 nr_targets_inc = true;
1095 }
1096 } else {
1097 cxl_rr = alloc_region_ref(port, cxlr, cxled);
1098 if (IS_ERR(cxl_rr)) {
1099 dev_dbg(&cxlr->dev,
1100 "%s: failed to allocate region reference\n",
1101 dev_name(&port->dev));
1102 return PTR_ERR(cxl_rr);
1103 }
1104 nr_targets_inc = true;
1105
1106 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
1107 if (rc)
1108 goto out_erase;
1109 }
1110 cxld = cxl_rr->decoder;
1111
1112 /*
1113 * the number of targets should not exceed the target_count
1114 * of the decoder
1115 */
1116 if (is_switch_decoder(&cxld->dev)) {
1117 struct cxl_switch_decoder *cxlsd;
1118
1119 cxlsd = to_cxl_switch_decoder(&cxld->dev);
1120 if (cxl_rr->nr_targets > cxlsd->nr_targets) {
1121 dev_dbg(&cxlr->dev,
1122 "%s:%s %s add: %s:%s @ %d overflows targets: %d\n",
1123 dev_name(port->uport_dev), dev_name(&port->dev),
1124 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1125 dev_name(&cxled->cxld.dev), pos,
1126 cxlsd->nr_targets);
1127 rc = -ENXIO;
1128 goto out_erase;
1129 }
1130 }
1131
1132 rc = cxl_rr_ep_add(cxl_rr, cxled);
1133 if (rc) {
1134 dev_dbg(&cxlr->dev,
1135 "%s: failed to track endpoint %s:%s reference\n",
1136 dev_name(&port->dev), dev_name(&cxlmd->dev),
1137 dev_name(&cxld->dev));
1138 goto out_erase;
1139 }
1140
1141 dev_dbg(&cxlr->dev,
1142 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
1143 dev_name(port->uport_dev), dev_name(&port->dev),
1144 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1145 dev_name(&cxled->cxld.dev), pos,
1146 ep ? ep->next ? dev_name(ep->next->uport_dev) :
1147 dev_name(&cxlmd->dev) :
1148 "none",
1149 cxl_rr->nr_eps, cxl_rr->nr_targets);
1150
1151 return 0;
1152out_erase:
1153 if (nr_targets_inc)
1154 cxl_rr->nr_targets--;
1155 if (cxl_rr->nr_eps == 0)
1156 free_region_ref(cxl_rr);
1157 return rc;
1158}
1159
1160static void cxl_port_detach_region(struct cxl_port *port,
1161 struct cxl_region *cxlr,
1162 struct cxl_endpoint_decoder *cxled)
1163{
1164 struct cxl_region_ref *cxl_rr;
1165 struct cxl_ep *ep = NULL;
1166
1167 lockdep_assert_held_write(&cxl_region_rwsem);
1168
1169 cxl_rr = cxl_rr_load(port, cxlr);
1170 if (!cxl_rr)
1171 return;
1172
1173 /*
1174 * Endpoint ports do not carry cxl_ep references, and they
1175 * never target more than one endpoint by definition
1176 */
1177 if (cxl_rr->decoder == &cxled->cxld)
1178 cxl_rr->nr_eps--;
1179 else
1180 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
1181 if (ep) {
1182 struct cxl_ep *ep_iter;
1183 unsigned long index;
1184 int found = 0;
1185
1186 cxl_rr->nr_eps--;
1187 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1188 if (ep_iter->next == ep->next) {
1189 found++;
1190 break;
1191 }
1192 }
1193 if (!found)
1194 cxl_rr->nr_targets--;
1195 }
1196
1197 if (cxl_rr->nr_eps == 0)
1198 free_region_ref(cxl_rr);
1199}
1200
1201static int check_last_peer(struct cxl_endpoint_decoder *cxled,
1202 struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
1203 int distance)
1204{
1205 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1206 struct cxl_region *cxlr = cxl_rr->region;
1207 struct cxl_region_params *p = &cxlr->params;
1208 struct cxl_endpoint_decoder *cxled_peer;
1209 struct cxl_port *port = cxl_rr->port;
1210 struct cxl_memdev *cxlmd_peer;
1211 struct cxl_ep *ep_peer;
1212 int pos = cxled->pos;
1213
1214 /*
1215 * If this position wants to share a dport with the last endpoint mapped
1216 * then that endpoint, at index 'position - distance', must also be
1217 * mapped by this dport.
1218 */
1219 if (pos < distance) {
1220 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
1221 dev_name(port->uport_dev), dev_name(&port->dev),
1222 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1223 return -ENXIO;
1224 }
1225 cxled_peer = p->targets[pos - distance];
1226 cxlmd_peer = cxled_to_memdev(cxled_peer);
1227 ep_peer = cxl_ep_load(port, cxlmd_peer);
1228 if (ep->dport != ep_peer->dport) {
1229 dev_dbg(&cxlr->dev,
1230 "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
1231 dev_name(port->uport_dev), dev_name(&port->dev),
1232 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
1233 dev_name(&cxlmd_peer->dev),
1234 dev_name(&cxled_peer->cxld.dev));
1235 return -ENXIO;
1236 }
1237
1238 return 0;
1239}
1240
1241static int check_interleave_cap(struct cxl_decoder *cxld, int iw, int ig)
1242{
1243 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
1244 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
1245 unsigned int interleave_mask;
1246 u8 eiw;
1247 u16 eig;
1248 int high_pos, low_pos;
1249
1250 if (!test_bit(iw, &cxlhdm->iw_cap_mask))
1251 return -ENXIO;
1252 /*
1253 * Per CXL specification r3.1(8.2.4.20.13 Decoder Protection),
1254 * if eiw < 8:
1255 * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + 8 + eiw]
1256 * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0]
1257 *
1258 * when the eiw is 0, all the bits of HPAOFFSET[51: 0] are used, the
1259 * interleave bits are none.
1260 *
1261 * if eiw >= 8:
1262 * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + eiw] / 3
1263 * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0]
1264 *
1265 * when the eiw is 8, all the bits of HPAOFFSET[51: 0] are used, the
1266 * interleave bits are none.
1267 */
1268 ways_to_eiw(iw, &eiw);
1269 if (eiw == 0 || eiw == 8)
1270 return 0;
1271
1272 granularity_to_eig(ig, &eig);
1273 if (eiw > 8)
1274 high_pos = eiw + eig - 1;
1275 else
1276 high_pos = eiw + eig + 7;
1277 low_pos = eig + 8;
1278 interleave_mask = GENMASK(high_pos, low_pos);
1279 if (interleave_mask & ~cxlhdm->interleave_mask)
1280 return -ENXIO;
1281
1282 return 0;
1283}
1284
1285static int cxl_port_setup_targets(struct cxl_port *port,
1286 struct cxl_region *cxlr,
1287 struct cxl_endpoint_decoder *cxled)
1288{
1289 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1290 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1291 struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1292 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1293 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1294 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1295 struct cxl_region_params *p = &cxlr->params;
1296 struct cxl_decoder *cxld = cxl_rr->decoder;
1297 struct cxl_switch_decoder *cxlsd;
1298 struct cxl_port *iter = port;
1299 u16 eig, peig;
1300 u8 eiw, peiw;
1301
1302 /*
1303 * While root level decoders support x3, x6, x12, switch level
1304 * decoders only support powers of 2 up to x16.
1305 */
1306 if (!is_power_of_2(cxl_rr->nr_targets)) {
1307 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1308 dev_name(port->uport_dev), dev_name(&port->dev),
1309 cxl_rr->nr_targets);
1310 return -EINVAL;
1311 }
1312
1313 cxlsd = to_cxl_switch_decoder(&cxld->dev);
1314 if (cxl_rr->nr_targets_set) {
1315 int i, distance = 1;
1316 struct cxl_region_ref *cxl_rr_iter;
1317
1318 /*
1319 * The "distance" between peer downstream ports represents which
1320 * endpoint positions in the region interleave a given port can
1321 * host.
1322 *
1323 * For example, at the root of a hierarchy the distance is
1324 * always 1 as every index targets a different host-bridge. At
1325 * each subsequent switch level those ports map every Nth region
1326 * position where N is the width of the switch == distance.
1327 */
1328 do {
1329 cxl_rr_iter = cxl_rr_load(iter, cxlr);
1330 distance *= cxl_rr_iter->nr_targets;
1331 iter = to_cxl_port(iter->dev.parent);
1332 } while (!is_cxl_root(iter));
1333 distance *= cxlrd->cxlsd.cxld.interleave_ways;
1334
1335 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1336 if (ep->dport == cxlsd->target[i]) {
1337 rc = check_last_peer(cxled, ep, cxl_rr,
1338 distance);
1339 if (rc)
1340 return rc;
1341 goto out_target_set;
1342 }
1343 goto add_target;
1344 }
1345
1346 if (is_cxl_root(parent_port)) {
1347 /*
1348 * Root decoder IG is always set to value in CFMWS which
1349 * may be different than this region's IG. We can use the
1350 * region's IG here since interleave_granularity_store()
1351 * does not allow interleaved host-bridges with
1352 * root IG != region IG.
1353 */
1354 parent_ig = p->interleave_granularity;
1355 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1356 /*
1357 * For purposes of address bit routing, use power-of-2 math for
1358 * switch ports.
1359 */
1360 if (!is_power_of_2(parent_iw))
1361 parent_iw /= 3;
1362 } else {
1363 struct cxl_region_ref *parent_rr;
1364 struct cxl_decoder *parent_cxld;
1365
1366 parent_rr = cxl_rr_load(parent_port, cxlr);
1367 parent_cxld = parent_rr->decoder;
1368 parent_ig = parent_cxld->interleave_granularity;
1369 parent_iw = parent_cxld->interleave_ways;
1370 }
1371
1372 rc = granularity_to_eig(parent_ig, &peig);
1373 if (rc) {
1374 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1375 dev_name(parent_port->uport_dev),
1376 dev_name(&parent_port->dev), parent_ig);
1377 return rc;
1378 }
1379
1380 rc = ways_to_eiw(parent_iw, &peiw);
1381 if (rc) {
1382 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1383 dev_name(parent_port->uport_dev),
1384 dev_name(&parent_port->dev), parent_iw);
1385 return rc;
1386 }
1387
1388 iw = cxl_rr->nr_targets;
1389 rc = ways_to_eiw(iw, &eiw);
1390 if (rc) {
1391 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1392 dev_name(port->uport_dev), dev_name(&port->dev), iw);
1393 return rc;
1394 }
1395
1396 /*
1397 * Interleave granularity is a multiple of @parent_port granularity.
1398 * Multiplier is the parent port interleave ways.
1399 */
1400 rc = granularity_to_eig(parent_ig * parent_iw, &eig);
1401 if (rc) {
1402 dev_dbg(&cxlr->dev,
1403 "%s: invalid granularity calculation (%d * %d)\n",
1404 dev_name(&parent_port->dev), parent_ig, parent_iw);
1405 return rc;
1406 }
1407
1408 rc = eig_to_granularity(eig, &ig);
1409 if (rc) {
1410 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1411 dev_name(port->uport_dev), dev_name(&port->dev),
1412 256 << eig);
1413 return rc;
1414 }
1415
1416 if (iw > 8 || iw > cxlsd->nr_targets) {
1417 dev_dbg(&cxlr->dev,
1418 "%s:%s:%s: ways: %d overflows targets: %d\n",
1419 dev_name(port->uport_dev), dev_name(&port->dev),
1420 dev_name(&cxld->dev), iw, cxlsd->nr_targets);
1421 return -ENXIO;
1422 }
1423
1424 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1425 if (cxld->interleave_ways != iw ||
1426 cxld->interleave_granularity != ig ||
1427 cxld->hpa_range.start != p->res->start ||
1428 cxld->hpa_range.end != p->res->end ||
1429 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1430 dev_err(&cxlr->dev,
1431 "%s:%s %s expected iw: %d ig: %d %pr\n",
1432 dev_name(port->uport_dev), dev_name(&port->dev),
1433 __func__, iw, ig, p->res);
1434 dev_err(&cxlr->dev,
1435 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1436 dev_name(port->uport_dev), dev_name(&port->dev),
1437 __func__, cxld->interleave_ways,
1438 cxld->interleave_granularity,
1439 (cxld->flags & CXL_DECODER_F_ENABLE) ?
1440 "enabled" :
1441 "disabled",
1442 cxld->hpa_range.start, cxld->hpa_range.end);
1443 return -ENXIO;
1444 }
1445 } else {
1446 rc = check_interleave_cap(cxld, iw, ig);
1447 if (rc) {
1448 dev_dbg(&cxlr->dev,
1449 "%s:%s iw: %d ig: %d is not supported\n",
1450 dev_name(port->uport_dev),
1451 dev_name(&port->dev), iw, ig);
1452 return rc;
1453 }
1454
1455 cxld->interleave_ways = iw;
1456 cxld->interleave_granularity = ig;
1457 cxld->hpa_range = (struct range) {
1458 .start = p->res->start,
1459 .end = p->res->end,
1460 };
1461 }
1462 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
1463 dev_name(&port->dev), iw, ig);
1464add_target:
1465 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1466 dev_dbg(&cxlr->dev,
1467 "%s:%s: targets full trying to add %s:%s at %d\n",
1468 dev_name(port->uport_dev), dev_name(&port->dev),
1469 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1470 return -ENXIO;
1471 }
1472 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1473 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1474 dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1475 dev_name(port->uport_dev), dev_name(&port->dev),
1476 dev_name(&cxlsd->cxld.dev),
1477 dev_name(ep->dport->dport_dev),
1478 cxl_rr->nr_targets_set);
1479 return -ENXIO;
1480 }
1481 } else
1482 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1483 inc = 1;
1484out_target_set:
1485 cxl_rr->nr_targets_set += inc;
1486 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1487 dev_name(port->uport_dev), dev_name(&port->dev),
1488 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
1489 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1490
1491 return 0;
1492}
1493
1494static void cxl_port_reset_targets(struct cxl_port *port,
1495 struct cxl_region *cxlr)
1496{
1497 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1498 struct cxl_decoder *cxld;
1499
1500 /*
1501 * After the last endpoint has been detached the entire cxl_rr may now
1502 * be gone.
1503 */
1504 if (!cxl_rr)
1505 return;
1506 cxl_rr->nr_targets_set = 0;
1507
1508 cxld = cxl_rr->decoder;
1509 cxld->hpa_range = (struct range) {
1510 .start = 0,
1511 .end = -1,
1512 };
1513}
1514
1515static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1516{
1517 struct cxl_region_params *p = &cxlr->params;
1518 struct cxl_endpoint_decoder *cxled;
1519 struct cxl_dev_state *cxlds;
1520 struct cxl_memdev *cxlmd;
1521 struct cxl_port *iter;
1522 struct cxl_ep *ep;
1523 int i;
1524
1525 /*
1526 * In the auto-discovery case skip automatic teardown since the
1527 * address space is already active
1528 */
1529 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1530 return;
1531
1532 for (i = 0; i < p->nr_targets; i++) {
1533 cxled = p->targets[i];
1534 cxlmd = cxled_to_memdev(cxled);
1535 cxlds = cxlmd->cxlds;
1536
1537 if (cxlds->rcd)
1538 continue;
1539
1540 iter = cxled_to_port(cxled);
1541 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1542 iter = to_cxl_port(iter->dev.parent);
1543
1544 for (ep = cxl_ep_load(iter, cxlmd); iter;
1545 iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1546 cxl_port_reset_targets(iter, cxlr);
1547 }
1548}
1549
1550static int cxl_region_setup_targets(struct cxl_region *cxlr)
1551{
1552 struct cxl_region_params *p = &cxlr->params;
1553 struct cxl_endpoint_decoder *cxled;
1554 struct cxl_dev_state *cxlds;
1555 int i, rc, rch = 0, vh = 0;
1556 struct cxl_memdev *cxlmd;
1557 struct cxl_port *iter;
1558 struct cxl_ep *ep;
1559
1560 for (i = 0; i < p->nr_targets; i++) {
1561 cxled = p->targets[i];
1562 cxlmd = cxled_to_memdev(cxled);
1563 cxlds = cxlmd->cxlds;
1564
1565 /* validate that all targets agree on topology */
1566 if (!cxlds->rcd) {
1567 vh++;
1568 } else {
1569 rch++;
1570 continue;
1571 }
1572
1573 iter = cxled_to_port(cxled);
1574 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1575 iter = to_cxl_port(iter->dev.parent);
1576
1577 /*
1578 * Descend the topology tree programming / validating
1579 * targets while looking for conflicts.
1580 */
1581 for (ep = cxl_ep_load(iter, cxlmd); iter;
1582 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1583 rc = cxl_port_setup_targets(iter, cxlr, cxled);
1584 if (rc) {
1585 cxl_region_teardown_targets(cxlr);
1586 return rc;
1587 }
1588 }
1589 }
1590
1591 if (rch && vh) {
1592 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1593 cxl_region_teardown_targets(cxlr);
1594 return -ENXIO;
1595 }
1596
1597 return 0;
1598}
1599
1600static int cxl_region_validate_position(struct cxl_region *cxlr,
1601 struct cxl_endpoint_decoder *cxled,
1602 int pos)
1603{
1604 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1605 struct cxl_region_params *p = &cxlr->params;
1606 int i;
1607
1608 if (pos < 0 || pos >= p->interleave_ways) {
1609 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1610 p->interleave_ways);
1611 return -ENXIO;
1612 }
1613
1614 if (p->targets[pos] == cxled)
1615 return 0;
1616
1617 if (p->targets[pos]) {
1618 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1619 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1620
1621 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1622 pos, dev_name(&cxlmd_target->dev),
1623 dev_name(&cxled_target->cxld.dev));
1624 return -EBUSY;
1625 }
1626
1627 for (i = 0; i < p->interleave_ways; i++) {
1628 struct cxl_endpoint_decoder *cxled_target;
1629 struct cxl_memdev *cxlmd_target;
1630
1631 cxled_target = p->targets[i];
1632 if (!cxled_target)
1633 continue;
1634
1635 cxlmd_target = cxled_to_memdev(cxled_target);
1636 if (cxlmd_target == cxlmd) {
1637 dev_dbg(&cxlr->dev,
1638 "%s already specified at position %d via: %s\n",
1639 dev_name(&cxlmd->dev), pos,
1640 dev_name(&cxled_target->cxld.dev));
1641 return -EBUSY;
1642 }
1643 }
1644
1645 return 0;
1646}
1647
1648static int cxl_region_attach_position(struct cxl_region *cxlr,
1649 struct cxl_root_decoder *cxlrd,
1650 struct cxl_endpoint_decoder *cxled,
1651 const struct cxl_dport *dport, int pos)
1652{
1653 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1654 struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd;
1655 struct cxl_decoder *cxld = &cxlsd->cxld;
1656 int iw = cxld->interleave_ways;
1657 struct cxl_port *iter;
1658 int rc;
1659
1660 if (dport != cxlrd->cxlsd.target[pos % iw]) {
1661 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1662 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1663 dev_name(&cxlrd->cxlsd.cxld.dev));
1664 return -ENXIO;
1665 }
1666
1667 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1668 iter = to_cxl_port(iter->dev.parent)) {
1669 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1670 if (rc)
1671 goto err;
1672 }
1673
1674 return 0;
1675
1676err:
1677 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1678 iter = to_cxl_port(iter->dev.parent))
1679 cxl_port_detach_region(iter, cxlr, cxled);
1680 return rc;
1681}
1682
1683static int cxl_region_attach_auto(struct cxl_region *cxlr,
1684 struct cxl_endpoint_decoder *cxled, int pos)
1685{
1686 struct cxl_region_params *p = &cxlr->params;
1687
1688 if (cxled->state != CXL_DECODER_STATE_AUTO) {
1689 dev_err(&cxlr->dev,
1690 "%s: unable to add decoder to autodetected region\n",
1691 dev_name(&cxled->cxld.dev));
1692 return -EINVAL;
1693 }
1694
1695 if (pos >= 0) {
1696 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1697 dev_name(&cxled->cxld.dev), pos);
1698 return -EINVAL;
1699 }
1700
1701 if (p->nr_targets >= p->interleave_ways) {
1702 dev_err(&cxlr->dev, "%s: no more target slots available\n",
1703 dev_name(&cxled->cxld.dev));
1704 return -ENXIO;
1705 }
1706
1707 /*
1708 * Temporarily record the endpoint decoder into the target array. Yes,
1709 * this means that userspace can view devices in the wrong position
1710 * before the region activates, and must be careful to understand when
1711 * it might be racing region autodiscovery.
1712 */
1713 pos = p->nr_targets;
1714 p->targets[pos] = cxled;
1715 cxled->pos = pos;
1716 p->nr_targets++;
1717
1718 return 0;
1719}
1720
1721static int cmp_interleave_pos(const void *a, const void *b)
1722{
1723 struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1724 struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1725
1726 return cxled_a->pos - cxled_b->pos;
1727}
1728
1729static struct cxl_port *next_port(struct cxl_port *port)
1730{
1731 if (!port->parent_dport)
1732 return NULL;
1733 return port->parent_dport->port;
1734}
1735
1736static int match_switch_decoder_by_range(struct device *dev, void *data)
1737{
1738 struct cxl_switch_decoder *cxlsd;
1739 struct range *r1, *r2 = data;
1740
1741 if (!is_switch_decoder(dev))
1742 return 0;
1743
1744 cxlsd = to_cxl_switch_decoder(dev);
1745 r1 = &cxlsd->cxld.hpa_range;
1746
1747 if (is_root_decoder(dev))
1748 return range_contains(r1, r2);
1749 return (r1->start == r2->start && r1->end == r2->end);
1750}
1751
1752static int find_pos_and_ways(struct cxl_port *port, struct range *range,
1753 int *pos, int *ways)
1754{
1755 struct cxl_switch_decoder *cxlsd;
1756 struct cxl_port *parent;
1757 struct device *dev;
1758 int rc = -ENXIO;
1759
1760 parent = next_port(port);
1761 if (!parent)
1762 return rc;
1763
1764 dev = device_find_child(&parent->dev, range,
1765 match_switch_decoder_by_range);
1766 if (!dev) {
1767 dev_err(port->uport_dev,
1768 "failed to find decoder mapping %#llx-%#llx\n",
1769 range->start, range->end);
1770 return rc;
1771 }
1772 cxlsd = to_cxl_switch_decoder(dev);
1773 *ways = cxlsd->cxld.interleave_ways;
1774
1775 for (int i = 0; i < *ways; i++) {
1776 if (cxlsd->target[i] == port->parent_dport) {
1777 *pos = i;
1778 rc = 0;
1779 break;
1780 }
1781 }
1782 put_device(dev);
1783
1784 return rc;
1785}
1786
1787/**
1788 * cxl_calc_interleave_pos() - calculate an endpoint position in a region
1789 * @cxled: endpoint decoder member of given region
1790 *
1791 * The endpoint position is calculated by traversing the topology from
1792 * the endpoint to the root decoder and iteratively applying this
1793 * calculation:
1794 *
1795 * position = position * parent_ways + parent_pos;
1796 *
1797 * ...where @position is inferred from switch and root decoder target lists.
1798 *
1799 * Return: position >= 0 on success
1800 * -ENXIO on failure
1801 */
1802static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled)
1803{
1804 struct cxl_port *iter, *port = cxled_to_port(cxled);
1805 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1806 struct range *range = &cxled->cxld.hpa_range;
1807 int parent_ways = 0, parent_pos = 0, pos = 0;
1808 int rc;
1809
1810 /*
1811 * Example: the expected interleave order of the 4-way region shown
1812 * below is: mem0, mem2, mem1, mem3
1813 *
1814 * root_port
1815 * / \
1816 * host_bridge_0 host_bridge_1
1817 * | | | |
1818 * mem0 mem1 mem2 mem3
1819 *
1820 * In the example the calculator will iterate twice. The first iteration
1821 * uses the mem position in the host-bridge and the ways of the host-
1822 * bridge to generate the first, or local, position. The second
1823 * iteration uses the host-bridge position in the root_port and the ways
1824 * of the root_port to refine the position.
1825 *
1826 * A trace of the calculation per endpoint looks like this:
1827 * mem0: pos = 0 * 2 + 0 mem2: pos = 0 * 2 + 0
1828 * pos = 0 * 2 + 0 pos = 0 * 2 + 1
1829 * pos: 0 pos: 1
1830 *
1831 * mem1: pos = 0 * 2 + 1 mem3: pos = 0 * 2 + 1
1832 * pos = 1 * 2 + 0 pos = 1 * 2 + 1
1833 * pos: 2 pos = 3
1834 *
1835 * Note that while this example is simple, the method applies to more
1836 * complex topologies, including those with switches.
1837 */
1838
1839 /* Iterate from endpoint to root_port refining the position */
1840 for (iter = port; iter; iter = next_port(iter)) {
1841 if (is_cxl_root(iter))
1842 break;
1843
1844 rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways);
1845 if (rc)
1846 return rc;
1847
1848 pos = pos * parent_ways + parent_pos;
1849 }
1850
1851 dev_dbg(&cxlmd->dev,
1852 "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n",
1853 dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent),
1854 dev_name(&port->dev), range->start, range->end, pos);
1855
1856 return pos;
1857}
1858
1859static int cxl_region_sort_targets(struct cxl_region *cxlr)
1860{
1861 struct cxl_region_params *p = &cxlr->params;
1862 int i, rc = 0;
1863
1864 for (i = 0; i < p->nr_targets; i++) {
1865 struct cxl_endpoint_decoder *cxled = p->targets[i];
1866
1867 cxled->pos = cxl_calc_interleave_pos(cxled);
1868 /*
1869 * Record that sorting failed, but still continue to calc
1870 * cxled->pos so that follow-on code paths can reliably
1871 * do p->targets[cxled->pos] to self-reference their entry.
1872 */
1873 if (cxled->pos < 0)
1874 rc = -ENXIO;
1875 }
1876 /* Keep the cxlr target list in interleave position order */
1877 sort(p->targets, p->nr_targets, sizeof(p->targets[0]),
1878 cmp_interleave_pos, NULL);
1879
1880 dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1881 return rc;
1882}
1883
1884static int cxl_region_attach(struct cxl_region *cxlr,
1885 struct cxl_endpoint_decoder *cxled, int pos)
1886{
1887 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1888 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1889 struct cxl_region_params *p = &cxlr->params;
1890 struct cxl_port *ep_port, *root_port;
1891 struct cxl_dport *dport;
1892 int rc = -ENXIO;
1893
1894 rc = check_interleave_cap(&cxled->cxld, p->interleave_ways,
1895 p->interleave_granularity);
1896 if (rc) {
1897 dev_dbg(&cxlr->dev, "%s iw: %d ig: %d is not supported\n",
1898 dev_name(&cxled->cxld.dev), p->interleave_ways,
1899 p->interleave_granularity);
1900 return rc;
1901 }
1902
1903 if (cxled->mode != cxlr->mode) {
1904 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
1905 dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
1906 return -EINVAL;
1907 }
1908
1909 if (cxled->mode == CXL_DECODER_DEAD) {
1910 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1911 return -ENODEV;
1912 }
1913
1914 /* all full of members, or interleave config not established? */
1915 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1916 dev_dbg(&cxlr->dev, "region already active\n");
1917 return -EBUSY;
1918 } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1919 dev_dbg(&cxlr->dev, "interleave config missing\n");
1920 return -ENXIO;
1921 }
1922
1923 if (p->nr_targets >= p->interleave_ways) {
1924 dev_dbg(&cxlr->dev, "region already has %d endpoints\n",
1925 p->nr_targets);
1926 return -EINVAL;
1927 }
1928
1929 ep_port = cxled_to_port(cxled);
1930 root_port = cxlrd_to_port(cxlrd);
1931 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1932 if (!dport) {
1933 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1934 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1935 dev_name(cxlr->dev.parent));
1936 return -ENXIO;
1937 }
1938
1939 if (cxled->cxld.target_type != cxlr->type) {
1940 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1941 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1942 cxled->cxld.target_type, cxlr->type);
1943 return -ENXIO;
1944 }
1945
1946 if (!cxled->dpa_res) {
1947 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1948 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1949 return -ENXIO;
1950 }
1951
1952 if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1953 resource_size(p->res)) {
1954 dev_dbg(&cxlr->dev,
1955 "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1956 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1957 (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1958 (u64)resource_size(p->res));
1959 return -EINVAL;
1960 }
1961
1962 cxl_region_perf_data_calculate(cxlr, cxled);
1963
1964 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1965 int i;
1966
1967 rc = cxl_region_attach_auto(cxlr, cxled, pos);
1968 if (rc)
1969 return rc;
1970
1971 /* await more targets to arrive... */
1972 if (p->nr_targets < p->interleave_ways)
1973 return 0;
1974
1975 /*
1976 * All targets are here, which implies all PCI enumeration that
1977 * affects this region has been completed. Walk the topology to
1978 * sort the devices into their relative region decode position.
1979 */
1980 rc = cxl_region_sort_targets(cxlr);
1981 if (rc)
1982 return rc;
1983
1984 for (i = 0; i < p->nr_targets; i++) {
1985 cxled = p->targets[i];
1986 ep_port = cxled_to_port(cxled);
1987 dport = cxl_find_dport_by_dev(root_port,
1988 ep_port->host_bridge);
1989 rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1990 dport, i);
1991 if (rc)
1992 return rc;
1993 }
1994
1995 rc = cxl_region_setup_targets(cxlr);
1996 if (rc)
1997 return rc;
1998
1999 /*
2000 * If target setup succeeds in the autodiscovery case
2001 * then the region is already committed.
2002 */
2003 p->state = CXL_CONFIG_COMMIT;
2004 cxl_region_shared_upstream_bandwidth_update(cxlr);
2005
2006 return 0;
2007 }
2008
2009 rc = cxl_region_validate_position(cxlr, cxled, pos);
2010 if (rc)
2011 return rc;
2012
2013 rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
2014 if (rc)
2015 return rc;
2016
2017 p->targets[pos] = cxled;
2018 cxled->pos = pos;
2019 p->nr_targets++;
2020
2021 if (p->nr_targets == p->interleave_ways) {
2022 rc = cxl_region_setup_targets(cxlr);
2023 if (rc)
2024 return rc;
2025 p->state = CXL_CONFIG_ACTIVE;
2026 cxl_region_shared_upstream_bandwidth_update(cxlr);
2027 }
2028
2029 cxled->cxld.interleave_ways = p->interleave_ways;
2030 cxled->cxld.interleave_granularity = p->interleave_granularity;
2031 cxled->cxld.hpa_range = (struct range) {
2032 .start = p->res->start,
2033 .end = p->res->end,
2034 };
2035
2036 if (p->nr_targets != p->interleave_ways)
2037 return 0;
2038
2039 /*
2040 * Test the auto-discovery position calculator function
2041 * against this successfully created user-defined region.
2042 * A fail message here means that this interleave config
2043 * will fail when presented as CXL_REGION_F_AUTO.
2044 */
2045 for (int i = 0; i < p->nr_targets; i++) {
2046 struct cxl_endpoint_decoder *cxled = p->targets[i];
2047 int test_pos;
2048
2049 test_pos = cxl_calc_interleave_pos(cxled);
2050 dev_dbg(&cxled->cxld.dev,
2051 "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n",
2052 (test_pos == cxled->pos) ? "success" : "fail",
2053 test_pos, cxled->pos);
2054 }
2055
2056 return 0;
2057}
2058
2059static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
2060{
2061 struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
2062 struct cxl_region *cxlr = cxled->cxld.region;
2063 struct cxl_region_params *p;
2064 int rc = 0;
2065
2066 lockdep_assert_held_write(&cxl_region_rwsem);
2067
2068 if (!cxlr)
2069 return 0;
2070
2071 p = &cxlr->params;
2072 get_device(&cxlr->dev);
2073
2074 if (p->state > CXL_CONFIG_ACTIVE) {
2075 cxl_region_decode_reset(cxlr, p->interleave_ways);
2076 p->state = CXL_CONFIG_ACTIVE;
2077 }
2078
2079 for (iter = ep_port; !is_cxl_root(iter);
2080 iter = to_cxl_port(iter->dev.parent))
2081 cxl_port_detach_region(iter, cxlr, cxled);
2082
2083 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
2084 p->targets[cxled->pos] != cxled) {
2085 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2086
2087 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
2088 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2089 cxled->pos);
2090 goto out;
2091 }
2092
2093 if (p->state == CXL_CONFIG_ACTIVE) {
2094 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2095 cxl_region_teardown_targets(cxlr);
2096 }
2097 p->targets[cxled->pos] = NULL;
2098 p->nr_targets--;
2099 cxled->cxld.hpa_range = (struct range) {
2100 .start = 0,
2101 .end = -1,
2102 };
2103
2104 /* notify the region driver that one of its targets has departed */
2105 up_write(&cxl_region_rwsem);
2106 device_release_driver(&cxlr->dev);
2107 down_write(&cxl_region_rwsem);
2108out:
2109 put_device(&cxlr->dev);
2110 return rc;
2111}
2112
2113void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
2114{
2115 down_write(&cxl_region_rwsem);
2116 cxled->mode = CXL_DECODER_DEAD;
2117 cxl_region_detach(cxled);
2118 up_write(&cxl_region_rwsem);
2119}
2120
2121static int attach_target(struct cxl_region *cxlr,
2122 struct cxl_endpoint_decoder *cxled, int pos,
2123 unsigned int state)
2124{
2125 int rc = 0;
2126
2127 if (state == TASK_INTERRUPTIBLE)
2128 rc = down_write_killable(&cxl_region_rwsem);
2129 else
2130 down_write(&cxl_region_rwsem);
2131 if (rc)
2132 return rc;
2133
2134 down_read(&cxl_dpa_rwsem);
2135 rc = cxl_region_attach(cxlr, cxled, pos);
2136 up_read(&cxl_dpa_rwsem);
2137 up_write(&cxl_region_rwsem);
2138 return rc;
2139}
2140
2141static int detach_target(struct cxl_region *cxlr, int pos)
2142{
2143 struct cxl_region_params *p = &cxlr->params;
2144 int rc;
2145
2146 rc = down_write_killable(&cxl_region_rwsem);
2147 if (rc)
2148 return rc;
2149
2150 if (pos >= p->interleave_ways) {
2151 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
2152 p->interleave_ways);
2153 rc = -ENXIO;
2154 goto out;
2155 }
2156
2157 if (!p->targets[pos]) {
2158 rc = 0;
2159 goto out;
2160 }
2161
2162 rc = cxl_region_detach(p->targets[pos]);
2163out:
2164 up_write(&cxl_region_rwsem);
2165 return rc;
2166}
2167
2168static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
2169 size_t len)
2170{
2171 int rc;
2172
2173 if (sysfs_streq(buf, "\n"))
2174 rc = detach_target(cxlr, pos);
2175 else {
2176 struct device *dev;
2177
2178 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
2179 if (!dev)
2180 return -ENODEV;
2181
2182 if (!is_endpoint_decoder(dev)) {
2183 rc = -EINVAL;
2184 goto out;
2185 }
2186
2187 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
2188 TASK_INTERRUPTIBLE);
2189out:
2190 put_device(dev);
2191 }
2192
2193 if (rc < 0)
2194 return rc;
2195 return len;
2196}
2197
2198#define TARGET_ATTR_RW(n) \
2199static ssize_t target##n##_show( \
2200 struct device *dev, struct device_attribute *attr, char *buf) \
2201{ \
2202 return show_targetN(to_cxl_region(dev), buf, (n)); \
2203} \
2204static ssize_t target##n##_store(struct device *dev, \
2205 struct device_attribute *attr, \
2206 const char *buf, size_t len) \
2207{ \
2208 return store_targetN(to_cxl_region(dev), buf, (n), len); \
2209} \
2210static DEVICE_ATTR_RW(target##n)
2211
2212TARGET_ATTR_RW(0);
2213TARGET_ATTR_RW(1);
2214TARGET_ATTR_RW(2);
2215TARGET_ATTR_RW(3);
2216TARGET_ATTR_RW(4);
2217TARGET_ATTR_RW(5);
2218TARGET_ATTR_RW(6);
2219TARGET_ATTR_RW(7);
2220TARGET_ATTR_RW(8);
2221TARGET_ATTR_RW(9);
2222TARGET_ATTR_RW(10);
2223TARGET_ATTR_RW(11);
2224TARGET_ATTR_RW(12);
2225TARGET_ATTR_RW(13);
2226TARGET_ATTR_RW(14);
2227TARGET_ATTR_RW(15);
2228
2229static struct attribute *target_attrs[] = {
2230 &dev_attr_target0.attr,
2231 &dev_attr_target1.attr,
2232 &dev_attr_target2.attr,
2233 &dev_attr_target3.attr,
2234 &dev_attr_target4.attr,
2235 &dev_attr_target5.attr,
2236 &dev_attr_target6.attr,
2237 &dev_attr_target7.attr,
2238 &dev_attr_target8.attr,
2239 &dev_attr_target9.attr,
2240 &dev_attr_target10.attr,
2241 &dev_attr_target11.attr,
2242 &dev_attr_target12.attr,
2243 &dev_attr_target13.attr,
2244 &dev_attr_target14.attr,
2245 &dev_attr_target15.attr,
2246 NULL,
2247};
2248
2249static umode_t cxl_region_target_visible(struct kobject *kobj,
2250 struct attribute *a, int n)
2251{
2252 struct device *dev = kobj_to_dev(kobj);
2253 struct cxl_region *cxlr = to_cxl_region(dev);
2254 struct cxl_region_params *p = &cxlr->params;
2255
2256 if (n < p->interleave_ways)
2257 return a->mode;
2258 return 0;
2259}
2260
2261static const struct attribute_group cxl_region_target_group = {
2262 .attrs = target_attrs,
2263 .is_visible = cxl_region_target_visible,
2264};
2265
2266static const struct attribute_group *get_cxl_region_target_group(void)
2267{
2268 return &cxl_region_target_group;
2269}
2270
2271static const struct attribute_group *region_groups[] = {
2272 &cxl_base_attribute_group,
2273 &cxl_region_group,
2274 &cxl_region_target_group,
2275 &cxl_region_access0_coordinate_group,
2276 &cxl_region_access1_coordinate_group,
2277 NULL,
2278};
2279
2280static void cxl_region_release(struct device *dev)
2281{
2282 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
2283 struct cxl_region *cxlr = to_cxl_region(dev);
2284 int id = atomic_read(&cxlrd->region_id);
2285
2286 /*
2287 * Try to reuse the recently idled id rather than the cached
2288 * next id to prevent the region id space from increasing
2289 * unnecessarily.
2290 */
2291 if (cxlr->id < id)
2292 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
2293 memregion_free(id);
2294 goto out;
2295 }
2296
2297 memregion_free(cxlr->id);
2298out:
2299 put_device(dev->parent);
2300 kfree(cxlr);
2301}
2302
2303const struct device_type cxl_region_type = {
2304 .name = "cxl_region",
2305 .release = cxl_region_release,
2306 .groups = region_groups
2307};
2308
2309bool is_cxl_region(struct device *dev)
2310{
2311 return dev->type == &cxl_region_type;
2312}
2313EXPORT_SYMBOL_NS_GPL(is_cxl_region, "CXL");
2314
2315static struct cxl_region *to_cxl_region(struct device *dev)
2316{
2317 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2318 "not a cxl_region device\n"))
2319 return NULL;
2320
2321 return container_of(dev, struct cxl_region, dev);
2322}
2323
2324static void unregister_region(void *_cxlr)
2325{
2326 struct cxl_region *cxlr = _cxlr;
2327 struct cxl_region_params *p = &cxlr->params;
2328 int i;
2329
2330 device_del(&cxlr->dev);
2331
2332 /*
2333 * Now that region sysfs is shutdown, the parameter block is now
2334 * read-only, so no need to hold the region rwsem to access the
2335 * region parameters.
2336 */
2337 for (i = 0; i < p->interleave_ways; i++)
2338 detach_target(cxlr, i);
2339
2340 cxl_region_iomem_release(cxlr);
2341 put_device(&cxlr->dev);
2342}
2343
2344static struct lock_class_key cxl_region_key;
2345
2346static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2347{
2348 struct cxl_region *cxlr;
2349 struct device *dev;
2350
2351 cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2352 if (!cxlr) {
2353 memregion_free(id);
2354 return ERR_PTR(-ENOMEM);
2355 }
2356
2357 dev = &cxlr->dev;
2358 device_initialize(dev);
2359 lockdep_set_class(&dev->mutex, &cxl_region_key);
2360 dev->parent = &cxlrd->cxlsd.cxld.dev;
2361 /*
2362 * Keep root decoder pinned through cxl_region_release to fixup
2363 * region id allocations
2364 */
2365 get_device(dev->parent);
2366 device_set_pm_not_required(dev);
2367 dev->bus = &cxl_bus_type;
2368 dev->type = &cxl_region_type;
2369 cxlr->id = id;
2370
2371 return cxlr;
2372}
2373
2374static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid)
2375{
2376 int cset = 0;
2377 int rc;
2378
2379 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2380 if (cxlr->coord[i].read_bandwidth) {
2381 rc = 0;
2382 if (cxl_need_node_perf_attrs_update(nid))
2383 node_set_perf_attrs(nid, &cxlr->coord[i], i);
2384 else
2385 rc = cxl_update_hmat_access_coordinates(nid, cxlr, i);
2386
2387 if (rc == 0)
2388 cset++;
2389 }
2390 }
2391
2392 if (!cset)
2393 return false;
2394
2395 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access0_group());
2396 if (rc)
2397 dev_dbg(&cxlr->dev, "Failed to update access0 group\n");
2398
2399 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access1_group());
2400 if (rc)
2401 dev_dbg(&cxlr->dev, "Failed to update access1 group\n");
2402
2403 return true;
2404}
2405
2406static int cxl_region_perf_attrs_callback(struct notifier_block *nb,
2407 unsigned long action, void *arg)
2408{
2409 struct cxl_region *cxlr = container_of(nb, struct cxl_region,
2410 memory_notifier);
2411 struct memory_notify *mnb = arg;
2412 int nid = mnb->status_change_nid;
2413 int region_nid;
2414
2415 if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
2416 return NOTIFY_DONE;
2417
2418 /*
2419 * No need to hold cxl_region_rwsem; region parameters are stable
2420 * within the cxl_region driver.
2421 */
2422 region_nid = phys_to_target_node(cxlr->params.res->start);
2423 if (nid != region_nid)
2424 return NOTIFY_DONE;
2425
2426 if (!cxl_region_update_coordinates(cxlr, nid))
2427 return NOTIFY_DONE;
2428
2429 return NOTIFY_OK;
2430}
2431
2432static int cxl_region_calculate_adistance(struct notifier_block *nb,
2433 unsigned long nid, void *data)
2434{
2435 struct cxl_region *cxlr = container_of(nb, struct cxl_region,
2436 adist_notifier);
2437 struct access_coordinate *perf;
2438 int *adist = data;
2439 int region_nid;
2440
2441 /*
2442 * No need to hold cxl_region_rwsem; region parameters are stable
2443 * within the cxl_region driver.
2444 */
2445 region_nid = phys_to_target_node(cxlr->params.res->start);
2446 if (nid != region_nid)
2447 return NOTIFY_OK;
2448
2449 perf = &cxlr->coord[ACCESS_COORDINATE_CPU];
2450
2451 if (mt_perf_to_adistance(perf, adist))
2452 return NOTIFY_OK;
2453
2454 return NOTIFY_STOP;
2455}
2456
2457/**
2458 * devm_cxl_add_region - Adds a region to a decoder
2459 * @cxlrd: root decoder
2460 * @id: memregion id to create, or memregion_free() on failure
2461 * @mode: mode for the endpoint decoders of this region
2462 * @type: select whether this is an expander or accelerator (type-2 or type-3)
2463 *
2464 * This is the second step of region initialization. Regions exist within an
2465 * address space which is mapped by a @cxlrd.
2466 *
2467 * Return: 0 if the region was added to the @cxlrd, else returns negative error
2468 * code. The region will be named "regionZ" where Z is the unique region number.
2469 */
2470static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2471 int id,
2472 enum cxl_decoder_mode mode,
2473 enum cxl_decoder_type type)
2474{
2475 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2476 struct cxl_region *cxlr;
2477 struct device *dev;
2478 int rc;
2479
2480 cxlr = cxl_region_alloc(cxlrd, id);
2481 if (IS_ERR(cxlr))
2482 return cxlr;
2483 cxlr->mode = mode;
2484 cxlr->type = type;
2485
2486 dev = &cxlr->dev;
2487 rc = dev_set_name(dev, "region%d", id);
2488 if (rc)
2489 goto err;
2490
2491 rc = device_add(dev);
2492 if (rc)
2493 goto err;
2494
2495 rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2496 if (rc)
2497 return ERR_PTR(rc);
2498
2499 dev_dbg(port->uport_dev, "%s: created %s\n",
2500 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2501 return cxlr;
2502
2503err:
2504 put_device(dev);
2505 return ERR_PTR(rc);
2506}
2507
2508static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2509{
2510 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2511}
2512
2513static ssize_t create_pmem_region_show(struct device *dev,
2514 struct device_attribute *attr, char *buf)
2515{
2516 return __create_region_show(to_cxl_root_decoder(dev), buf);
2517}
2518
2519static ssize_t create_ram_region_show(struct device *dev,
2520 struct device_attribute *attr, char *buf)
2521{
2522 return __create_region_show(to_cxl_root_decoder(dev), buf);
2523}
2524
2525static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2526 enum cxl_decoder_mode mode, int id)
2527{
2528 int rc;
2529
2530 switch (mode) {
2531 case CXL_DECODER_RAM:
2532 case CXL_DECODER_PMEM:
2533 break;
2534 default:
2535 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2536 return ERR_PTR(-EINVAL);
2537 }
2538
2539 rc = memregion_alloc(GFP_KERNEL);
2540 if (rc < 0)
2541 return ERR_PTR(rc);
2542
2543 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2544 memregion_free(rc);
2545 return ERR_PTR(-EBUSY);
2546 }
2547
2548 return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
2549}
2550
2551static ssize_t create_region_store(struct device *dev, const char *buf,
2552 size_t len, enum cxl_decoder_mode mode)
2553{
2554 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2555 struct cxl_region *cxlr;
2556 int rc, id;
2557
2558 rc = sscanf(buf, "region%d\n", &id);
2559 if (rc != 1)
2560 return -EINVAL;
2561
2562 cxlr = __create_region(cxlrd, mode, id);
2563 if (IS_ERR(cxlr))
2564 return PTR_ERR(cxlr);
2565
2566 return len;
2567}
2568
2569static ssize_t create_pmem_region_store(struct device *dev,
2570 struct device_attribute *attr,
2571 const char *buf, size_t len)
2572{
2573 return create_region_store(dev, buf, len, CXL_DECODER_PMEM);
2574}
2575DEVICE_ATTR_RW(create_pmem_region);
2576
2577static ssize_t create_ram_region_store(struct device *dev,
2578 struct device_attribute *attr,
2579 const char *buf, size_t len)
2580{
2581 return create_region_store(dev, buf, len, CXL_DECODER_RAM);
2582}
2583DEVICE_ATTR_RW(create_ram_region);
2584
2585static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2586 char *buf)
2587{
2588 struct cxl_decoder *cxld = to_cxl_decoder(dev);
2589 ssize_t rc;
2590
2591 rc = down_read_interruptible(&cxl_region_rwsem);
2592 if (rc)
2593 return rc;
2594
2595 if (cxld->region)
2596 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2597 else
2598 rc = sysfs_emit(buf, "\n");
2599 up_read(&cxl_region_rwsem);
2600
2601 return rc;
2602}
2603DEVICE_ATTR_RO(region);
2604
2605static struct cxl_region *
2606cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2607{
2608 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2609 struct device *region_dev;
2610
2611 region_dev = device_find_child_by_name(&cxld->dev, name);
2612 if (!region_dev)
2613 return ERR_PTR(-ENODEV);
2614
2615 return to_cxl_region(region_dev);
2616}
2617
2618static ssize_t delete_region_store(struct device *dev,
2619 struct device_attribute *attr,
2620 const char *buf, size_t len)
2621{
2622 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2623 struct cxl_port *port = to_cxl_port(dev->parent);
2624 struct cxl_region *cxlr;
2625
2626 cxlr = cxl_find_region_by_name(cxlrd, buf);
2627 if (IS_ERR(cxlr))
2628 return PTR_ERR(cxlr);
2629
2630 devm_release_action(port->uport_dev, unregister_region, cxlr);
2631 put_device(&cxlr->dev);
2632
2633 return len;
2634}
2635DEVICE_ATTR_WO(delete_region);
2636
2637static void cxl_pmem_region_release(struct device *dev)
2638{
2639 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2640 int i;
2641
2642 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2643 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2644
2645 put_device(&cxlmd->dev);
2646 }
2647
2648 kfree(cxlr_pmem);
2649}
2650
2651static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2652 &cxl_base_attribute_group,
2653 NULL,
2654};
2655
2656const struct device_type cxl_pmem_region_type = {
2657 .name = "cxl_pmem_region",
2658 .release = cxl_pmem_region_release,
2659 .groups = cxl_pmem_region_attribute_groups,
2660};
2661
2662bool is_cxl_pmem_region(struct device *dev)
2663{
2664 return dev->type == &cxl_pmem_region_type;
2665}
2666EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, "CXL");
2667
2668struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2669{
2670 if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2671 "not a cxl_pmem_region device\n"))
2672 return NULL;
2673 return container_of(dev, struct cxl_pmem_region, dev);
2674}
2675EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, "CXL");
2676
2677struct cxl_poison_context {
2678 struct cxl_port *port;
2679 enum cxl_decoder_mode mode;
2680 u64 offset;
2681};
2682
2683static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2684 struct cxl_poison_context *ctx)
2685{
2686 struct cxl_dev_state *cxlds = cxlmd->cxlds;
2687 u64 offset, length;
2688 int rc = 0;
2689
2690 /*
2691 * Collect poison for the remaining unmapped resources
2692 * after poison is collected by committed endpoints.
2693 *
2694 * Knowing that PMEM must always follow RAM, get poison
2695 * for unmapped resources based on the last decoder's mode:
2696 * ram: scan remains of ram range, then any pmem range
2697 * pmem: scan remains of pmem range
2698 */
2699
2700 if (ctx->mode == CXL_DECODER_RAM) {
2701 offset = ctx->offset;
2702 length = resource_size(&cxlds->ram_res) - offset;
2703 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2704 if (rc == -EFAULT)
2705 rc = 0;
2706 if (rc)
2707 return rc;
2708 }
2709 if (ctx->mode == CXL_DECODER_PMEM) {
2710 offset = ctx->offset;
2711 length = resource_size(&cxlds->dpa_res) - offset;
2712 if (!length)
2713 return 0;
2714 } else if (resource_size(&cxlds->pmem_res)) {
2715 offset = cxlds->pmem_res.start;
2716 length = resource_size(&cxlds->pmem_res);
2717 } else {
2718 return 0;
2719 }
2720
2721 return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2722}
2723
2724static int poison_by_decoder(struct device *dev, void *arg)
2725{
2726 struct cxl_poison_context *ctx = arg;
2727 struct cxl_endpoint_decoder *cxled;
2728 struct cxl_memdev *cxlmd;
2729 u64 offset, length;
2730 int rc = 0;
2731
2732 if (!is_endpoint_decoder(dev))
2733 return rc;
2734
2735 cxled = to_cxl_endpoint_decoder(dev);
2736 if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2737 return rc;
2738
2739 /*
2740 * Regions are only created with single mode decoders: pmem or ram.
2741 * Linux does not support mixed mode decoders. This means that
2742 * reading poison per endpoint decoder adheres to the requirement
2743 * that poison reads of pmem and ram must be separated.
2744 * CXL 3.0 Spec 8.2.9.8.4.1
2745 */
2746 if (cxled->mode == CXL_DECODER_MIXED) {
2747 dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2748 return rc;
2749 }
2750
2751 cxlmd = cxled_to_memdev(cxled);
2752 if (cxled->skip) {
2753 offset = cxled->dpa_res->start - cxled->skip;
2754 length = cxled->skip;
2755 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2756 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2757 rc = 0;
2758 if (rc)
2759 return rc;
2760 }
2761
2762 offset = cxled->dpa_res->start;
2763 length = cxled->dpa_res->end - offset + 1;
2764 rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2765 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2766 rc = 0;
2767 if (rc)
2768 return rc;
2769
2770 /* Iterate until commit_end is reached */
2771 if (cxled->cxld.id == ctx->port->commit_end) {
2772 ctx->offset = cxled->dpa_res->end + 1;
2773 ctx->mode = cxled->mode;
2774 return 1;
2775 }
2776
2777 return 0;
2778}
2779
2780int cxl_get_poison_by_endpoint(struct cxl_port *port)
2781{
2782 struct cxl_poison_context ctx;
2783 int rc = 0;
2784
2785 ctx = (struct cxl_poison_context) {
2786 .port = port
2787 };
2788
2789 rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2790 if (rc == 1)
2791 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
2792 &ctx);
2793
2794 return rc;
2795}
2796
2797struct cxl_dpa_to_region_context {
2798 struct cxl_region *cxlr;
2799 u64 dpa;
2800};
2801
2802static int __cxl_dpa_to_region(struct device *dev, void *arg)
2803{
2804 struct cxl_dpa_to_region_context *ctx = arg;
2805 struct cxl_endpoint_decoder *cxled;
2806 struct cxl_region *cxlr;
2807 u64 dpa = ctx->dpa;
2808
2809 if (!is_endpoint_decoder(dev))
2810 return 0;
2811
2812 cxled = to_cxl_endpoint_decoder(dev);
2813 if (!cxled || !cxled->dpa_res || !resource_size(cxled->dpa_res))
2814 return 0;
2815
2816 if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start)
2817 return 0;
2818
2819 /*
2820 * Stop the region search (return 1) when an endpoint mapping is
2821 * found. The region may not be fully constructed so offering
2822 * the cxlr in the context structure is not guaranteed.
2823 */
2824 cxlr = cxled->cxld.region;
2825 if (cxlr)
2826 dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa,
2827 dev_name(&cxlr->dev));
2828 else
2829 dev_dbg(dev, "dpa:0x%llx mapped in endpoint:%s\n", dpa,
2830 dev_name(dev));
2831
2832 ctx->cxlr = cxlr;
2833
2834 return 1;
2835}
2836
2837struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa)
2838{
2839 struct cxl_dpa_to_region_context ctx;
2840 struct cxl_port *port;
2841
2842 ctx = (struct cxl_dpa_to_region_context) {
2843 .dpa = dpa,
2844 };
2845 port = cxlmd->endpoint;
2846 if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port))
2847 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region);
2848
2849 return ctx.cxlr;
2850}
2851
2852static bool cxl_is_hpa_in_chunk(u64 hpa, struct cxl_region *cxlr, int pos)
2853{
2854 struct cxl_region_params *p = &cxlr->params;
2855 int gran = p->interleave_granularity;
2856 int ways = p->interleave_ways;
2857 u64 offset;
2858
2859 /* Is the hpa in an expected chunk for its pos(-ition) */
2860 offset = hpa - p->res->start;
2861 offset = do_div(offset, gran * ways);
2862 if ((offset >= pos * gran) && (offset < (pos + 1) * gran))
2863 return true;
2864
2865 dev_dbg(&cxlr->dev,
2866 "Addr trans fail: hpa 0x%llx not in expected chunk\n", hpa);
2867
2868 return false;
2869}
2870
2871u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd,
2872 u64 dpa)
2873{
2874 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
2875 u64 dpa_offset, hpa_offset, bits_upper, mask_upper, hpa;
2876 struct cxl_region_params *p = &cxlr->params;
2877 struct cxl_endpoint_decoder *cxled = NULL;
2878 u16 eig = 0;
2879 u8 eiw = 0;
2880 int pos;
2881
2882 for (int i = 0; i < p->nr_targets; i++) {
2883 cxled = p->targets[i];
2884 if (cxlmd == cxled_to_memdev(cxled))
2885 break;
2886 }
2887 if (!cxled || cxlmd != cxled_to_memdev(cxled))
2888 return ULLONG_MAX;
2889
2890 pos = cxled->pos;
2891 ways_to_eiw(p->interleave_ways, &eiw);
2892 granularity_to_eig(p->interleave_granularity, &eig);
2893
2894 /*
2895 * The device position in the region interleave set was removed
2896 * from the offset at HPA->DPA translation. To reconstruct the
2897 * HPA, place the 'pos' in the offset.
2898 *
2899 * The placement of 'pos' in the HPA is determined by interleave
2900 * ways and granularity and is defined in the CXL Spec 3.0 Section
2901 * 8.2.4.19.13 Implementation Note: Device Decode Logic
2902 */
2903
2904 /* Remove the dpa base */
2905 dpa_offset = dpa - cxl_dpa_resource_start(cxled);
2906
2907 mask_upper = GENMASK_ULL(51, eig + 8);
2908
2909 if (eiw < 8) {
2910 hpa_offset = (dpa_offset & mask_upper) << eiw;
2911 hpa_offset |= pos << (eig + 8);
2912 } else {
2913 bits_upper = (dpa_offset & mask_upper) >> (eig + 8);
2914 bits_upper = bits_upper * 3;
2915 hpa_offset = ((bits_upper << (eiw - 8)) + pos) << (eig + 8);
2916 }
2917
2918 /* The lower bits remain unchanged */
2919 hpa_offset |= dpa_offset & GENMASK_ULL(eig + 7, 0);
2920
2921 /* Apply the hpa_offset to the region base address */
2922 hpa = hpa_offset + p->res->start;
2923
2924 /* Root decoder translation overrides typical modulo decode */
2925 if (cxlrd->hpa_to_spa)
2926 hpa = cxlrd->hpa_to_spa(cxlrd, hpa);
2927
2928 if (hpa < p->res->start || hpa > p->res->end) {
2929 dev_dbg(&cxlr->dev,
2930 "Addr trans fail: hpa 0x%llx not in region\n", hpa);
2931 return ULLONG_MAX;
2932 }
2933
2934 /* Simple chunk check, by pos & gran, only applies to modulo decodes */
2935 if (!cxlrd->hpa_to_spa && (!cxl_is_hpa_in_chunk(hpa, cxlr, pos)))
2936 return ULLONG_MAX;
2937
2938 return hpa;
2939}
2940
2941static struct lock_class_key cxl_pmem_region_key;
2942
2943static int cxl_pmem_region_alloc(struct cxl_region *cxlr)
2944{
2945 struct cxl_region_params *p = &cxlr->params;
2946 struct cxl_nvdimm_bridge *cxl_nvb;
2947 struct device *dev;
2948 int i;
2949
2950 guard(rwsem_read)(&cxl_region_rwsem);
2951 if (p->state != CXL_CONFIG_COMMIT)
2952 return -ENXIO;
2953
2954 struct cxl_pmem_region *cxlr_pmem __free(kfree) =
2955 kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets), GFP_KERNEL);
2956 if (!cxlr_pmem)
2957 return -ENOMEM;
2958
2959 cxlr_pmem->hpa_range.start = p->res->start;
2960 cxlr_pmem->hpa_range.end = p->res->end;
2961
2962 /* Snapshot the region configuration underneath the cxl_region_rwsem */
2963 cxlr_pmem->nr_mappings = p->nr_targets;
2964 for (i = 0; i < p->nr_targets; i++) {
2965 struct cxl_endpoint_decoder *cxled = p->targets[i];
2966 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2967 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
2968
2969 /*
2970 * Regions never span CXL root devices, so by definition the
2971 * bridge for one device is the same for all.
2972 */
2973 if (i == 0) {
2974 cxl_nvb = cxl_find_nvdimm_bridge(cxlmd->endpoint);
2975 if (!cxl_nvb)
2976 return -ENODEV;
2977 cxlr->cxl_nvb = cxl_nvb;
2978 }
2979 m->cxlmd = cxlmd;
2980 get_device(&cxlmd->dev);
2981 m->start = cxled->dpa_res->start;
2982 m->size = resource_size(cxled->dpa_res);
2983 m->position = i;
2984 }
2985
2986 dev = &cxlr_pmem->dev;
2987 device_initialize(dev);
2988 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
2989 device_set_pm_not_required(dev);
2990 dev->parent = &cxlr->dev;
2991 dev->bus = &cxl_bus_type;
2992 dev->type = &cxl_pmem_region_type;
2993 cxlr_pmem->cxlr = cxlr;
2994 cxlr->cxlr_pmem = no_free_ptr(cxlr_pmem);
2995
2996 return 0;
2997}
2998
2999static void cxl_dax_region_release(struct device *dev)
3000{
3001 struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
3002
3003 kfree(cxlr_dax);
3004}
3005
3006static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
3007 &cxl_base_attribute_group,
3008 NULL,
3009};
3010
3011const struct device_type cxl_dax_region_type = {
3012 .name = "cxl_dax_region",
3013 .release = cxl_dax_region_release,
3014 .groups = cxl_dax_region_attribute_groups,
3015};
3016
3017static bool is_cxl_dax_region(struct device *dev)
3018{
3019 return dev->type == &cxl_dax_region_type;
3020}
3021
3022struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
3023{
3024 if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
3025 "not a cxl_dax_region device\n"))
3026 return NULL;
3027 return container_of(dev, struct cxl_dax_region, dev);
3028}
3029EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, "CXL");
3030
3031static struct lock_class_key cxl_dax_region_key;
3032
3033static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
3034{
3035 struct cxl_region_params *p = &cxlr->params;
3036 struct cxl_dax_region *cxlr_dax;
3037 struct device *dev;
3038
3039 down_read(&cxl_region_rwsem);
3040 if (p->state != CXL_CONFIG_COMMIT) {
3041 cxlr_dax = ERR_PTR(-ENXIO);
3042 goto out;
3043 }
3044
3045 cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
3046 if (!cxlr_dax) {
3047 cxlr_dax = ERR_PTR(-ENOMEM);
3048 goto out;
3049 }
3050
3051 cxlr_dax->hpa_range.start = p->res->start;
3052 cxlr_dax->hpa_range.end = p->res->end;
3053
3054 dev = &cxlr_dax->dev;
3055 cxlr_dax->cxlr = cxlr;
3056 device_initialize(dev);
3057 lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
3058 device_set_pm_not_required(dev);
3059 dev->parent = &cxlr->dev;
3060 dev->bus = &cxl_bus_type;
3061 dev->type = &cxl_dax_region_type;
3062out:
3063 up_read(&cxl_region_rwsem);
3064
3065 return cxlr_dax;
3066}
3067
3068static void cxlr_pmem_unregister(void *_cxlr_pmem)
3069{
3070 struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
3071 struct cxl_region *cxlr = cxlr_pmem->cxlr;
3072 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
3073
3074 /*
3075 * Either the bridge is in ->remove() context under the device_lock(),
3076 * or cxlr_release_nvdimm() is cancelling the bridge's release action
3077 * for @cxlr_pmem and doing it itself (while manually holding the bridge
3078 * lock).
3079 */
3080 device_lock_assert(&cxl_nvb->dev);
3081 cxlr->cxlr_pmem = NULL;
3082 cxlr_pmem->cxlr = NULL;
3083 device_unregister(&cxlr_pmem->dev);
3084}
3085
3086static void cxlr_release_nvdimm(void *_cxlr)
3087{
3088 struct cxl_region *cxlr = _cxlr;
3089 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
3090
3091 scoped_guard(device, &cxl_nvb->dev) {
3092 if (cxlr->cxlr_pmem)
3093 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
3094 cxlr->cxlr_pmem);
3095 }
3096 cxlr->cxl_nvb = NULL;
3097 put_device(&cxl_nvb->dev);
3098}
3099
3100/**
3101 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
3102 * @cxlr: parent CXL region for this pmem region bridge device
3103 *
3104 * Return: 0 on success negative error code on failure.
3105 */
3106static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
3107{
3108 struct cxl_pmem_region *cxlr_pmem;
3109 struct cxl_nvdimm_bridge *cxl_nvb;
3110 struct device *dev;
3111 int rc;
3112
3113 rc = cxl_pmem_region_alloc(cxlr);
3114 if (rc)
3115 return rc;
3116 cxlr_pmem = cxlr->cxlr_pmem;
3117 cxl_nvb = cxlr->cxl_nvb;
3118
3119 dev = &cxlr_pmem->dev;
3120 rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
3121 if (rc)
3122 goto err;
3123
3124 rc = device_add(dev);
3125 if (rc)
3126 goto err;
3127
3128 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
3129 dev_name(dev));
3130
3131 scoped_guard(device, &cxl_nvb->dev) {
3132 if (cxl_nvb->dev.driver)
3133 rc = devm_add_action_or_reset(&cxl_nvb->dev,
3134 cxlr_pmem_unregister,
3135 cxlr_pmem);
3136 else
3137 rc = -ENXIO;
3138 }
3139
3140 if (rc)
3141 goto err_bridge;
3142
3143 /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
3144 return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
3145
3146err:
3147 put_device(dev);
3148err_bridge:
3149 put_device(&cxl_nvb->dev);
3150 cxlr->cxl_nvb = NULL;
3151 return rc;
3152}
3153
3154static void cxlr_dax_unregister(void *_cxlr_dax)
3155{
3156 struct cxl_dax_region *cxlr_dax = _cxlr_dax;
3157
3158 device_unregister(&cxlr_dax->dev);
3159}
3160
3161static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
3162{
3163 struct cxl_dax_region *cxlr_dax;
3164 struct device *dev;
3165 int rc;
3166
3167 cxlr_dax = cxl_dax_region_alloc(cxlr);
3168 if (IS_ERR(cxlr_dax))
3169 return PTR_ERR(cxlr_dax);
3170
3171 dev = &cxlr_dax->dev;
3172 rc = dev_set_name(dev, "dax_region%d", cxlr->id);
3173 if (rc)
3174 goto err;
3175
3176 rc = device_add(dev);
3177 if (rc)
3178 goto err;
3179
3180 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
3181 dev_name(dev));
3182
3183 return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
3184 cxlr_dax);
3185err:
3186 put_device(dev);
3187 return rc;
3188}
3189
3190static int match_root_decoder_by_range(struct device *dev, void *data)
3191{
3192 struct range *r1, *r2 = data;
3193 struct cxl_root_decoder *cxlrd;
3194
3195 if (!is_root_decoder(dev))
3196 return 0;
3197
3198 cxlrd = to_cxl_root_decoder(dev);
3199 r1 = &cxlrd->cxlsd.cxld.hpa_range;
3200 return range_contains(r1, r2);
3201}
3202
3203static int match_region_by_range(struct device *dev, void *data)
3204{
3205 struct cxl_region_params *p;
3206 struct cxl_region *cxlr;
3207 struct range *r = data;
3208 int rc = 0;
3209
3210 if (!is_cxl_region(dev))
3211 return 0;
3212
3213 cxlr = to_cxl_region(dev);
3214 p = &cxlr->params;
3215
3216 down_read(&cxl_region_rwsem);
3217 if (p->res && p->res->start == r->start && p->res->end == r->end)
3218 rc = 1;
3219 up_read(&cxl_region_rwsem);
3220
3221 return rc;
3222}
3223
3224/* Establish an empty region covering the given HPA range */
3225static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
3226 struct cxl_endpoint_decoder *cxled)
3227{
3228 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3229 struct cxl_port *port = cxlrd_to_port(cxlrd);
3230 struct range *hpa = &cxled->cxld.hpa_range;
3231 struct cxl_region_params *p;
3232 struct cxl_region *cxlr;
3233 struct resource *res;
3234 int rc;
3235
3236 do {
3237 cxlr = __create_region(cxlrd, cxled->mode,
3238 atomic_read(&cxlrd->region_id));
3239 } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
3240
3241 if (IS_ERR(cxlr)) {
3242 dev_err(cxlmd->dev.parent,
3243 "%s:%s: %s failed assign region: %ld\n",
3244 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3245 __func__, PTR_ERR(cxlr));
3246 return cxlr;
3247 }
3248
3249 down_write(&cxl_region_rwsem);
3250 p = &cxlr->params;
3251 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
3252 dev_err(cxlmd->dev.parent,
3253 "%s:%s: %s autodiscovery interrupted\n",
3254 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3255 __func__);
3256 rc = -EBUSY;
3257 goto err;
3258 }
3259
3260 set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
3261
3262 res = kmalloc(sizeof(*res), GFP_KERNEL);
3263 if (!res) {
3264 rc = -ENOMEM;
3265 goto err;
3266 }
3267
3268 *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
3269 dev_name(&cxlr->dev));
3270 rc = insert_resource(cxlrd->res, res);
3271 if (rc) {
3272 /*
3273 * Platform-firmware may not have split resources like "System
3274 * RAM" on CXL window boundaries see cxl_region_iomem_release()
3275 */
3276 dev_warn(cxlmd->dev.parent,
3277 "%s:%s: %s %s cannot insert resource\n",
3278 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3279 __func__, dev_name(&cxlr->dev));
3280 }
3281
3282 p->res = res;
3283 p->interleave_ways = cxled->cxld.interleave_ways;
3284 p->interleave_granularity = cxled->cxld.interleave_granularity;
3285 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
3286
3287 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
3288 if (rc)
3289 goto err;
3290
3291 dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
3292 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
3293 dev_name(&cxlr->dev), p->res, p->interleave_ways,
3294 p->interleave_granularity);
3295
3296 /* ...to match put_device() in cxl_add_to_region() */
3297 get_device(&cxlr->dev);
3298 up_write(&cxl_region_rwsem);
3299
3300 return cxlr;
3301
3302err:
3303 up_write(&cxl_region_rwsem);
3304 devm_release_action(port->uport_dev, unregister_region, cxlr);
3305 return ERR_PTR(rc);
3306}
3307
3308int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
3309{
3310 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3311 struct range *hpa = &cxled->cxld.hpa_range;
3312 struct cxl_decoder *cxld = &cxled->cxld;
3313 struct device *cxlrd_dev, *region_dev;
3314 struct cxl_root_decoder *cxlrd;
3315 struct cxl_region_params *p;
3316 struct cxl_region *cxlr;
3317 bool attach = false;
3318 int rc;
3319
3320 cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
3321 match_root_decoder_by_range);
3322 if (!cxlrd_dev) {
3323 dev_err(cxlmd->dev.parent,
3324 "%s:%s no CXL window for range %#llx:%#llx\n",
3325 dev_name(&cxlmd->dev), dev_name(&cxld->dev),
3326 cxld->hpa_range.start, cxld->hpa_range.end);
3327 return -ENXIO;
3328 }
3329
3330 cxlrd = to_cxl_root_decoder(cxlrd_dev);
3331
3332 /*
3333 * Ensure that if multiple threads race to construct_region() for @hpa
3334 * one does the construction and the others add to that.
3335 */
3336 mutex_lock(&cxlrd->range_lock);
3337 region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
3338 match_region_by_range);
3339 if (!region_dev) {
3340 cxlr = construct_region(cxlrd, cxled);
3341 region_dev = &cxlr->dev;
3342 } else
3343 cxlr = to_cxl_region(region_dev);
3344 mutex_unlock(&cxlrd->range_lock);
3345
3346 rc = PTR_ERR_OR_ZERO(cxlr);
3347 if (rc)
3348 goto out;
3349
3350 attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
3351
3352 down_read(&cxl_region_rwsem);
3353 p = &cxlr->params;
3354 attach = p->state == CXL_CONFIG_COMMIT;
3355 up_read(&cxl_region_rwsem);
3356
3357 if (attach) {
3358 /*
3359 * If device_attach() fails the range may still be active via
3360 * the platform-firmware memory map, otherwise the driver for
3361 * regions is local to this file, so driver matching can't fail.
3362 */
3363 if (device_attach(&cxlr->dev) < 0)
3364 dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
3365 p->res);
3366 }
3367
3368 put_device(region_dev);
3369out:
3370 put_device(cxlrd_dev);
3371 return rc;
3372}
3373EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, "CXL");
3374
3375static int is_system_ram(struct resource *res, void *arg)
3376{
3377 struct cxl_region *cxlr = arg;
3378 struct cxl_region_params *p = &cxlr->params;
3379
3380 dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
3381 return 1;
3382}
3383
3384static void shutdown_notifiers(void *_cxlr)
3385{
3386 struct cxl_region *cxlr = _cxlr;
3387
3388 unregister_memory_notifier(&cxlr->memory_notifier);
3389 unregister_mt_adistance_algorithm(&cxlr->adist_notifier);
3390}
3391
3392static int cxl_region_probe(struct device *dev)
3393{
3394 struct cxl_region *cxlr = to_cxl_region(dev);
3395 struct cxl_region_params *p = &cxlr->params;
3396 int rc;
3397
3398 rc = down_read_interruptible(&cxl_region_rwsem);
3399 if (rc) {
3400 dev_dbg(&cxlr->dev, "probe interrupted\n");
3401 return rc;
3402 }
3403
3404 if (p->state < CXL_CONFIG_COMMIT) {
3405 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
3406 rc = -ENXIO;
3407 goto out;
3408 }
3409
3410 if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
3411 dev_err(&cxlr->dev,
3412 "failed to activate, re-commit region and retry\n");
3413 rc = -ENXIO;
3414 goto out;
3415 }
3416
3417 /*
3418 * From this point on any path that changes the region's state away from
3419 * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
3420 */
3421out:
3422 up_read(&cxl_region_rwsem);
3423
3424 if (rc)
3425 return rc;
3426
3427 cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback;
3428 cxlr->memory_notifier.priority = CXL_CALLBACK_PRI;
3429 register_memory_notifier(&cxlr->memory_notifier);
3430
3431 cxlr->adist_notifier.notifier_call = cxl_region_calculate_adistance;
3432 cxlr->adist_notifier.priority = 100;
3433 register_mt_adistance_algorithm(&cxlr->adist_notifier);
3434
3435 rc = devm_add_action_or_reset(&cxlr->dev, shutdown_notifiers, cxlr);
3436 if (rc)
3437 return rc;
3438
3439 switch (cxlr->mode) {
3440 case CXL_DECODER_PMEM:
3441 return devm_cxl_add_pmem_region(cxlr);
3442 case CXL_DECODER_RAM:
3443 /*
3444 * The region can not be manged by CXL if any portion of
3445 * it is already online as 'System RAM'
3446 */
3447 if (walk_iomem_res_desc(IORES_DESC_NONE,
3448 IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
3449 p->res->start, p->res->end, cxlr,
3450 is_system_ram) > 0)
3451 return 0;
3452 return devm_cxl_add_dax_region(cxlr);
3453 default:
3454 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
3455 cxlr->mode);
3456 return -ENXIO;
3457 }
3458}
3459
3460static struct cxl_driver cxl_region_driver = {
3461 .name = "cxl_region",
3462 .probe = cxl_region_probe,
3463 .id = CXL_DEVICE_REGION,
3464};
3465
3466int cxl_region_init(void)
3467{
3468 return cxl_driver_register(&cxl_region_driver);
3469}
3470
3471void cxl_region_exit(void)
3472{
3473 cxl_driver_unregister(&cxl_region_driver);
3474}
3475
3476MODULE_IMPORT_NS("CXL");
3477MODULE_IMPORT_NS("DEVMEM");
3478MODULE_ALIAS_CXL(CXL_DEVICE_REGION);