Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3#include <linux/io-64-nonatomic-hi-lo.h>
4#include <linux/seq_file.h>
5#include <linux/device.h>
6#include <linux/delay.h>
7
8#include "cxlmem.h"
9#include "core.h"
10
11/**
12 * DOC: cxl core hdm
13 *
14 * Compute Express Link Host Managed Device Memory, starting with the
15 * CXL 2.0 specification, is managed by an array of HDM Decoder register
16 * instances per CXL port and per CXL endpoint. Define common helpers
17 * for enumerating these registers and capabilities.
18 */
19
20DECLARE_RWSEM(cxl_dpa_rwsem);
21
22static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
23 int *target_map)
24{
25 int rc;
26
27 rc = cxl_decoder_add_locked(cxld, target_map);
28 if (rc) {
29 put_device(&cxld->dev);
30 dev_err(&port->dev, "Failed to add decoder\n");
31 return rc;
32 }
33
34 rc = cxl_decoder_autoremove(&port->dev, cxld);
35 if (rc)
36 return rc;
37
38 dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev));
39
40 return 0;
41}
42
43/*
44 * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
45 * single ported host-bridges need not publish a decoder capability when a
46 * passthrough decode can be assumed, i.e. all transactions that the uport sees
47 * are claimed and passed to the single dport. Disable the range until the first
48 * CXL region is enumerated / activated.
49 */
50int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
51{
52 struct cxl_switch_decoder *cxlsd;
53 struct cxl_dport *dport = NULL;
54 int single_port_map[1];
55 unsigned long index;
56
57 cxlsd = cxl_switch_decoder_alloc(port, 1);
58 if (IS_ERR(cxlsd))
59 return PTR_ERR(cxlsd);
60
61 device_lock_assert(&port->dev);
62
63 xa_for_each(&port->dports, index, dport)
64 break;
65 single_port_map[0] = dport->port_id;
66
67 return add_hdm_decoder(port, &cxlsd->cxld, single_port_map);
68}
69EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL);
70
71static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
72{
73 u32 hdm_cap;
74
75 hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
76 cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
77 cxlhdm->target_count =
78 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
79 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
80 cxlhdm->interleave_mask |= GENMASK(11, 8);
81 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap))
82 cxlhdm->interleave_mask |= GENMASK(14, 12);
83}
84
85static int map_hdm_decoder_regs(struct cxl_port *port, void __iomem *crb,
86 struct cxl_component_regs *regs)
87{
88 struct cxl_register_map map = {
89 .resource = port->component_reg_phys,
90 .base = crb,
91 .max_size = CXL_COMPONENT_REG_BLOCK_SIZE,
92 };
93
94 cxl_probe_component_regs(&port->dev, crb, &map.component_map);
95 if (!map.component_map.hdm_decoder.valid) {
96 dev_err(&port->dev, "HDM decoder registers invalid\n");
97 return -ENXIO;
98 }
99
100 return cxl_map_component_regs(&port->dev, regs, &map,
101 BIT(CXL_CM_CAP_CAP_ID_HDM));
102}
103
104/**
105 * devm_cxl_setup_hdm - map HDM decoder component registers
106 * @port: cxl_port to map
107 */
108struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port)
109{
110 struct device *dev = &port->dev;
111 struct cxl_hdm *cxlhdm;
112 void __iomem *crb;
113 int rc;
114
115 cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);
116 if (!cxlhdm)
117 return ERR_PTR(-ENOMEM);
118
119 cxlhdm->port = port;
120 crb = ioremap(port->component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE);
121 if (!crb) {
122 dev_err(dev, "No component registers mapped\n");
123 return ERR_PTR(-ENXIO);
124 }
125
126 rc = map_hdm_decoder_regs(port, crb, &cxlhdm->regs);
127 iounmap(crb);
128 if (rc)
129 return ERR_PTR(rc);
130
131 parse_hdm_decoder_caps(cxlhdm);
132 if (cxlhdm->decoder_count == 0) {
133 dev_err(dev, "Spec violation. Caps invalid\n");
134 return ERR_PTR(-ENXIO);
135 }
136
137 dev_set_drvdata(dev, cxlhdm);
138
139 return cxlhdm;
140}
141EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL);
142
143static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth)
144{
145 unsigned long long start = r->start, end = r->end;
146
147 seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end,
148 r->name);
149}
150
151void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds)
152{
153 struct resource *p1, *p2;
154
155 down_read(&cxl_dpa_rwsem);
156 for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {
157 __cxl_dpa_debug(file, p1, 0);
158 for (p2 = p1->child; p2; p2 = p2->sibling)
159 __cxl_dpa_debug(file, p2, 1);
160 }
161 up_read(&cxl_dpa_rwsem);
162}
163EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, CXL);
164
165/*
166 * Must be called in a context that synchronizes against this decoder's
167 * port ->remove() callback (like an endpoint decoder sysfs attribute)
168 */
169static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
170{
171 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
172 struct cxl_port *port = cxled_to_port(cxled);
173 struct cxl_dev_state *cxlds = cxlmd->cxlds;
174 struct resource *res = cxled->dpa_res;
175 resource_size_t skip_start;
176
177 lockdep_assert_held_write(&cxl_dpa_rwsem);
178
179 /* save @skip_start, before @res is released */
180 skip_start = res->start - cxled->skip;
181 __release_region(&cxlds->dpa_res, res->start, resource_size(res));
182 if (cxled->skip)
183 __release_region(&cxlds->dpa_res, skip_start, cxled->skip);
184 cxled->skip = 0;
185 cxled->dpa_res = NULL;
186 put_device(&cxled->cxld.dev);
187 port->hdm_end--;
188}
189
190static void cxl_dpa_release(void *cxled)
191{
192 down_write(&cxl_dpa_rwsem);
193 __cxl_dpa_release(cxled);
194 up_write(&cxl_dpa_rwsem);
195}
196
197/*
198 * Must be called from context that will not race port device
199 * unregistration, like decoder sysfs attribute methods
200 */
201static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
202{
203 struct cxl_port *port = cxled_to_port(cxled);
204
205 lockdep_assert_held_write(&cxl_dpa_rwsem);
206 devm_remove_action(&port->dev, cxl_dpa_release, cxled);
207 __cxl_dpa_release(cxled);
208}
209
210static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
211 resource_size_t base, resource_size_t len,
212 resource_size_t skipped)
213{
214 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
215 struct cxl_port *port = cxled_to_port(cxled);
216 struct cxl_dev_state *cxlds = cxlmd->cxlds;
217 struct device *dev = &port->dev;
218 struct resource *res;
219
220 lockdep_assert_held_write(&cxl_dpa_rwsem);
221
222 if (!len)
223 goto success;
224
225 if (cxled->dpa_res) {
226 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",
227 port->id, cxled->cxld.id, cxled->dpa_res);
228 return -EBUSY;
229 }
230
231 if (port->hdm_end + 1 != cxled->cxld.id) {
232 /*
233 * Assumes alloc and commit order is always in hardware instance
234 * order per expectations from 8.2.5.12.20 Committing Decoder
235 * Programming that enforce decoder[m] committed before
236 * decoder[m+1] commit start.
237 */
238 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id,
239 cxled->cxld.id, port->id, port->hdm_end + 1);
240 return -EBUSY;
241 }
242
243 if (skipped) {
244 res = __request_region(&cxlds->dpa_res, base - skipped, skipped,
245 dev_name(&cxled->cxld.dev), 0);
246 if (!res) {
247 dev_dbg(dev,
248 "decoder%d.%d: failed to reserve skipped space\n",
249 port->id, cxled->cxld.id);
250 return -EBUSY;
251 }
252 }
253 res = __request_region(&cxlds->dpa_res, base, len,
254 dev_name(&cxled->cxld.dev), 0);
255 if (!res) {
256 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",
257 port->id, cxled->cxld.id);
258 if (skipped)
259 __release_region(&cxlds->dpa_res, base - skipped,
260 skipped);
261 return -EBUSY;
262 }
263 cxled->dpa_res = res;
264 cxled->skip = skipped;
265
266 if (resource_contains(&cxlds->pmem_res, res))
267 cxled->mode = CXL_DECODER_PMEM;
268 else if (resource_contains(&cxlds->ram_res, res))
269 cxled->mode = CXL_DECODER_RAM;
270 else {
271 dev_dbg(dev, "decoder%d.%d: %pr mixed\n", port->id,
272 cxled->cxld.id, cxled->dpa_res);
273 cxled->mode = CXL_DECODER_MIXED;
274 }
275
276success:
277 port->hdm_end++;
278 get_device(&cxled->cxld.dev);
279 return 0;
280}
281
282static int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
283 resource_size_t base, resource_size_t len,
284 resource_size_t skipped)
285{
286 struct cxl_port *port = cxled_to_port(cxled);
287 int rc;
288
289 down_write(&cxl_dpa_rwsem);
290 rc = __cxl_dpa_reserve(cxled, base, len, skipped);
291 up_write(&cxl_dpa_rwsem);
292
293 if (rc)
294 return rc;
295
296 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
297}
298
299resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)
300{
301 resource_size_t size = 0;
302
303 down_read(&cxl_dpa_rwsem);
304 if (cxled->dpa_res)
305 size = resource_size(cxled->dpa_res);
306 up_read(&cxl_dpa_rwsem);
307
308 return size;
309}
310
311resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled)
312{
313 resource_size_t base = -1;
314
315 down_read(&cxl_dpa_rwsem);
316 if (cxled->dpa_res)
317 base = cxled->dpa_res->start;
318 up_read(&cxl_dpa_rwsem);
319
320 return base;
321}
322
323int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
324{
325 struct cxl_port *port = cxled_to_port(cxled);
326 struct device *dev = &cxled->cxld.dev;
327 int rc;
328
329 down_write(&cxl_dpa_rwsem);
330 if (!cxled->dpa_res) {
331 rc = 0;
332 goto out;
333 }
334 if (cxled->cxld.region) {
335 dev_dbg(dev, "decoder assigned to: %s\n",
336 dev_name(&cxled->cxld.region->dev));
337 rc = -EBUSY;
338 goto out;
339 }
340 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
341 dev_dbg(dev, "decoder enabled\n");
342 rc = -EBUSY;
343 goto out;
344 }
345 if (cxled->cxld.id != port->hdm_end) {
346 dev_dbg(dev, "expected decoder%d.%d\n", port->id,
347 port->hdm_end);
348 rc = -EBUSY;
349 goto out;
350 }
351 devm_cxl_dpa_release(cxled);
352 rc = 0;
353out:
354 up_write(&cxl_dpa_rwsem);
355 return rc;
356}
357
358int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled,
359 enum cxl_decoder_mode mode)
360{
361 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
362 struct cxl_dev_state *cxlds = cxlmd->cxlds;
363 struct device *dev = &cxled->cxld.dev;
364 int rc;
365
366 switch (mode) {
367 case CXL_DECODER_RAM:
368 case CXL_DECODER_PMEM:
369 break;
370 default:
371 dev_dbg(dev, "unsupported mode: %d\n", mode);
372 return -EINVAL;
373 }
374
375 down_write(&cxl_dpa_rwsem);
376 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
377 rc = -EBUSY;
378 goto out;
379 }
380
381 /*
382 * Only allow modes that are supported by the current partition
383 * configuration
384 */
385 if (mode == CXL_DECODER_PMEM && !resource_size(&cxlds->pmem_res)) {
386 dev_dbg(dev, "no available pmem capacity\n");
387 rc = -ENXIO;
388 goto out;
389 }
390 if (mode == CXL_DECODER_RAM && !resource_size(&cxlds->ram_res)) {
391 dev_dbg(dev, "no available ram capacity\n");
392 rc = -ENXIO;
393 goto out;
394 }
395
396 cxled->mode = mode;
397 rc = 0;
398out:
399 up_write(&cxl_dpa_rwsem);
400
401 return rc;
402}
403
404int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
405{
406 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
407 resource_size_t free_ram_start, free_pmem_start;
408 struct cxl_port *port = cxled_to_port(cxled);
409 struct cxl_dev_state *cxlds = cxlmd->cxlds;
410 struct device *dev = &cxled->cxld.dev;
411 resource_size_t start, avail, skip;
412 struct resource *p, *last;
413 int rc;
414
415 down_write(&cxl_dpa_rwsem);
416 if (cxled->cxld.region) {
417 dev_dbg(dev, "decoder attached to %s\n",
418 dev_name(&cxled->cxld.region->dev));
419 rc = -EBUSY;
420 goto out;
421 }
422
423 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
424 dev_dbg(dev, "decoder enabled\n");
425 rc = -EBUSY;
426 goto out;
427 }
428
429 for (p = cxlds->ram_res.child, last = NULL; p; p = p->sibling)
430 last = p;
431 if (last)
432 free_ram_start = last->end + 1;
433 else
434 free_ram_start = cxlds->ram_res.start;
435
436 for (p = cxlds->pmem_res.child, last = NULL; p; p = p->sibling)
437 last = p;
438 if (last)
439 free_pmem_start = last->end + 1;
440 else
441 free_pmem_start = cxlds->pmem_res.start;
442
443 if (cxled->mode == CXL_DECODER_RAM) {
444 start = free_ram_start;
445 avail = cxlds->ram_res.end - start + 1;
446 skip = 0;
447 } else if (cxled->mode == CXL_DECODER_PMEM) {
448 resource_size_t skip_start, skip_end;
449
450 start = free_pmem_start;
451 avail = cxlds->pmem_res.end - start + 1;
452 skip_start = free_ram_start;
453
454 /*
455 * If some pmem is already allocated, then that allocation
456 * already handled the skip.
457 */
458 if (cxlds->pmem_res.child &&
459 skip_start == cxlds->pmem_res.child->start)
460 skip_end = skip_start - 1;
461 else
462 skip_end = start - 1;
463 skip = skip_end - skip_start + 1;
464 } else {
465 dev_dbg(dev, "mode not set\n");
466 rc = -EINVAL;
467 goto out;
468 }
469
470 if (size > avail) {
471 dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size,
472 cxled->mode == CXL_DECODER_RAM ? "ram" : "pmem",
473 &avail);
474 rc = -ENOSPC;
475 goto out;
476 }
477
478 rc = __cxl_dpa_reserve(cxled, start, size, skip);
479out:
480 up_write(&cxl_dpa_rwsem);
481
482 if (rc)
483 return rc;
484
485 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
486}
487
488static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
489{
490 u16 eig;
491 u8 eiw;
492
493 /*
494 * Input validation ensures these warns never fire, but otherwise
495 * suppress unititalized variable usage warnings.
496 */
497 if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw),
498 "invalid interleave_ways: %d\n", cxld->interleave_ways))
499 return;
500 if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig),
501 "invalid interleave_granularity: %d\n",
502 cxld->interleave_granularity))
503 return;
504
505 u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK);
506 u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK);
507 *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT;
508}
509
510static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl)
511{
512 u32p_replace_bits(ctrl, !!(cxld->target_type == 3),
513 CXL_HDM_DECODER0_CTRL_TYPE);
514}
515
516static int cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt)
517{
518 struct cxl_dport **t = &cxlsd->target[0];
519 int ways = cxlsd->cxld.interleave_ways;
520
521 if (dev_WARN_ONCE(&cxlsd->cxld.dev,
522 ways > 8 || ways > cxlsd->nr_targets,
523 "ways: %d overflows targets: %d\n", ways,
524 cxlsd->nr_targets))
525 return -ENXIO;
526
527 *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id);
528 if (ways > 1)
529 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id);
530 if (ways > 2)
531 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id);
532 if (ways > 3)
533 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id);
534 if (ways > 4)
535 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id);
536 if (ways > 5)
537 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id);
538 if (ways > 6)
539 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id);
540 if (ways > 7)
541 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id);
542
543 return 0;
544}
545
546/*
547 * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set
548 * committed or error within 10ms, but just be generous with 20ms to account for
549 * clock skew and other marginal behavior
550 */
551#define COMMIT_TIMEOUT_MS 20
552static int cxld_await_commit(void __iomem *hdm, int id)
553{
554 u32 ctrl;
555 int i;
556
557 for (i = 0; i < COMMIT_TIMEOUT_MS; i++) {
558 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
559 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) {
560 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
561 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
562 return -EIO;
563 }
564 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
565 return 0;
566 fsleep(1000);
567 }
568
569 return -ETIMEDOUT;
570}
571
572static int cxl_decoder_commit(struct cxl_decoder *cxld)
573{
574 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
575 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
576 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
577 int id = cxld->id, rc;
578 u64 base, size;
579 u32 ctrl;
580
581 if (cxld->flags & CXL_DECODER_F_ENABLE)
582 return 0;
583
584 if (port->commit_end + 1 != id) {
585 dev_dbg(&port->dev,
586 "%s: out of order commit, expected decoder%d.%d\n",
587 dev_name(&cxld->dev), port->id, port->commit_end + 1);
588 return -EBUSY;
589 }
590
591 down_read(&cxl_dpa_rwsem);
592 /* common decoder settings */
593 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
594 cxld_set_interleave(cxld, &ctrl);
595 cxld_set_type(cxld, &ctrl);
596 base = cxld->hpa_range.start;
597 size = range_len(&cxld->hpa_range);
598
599 writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
600 writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
601 writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
602 writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
603
604 if (is_switch_decoder(&cxld->dev)) {
605 struct cxl_switch_decoder *cxlsd =
606 to_cxl_switch_decoder(&cxld->dev);
607 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id);
608 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id);
609 u64 targets;
610
611 rc = cxlsd_set_targets(cxlsd, &targets);
612 if (rc) {
613 dev_dbg(&port->dev, "%s: target configuration error\n",
614 dev_name(&cxld->dev));
615 goto err;
616 }
617
618 writel(upper_32_bits(targets), tl_hi);
619 writel(lower_32_bits(targets), tl_lo);
620 } else {
621 struct cxl_endpoint_decoder *cxled =
622 to_cxl_endpoint_decoder(&cxld->dev);
623 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id);
624 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id);
625
626 writel(upper_32_bits(cxled->skip), sk_hi);
627 writel(lower_32_bits(cxled->skip), sk_lo);
628 }
629
630 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
631 up_read(&cxl_dpa_rwsem);
632
633 port->commit_end++;
634 rc = cxld_await_commit(hdm, cxld->id);
635err:
636 if (rc) {
637 dev_dbg(&port->dev, "%s: error %d committing decoder\n",
638 dev_name(&cxld->dev), rc);
639 cxld->reset(cxld);
640 return rc;
641 }
642 cxld->flags |= CXL_DECODER_F_ENABLE;
643
644 return 0;
645}
646
647static int cxl_decoder_reset(struct cxl_decoder *cxld)
648{
649 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
650 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
651 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
652 int id = cxld->id;
653 u32 ctrl;
654
655 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
656 return 0;
657
658 if (port->commit_end != id) {
659 dev_dbg(&port->dev,
660 "%s: out of order reset, expected decoder%d.%d\n",
661 dev_name(&cxld->dev), port->id, port->commit_end);
662 return -EBUSY;
663 }
664
665 down_read(&cxl_dpa_rwsem);
666 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
667 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
668 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
669
670 writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
671 writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
672 writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
673 writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
674 up_read(&cxl_dpa_rwsem);
675
676 port->commit_end--;
677 cxld->flags &= ~CXL_DECODER_F_ENABLE;
678
679 return 0;
680}
681
682static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
683 int *target_map, void __iomem *hdm, int which,
684 u64 *dpa_base)
685{
686 struct cxl_endpoint_decoder *cxled = NULL;
687 u64 size, base, skip, dpa_size;
688 bool committed;
689 u32 remainder;
690 int i, rc;
691 u32 ctrl;
692 union {
693 u64 value;
694 unsigned char target_id[8];
695 } target_list;
696
697 if (is_endpoint_decoder(&cxld->dev))
698 cxled = to_cxl_endpoint_decoder(&cxld->dev);
699
700 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
701 base = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
702 size = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
703 committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED);
704 cxld->commit = cxl_decoder_commit;
705 cxld->reset = cxl_decoder_reset;
706
707 if (!committed)
708 size = 0;
709 if (base == U64_MAX || size == U64_MAX) {
710 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
711 port->id, cxld->id);
712 return -ENXIO;
713 }
714
715 cxld->hpa_range = (struct range) {
716 .start = base,
717 .end = base + size - 1,
718 };
719
720 /* decoders are enabled if committed */
721 if (committed) {
722 cxld->flags |= CXL_DECODER_F_ENABLE;
723 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
724 cxld->flags |= CXL_DECODER_F_LOCK;
725 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl))
726 cxld->target_type = CXL_DECODER_EXPANDER;
727 else
728 cxld->target_type = CXL_DECODER_ACCELERATOR;
729 if (cxld->id != port->commit_end + 1) {
730 dev_warn(&port->dev,
731 "decoder%d.%d: Committed out of order\n",
732 port->id, cxld->id);
733 return -ENXIO;
734 }
735 port->commit_end = cxld->id;
736 } else {
737 /* unless / until type-2 drivers arrive, assume type-3 */
738 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl) == 0) {
739 ctrl |= CXL_HDM_DECODER0_CTRL_TYPE;
740 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
741 }
742 cxld->target_type = CXL_DECODER_EXPANDER;
743 }
744 rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl),
745 &cxld->interleave_ways);
746 if (rc) {
747 dev_warn(&port->dev,
748 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
749 port->id, cxld->id, ctrl);
750 return rc;
751 }
752 rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl),
753 &cxld->interleave_granularity);
754 if (rc)
755 return rc;
756
757 if (!cxled) {
758 target_list.value =
759 ioread64_hi_lo(hdm + CXL_HDM_DECODER0_TL_LOW(which));
760 for (i = 0; i < cxld->interleave_ways; i++)
761 target_map[i] = target_list.target_id[i];
762
763 return 0;
764 }
765
766 if (!committed)
767 return 0;
768
769 dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder);
770 if (remainder) {
771 dev_err(&port->dev,
772 "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n",
773 port->id, cxld->id, size, cxld->interleave_ways);
774 return -ENXIO;
775 }
776 skip = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_SKIP_LOW(which));
777 rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip);
778 if (rc) {
779 dev_err(&port->dev,
780 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
781 port->id, cxld->id, *dpa_base,
782 *dpa_base + dpa_size + skip - 1, rc);
783 return rc;
784 }
785 *dpa_base += dpa_size + skip;
786 return 0;
787}
788
789/**
790 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set
791 * @cxlhdm: Structure to populate with HDM capabilities
792 */
793int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm)
794{
795 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
796 struct cxl_port *port = cxlhdm->port;
797 int i, committed;
798 u64 dpa_base = 0;
799 u32 ctrl;
800
801 /*
802 * Since the register resource was recently claimed via request_region()
803 * be careful about trusting the "not-committed" status until the commit
804 * timeout has elapsed. The commit timeout is 10ms (CXL 2.0
805 * 8.2.5.12.20), but double it to be tolerant of any clock skew between
806 * host and target.
807 */
808 for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) {
809 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
810 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED)
811 committed++;
812 }
813
814 /* ensure that future checks of committed can be trusted */
815 if (committed != cxlhdm->decoder_count)
816 msleep(20);
817
818 for (i = 0; i < cxlhdm->decoder_count; i++) {
819 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
820 int rc, target_count = cxlhdm->target_count;
821 struct cxl_decoder *cxld;
822
823 if (is_cxl_endpoint(port)) {
824 struct cxl_endpoint_decoder *cxled;
825
826 cxled = cxl_endpoint_decoder_alloc(port);
827 if (IS_ERR(cxled)) {
828 dev_warn(&port->dev,
829 "Failed to allocate the decoder\n");
830 return PTR_ERR(cxled);
831 }
832 cxld = &cxled->cxld;
833 } else {
834 struct cxl_switch_decoder *cxlsd;
835
836 cxlsd = cxl_switch_decoder_alloc(port, target_count);
837 if (IS_ERR(cxlsd)) {
838 dev_warn(&port->dev,
839 "Failed to allocate the decoder\n");
840 return PTR_ERR(cxlsd);
841 }
842 cxld = &cxlsd->cxld;
843 }
844
845 rc = init_hdm_decoder(port, cxld, target_map, hdm, i, &dpa_base);
846 if (rc) {
847 put_device(&cxld->dev);
848 return rc;
849 }
850 rc = add_hdm_decoder(port, cxld, target_map);
851 if (rc) {
852 dev_warn(&port->dev,
853 "Failed to add decoder to port\n");
854 return rc;
855 }
856 }
857
858 return 0;
859}
860EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL);
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3#include <linux/seq_file.h>
4#include <linux/device.h>
5#include <linux/delay.h>
6
7#include "cxlmem.h"
8#include "core.h"
9
10/**
11 * DOC: cxl core hdm
12 *
13 * Compute Express Link Host Managed Device Memory, starting with the
14 * CXL 2.0 specification, is managed by an array of HDM Decoder register
15 * instances per CXL port and per CXL endpoint. Define common helpers
16 * for enumerating these registers and capabilities.
17 */
18
19DECLARE_RWSEM(cxl_dpa_rwsem);
20
21static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
22 int *target_map)
23{
24 int rc;
25
26 rc = cxl_decoder_add_locked(cxld, target_map);
27 if (rc) {
28 put_device(&cxld->dev);
29 dev_err(&port->dev, "Failed to add decoder\n");
30 return rc;
31 }
32
33 rc = cxl_decoder_autoremove(&port->dev, cxld);
34 if (rc)
35 return rc;
36
37 dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev));
38
39 return 0;
40}
41
42/*
43 * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
44 * single ported host-bridges need not publish a decoder capability when a
45 * passthrough decode can be assumed, i.e. all transactions that the uport sees
46 * are claimed and passed to the single dport. Disable the range until the first
47 * CXL region is enumerated / activated.
48 */
49int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
50{
51 struct cxl_switch_decoder *cxlsd;
52 struct cxl_dport *dport = NULL;
53 int single_port_map[1];
54 unsigned long index;
55 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
56
57 /*
58 * Capability checks are moot for passthrough decoders, support
59 * any and all possibilities.
60 */
61 cxlhdm->interleave_mask = ~0U;
62 cxlhdm->iw_cap_mask = ~0UL;
63
64 cxlsd = cxl_switch_decoder_alloc(port, 1);
65 if (IS_ERR(cxlsd))
66 return PTR_ERR(cxlsd);
67
68 device_lock_assert(&port->dev);
69
70 xa_for_each(&port->dports, index, dport)
71 break;
72 single_port_map[0] = dport->port_id;
73
74 return add_hdm_decoder(port, &cxlsd->cxld, single_port_map);
75}
76EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, "CXL");
77
78static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
79{
80 u32 hdm_cap;
81
82 hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
83 cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
84 cxlhdm->target_count =
85 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
86 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
87 cxlhdm->interleave_mask |= GENMASK(11, 8);
88 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap))
89 cxlhdm->interleave_mask |= GENMASK(14, 12);
90 cxlhdm->iw_cap_mask = BIT(1) | BIT(2) | BIT(4) | BIT(8);
91 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_3_6_12_WAY, hdm_cap))
92 cxlhdm->iw_cap_mask |= BIT(3) | BIT(6) | BIT(12);
93 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_16_WAY, hdm_cap))
94 cxlhdm->iw_cap_mask |= BIT(16);
95}
96
97static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info)
98{
99 struct cxl_hdm *cxlhdm;
100 void __iomem *hdm;
101 u32 ctrl;
102 int i;
103
104 if (!info)
105 return false;
106
107 cxlhdm = dev_get_drvdata(&info->port->dev);
108 hdm = cxlhdm->regs.hdm_decoder;
109
110 if (!hdm)
111 return true;
112
113 /*
114 * If HDM decoders are present and the driver is in control of
115 * Mem_Enable skip DVSEC based emulation
116 */
117 if (!info->mem_enabled)
118 return false;
119
120 /*
121 * If any decoders are committed already, there should not be any
122 * emulated DVSEC decoders.
123 */
124 for (i = 0; i < cxlhdm->decoder_count; i++) {
125 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
126 dev_dbg(&info->port->dev,
127 "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n",
128 info->port->id, i,
129 FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl),
130 readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)),
131 readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)),
132 readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)),
133 readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i)));
134 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
135 return false;
136 }
137
138 return true;
139}
140
141/**
142 * devm_cxl_setup_hdm - map HDM decoder component registers
143 * @port: cxl_port to map
144 * @info: cached DVSEC range register info
145 */
146struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port,
147 struct cxl_endpoint_dvsec_info *info)
148{
149 struct cxl_register_map *reg_map = &port->reg_map;
150 struct device *dev = &port->dev;
151 struct cxl_hdm *cxlhdm;
152 int rc;
153
154 cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);
155 if (!cxlhdm)
156 return ERR_PTR(-ENOMEM);
157 cxlhdm->port = port;
158 dev_set_drvdata(dev, cxlhdm);
159
160 /* Memory devices can configure device HDM using DVSEC range regs. */
161 if (reg_map->resource == CXL_RESOURCE_NONE) {
162 if (!info || !info->mem_enabled) {
163 dev_err(dev, "No component registers mapped\n");
164 return ERR_PTR(-ENXIO);
165 }
166
167 cxlhdm->decoder_count = info->ranges;
168 return cxlhdm;
169 }
170
171 if (!reg_map->component_map.hdm_decoder.valid) {
172 dev_dbg(&port->dev, "HDM decoder registers not implemented\n");
173 /* unique error code to indicate no HDM decoder capability */
174 return ERR_PTR(-ENODEV);
175 }
176
177 rc = cxl_map_component_regs(reg_map, &cxlhdm->regs,
178 BIT(CXL_CM_CAP_CAP_ID_HDM));
179 if (rc) {
180 dev_err(dev, "Failed to map HDM capability.\n");
181 return ERR_PTR(rc);
182 }
183
184 parse_hdm_decoder_caps(cxlhdm);
185 if (cxlhdm->decoder_count == 0) {
186 dev_err(dev, "Spec violation. Caps invalid\n");
187 return ERR_PTR(-ENXIO);
188 }
189
190 /*
191 * Now that the hdm capability is parsed, decide if range
192 * register emulation is needed and fixup cxlhdm accordingly.
193 */
194 if (should_emulate_decoders(info)) {
195 dev_dbg(dev, "Fallback map %d range register%s\n", info->ranges,
196 info->ranges > 1 ? "s" : "");
197 cxlhdm->decoder_count = info->ranges;
198 }
199
200 return cxlhdm;
201}
202EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, "CXL");
203
204static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth)
205{
206 unsigned long long start = r->start, end = r->end;
207
208 seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end,
209 r->name);
210}
211
212void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds)
213{
214 struct resource *p1, *p2;
215
216 down_read(&cxl_dpa_rwsem);
217 for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {
218 __cxl_dpa_debug(file, p1, 0);
219 for (p2 = p1->child; p2; p2 = p2->sibling)
220 __cxl_dpa_debug(file, p2, 1);
221 }
222 up_read(&cxl_dpa_rwsem);
223}
224EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, "CXL");
225
226/*
227 * Must be called in a context that synchronizes against this decoder's
228 * port ->remove() callback (like an endpoint decoder sysfs attribute)
229 */
230static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
231{
232 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
233 struct cxl_port *port = cxled_to_port(cxled);
234 struct cxl_dev_state *cxlds = cxlmd->cxlds;
235 struct resource *res = cxled->dpa_res;
236 resource_size_t skip_start;
237
238 lockdep_assert_held_write(&cxl_dpa_rwsem);
239
240 /* save @skip_start, before @res is released */
241 skip_start = res->start - cxled->skip;
242 __release_region(&cxlds->dpa_res, res->start, resource_size(res));
243 if (cxled->skip)
244 __release_region(&cxlds->dpa_res, skip_start, cxled->skip);
245 cxled->skip = 0;
246 cxled->dpa_res = NULL;
247 put_device(&cxled->cxld.dev);
248 port->hdm_end--;
249}
250
251static void cxl_dpa_release(void *cxled)
252{
253 down_write(&cxl_dpa_rwsem);
254 __cxl_dpa_release(cxled);
255 up_write(&cxl_dpa_rwsem);
256}
257
258/*
259 * Must be called from context that will not race port device
260 * unregistration, like decoder sysfs attribute methods
261 */
262static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
263{
264 struct cxl_port *port = cxled_to_port(cxled);
265
266 lockdep_assert_held_write(&cxl_dpa_rwsem);
267 devm_remove_action(&port->dev, cxl_dpa_release, cxled);
268 __cxl_dpa_release(cxled);
269}
270
271static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
272 resource_size_t base, resource_size_t len,
273 resource_size_t skipped)
274{
275 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
276 struct cxl_port *port = cxled_to_port(cxled);
277 struct cxl_dev_state *cxlds = cxlmd->cxlds;
278 struct device *dev = &port->dev;
279 struct resource *res;
280
281 lockdep_assert_held_write(&cxl_dpa_rwsem);
282
283 if (!len) {
284 dev_warn(dev, "decoder%d.%d: empty reservation attempted\n",
285 port->id, cxled->cxld.id);
286 return -EINVAL;
287 }
288
289 if (cxled->dpa_res) {
290 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",
291 port->id, cxled->cxld.id, cxled->dpa_res);
292 return -EBUSY;
293 }
294
295 if (port->hdm_end + 1 != cxled->cxld.id) {
296 /*
297 * Assumes alloc and commit order is always in hardware instance
298 * order per expectations from 8.2.5.12.20 Committing Decoder
299 * Programming that enforce decoder[m] committed before
300 * decoder[m+1] commit start.
301 */
302 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id,
303 cxled->cxld.id, port->id, port->hdm_end + 1);
304 return -EBUSY;
305 }
306
307 if (skipped) {
308 res = __request_region(&cxlds->dpa_res, base - skipped, skipped,
309 dev_name(&cxled->cxld.dev), 0);
310 if (!res) {
311 dev_dbg(dev,
312 "decoder%d.%d: failed to reserve skipped space\n",
313 port->id, cxled->cxld.id);
314 return -EBUSY;
315 }
316 }
317 res = __request_region(&cxlds->dpa_res, base, len,
318 dev_name(&cxled->cxld.dev), 0);
319 if (!res) {
320 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",
321 port->id, cxled->cxld.id);
322 if (skipped)
323 __release_region(&cxlds->dpa_res, base - skipped,
324 skipped);
325 return -EBUSY;
326 }
327 cxled->dpa_res = res;
328 cxled->skip = skipped;
329
330 if (resource_contains(&cxlds->pmem_res, res))
331 cxled->mode = CXL_DECODER_PMEM;
332 else if (resource_contains(&cxlds->ram_res, res))
333 cxled->mode = CXL_DECODER_RAM;
334 else {
335 dev_warn(dev, "decoder%d.%d: %pr mixed mode not supported\n",
336 port->id, cxled->cxld.id, cxled->dpa_res);
337 cxled->mode = CXL_DECODER_MIXED;
338 }
339
340 port->hdm_end++;
341 get_device(&cxled->cxld.dev);
342 return 0;
343}
344
345int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
346 resource_size_t base, resource_size_t len,
347 resource_size_t skipped)
348{
349 struct cxl_port *port = cxled_to_port(cxled);
350 int rc;
351
352 down_write(&cxl_dpa_rwsem);
353 rc = __cxl_dpa_reserve(cxled, base, len, skipped);
354 up_write(&cxl_dpa_rwsem);
355
356 if (rc)
357 return rc;
358
359 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
360}
361EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, "CXL");
362
363resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)
364{
365 resource_size_t size = 0;
366
367 down_read(&cxl_dpa_rwsem);
368 if (cxled->dpa_res)
369 size = resource_size(cxled->dpa_res);
370 up_read(&cxl_dpa_rwsem);
371
372 return size;
373}
374
375resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled)
376{
377 resource_size_t base = -1;
378
379 lockdep_assert_held(&cxl_dpa_rwsem);
380 if (cxled->dpa_res)
381 base = cxled->dpa_res->start;
382
383 return base;
384}
385
386int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
387{
388 struct cxl_port *port = cxled_to_port(cxled);
389 struct device *dev = &cxled->cxld.dev;
390 int rc;
391
392 down_write(&cxl_dpa_rwsem);
393 if (!cxled->dpa_res) {
394 rc = 0;
395 goto out;
396 }
397 if (cxled->cxld.region) {
398 dev_dbg(dev, "decoder assigned to: %s\n",
399 dev_name(&cxled->cxld.region->dev));
400 rc = -EBUSY;
401 goto out;
402 }
403 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
404 dev_dbg(dev, "decoder enabled\n");
405 rc = -EBUSY;
406 goto out;
407 }
408 if (cxled->cxld.id != port->hdm_end) {
409 dev_dbg(dev, "expected decoder%d.%d\n", port->id,
410 port->hdm_end);
411 rc = -EBUSY;
412 goto out;
413 }
414 devm_cxl_dpa_release(cxled);
415 rc = 0;
416out:
417 up_write(&cxl_dpa_rwsem);
418 return rc;
419}
420
421int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled,
422 enum cxl_decoder_mode mode)
423{
424 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
425 struct cxl_dev_state *cxlds = cxlmd->cxlds;
426 struct device *dev = &cxled->cxld.dev;
427
428 switch (mode) {
429 case CXL_DECODER_RAM:
430 case CXL_DECODER_PMEM:
431 break;
432 default:
433 dev_dbg(dev, "unsupported mode: %d\n", mode);
434 return -EINVAL;
435 }
436
437 guard(rwsem_write)(&cxl_dpa_rwsem);
438 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE)
439 return -EBUSY;
440
441 /*
442 * Only allow modes that are supported by the current partition
443 * configuration
444 */
445 if (mode == CXL_DECODER_PMEM && !resource_size(&cxlds->pmem_res)) {
446 dev_dbg(dev, "no available pmem capacity\n");
447 return -ENXIO;
448 }
449 if (mode == CXL_DECODER_RAM && !resource_size(&cxlds->ram_res)) {
450 dev_dbg(dev, "no available ram capacity\n");
451 return -ENXIO;
452 }
453
454 cxled->mode = mode;
455 return 0;
456}
457
458int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
459{
460 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
461 resource_size_t free_ram_start, free_pmem_start;
462 struct cxl_port *port = cxled_to_port(cxled);
463 struct cxl_dev_state *cxlds = cxlmd->cxlds;
464 struct device *dev = &cxled->cxld.dev;
465 resource_size_t start, avail, skip;
466 struct resource *p, *last;
467 int rc;
468
469 down_write(&cxl_dpa_rwsem);
470 if (cxled->cxld.region) {
471 dev_dbg(dev, "decoder attached to %s\n",
472 dev_name(&cxled->cxld.region->dev));
473 rc = -EBUSY;
474 goto out;
475 }
476
477 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
478 dev_dbg(dev, "decoder enabled\n");
479 rc = -EBUSY;
480 goto out;
481 }
482
483 for (p = cxlds->ram_res.child, last = NULL; p; p = p->sibling)
484 last = p;
485 if (last)
486 free_ram_start = last->end + 1;
487 else
488 free_ram_start = cxlds->ram_res.start;
489
490 for (p = cxlds->pmem_res.child, last = NULL; p; p = p->sibling)
491 last = p;
492 if (last)
493 free_pmem_start = last->end + 1;
494 else
495 free_pmem_start = cxlds->pmem_res.start;
496
497 if (cxled->mode == CXL_DECODER_RAM) {
498 start = free_ram_start;
499 avail = cxlds->ram_res.end - start + 1;
500 skip = 0;
501 } else if (cxled->mode == CXL_DECODER_PMEM) {
502 resource_size_t skip_start, skip_end;
503
504 start = free_pmem_start;
505 avail = cxlds->pmem_res.end - start + 1;
506 skip_start = free_ram_start;
507
508 /*
509 * If some pmem is already allocated, then that allocation
510 * already handled the skip.
511 */
512 if (cxlds->pmem_res.child &&
513 skip_start == cxlds->pmem_res.child->start)
514 skip_end = skip_start - 1;
515 else
516 skip_end = start - 1;
517 skip = skip_end - skip_start + 1;
518 } else {
519 dev_dbg(dev, "mode not set\n");
520 rc = -EINVAL;
521 goto out;
522 }
523
524 if (size > avail) {
525 dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size,
526 cxl_decoder_mode_name(cxled->mode), &avail);
527 rc = -ENOSPC;
528 goto out;
529 }
530
531 rc = __cxl_dpa_reserve(cxled, start, size, skip);
532out:
533 up_write(&cxl_dpa_rwsem);
534
535 if (rc)
536 return rc;
537
538 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
539}
540
541static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
542{
543 u16 eig;
544 u8 eiw;
545
546 /*
547 * Input validation ensures these warns never fire, but otherwise
548 * suppress unititalized variable usage warnings.
549 */
550 if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw),
551 "invalid interleave_ways: %d\n", cxld->interleave_ways))
552 return;
553 if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig),
554 "invalid interleave_granularity: %d\n",
555 cxld->interleave_granularity))
556 return;
557
558 u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK);
559 u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK);
560 *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT;
561}
562
563static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl)
564{
565 u32p_replace_bits(ctrl,
566 !!(cxld->target_type == CXL_DECODER_HOSTONLYMEM),
567 CXL_HDM_DECODER0_CTRL_HOSTONLY);
568}
569
570static void cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt)
571{
572 struct cxl_dport **t = &cxlsd->target[0];
573 int ways = cxlsd->cxld.interleave_ways;
574
575 *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id);
576 if (ways > 1)
577 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id);
578 if (ways > 2)
579 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id);
580 if (ways > 3)
581 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id);
582 if (ways > 4)
583 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id);
584 if (ways > 5)
585 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id);
586 if (ways > 6)
587 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id);
588 if (ways > 7)
589 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id);
590}
591
592/*
593 * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set
594 * committed or error within 10ms, but just be generous with 20ms to account for
595 * clock skew and other marginal behavior
596 */
597#define COMMIT_TIMEOUT_MS 20
598static int cxld_await_commit(void __iomem *hdm, int id)
599{
600 u32 ctrl;
601 int i;
602
603 for (i = 0; i < COMMIT_TIMEOUT_MS; i++) {
604 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
605 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) {
606 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
607 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
608 return -EIO;
609 }
610 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
611 return 0;
612 fsleep(1000);
613 }
614
615 return -ETIMEDOUT;
616}
617
618static int cxl_decoder_commit(struct cxl_decoder *cxld)
619{
620 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
621 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
622 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
623 int id = cxld->id, rc;
624 u64 base, size;
625 u32 ctrl;
626
627 if (cxld->flags & CXL_DECODER_F_ENABLE)
628 return 0;
629
630 if (cxl_num_decoders_committed(port) != id) {
631 dev_dbg(&port->dev,
632 "%s: out of order commit, expected decoder%d.%d\n",
633 dev_name(&cxld->dev), port->id,
634 cxl_num_decoders_committed(port));
635 return -EBUSY;
636 }
637
638 /*
639 * For endpoint decoders hosted on CXL memory devices that
640 * support the sanitize operation, make sure sanitize is not in-flight.
641 */
642 if (is_endpoint_decoder(&cxld->dev)) {
643 struct cxl_endpoint_decoder *cxled =
644 to_cxl_endpoint_decoder(&cxld->dev);
645 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
646 struct cxl_memdev_state *mds =
647 to_cxl_memdev_state(cxlmd->cxlds);
648
649 if (mds && mds->security.sanitize_active) {
650 dev_dbg(&cxlmd->dev,
651 "attempted to commit %s during sanitize\n",
652 dev_name(&cxld->dev));
653 return -EBUSY;
654 }
655 }
656
657 down_read(&cxl_dpa_rwsem);
658 /* common decoder settings */
659 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
660 cxld_set_interleave(cxld, &ctrl);
661 cxld_set_type(cxld, &ctrl);
662 base = cxld->hpa_range.start;
663 size = range_len(&cxld->hpa_range);
664
665 writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
666 writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
667 writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
668 writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
669
670 if (is_switch_decoder(&cxld->dev)) {
671 struct cxl_switch_decoder *cxlsd =
672 to_cxl_switch_decoder(&cxld->dev);
673 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id);
674 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id);
675 u64 targets;
676
677 cxlsd_set_targets(cxlsd, &targets);
678 writel(upper_32_bits(targets), tl_hi);
679 writel(lower_32_bits(targets), tl_lo);
680 } else {
681 struct cxl_endpoint_decoder *cxled =
682 to_cxl_endpoint_decoder(&cxld->dev);
683 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id);
684 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id);
685
686 writel(upper_32_bits(cxled->skip), sk_hi);
687 writel(lower_32_bits(cxled->skip), sk_lo);
688 }
689
690 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
691 up_read(&cxl_dpa_rwsem);
692
693 port->commit_end++;
694 rc = cxld_await_commit(hdm, cxld->id);
695 if (rc) {
696 dev_dbg(&port->dev, "%s: error %d committing decoder\n",
697 dev_name(&cxld->dev), rc);
698 cxld->reset(cxld);
699 return rc;
700 }
701 cxld->flags |= CXL_DECODER_F_ENABLE;
702
703 return 0;
704}
705
706static int commit_reap(struct device *dev, const void *data)
707{
708 struct cxl_port *port = to_cxl_port(dev->parent);
709 struct cxl_decoder *cxld;
710
711 if (!is_switch_decoder(dev) && !is_endpoint_decoder(dev))
712 return 0;
713
714 cxld = to_cxl_decoder(dev);
715 if (port->commit_end == cxld->id &&
716 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
717 port->commit_end--;
718 dev_dbg(&port->dev, "reap: %s commit_end: %d\n",
719 dev_name(&cxld->dev), port->commit_end);
720 }
721
722 return 0;
723}
724
725void cxl_port_commit_reap(struct cxl_decoder *cxld)
726{
727 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
728
729 lockdep_assert_held_write(&cxl_region_rwsem);
730
731 /*
732 * Once the highest committed decoder is disabled, free any other
733 * decoders that were pinned allocated by out-of-order release.
734 */
735 port->commit_end--;
736 dev_dbg(&port->dev, "reap: %s commit_end: %d\n", dev_name(&cxld->dev),
737 port->commit_end);
738 device_for_each_child_reverse_from(&port->dev, &cxld->dev, NULL,
739 commit_reap);
740}
741EXPORT_SYMBOL_NS_GPL(cxl_port_commit_reap, "CXL");
742
743static void cxl_decoder_reset(struct cxl_decoder *cxld)
744{
745 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
746 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
747 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
748 int id = cxld->id;
749 u32 ctrl;
750
751 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
752 return;
753
754 if (port->commit_end == id)
755 cxl_port_commit_reap(cxld);
756 else
757 dev_dbg(&port->dev,
758 "%s: out of order reset, expected decoder%d.%d\n",
759 dev_name(&cxld->dev), port->id, port->commit_end);
760
761 down_read(&cxl_dpa_rwsem);
762 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
763 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
764 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
765
766 writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
767 writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
768 writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
769 writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
770 up_read(&cxl_dpa_rwsem);
771
772 cxld->flags &= ~CXL_DECODER_F_ENABLE;
773
774 /* Userspace is now responsible for reconfiguring this decoder */
775 if (is_endpoint_decoder(&cxld->dev)) {
776 struct cxl_endpoint_decoder *cxled;
777
778 cxled = to_cxl_endpoint_decoder(&cxld->dev);
779 cxled->state = CXL_DECODER_STATE_MANUAL;
780 }
781}
782
783static int cxl_setup_hdm_decoder_from_dvsec(
784 struct cxl_port *port, struct cxl_decoder *cxld, u64 *dpa_base,
785 int which, struct cxl_endpoint_dvsec_info *info)
786{
787 struct cxl_endpoint_decoder *cxled;
788 u64 len;
789 int rc;
790
791 if (!is_cxl_endpoint(port))
792 return -EOPNOTSUPP;
793
794 cxled = to_cxl_endpoint_decoder(&cxld->dev);
795 len = range_len(&info->dvsec_range[which]);
796 if (!len)
797 return -ENOENT;
798
799 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
800 cxld->commit = NULL;
801 cxld->reset = NULL;
802 cxld->hpa_range = info->dvsec_range[which];
803
804 /*
805 * Set the emulated decoder as locked pending additional support to
806 * change the range registers at run time.
807 */
808 cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK;
809 port->commit_end = cxld->id;
810
811 rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0);
812 if (rc) {
813 dev_err(&port->dev,
814 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
815 port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc);
816 return rc;
817 }
818 *dpa_base += len;
819 cxled->state = CXL_DECODER_STATE_AUTO;
820
821 return 0;
822}
823
824static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
825 int *target_map, void __iomem *hdm, int which,
826 u64 *dpa_base, struct cxl_endpoint_dvsec_info *info)
827{
828 struct cxl_endpoint_decoder *cxled = NULL;
829 u64 size, base, skip, dpa_size, lo, hi;
830 bool committed;
831 u32 remainder;
832 int i, rc;
833 u32 ctrl;
834 union {
835 u64 value;
836 unsigned char target_id[8];
837 } target_list;
838
839 if (should_emulate_decoders(info))
840 return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base,
841 which, info);
842
843 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
844 lo = readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
845 hi = readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(which));
846 base = (hi << 32) + lo;
847 lo = readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
848 hi = readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(which));
849 size = (hi << 32) + lo;
850 committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED);
851 cxld->commit = cxl_decoder_commit;
852 cxld->reset = cxl_decoder_reset;
853
854 if (!committed)
855 size = 0;
856 if (base == U64_MAX || size == U64_MAX) {
857 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
858 port->id, cxld->id);
859 return -ENXIO;
860 }
861
862 if (info)
863 cxled = to_cxl_endpoint_decoder(&cxld->dev);
864 cxld->hpa_range = (struct range) {
865 .start = base,
866 .end = base + size - 1,
867 };
868
869 /* decoders are enabled if committed */
870 if (committed) {
871 cxld->flags |= CXL_DECODER_F_ENABLE;
872 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
873 cxld->flags |= CXL_DECODER_F_LOCK;
874 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl))
875 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
876 else
877 cxld->target_type = CXL_DECODER_DEVMEM;
878
879 guard(rwsem_write)(&cxl_region_rwsem);
880 if (cxld->id != cxl_num_decoders_committed(port)) {
881 dev_warn(&port->dev,
882 "decoder%d.%d: Committed out of order\n",
883 port->id, cxld->id);
884 return -ENXIO;
885 }
886
887 if (size == 0) {
888 dev_warn(&port->dev,
889 "decoder%d.%d: Committed with zero size\n",
890 port->id, cxld->id);
891 return -ENXIO;
892 }
893 port->commit_end = cxld->id;
894 } else {
895 if (cxled) {
896 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
897 struct cxl_dev_state *cxlds = cxlmd->cxlds;
898
899 /*
900 * Default by devtype until a device arrives that needs
901 * more precision.
902 */
903 if (cxlds->type == CXL_DEVTYPE_CLASSMEM)
904 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
905 else
906 cxld->target_type = CXL_DECODER_DEVMEM;
907 } else {
908 /* To be overridden by region type at commit time */
909 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
910 }
911
912 if (!FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl) &&
913 cxld->target_type == CXL_DECODER_HOSTONLYMEM) {
914 ctrl |= CXL_HDM_DECODER0_CTRL_HOSTONLY;
915 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
916 }
917 }
918 rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl),
919 &cxld->interleave_ways);
920 if (rc) {
921 dev_warn(&port->dev,
922 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
923 port->id, cxld->id, ctrl);
924 return rc;
925 }
926 rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl),
927 &cxld->interleave_granularity);
928 if (rc) {
929 dev_warn(&port->dev,
930 "decoder%d.%d: Invalid interleave granularity (ctrl: %#x)\n",
931 port->id, cxld->id, ctrl);
932 return rc;
933 }
934
935 dev_dbg(&port->dev, "decoder%d.%d: range: %#llx-%#llx iw: %d ig: %d\n",
936 port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end,
937 cxld->interleave_ways, cxld->interleave_granularity);
938
939 if (!cxled) {
940 lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which));
941 hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which));
942 target_list.value = (hi << 32) + lo;
943 for (i = 0; i < cxld->interleave_ways; i++)
944 target_map[i] = target_list.target_id[i];
945
946 return 0;
947 }
948
949 if (!committed)
950 return 0;
951
952 dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder);
953 if (remainder) {
954 dev_err(&port->dev,
955 "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n",
956 port->id, cxld->id, size, cxld->interleave_ways);
957 return -ENXIO;
958 }
959 lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which));
960 hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which));
961 skip = (hi << 32) + lo;
962 rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip);
963 if (rc) {
964 dev_err(&port->dev,
965 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
966 port->id, cxld->id, *dpa_base,
967 *dpa_base + dpa_size + skip - 1, rc);
968 return rc;
969 }
970 *dpa_base += dpa_size + skip;
971
972 cxled->state = CXL_DECODER_STATE_AUTO;
973
974 return 0;
975}
976
977static void cxl_settle_decoders(struct cxl_hdm *cxlhdm)
978{
979 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
980 int committed, i;
981 u32 ctrl;
982
983 if (!hdm)
984 return;
985
986 /*
987 * Since the register resource was recently claimed via request_region()
988 * be careful about trusting the "not-committed" status until the commit
989 * timeout has elapsed. The commit timeout is 10ms (CXL 2.0
990 * 8.2.5.12.20), but double it to be tolerant of any clock skew between
991 * host and target.
992 */
993 for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) {
994 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
995 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED)
996 committed++;
997 }
998
999 /* ensure that future checks of committed can be trusted */
1000 if (committed != cxlhdm->decoder_count)
1001 msleep(20);
1002}
1003
1004/**
1005 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set
1006 * @cxlhdm: Structure to populate with HDM capabilities
1007 * @info: cached DVSEC range register info
1008 */
1009int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
1010 struct cxl_endpoint_dvsec_info *info)
1011{
1012 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
1013 struct cxl_port *port = cxlhdm->port;
1014 int i;
1015 u64 dpa_base = 0;
1016
1017 cxl_settle_decoders(cxlhdm);
1018
1019 for (i = 0; i < cxlhdm->decoder_count; i++) {
1020 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
1021 int rc, target_count = cxlhdm->target_count;
1022 struct cxl_decoder *cxld;
1023
1024 if (is_cxl_endpoint(port)) {
1025 struct cxl_endpoint_decoder *cxled;
1026
1027 cxled = cxl_endpoint_decoder_alloc(port);
1028 if (IS_ERR(cxled)) {
1029 dev_warn(&port->dev,
1030 "Failed to allocate decoder%d.%d\n",
1031 port->id, i);
1032 return PTR_ERR(cxled);
1033 }
1034 cxld = &cxled->cxld;
1035 } else {
1036 struct cxl_switch_decoder *cxlsd;
1037
1038 cxlsd = cxl_switch_decoder_alloc(port, target_count);
1039 if (IS_ERR(cxlsd)) {
1040 dev_warn(&port->dev,
1041 "Failed to allocate decoder%d.%d\n",
1042 port->id, i);
1043 return PTR_ERR(cxlsd);
1044 }
1045 cxld = &cxlsd->cxld;
1046 }
1047
1048 rc = init_hdm_decoder(port, cxld, target_map, hdm, i,
1049 &dpa_base, info);
1050 if (rc) {
1051 dev_warn(&port->dev,
1052 "Failed to initialize decoder%d.%d\n",
1053 port->id, i);
1054 put_device(&cxld->dev);
1055 return rc;
1056 }
1057 rc = add_hdm_decoder(port, cxld, target_map);
1058 if (rc) {
1059 dev_warn(&port->dev,
1060 "Failed to add decoder%d.%d\n", port->id, i);
1061 return rc;
1062 }
1063 }
1064
1065 return 0;
1066}
1067EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, "CXL");