Loading...
Note: File does not exist in v5.14.15.
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
3#include <linux/io-64-nonatomic-lo-hi.h>
4#include <linux/device.h>
5#include <linux/delay.h>
6#include <linux/pci.h>
7#include <linux/pci-doe.h>
8#include <cxlpci.h>
9#include <cxlmem.h>
10#include <cxl.h>
11#include "core.h"
12
13/**
14 * DOC: cxl core pci
15 *
16 * Compute Express Link protocols are layered on top of PCIe. CXL core provides
17 * a set of helpers for CXL interactions which occur via PCIe.
18 */
19
20static unsigned short media_ready_timeout = 60;
21module_param(media_ready_timeout, ushort, 0644);
22MODULE_PARM_DESC(media_ready_timeout, "seconds to wait for media ready");
23
24struct cxl_walk_context {
25 struct pci_bus *bus;
26 struct cxl_port *port;
27 int type;
28 int error;
29 int count;
30};
31
32static int match_add_dports(struct pci_dev *pdev, void *data)
33{
34 struct cxl_walk_context *ctx = data;
35 struct cxl_port *port = ctx->port;
36 int type = pci_pcie_type(pdev);
37 struct cxl_register_map map;
38 struct cxl_dport *dport;
39 u32 lnkcap, port_num;
40 int rc;
41
42 if (pdev->bus != ctx->bus)
43 return 0;
44 if (!pci_is_pcie(pdev))
45 return 0;
46 if (type != ctx->type)
47 return 0;
48 if (pci_read_config_dword(pdev, pci_pcie_cap(pdev) + PCI_EXP_LNKCAP,
49 &lnkcap))
50 return 0;
51
52 rc = cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
53 if (rc)
54 dev_dbg(&port->dev, "failed to find component registers\n");
55
56 port_num = FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap);
57 dport = devm_cxl_add_dport(port, &pdev->dev, port_num, map.resource);
58 if (IS_ERR(dport)) {
59 ctx->error = PTR_ERR(dport);
60 return PTR_ERR(dport);
61 }
62 ctx->count++;
63
64 return 0;
65}
66
67/**
68 * devm_cxl_port_enumerate_dports - enumerate downstream ports of the upstream port
69 * @port: cxl_port whose ->uport is the upstream of dports to be enumerated
70 *
71 * Returns a positive number of dports enumerated or a negative error
72 * code.
73 */
74int devm_cxl_port_enumerate_dports(struct cxl_port *port)
75{
76 struct pci_bus *bus = cxl_port_to_pci_bus(port);
77 struct cxl_walk_context ctx;
78 int type;
79
80 if (!bus)
81 return -ENXIO;
82
83 if (pci_is_root_bus(bus))
84 type = PCI_EXP_TYPE_ROOT_PORT;
85 else
86 type = PCI_EXP_TYPE_DOWNSTREAM;
87
88 ctx = (struct cxl_walk_context) {
89 .port = port,
90 .bus = bus,
91 .type = type,
92 };
93 pci_walk_bus(bus, match_add_dports, &ctx);
94
95 if (ctx.count == 0)
96 return -ENODEV;
97 if (ctx.error)
98 return ctx.error;
99 return ctx.count;
100}
101EXPORT_SYMBOL_NS_GPL(devm_cxl_port_enumerate_dports, CXL);
102
103/*
104 * Wait up to @media_ready_timeout for the device to report memory
105 * active.
106 */
107int cxl_await_media_ready(struct cxl_dev_state *cxlds)
108{
109 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
110 int d = cxlds->cxl_dvsec;
111 bool active = false;
112 u64 md_status;
113 int rc, i;
114
115 for (i = media_ready_timeout; i; i--) {
116 u32 temp;
117
118 rc = pci_read_config_dword(
119 pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &temp);
120 if (rc)
121 return rc;
122
123 active = FIELD_GET(CXL_DVSEC_MEM_ACTIVE, temp);
124 if (active)
125 break;
126 msleep(1000);
127 }
128
129 if (!active) {
130 dev_err(&pdev->dev,
131 "timeout awaiting memory active after %d seconds\n",
132 media_ready_timeout);
133 return -ETIMEDOUT;
134 }
135
136 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
137 if (!CXLMDEV_READY(md_status))
138 return -EIO;
139
140 return 0;
141}
142EXPORT_SYMBOL_NS_GPL(cxl_await_media_ready, CXL);
143
144static int wait_for_valid(struct cxl_dev_state *cxlds)
145{
146 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
147 int d = cxlds->cxl_dvsec, rc;
148 u32 val;
149
150 /*
151 * Memory_Info_Valid: When set, indicates that the CXL Range 1 Size high
152 * and Size Low registers are valid. Must be set within 1 second of
153 * deassertion of reset to CXL device. Likely it is already set by the
154 * time this runs, but otherwise give a 1.5 second timeout in case of
155 * clock skew.
156 */
157 rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val);
158 if (rc)
159 return rc;
160
161 if (val & CXL_DVSEC_MEM_INFO_VALID)
162 return 0;
163
164 msleep(1500);
165
166 rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val);
167 if (rc)
168 return rc;
169
170 if (val & CXL_DVSEC_MEM_INFO_VALID)
171 return 0;
172
173 return -ETIMEDOUT;
174}
175
176static int cxl_set_mem_enable(struct cxl_dev_state *cxlds, u16 val)
177{
178 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
179 int d = cxlds->cxl_dvsec;
180 u16 ctrl;
181 int rc;
182
183 rc = pci_read_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, &ctrl);
184 if (rc < 0)
185 return rc;
186
187 if ((ctrl & CXL_DVSEC_MEM_ENABLE) == val)
188 return 1;
189 ctrl &= ~CXL_DVSEC_MEM_ENABLE;
190 ctrl |= val;
191
192 rc = pci_write_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, ctrl);
193 if (rc < 0)
194 return rc;
195
196 return 0;
197}
198
199static void clear_mem_enable(void *cxlds)
200{
201 cxl_set_mem_enable(cxlds, 0);
202}
203
204static int devm_cxl_enable_mem(struct device *host, struct cxl_dev_state *cxlds)
205{
206 int rc;
207
208 rc = cxl_set_mem_enable(cxlds, CXL_DVSEC_MEM_ENABLE);
209 if (rc < 0)
210 return rc;
211 if (rc > 0)
212 return 0;
213 return devm_add_action_or_reset(host, clear_mem_enable, cxlds);
214}
215
216static bool range_contains(struct range *r1, struct range *r2)
217{
218 return r1->start <= r2->start && r1->end >= r2->end;
219}
220
221/* require dvsec ranges to be covered by a locked platform window */
222static int dvsec_range_allowed(struct device *dev, void *arg)
223{
224 struct range *dev_range = arg;
225 struct cxl_decoder *cxld;
226
227 if (!is_root_decoder(dev))
228 return 0;
229
230 cxld = to_cxl_decoder(dev);
231
232 if (!(cxld->flags & CXL_DECODER_F_LOCK))
233 return 0;
234 if (!(cxld->flags & CXL_DECODER_F_RAM))
235 return 0;
236
237 return range_contains(&cxld->hpa_range, dev_range);
238}
239
240static void disable_hdm(void *_cxlhdm)
241{
242 u32 global_ctrl;
243 struct cxl_hdm *cxlhdm = _cxlhdm;
244 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
245
246 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);
247 writel(global_ctrl & ~CXL_HDM_DECODER_ENABLE,
248 hdm + CXL_HDM_DECODER_CTRL_OFFSET);
249}
250
251static int devm_cxl_enable_hdm(struct device *host, struct cxl_hdm *cxlhdm)
252{
253 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
254 u32 global_ctrl;
255
256 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);
257 writel(global_ctrl | CXL_HDM_DECODER_ENABLE,
258 hdm + CXL_HDM_DECODER_CTRL_OFFSET);
259
260 return devm_add_action_or_reset(host, disable_hdm, cxlhdm);
261}
262
263static bool __cxl_hdm_decode_init(struct cxl_dev_state *cxlds,
264 struct cxl_hdm *cxlhdm,
265 struct cxl_endpoint_dvsec_info *info)
266{
267 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
268 struct cxl_port *port = cxlhdm->port;
269 struct device *dev = cxlds->dev;
270 struct cxl_port *root;
271 int i, rc, allowed;
272 u32 global_ctrl;
273
274 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);
275
276 /*
277 * If the HDM Decoder Capability is already enabled then assume
278 * that some other agent like platform firmware set it up.
279 */
280 if (global_ctrl & CXL_HDM_DECODER_ENABLE) {
281 rc = devm_cxl_enable_mem(&port->dev, cxlds);
282 if (rc)
283 return false;
284 return true;
285 }
286
287 root = to_cxl_port(port->dev.parent);
288 while (!is_cxl_root(root) && is_cxl_port(root->dev.parent))
289 root = to_cxl_port(root->dev.parent);
290 if (!is_cxl_root(root)) {
291 dev_err(dev, "Failed to acquire root port for HDM enable\n");
292 return false;
293 }
294
295 for (i = 0, allowed = 0; info->mem_enabled && i < info->ranges; i++) {
296 struct device *cxld_dev;
297
298 cxld_dev = device_find_child(&root->dev, &info->dvsec_range[i],
299 dvsec_range_allowed);
300 if (!cxld_dev) {
301 dev_dbg(dev, "DVSEC Range%d denied by platform\n", i);
302 continue;
303 }
304 dev_dbg(dev, "DVSEC Range%d allowed by platform\n", i);
305 put_device(cxld_dev);
306 allowed++;
307 }
308
309 if (!allowed) {
310 cxl_set_mem_enable(cxlds, 0);
311 info->mem_enabled = 0;
312 }
313
314 /*
315 * Per CXL 2.0 Section 8.1.3.8.3 and 8.1.3.8.4 DVSEC CXL Range 1 Base
316 * [High,Low] when HDM operation is enabled the range register values
317 * are ignored by the device, but the spec also recommends matching the
318 * DVSEC Range 1,2 to HDM Decoder Range 0,1. So, non-zero info->ranges
319 * are expected even though Linux does not require or maintain that
320 * match. If at least one DVSEC range is enabled and allowed, skip HDM
321 * Decoder Capability Enable.
322 */
323 if (info->mem_enabled)
324 return false;
325
326 rc = devm_cxl_enable_hdm(&port->dev, cxlhdm);
327 if (rc)
328 return false;
329
330 rc = devm_cxl_enable_mem(&port->dev, cxlds);
331 if (rc)
332 return false;
333
334 return true;
335}
336
337/**
338 * cxl_hdm_decode_init() - Setup HDM decoding for the endpoint
339 * @cxlds: Device state
340 * @cxlhdm: Mapped HDM decoder Capability
341 *
342 * Try to enable the endpoint's HDM Decoder Capability
343 */
344int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm)
345{
346 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
347 struct cxl_endpoint_dvsec_info info = { 0 };
348 int hdm_count, rc, i, ranges = 0;
349 struct device *dev = &pdev->dev;
350 int d = cxlds->cxl_dvsec;
351 u16 cap, ctrl;
352
353 if (!d) {
354 dev_dbg(dev, "No DVSEC Capability\n");
355 return -ENXIO;
356 }
357
358 rc = pci_read_config_word(pdev, d + CXL_DVSEC_CAP_OFFSET, &cap);
359 if (rc)
360 return rc;
361
362 rc = pci_read_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, &ctrl);
363 if (rc)
364 return rc;
365
366 if (!(cap & CXL_DVSEC_MEM_CAPABLE)) {
367 dev_dbg(dev, "Not MEM Capable\n");
368 return -ENXIO;
369 }
370
371 /*
372 * It is not allowed by spec for MEM.capable to be set and have 0 legacy
373 * HDM decoders (values > 2 are also undefined as of CXL 2.0). As this
374 * driver is for a spec defined class code which must be CXL.mem
375 * capable, there is no point in continuing to enable CXL.mem.
376 */
377 hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap);
378 if (!hdm_count || hdm_count > 2)
379 return -EINVAL;
380
381 rc = wait_for_valid(cxlds);
382 if (rc) {
383 dev_dbg(dev, "Failure awaiting MEM_INFO_VALID (%d)\n", rc);
384 return rc;
385 }
386
387 /*
388 * The current DVSEC values are moot if the memory capability is
389 * disabled, and they will remain moot after the HDM Decoder
390 * capability is enabled.
391 */
392 info.mem_enabled = FIELD_GET(CXL_DVSEC_MEM_ENABLE, ctrl);
393 if (!info.mem_enabled)
394 goto hdm_init;
395
396 for (i = 0; i < hdm_count; i++) {
397 u64 base, size;
398 u32 temp;
399
400 rc = pci_read_config_dword(
401 pdev, d + CXL_DVSEC_RANGE_SIZE_HIGH(i), &temp);
402 if (rc)
403 return rc;
404
405 size = (u64)temp << 32;
406
407 rc = pci_read_config_dword(
408 pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(i), &temp);
409 if (rc)
410 return rc;
411
412 size |= temp & CXL_DVSEC_MEM_SIZE_LOW_MASK;
413
414 rc = pci_read_config_dword(
415 pdev, d + CXL_DVSEC_RANGE_BASE_HIGH(i), &temp);
416 if (rc)
417 return rc;
418
419 base = (u64)temp << 32;
420
421 rc = pci_read_config_dword(
422 pdev, d + CXL_DVSEC_RANGE_BASE_LOW(i), &temp);
423 if (rc)
424 return rc;
425
426 base |= temp & CXL_DVSEC_MEM_BASE_LOW_MASK;
427
428 info.dvsec_range[i] = (struct range) {
429 .start = base,
430 .end = base + size - 1
431 };
432
433 if (size)
434 ranges++;
435 }
436
437 info.ranges = ranges;
438
439 /*
440 * If DVSEC ranges are being used instead of HDM decoder registers there
441 * is no use in trying to manage those.
442 */
443hdm_init:
444 if (!__cxl_hdm_decode_init(cxlds, cxlhdm, &info)) {
445 dev_err(dev,
446 "Legacy range registers configuration prevents HDM operation.\n");
447 return -EBUSY;
448 }
449
450 return 0;
451}
452EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, CXL);
453
454#define CXL_DOE_TABLE_ACCESS_REQ_CODE 0x000000ff
455#define CXL_DOE_TABLE_ACCESS_REQ_CODE_READ 0
456#define CXL_DOE_TABLE_ACCESS_TABLE_TYPE 0x0000ff00
457#define CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA 0
458#define CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE 0xffff0000
459#define CXL_DOE_TABLE_ACCESS_LAST_ENTRY 0xffff
460#define CXL_DOE_PROTOCOL_TABLE_ACCESS 2
461
462static struct pci_doe_mb *find_cdat_doe(struct device *uport)
463{
464 struct cxl_memdev *cxlmd;
465 struct cxl_dev_state *cxlds;
466 unsigned long index;
467 void *entry;
468
469 cxlmd = to_cxl_memdev(uport);
470 cxlds = cxlmd->cxlds;
471
472 xa_for_each(&cxlds->doe_mbs, index, entry) {
473 struct pci_doe_mb *cur = entry;
474
475 if (pci_doe_supports_prot(cur, PCI_DVSEC_VENDOR_ID_CXL,
476 CXL_DOE_PROTOCOL_TABLE_ACCESS))
477 return cur;
478 }
479
480 return NULL;
481}
482
483#define CDAT_DOE_REQ(entry_handle) \
484 (FIELD_PREP(CXL_DOE_TABLE_ACCESS_REQ_CODE, \
485 CXL_DOE_TABLE_ACCESS_REQ_CODE_READ) | \
486 FIELD_PREP(CXL_DOE_TABLE_ACCESS_TABLE_TYPE, \
487 CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA) | \
488 FIELD_PREP(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, (entry_handle)))
489
490static void cxl_doe_task_complete(struct pci_doe_task *task)
491{
492 complete(task->private);
493}
494
495struct cdat_doe_task {
496 u32 request_pl;
497 u32 response_pl[32];
498 struct completion c;
499 struct pci_doe_task task;
500};
501
502#define DECLARE_CDAT_DOE_TASK(req, cdt) \
503struct cdat_doe_task cdt = { \
504 .c = COMPLETION_INITIALIZER_ONSTACK(cdt.c), \
505 .request_pl = req, \
506 .task = { \
507 .prot.vid = PCI_DVSEC_VENDOR_ID_CXL, \
508 .prot.type = CXL_DOE_PROTOCOL_TABLE_ACCESS, \
509 .request_pl = &cdt.request_pl, \
510 .request_pl_sz = sizeof(cdt.request_pl), \
511 .response_pl = cdt.response_pl, \
512 .response_pl_sz = sizeof(cdt.response_pl), \
513 .complete = cxl_doe_task_complete, \
514 .private = &cdt.c, \
515 } \
516}
517
518static int cxl_cdat_get_length(struct device *dev,
519 struct pci_doe_mb *cdat_doe,
520 size_t *length)
521{
522 DECLARE_CDAT_DOE_TASK(CDAT_DOE_REQ(0), t);
523 int rc;
524
525 rc = pci_doe_submit_task(cdat_doe, &t.task);
526 if (rc < 0) {
527 dev_err(dev, "DOE submit failed: %d", rc);
528 return rc;
529 }
530 wait_for_completion(&t.c);
531 if (t.task.rv < sizeof(u32))
532 return -EIO;
533
534 *length = t.response_pl[1];
535 dev_dbg(dev, "CDAT length %zu\n", *length);
536
537 return 0;
538}
539
540static int cxl_cdat_read_table(struct device *dev,
541 struct pci_doe_mb *cdat_doe,
542 struct cxl_cdat *cdat)
543{
544 size_t length = cdat->length;
545 u32 *data = cdat->table;
546 int entry_handle = 0;
547
548 do {
549 DECLARE_CDAT_DOE_TASK(CDAT_DOE_REQ(entry_handle), t);
550 size_t entry_dw;
551 u32 *entry;
552 int rc;
553
554 rc = pci_doe_submit_task(cdat_doe, &t.task);
555 if (rc < 0) {
556 dev_err(dev, "DOE submit failed: %d", rc);
557 return rc;
558 }
559 wait_for_completion(&t.c);
560 /* 1 DW header + 1 DW data min */
561 if (t.task.rv < (2 * sizeof(u32)))
562 return -EIO;
563
564 /* Get the CXL table access header entry handle */
565 entry_handle = FIELD_GET(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE,
566 t.response_pl[0]);
567 entry = t.response_pl + 1;
568 entry_dw = t.task.rv / sizeof(u32);
569 /* Skip Header */
570 entry_dw -= 1;
571 entry_dw = min(length / sizeof(u32), entry_dw);
572 /* Prevent length < 1 DW from causing a buffer overflow */
573 if (entry_dw) {
574 memcpy(data, entry, entry_dw * sizeof(u32));
575 length -= entry_dw * sizeof(u32);
576 data += entry_dw;
577 }
578 } while (entry_handle != CXL_DOE_TABLE_ACCESS_LAST_ENTRY);
579
580 return 0;
581}
582
583/**
584 * read_cdat_data - Read the CDAT data on this port
585 * @port: Port to read data from
586 *
587 * This call will sleep waiting for responses from the DOE mailbox.
588 */
589void read_cdat_data(struct cxl_port *port)
590{
591 struct pci_doe_mb *cdat_doe;
592 struct device *dev = &port->dev;
593 struct device *uport = port->uport;
594 size_t cdat_length;
595 int rc;
596
597 cdat_doe = find_cdat_doe(uport);
598 if (!cdat_doe) {
599 dev_dbg(dev, "No CDAT mailbox\n");
600 return;
601 }
602
603 port->cdat_available = true;
604
605 if (cxl_cdat_get_length(dev, cdat_doe, &cdat_length)) {
606 dev_dbg(dev, "No CDAT length\n");
607 return;
608 }
609
610 port->cdat.table = devm_kzalloc(dev, cdat_length, GFP_KERNEL);
611 if (!port->cdat.table)
612 return;
613
614 port->cdat.length = cdat_length;
615 rc = cxl_cdat_read_table(dev, cdat_doe, &port->cdat);
616 if (rc) {
617 /* Don't leave table data allocated on error */
618 devm_kfree(dev, port->cdat.table);
619 port->cdat.table = NULL;
620 port->cdat.length = 0;
621 dev_err(dev, "CDAT data read error\n");
622 }
623}
624EXPORT_SYMBOL_NS_GPL(read_cdat_data, CXL);