Loading...
Note: File does not exist in v5.9.
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2020 Intel Corporation. */
3
4#include <linux/io-64-nonatomic-lo-hi.h>
5#include <linux/firmware.h>
6#include <linux/device.h>
7#include <linux/slab.h>
8#include <linux/idr.h>
9#include <linux/pci.h>
10#include <cxlmem.h>
11#include "trace.h"
12#include "core.h"
13
14static DECLARE_RWSEM(cxl_memdev_rwsem);
15
16/*
17 * An entire PCI topology full of devices should be enough for any
18 * config
19 */
20#define CXL_MEM_MAX_DEVS 65536
21
22static int cxl_mem_major;
23static DEFINE_IDA(cxl_memdev_ida);
24
25static void cxl_memdev_release(struct device *dev)
26{
27 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
28
29 ida_free(&cxl_memdev_ida, cxlmd->id);
30 kfree(cxlmd);
31}
32
33static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid,
34 kgid_t *gid)
35{
36 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev));
37}
38
39static ssize_t firmware_version_show(struct device *dev,
40 struct device_attribute *attr, char *buf)
41{
42 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
43 struct cxl_dev_state *cxlds = cxlmd->cxlds;
44 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
45
46 if (!mds)
47 return sysfs_emit(buf, "\n");
48 return sysfs_emit(buf, "%.16s\n", mds->firmware_version);
49}
50static DEVICE_ATTR_RO(firmware_version);
51
52static ssize_t payload_max_show(struct device *dev,
53 struct device_attribute *attr, char *buf)
54{
55 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
56 struct cxl_dev_state *cxlds = cxlmd->cxlds;
57 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
58
59 if (!mds)
60 return sysfs_emit(buf, "\n");
61 return sysfs_emit(buf, "%zu\n", mds->payload_size);
62}
63static DEVICE_ATTR_RO(payload_max);
64
65static ssize_t label_storage_size_show(struct device *dev,
66 struct device_attribute *attr, char *buf)
67{
68 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
69 struct cxl_dev_state *cxlds = cxlmd->cxlds;
70 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
71
72 if (!mds)
73 return sysfs_emit(buf, "\n");
74 return sysfs_emit(buf, "%zu\n", mds->lsa_size);
75}
76static DEVICE_ATTR_RO(label_storage_size);
77
78static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr,
79 char *buf)
80{
81 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
82 struct cxl_dev_state *cxlds = cxlmd->cxlds;
83 unsigned long long len = resource_size(&cxlds->ram_res);
84
85 return sysfs_emit(buf, "%#llx\n", len);
86}
87
88static struct device_attribute dev_attr_ram_size =
89 __ATTR(size, 0444, ram_size_show, NULL);
90
91static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr,
92 char *buf)
93{
94 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
95 struct cxl_dev_state *cxlds = cxlmd->cxlds;
96 unsigned long long len = resource_size(&cxlds->pmem_res);
97
98 return sysfs_emit(buf, "%#llx\n", len);
99}
100
101static struct device_attribute dev_attr_pmem_size =
102 __ATTR(size, 0444, pmem_size_show, NULL);
103
104static ssize_t serial_show(struct device *dev, struct device_attribute *attr,
105 char *buf)
106{
107 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
108 struct cxl_dev_state *cxlds = cxlmd->cxlds;
109
110 return sysfs_emit(buf, "%#llx\n", cxlds->serial);
111}
112static DEVICE_ATTR_RO(serial);
113
114static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr,
115 char *buf)
116{
117 return sysfs_emit(buf, "%d\n", dev_to_node(dev));
118}
119static DEVICE_ATTR_RO(numa_node);
120
121static ssize_t security_state_show(struct device *dev,
122 struct device_attribute *attr,
123 char *buf)
124{
125 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
126 struct cxl_dev_state *cxlds = cxlmd->cxlds;
127 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
128 unsigned long state = mds->security.state;
129 int rc = 0;
130
131 /* sync with latest submission state */
132 mutex_lock(&mds->mbox_mutex);
133 if (mds->security.sanitize_active)
134 rc = sysfs_emit(buf, "sanitize\n");
135 mutex_unlock(&mds->mbox_mutex);
136 if (rc)
137 return rc;
138
139 if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET))
140 return sysfs_emit(buf, "disabled\n");
141 if (state & CXL_PMEM_SEC_STATE_FROZEN ||
142 state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT ||
143 state & CXL_PMEM_SEC_STATE_USER_PLIMIT)
144 return sysfs_emit(buf, "frozen\n");
145 if (state & CXL_PMEM_SEC_STATE_LOCKED)
146 return sysfs_emit(buf, "locked\n");
147 else
148 return sysfs_emit(buf, "unlocked\n");
149}
150static struct device_attribute dev_attr_security_state =
151 __ATTR(state, 0444, security_state_show, NULL);
152
153static ssize_t security_sanitize_store(struct device *dev,
154 struct device_attribute *attr,
155 const char *buf, size_t len)
156{
157 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
158 bool sanitize;
159 ssize_t rc;
160
161 if (kstrtobool(buf, &sanitize) || !sanitize)
162 return -EINVAL;
163
164 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE);
165 if (rc)
166 return rc;
167
168 return len;
169}
170static struct device_attribute dev_attr_security_sanitize =
171 __ATTR(sanitize, 0200, NULL, security_sanitize_store);
172
173static ssize_t security_erase_store(struct device *dev,
174 struct device_attribute *attr,
175 const char *buf, size_t len)
176{
177 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
178 ssize_t rc;
179 bool erase;
180
181 if (kstrtobool(buf, &erase) || !erase)
182 return -EINVAL;
183
184 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE);
185 if (rc)
186 return rc;
187
188 return len;
189}
190static struct device_attribute dev_attr_security_erase =
191 __ATTR(erase, 0200, NULL, security_erase_store);
192
193static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd)
194{
195 struct cxl_dev_state *cxlds = cxlmd->cxlds;
196 u64 offset, length;
197 int rc = 0;
198
199 /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */
200 if (resource_size(&cxlds->pmem_res)) {
201 offset = cxlds->pmem_res.start;
202 length = resource_size(&cxlds->pmem_res);
203 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
204 if (rc)
205 return rc;
206 }
207 if (resource_size(&cxlds->ram_res)) {
208 offset = cxlds->ram_res.start;
209 length = resource_size(&cxlds->ram_res);
210 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
211 /*
212 * Invalid Physical Address is not an error for
213 * volatile addresses. Device support is optional.
214 */
215 if (rc == -EFAULT)
216 rc = 0;
217 }
218 return rc;
219}
220
221int cxl_trigger_poison_list(struct cxl_memdev *cxlmd)
222{
223 struct cxl_port *port;
224 int rc;
225
226 port = cxlmd->endpoint;
227 if (!port || !is_cxl_endpoint(port))
228 return -EINVAL;
229
230 rc = down_read_interruptible(&cxl_region_rwsem);
231 if (rc)
232 return rc;
233
234 rc = down_read_interruptible(&cxl_dpa_rwsem);
235 if (rc) {
236 up_read(&cxl_region_rwsem);
237 return rc;
238 }
239
240 if (cxl_num_decoders_committed(port) == 0) {
241 /* No regions mapped to this memdev */
242 rc = cxl_get_poison_by_memdev(cxlmd);
243 } else {
244 /* Regions mapped, collect poison by endpoint */
245 rc = cxl_get_poison_by_endpoint(port);
246 }
247 up_read(&cxl_dpa_rwsem);
248 up_read(&cxl_region_rwsem);
249
250 return rc;
251}
252EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, CXL);
253
254struct cxl_dpa_to_region_context {
255 struct cxl_region *cxlr;
256 u64 dpa;
257};
258
259static int __cxl_dpa_to_region(struct device *dev, void *arg)
260{
261 struct cxl_dpa_to_region_context *ctx = arg;
262 struct cxl_endpoint_decoder *cxled;
263 u64 dpa = ctx->dpa;
264
265 if (!is_endpoint_decoder(dev))
266 return 0;
267
268 cxled = to_cxl_endpoint_decoder(dev);
269 if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
270 return 0;
271
272 if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start)
273 return 0;
274
275 dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa,
276 dev_name(&cxled->cxld.region->dev));
277
278 ctx->cxlr = cxled->cxld.region;
279
280 return 1;
281}
282
283static struct cxl_region *cxl_dpa_to_region(struct cxl_memdev *cxlmd, u64 dpa)
284{
285 struct cxl_dpa_to_region_context ctx;
286 struct cxl_port *port;
287
288 ctx = (struct cxl_dpa_to_region_context) {
289 .dpa = dpa,
290 };
291 port = cxlmd->endpoint;
292 if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port))
293 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region);
294
295 return ctx.cxlr;
296}
297
298static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa)
299{
300 struct cxl_dev_state *cxlds = cxlmd->cxlds;
301
302 if (!IS_ENABLED(CONFIG_DEBUG_FS))
303 return 0;
304
305 if (!resource_size(&cxlds->dpa_res)) {
306 dev_dbg(cxlds->dev, "device has no dpa resource\n");
307 return -EINVAL;
308 }
309 if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) {
310 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n",
311 dpa, &cxlds->dpa_res);
312 return -EINVAL;
313 }
314 if (!IS_ALIGNED(dpa, 64)) {
315 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa);
316 return -EINVAL;
317 }
318
319 return 0;
320}
321
322int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
323{
324 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
325 struct cxl_mbox_inject_poison inject;
326 struct cxl_poison_record record;
327 struct cxl_mbox_cmd mbox_cmd;
328 struct cxl_region *cxlr;
329 int rc;
330
331 if (!IS_ENABLED(CONFIG_DEBUG_FS))
332 return 0;
333
334 rc = down_read_interruptible(&cxl_region_rwsem);
335 if (rc)
336 return rc;
337
338 rc = down_read_interruptible(&cxl_dpa_rwsem);
339 if (rc) {
340 up_read(&cxl_region_rwsem);
341 return rc;
342 }
343
344 rc = cxl_validate_poison_dpa(cxlmd, dpa);
345 if (rc)
346 goto out;
347
348 inject.address = cpu_to_le64(dpa);
349 mbox_cmd = (struct cxl_mbox_cmd) {
350 .opcode = CXL_MBOX_OP_INJECT_POISON,
351 .size_in = sizeof(inject),
352 .payload_in = &inject,
353 };
354 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
355 if (rc)
356 goto out;
357
358 cxlr = cxl_dpa_to_region(cxlmd, dpa);
359 if (cxlr)
360 dev_warn_once(mds->cxlds.dev,
361 "poison inject dpa:%#llx region: %s\n", dpa,
362 dev_name(&cxlr->dev));
363
364 record = (struct cxl_poison_record) {
365 .address = cpu_to_le64(dpa),
366 .length = cpu_to_le32(1),
367 };
368 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT);
369out:
370 up_read(&cxl_dpa_rwsem);
371 up_read(&cxl_region_rwsem);
372
373 return rc;
374}
375EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL);
376
377int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
378{
379 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
380 struct cxl_mbox_clear_poison clear;
381 struct cxl_poison_record record;
382 struct cxl_mbox_cmd mbox_cmd;
383 struct cxl_region *cxlr;
384 int rc;
385
386 if (!IS_ENABLED(CONFIG_DEBUG_FS))
387 return 0;
388
389 rc = down_read_interruptible(&cxl_region_rwsem);
390 if (rc)
391 return rc;
392
393 rc = down_read_interruptible(&cxl_dpa_rwsem);
394 if (rc) {
395 up_read(&cxl_region_rwsem);
396 return rc;
397 }
398
399 rc = cxl_validate_poison_dpa(cxlmd, dpa);
400 if (rc)
401 goto out;
402
403 /*
404 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command
405 * is defined to accept 64 bytes of write-data, along with the
406 * address to clear. This driver uses zeroes as write-data.
407 */
408 clear = (struct cxl_mbox_clear_poison) {
409 .address = cpu_to_le64(dpa)
410 };
411
412 mbox_cmd = (struct cxl_mbox_cmd) {
413 .opcode = CXL_MBOX_OP_CLEAR_POISON,
414 .size_in = sizeof(clear),
415 .payload_in = &clear,
416 };
417
418 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
419 if (rc)
420 goto out;
421
422 cxlr = cxl_dpa_to_region(cxlmd, dpa);
423 if (cxlr)
424 dev_warn_once(mds->cxlds.dev,
425 "poison clear dpa:%#llx region: %s\n", dpa,
426 dev_name(&cxlr->dev));
427
428 record = (struct cxl_poison_record) {
429 .address = cpu_to_le64(dpa),
430 .length = cpu_to_le32(1),
431 };
432 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR);
433out:
434 up_read(&cxl_dpa_rwsem);
435 up_read(&cxl_region_rwsem);
436
437 return rc;
438}
439EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, CXL);
440
441static struct attribute *cxl_memdev_attributes[] = {
442 &dev_attr_serial.attr,
443 &dev_attr_firmware_version.attr,
444 &dev_attr_payload_max.attr,
445 &dev_attr_label_storage_size.attr,
446 &dev_attr_numa_node.attr,
447 NULL,
448};
449
450static ssize_t pmem_qos_class_show(struct device *dev,
451 struct device_attribute *attr, char *buf)
452{
453 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
454 struct cxl_dev_state *cxlds = cxlmd->cxlds;
455 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
456
457 return sysfs_emit(buf, "%d\n", mds->pmem_perf.qos_class);
458}
459
460static struct device_attribute dev_attr_pmem_qos_class =
461 __ATTR(qos_class, 0444, pmem_qos_class_show, NULL);
462
463static struct attribute *cxl_memdev_pmem_attributes[] = {
464 &dev_attr_pmem_size.attr,
465 &dev_attr_pmem_qos_class.attr,
466 NULL,
467};
468
469static ssize_t ram_qos_class_show(struct device *dev,
470 struct device_attribute *attr, char *buf)
471{
472 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
473 struct cxl_dev_state *cxlds = cxlmd->cxlds;
474 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
475
476 return sysfs_emit(buf, "%d\n", mds->ram_perf.qos_class);
477}
478
479static struct device_attribute dev_attr_ram_qos_class =
480 __ATTR(qos_class, 0444, ram_qos_class_show, NULL);
481
482static struct attribute *cxl_memdev_ram_attributes[] = {
483 &dev_attr_ram_size.attr,
484 &dev_attr_ram_qos_class.attr,
485 NULL,
486};
487
488static struct attribute *cxl_memdev_security_attributes[] = {
489 &dev_attr_security_state.attr,
490 &dev_attr_security_sanitize.attr,
491 &dev_attr_security_erase.attr,
492 NULL,
493};
494
495static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a,
496 int n)
497{
498 if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr)
499 return 0;
500 return a->mode;
501}
502
503static struct attribute_group cxl_memdev_attribute_group = {
504 .attrs = cxl_memdev_attributes,
505 .is_visible = cxl_memdev_visible,
506};
507
508static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n)
509{
510 struct device *dev = kobj_to_dev(kobj);
511 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
512 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
513
514 if (a == &dev_attr_ram_qos_class.attr)
515 if (mds->ram_perf.qos_class == CXL_QOS_CLASS_INVALID)
516 return 0;
517
518 return a->mode;
519}
520
521static struct attribute_group cxl_memdev_ram_attribute_group = {
522 .name = "ram",
523 .attrs = cxl_memdev_ram_attributes,
524 .is_visible = cxl_ram_visible,
525};
526
527static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n)
528{
529 struct device *dev = kobj_to_dev(kobj);
530 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
531 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
532
533 if (a == &dev_attr_pmem_qos_class.attr)
534 if (mds->pmem_perf.qos_class == CXL_QOS_CLASS_INVALID)
535 return 0;
536
537 return a->mode;
538}
539
540static struct attribute_group cxl_memdev_pmem_attribute_group = {
541 .name = "pmem",
542 .attrs = cxl_memdev_pmem_attributes,
543 .is_visible = cxl_pmem_visible,
544};
545
546static umode_t cxl_memdev_security_visible(struct kobject *kobj,
547 struct attribute *a, int n)
548{
549 struct device *dev = kobj_to_dev(kobj);
550 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
551 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
552
553 if (a == &dev_attr_security_sanitize.attr &&
554 !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
555 return 0;
556
557 if (a == &dev_attr_security_erase.attr &&
558 !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds))
559 return 0;
560
561 return a->mode;
562}
563
564static struct attribute_group cxl_memdev_security_attribute_group = {
565 .name = "security",
566 .attrs = cxl_memdev_security_attributes,
567 .is_visible = cxl_memdev_security_visible,
568};
569
570static const struct attribute_group *cxl_memdev_attribute_groups[] = {
571 &cxl_memdev_attribute_group,
572 &cxl_memdev_ram_attribute_group,
573 &cxl_memdev_pmem_attribute_group,
574 &cxl_memdev_security_attribute_group,
575 NULL,
576};
577
578void cxl_memdev_update_perf(struct cxl_memdev *cxlmd)
579{
580 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group);
581 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group);
582}
583EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, CXL);
584
585static const struct device_type cxl_memdev_type = {
586 .name = "cxl_memdev",
587 .release = cxl_memdev_release,
588 .devnode = cxl_memdev_devnode,
589 .groups = cxl_memdev_attribute_groups,
590};
591
592bool is_cxl_memdev(const struct device *dev)
593{
594 return dev->type == &cxl_memdev_type;
595}
596EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, CXL);
597
598/**
599 * set_exclusive_cxl_commands() - atomically disable user cxl commands
600 * @mds: The device state to operate on
601 * @cmds: bitmap of commands to mark exclusive
602 *
603 * Grab the cxl_memdev_rwsem in write mode to flush in-flight
604 * invocations of the ioctl path and then disable future execution of
605 * commands with the command ids set in @cmds.
606 */
607void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
608 unsigned long *cmds)
609{
610 down_write(&cxl_memdev_rwsem);
611 bitmap_or(mds->exclusive_cmds, mds->exclusive_cmds, cmds,
612 CXL_MEM_COMMAND_ID_MAX);
613 up_write(&cxl_memdev_rwsem);
614}
615EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, CXL);
616
617/**
618 * clear_exclusive_cxl_commands() - atomically enable user cxl commands
619 * @mds: The device state to modify
620 * @cmds: bitmap of commands to mark available for userspace
621 */
622void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds,
623 unsigned long *cmds)
624{
625 down_write(&cxl_memdev_rwsem);
626 bitmap_andnot(mds->exclusive_cmds, mds->exclusive_cmds, cmds,
627 CXL_MEM_COMMAND_ID_MAX);
628 up_write(&cxl_memdev_rwsem);
629}
630EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, CXL);
631
632static void cxl_memdev_shutdown(struct device *dev)
633{
634 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
635
636 down_write(&cxl_memdev_rwsem);
637 cxlmd->cxlds = NULL;
638 up_write(&cxl_memdev_rwsem);
639}
640
641static void cxl_memdev_unregister(void *_cxlmd)
642{
643 struct cxl_memdev *cxlmd = _cxlmd;
644 struct device *dev = &cxlmd->dev;
645
646 cdev_device_del(&cxlmd->cdev, dev);
647 cxl_memdev_shutdown(dev);
648 put_device(dev);
649}
650
651static void detach_memdev(struct work_struct *work)
652{
653 struct cxl_memdev *cxlmd;
654
655 cxlmd = container_of(work, typeof(*cxlmd), detach_work);
656 device_release_driver(&cxlmd->dev);
657 put_device(&cxlmd->dev);
658}
659
660static struct lock_class_key cxl_memdev_key;
661
662static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
663 const struct file_operations *fops)
664{
665 struct cxl_memdev *cxlmd;
666 struct device *dev;
667 struct cdev *cdev;
668 int rc;
669
670 cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL);
671 if (!cxlmd)
672 return ERR_PTR(-ENOMEM);
673
674 rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL);
675 if (rc < 0)
676 goto err;
677 cxlmd->id = rc;
678 cxlmd->depth = -1;
679
680 dev = &cxlmd->dev;
681 device_initialize(dev);
682 lockdep_set_class(&dev->mutex, &cxl_memdev_key);
683 dev->parent = cxlds->dev;
684 dev->bus = &cxl_bus_type;
685 dev->devt = MKDEV(cxl_mem_major, cxlmd->id);
686 dev->type = &cxl_memdev_type;
687 device_set_pm_not_required(dev);
688 INIT_WORK(&cxlmd->detach_work, detach_memdev);
689
690 cdev = &cxlmd->cdev;
691 cdev_init(cdev, fops);
692 return cxlmd;
693
694err:
695 kfree(cxlmd);
696 return ERR_PTR(rc);
697}
698
699static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,
700 unsigned long arg)
701{
702 switch (cmd) {
703 case CXL_MEM_QUERY_COMMANDS:
704 return cxl_query_cmd(cxlmd, (void __user *)arg);
705 case CXL_MEM_SEND_COMMAND:
706 return cxl_send_cmd(cxlmd, (void __user *)arg);
707 default:
708 return -ENOTTY;
709 }
710}
711
712static long cxl_memdev_ioctl(struct file *file, unsigned int cmd,
713 unsigned long arg)
714{
715 struct cxl_memdev *cxlmd = file->private_data;
716 struct cxl_dev_state *cxlds;
717 int rc = -ENXIO;
718
719 down_read(&cxl_memdev_rwsem);
720 cxlds = cxlmd->cxlds;
721 if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM)
722 rc = __cxl_memdev_ioctl(cxlmd, cmd, arg);
723 up_read(&cxl_memdev_rwsem);
724
725 return rc;
726}
727
728static int cxl_memdev_open(struct inode *inode, struct file *file)
729{
730 struct cxl_memdev *cxlmd =
731 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
732
733 get_device(&cxlmd->dev);
734 file->private_data = cxlmd;
735
736 return 0;
737}
738
739static int cxl_memdev_release_file(struct inode *inode, struct file *file)
740{
741 struct cxl_memdev *cxlmd =
742 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
743
744 put_device(&cxlmd->dev);
745
746 return 0;
747}
748
749/**
750 * cxl_mem_get_fw_info - Get Firmware info
751 * @mds: The device data for the operation
752 *
753 * Retrieve firmware info for the device specified.
754 *
755 * Return: 0 if no error: or the result of the mailbox command.
756 *
757 * See CXL-3.0 8.2.9.3.1 Get FW Info
758 */
759static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds)
760{
761 struct cxl_mbox_get_fw_info info;
762 struct cxl_mbox_cmd mbox_cmd;
763 int rc;
764
765 mbox_cmd = (struct cxl_mbox_cmd) {
766 .opcode = CXL_MBOX_OP_GET_FW_INFO,
767 .size_out = sizeof(info),
768 .payload_out = &info,
769 };
770
771 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
772 if (rc < 0)
773 return rc;
774
775 mds->fw.num_slots = info.num_slots;
776 mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK,
777 info.slot_info);
778
779 return 0;
780}
781
782/**
783 * cxl_mem_activate_fw - Activate Firmware
784 * @mds: The device data for the operation
785 * @slot: slot number to activate
786 *
787 * Activate firmware in a given slot for the device specified.
788 *
789 * Return: 0 if no error: or the result of the mailbox command.
790 *
791 * See CXL-3.0 8.2.9.3.3 Activate FW
792 */
793static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot)
794{
795 struct cxl_mbox_activate_fw activate;
796 struct cxl_mbox_cmd mbox_cmd;
797
798 if (slot == 0 || slot > mds->fw.num_slots)
799 return -EINVAL;
800
801 mbox_cmd = (struct cxl_mbox_cmd) {
802 .opcode = CXL_MBOX_OP_ACTIVATE_FW,
803 .size_in = sizeof(activate),
804 .payload_in = &activate,
805 };
806
807 /* Only offline activation supported for now */
808 activate.action = CXL_FW_ACTIVATE_OFFLINE;
809 activate.slot = slot;
810
811 return cxl_internal_send_cmd(mds, &mbox_cmd);
812}
813
814/**
815 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer
816 * @mds: The device data for the operation
817 *
818 * Abort an in-progress firmware transfer for the device specified.
819 *
820 * Return: 0 if no error: or the result of the mailbox command.
821 *
822 * See CXL-3.0 8.2.9.3.2 Transfer FW
823 */
824static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds)
825{
826 struct cxl_mbox_transfer_fw *transfer;
827 struct cxl_mbox_cmd mbox_cmd;
828 int rc;
829
830 transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL);
831 if (!transfer)
832 return -ENOMEM;
833
834 /* Set a 1s poll interval and a total wait time of 30s */
835 mbox_cmd = (struct cxl_mbox_cmd) {
836 .opcode = CXL_MBOX_OP_TRANSFER_FW,
837 .size_in = sizeof(*transfer),
838 .payload_in = transfer,
839 .poll_interval_ms = 1000,
840 .poll_count = 30,
841 };
842
843 transfer->action = CXL_FW_TRANSFER_ACTION_ABORT;
844
845 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
846 kfree(transfer);
847 return rc;
848}
849
850static void cxl_fw_cleanup(struct fw_upload *fwl)
851{
852 struct cxl_memdev_state *mds = fwl->dd_handle;
853
854 mds->fw.next_slot = 0;
855}
856
857static int cxl_fw_do_cancel(struct fw_upload *fwl)
858{
859 struct cxl_memdev_state *mds = fwl->dd_handle;
860 struct cxl_dev_state *cxlds = &mds->cxlds;
861 struct cxl_memdev *cxlmd = cxlds->cxlmd;
862 int rc;
863
864 rc = cxl_mem_abort_fw_xfer(mds);
865 if (rc < 0)
866 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc);
867
868 return FW_UPLOAD_ERR_CANCELED;
869}
870
871static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,
872 u32 size)
873{
874 struct cxl_memdev_state *mds = fwl->dd_handle;
875 struct cxl_mbox_transfer_fw *transfer;
876
877 if (!size)
878 return FW_UPLOAD_ERR_INVALID_SIZE;
879
880 mds->fw.oneshot = struct_size(transfer, data, size) <
881 mds->payload_size;
882
883 if (cxl_mem_get_fw_info(mds))
884 return FW_UPLOAD_ERR_HW_ERROR;
885
886 /*
887 * So far no state has been changed, hence no other cleanup is
888 * necessary. Simply return the cancelled status.
889 */
890 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
891 return FW_UPLOAD_ERR_CANCELED;
892
893 return FW_UPLOAD_ERR_NONE;
894}
895
896static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
897 u32 offset, u32 size, u32 *written)
898{
899 struct cxl_memdev_state *mds = fwl->dd_handle;
900 struct cxl_dev_state *cxlds = &mds->cxlds;
901 struct cxl_memdev *cxlmd = cxlds->cxlmd;
902 struct cxl_mbox_transfer_fw *transfer;
903 struct cxl_mbox_cmd mbox_cmd;
904 u32 cur_size, remaining;
905 size_t size_in;
906 int rc;
907
908 *written = 0;
909
910 /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */
911 if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) {
912 dev_err(&cxlmd->dev,
913 "misaligned offset for FW transfer slice (%u)\n",
914 offset);
915 return FW_UPLOAD_ERR_RW_ERROR;
916 }
917
918 /*
919 * Pick transfer size based on mds->payload_size @size must bw 128-byte
920 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and
921 * sizeof(*transfer) is 128. These constraints imply that @cur_size
922 * will always be 128b aligned.
923 */
924 cur_size = min_t(size_t, size, mds->payload_size - sizeof(*transfer));
925
926 remaining = size - cur_size;
927 size_in = struct_size(transfer, data, cur_size);
928
929 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
930 return cxl_fw_do_cancel(fwl);
931
932 /*
933 * Slot numbers are 1-indexed
934 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1')
935 * Check for rollover using modulo, and 1-index it by adding 1
936 */
937 mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1;
938
939 /* Do the transfer via mailbox cmd */
940 transfer = kzalloc(size_in, GFP_KERNEL);
941 if (!transfer)
942 return FW_UPLOAD_ERR_RW_ERROR;
943
944 transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT);
945 memcpy(transfer->data, data + offset, cur_size);
946 if (mds->fw.oneshot) {
947 transfer->action = CXL_FW_TRANSFER_ACTION_FULL;
948 transfer->slot = mds->fw.next_slot;
949 } else {
950 if (offset == 0) {
951 transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE;
952 } else if (remaining == 0) {
953 transfer->action = CXL_FW_TRANSFER_ACTION_END;
954 transfer->slot = mds->fw.next_slot;
955 } else {
956 transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE;
957 }
958 }
959
960 mbox_cmd = (struct cxl_mbox_cmd) {
961 .opcode = CXL_MBOX_OP_TRANSFER_FW,
962 .size_in = size_in,
963 .payload_in = transfer,
964 .poll_interval_ms = 1000,
965 .poll_count = 30,
966 };
967
968 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
969 if (rc < 0) {
970 rc = FW_UPLOAD_ERR_RW_ERROR;
971 goto out_free;
972 }
973
974 *written = cur_size;
975
976 /* Activate FW if oneshot or if the last slice was written */
977 if (mds->fw.oneshot || remaining == 0) {
978 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n",
979 mds->fw.next_slot);
980 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot);
981 if (rc < 0) {
982 dev_err(&cxlmd->dev, "Error activating firmware: %d\n",
983 rc);
984 rc = FW_UPLOAD_ERR_HW_ERROR;
985 goto out_free;
986 }
987 }
988
989 rc = FW_UPLOAD_ERR_NONE;
990
991out_free:
992 kfree(transfer);
993 return rc;
994}
995
996static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl)
997{
998 struct cxl_memdev_state *mds = fwl->dd_handle;
999
1000 /*
1001 * cxl_internal_send_cmd() handles background operations synchronously.
1002 * No need to wait for completions here - any errors would've been
1003 * reported and handled during the ->write() call(s).
1004 * Just check if a cancel request was received, and return success.
1005 */
1006 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
1007 return cxl_fw_do_cancel(fwl);
1008
1009 return FW_UPLOAD_ERR_NONE;
1010}
1011
1012static void cxl_fw_cancel(struct fw_upload *fwl)
1013{
1014 struct cxl_memdev_state *mds = fwl->dd_handle;
1015
1016 set_bit(CXL_FW_CANCEL, mds->fw.state);
1017}
1018
1019static const struct fw_upload_ops cxl_memdev_fw_ops = {
1020 .prepare = cxl_fw_prepare,
1021 .write = cxl_fw_write,
1022 .poll_complete = cxl_fw_poll_complete,
1023 .cancel = cxl_fw_cancel,
1024 .cleanup = cxl_fw_cleanup,
1025};
1026
1027static void cxl_remove_fw_upload(void *fwl)
1028{
1029 firmware_upload_unregister(fwl);
1030}
1031
1032int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds)
1033{
1034 struct cxl_dev_state *cxlds = &mds->cxlds;
1035 struct device *dev = &cxlds->cxlmd->dev;
1036 struct fw_upload *fwl;
1037
1038 if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, mds->enabled_cmds))
1039 return 0;
1040
1041 fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev),
1042 &cxl_memdev_fw_ops, mds);
1043 if (IS_ERR(fwl))
1044 return PTR_ERR(fwl);
1045 return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl);
1046}
1047EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, CXL);
1048
1049static const struct file_operations cxl_memdev_fops = {
1050 .owner = THIS_MODULE,
1051 .unlocked_ioctl = cxl_memdev_ioctl,
1052 .open = cxl_memdev_open,
1053 .release = cxl_memdev_release_file,
1054 .compat_ioctl = compat_ptr_ioctl,
1055 .llseek = noop_llseek,
1056};
1057
1058struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
1059 struct cxl_dev_state *cxlds)
1060{
1061 struct cxl_memdev *cxlmd;
1062 struct device *dev;
1063 struct cdev *cdev;
1064 int rc;
1065
1066 cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops);
1067 if (IS_ERR(cxlmd))
1068 return cxlmd;
1069
1070 dev = &cxlmd->dev;
1071 rc = dev_set_name(dev, "mem%d", cxlmd->id);
1072 if (rc)
1073 goto err;
1074
1075 /*
1076 * Activate ioctl operations, no cxl_memdev_rwsem manipulation
1077 * needed as this is ordered with cdev_add() publishing the device.
1078 */
1079 cxlmd->cxlds = cxlds;
1080 cxlds->cxlmd = cxlmd;
1081
1082 cdev = &cxlmd->cdev;
1083 rc = cdev_device_add(cdev, dev);
1084 if (rc)
1085 goto err;
1086
1087 rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd);
1088 if (rc)
1089 return ERR_PTR(rc);
1090 return cxlmd;
1091
1092err:
1093 /*
1094 * The cdev was briefly live, shutdown any ioctl operations that
1095 * saw that state.
1096 */
1097 cxl_memdev_shutdown(dev);
1098 put_device(dev);
1099 return ERR_PTR(rc);
1100}
1101EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
1102
1103static void sanitize_teardown_notifier(void *data)
1104{
1105 struct cxl_memdev_state *mds = data;
1106 struct kernfs_node *state;
1107
1108 /*
1109 * Prevent new irq triggered invocations of the workqueue and
1110 * flush inflight invocations.
1111 */
1112 mutex_lock(&mds->mbox_mutex);
1113 state = mds->security.sanitize_node;
1114 mds->security.sanitize_node = NULL;
1115 mutex_unlock(&mds->mbox_mutex);
1116
1117 cancel_delayed_work_sync(&mds->security.poll_dwork);
1118 sysfs_put(state);
1119}
1120
1121int devm_cxl_sanitize_setup_notifier(struct device *host,
1122 struct cxl_memdev *cxlmd)
1123{
1124 struct cxl_dev_state *cxlds = cxlmd->cxlds;
1125 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
1126 struct kernfs_node *sec;
1127
1128 if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
1129 return 0;
1130
1131 /*
1132 * Note, the expectation is that @cxlmd would have failed to be
1133 * created if these sysfs_get_dirent calls fail.
1134 */
1135 sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security");
1136 if (!sec)
1137 return -ENOENT;
1138 mds->security.sanitize_node = sysfs_get_dirent(sec, "state");
1139 sysfs_put(sec);
1140 if (!mds->security.sanitize_node)
1141 return -ENOENT;
1142
1143 return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds);
1144}
1145EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, CXL);
1146
1147__init int cxl_memdev_init(void)
1148{
1149 dev_t devt;
1150 int rc;
1151
1152 rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl");
1153 if (rc)
1154 return rc;
1155
1156 cxl_mem_major = MAJOR(devt);
1157
1158 return 0;
1159}
1160
1161void cxl_memdev_exit(void)
1162{
1163 unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS);
1164}