Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4 */
5#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6#include <linux/libnvdimm.h>
7#include <linux/sched/mm.h>
8#include <linux/vmalloc.h>
9#include <linux/uaccess.h>
10#include <linux/module.h>
11#include <linux/blkdev.h>
12#include <linux/fcntl.h>
13#include <linux/async.h>
14#include <linux/ndctl.h>
15#include <linux/sched.h>
16#include <linux/slab.h>
17#include <linux/cpu.h>
18#include <linux/fs.h>
19#include <linux/io.h>
20#include <linux/mm.h>
21#include <linux/nd.h>
22#include "nd-core.h"
23#include "nd.h"
24#include "pfn.h"
25
26int nvdimm_major;
27static int nvdimm_bus_major;
28static struct class *nd_class;
29static DEFINE_IDA(nd_ida);
30
31static int to_nd_device_type(const struct device *dev)
32{
33 if (is_nvdimm(dev))
34 return ND_DEVICE_DIMM;
35 else if (is_memory(dev))
36 return ND_DEVICE_REGION_PMEM;
37 else if (is_nd_dax(dev))
38 return ND_DEVICE_DAX_PMEM;
39 else if (is_nd_region(dev->parent))
40 return nd_region_to_nstype(to_nd_region(dev->parent));
41
42 return 0;
43}
44
45static int nvdimm_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
46{
47 return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT,
48 to_nd_device_type(dev));
49}
50
51static struct module *to_bus_provider(struct device *dev)
52{
53 /* pin bus providers while regions are enabled */
54 if (is_nd_region(dev)) {
55 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
56
57 return nvdimm_bus->nd_desc->module;
58 }
59 return NULL;
60}
61
62static void nvdimm_bus_probe_start(struct nvdimm_bus *nvdimm_bus)
63{
64 nvdimm_bus_lock(&nvdimm_bus->dev);
65 nvdimm_bus->probe_active++;
66 nvdimm_bus_unlock(&nvdimm_bus->dev);
67}
68
69static void nvdimm_bus_probe_end(struct nvdimm_bus *nvdimm_bus)
70{
71 nvdimm_bus_lock(&nvdimm_bus->dev);
72 if (--nvdimm_bus->probe_active == 0)
73 wake_up(&nvdimm_bus->wait);
74 nvdimm_bus_unlock(&nvdimm_bus->dev);
75}
76
77static int nvdimm_bus_probe(struct device *dev)
78{
79 struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
80 struct module *provider = to_bus_provider(dev);
81 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
82 int rc;
83
84 if (!try_module_get(provider))
85 return -ENXIO;
86
87 dev_dbg(&nvdimm_bus->dev, "START: %s.probe(%s)\n",
88 dev->driver->name, dev_name(dev));
89
90 nvdimm_bus_probe_start(nvdimm_bus);
91 rc = nd_drv->probe(dev);
92 if ((rc == 0 || rc == -EOPNOTSUPP) &&
93 dev->parent && is_nd_region(dev->parent))
94 nd_region_advance_seeds(to_nd_region(dev->parent), dev);
95 nvdimm_bus_probe_end(nvdimm_bus);
96
97 dev_dbg(&nvdimm_bus->dev, "END: %s.probe(%s) = %d\n", dev->driver->name,
98 dev_name(dev), rc);
99
100 if (rc != 0)
101 module_put(provider);
102 return rc;
103}
104
105static void nvdimm_bus_remove(struct device *dev)
106{
107 struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
108 struct module *provider = to_bus_provider(dev);
109 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
110
111 if (nd_drv->remove)
112 nd_drv->remove(dev);
113
114 dev_dbg(&nvdimm_bus->dev, "%s.remove(%s)\n", dev->driver->name,
115 dev_name(dev));
116 module_put(provider);
117}
118
119static void nvdimm_bus_shutdown(struct device *dev)
120{
121 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
122 struct nd_device_driver *nd_drv = NULL;
123
124 if (dev->driver)
125 nd_drv = to_nd_device_driver(dev->driver);
126
127 if (nd_drv && nd_drv->shutdown) {
128 nd_drv->shutdown(dev);
129 dev_dbg(&nvdimm_bus->dev, "%s.shutdown(%s)\n",
130 dev->driver->name, dev_name(dev));
131 }
132}
133
134void nd_device_notify(struct device *dev, enum nvdimm_event event)
135{
136 device_lock(dev);
137 if (dev->driver) {
138 struct nd_device_driver *nd_drv;
139
140 nd_drv = to_nd_device_driver(dev->driver);
141 if (nd_drv->notify)
142 nd_drv->notify(dev, event);
143 }
144 device_unlock(dev);
145}
146EXPORT_SYMBOL(nd_device_notify);
147
148void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event)
149{
150 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);
151
152 if (!nvdimm_bus)
153 return;
154
155 /* caller is responsible for holding a reference on the device */
156 nd_device_notify(&nd_region->dev, event);
157}
158EXPORT_SYMBOL_GPL(nvdimm_region_notify);
159
160struct clear_badblocks_context {
161 resource_size_t phys, cleared;
162};
163
164static int nvdimm_clear_badblocks_region(struct device *dev, void *data)
165{
166 struct clear_badblocks_context *ctx = data;
167 struct nd_region *nd_region;
168 resource_size_t ndr_end;
169 sector_t sector;
170
171 /* make sure device is a region */
172 if (!is_memory(dev))
173 return 0;
174
175 nd_region = to_nd_region(dev);
176 ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1;
177
178 /* make sure we are in the region */
179 if (ctx->phys < nd_region->ndr_start ||
180 (ctx->phys + ctx->cleared - 1) > ndr_end)
181 return 0;
182
183 sector = (ctx->phys - nd_region->ndr_start) / 512;
184 badblocks_clear(&nd_region->bb, sector, ctx->cleared / 512);
185
186 if (nd_region->bb_state)
187 sysfs_notify_dirent(nd_region->bb_state);
188
189 return 0;
190}
191
192static void nvdimm_clear_badblocks_regions(struct nvdimm_bus *nvdimm_bus,
193 phys_addr_t phys, u64 cleared)
194{
195 struct clear_badblocks_context ctx = {
196 .phys = phys,
197 .cleared = cleared,
198 };
199
200 device_for_each_child(&nvdimm_bus->dev, &ctx,
201 nvdimm_clear_badblocks_region);
202}
203
204static void nvdimm_account_cleared_poison(struct nvdimm_bus *nvdimm_bus,
205 phys_addr_t phys, u64 cleared)
206{
207 if (cleared > 0)
208 badrange_forget(&nvdimm_bus->badrange, phys, cleared);
209
210 if (cleared > 0 && cleared / 512)
211 nvdimm_clear_badblocks_regions(nvdimm_bus, phys, cleared);
212}
213
214long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
215 unsigned int len)
216{
217 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
218 struct nvdimm_bus_descriptor *nd_desc;
219 struct nd_cmd_clear_error clear_err;
220 struct nd_cmd_ars_cap ars_cap;
221 u32 clear_err_unit, mask;
222 unsigned int noio_flag;
223 int cmd_rc, rc;
224
225 if (!nvdimm_bus)
226 return -ENXIO;
227
228 nd_desc = nvdimm_bus->nd_desc;
229 /*
230 * if ndctl does not exist, it's PMEM_LEGACY and
231 * we want to just pretend everything is handled.
232 */
233 if (!nd_desc->ndctl)
234 return len;
235
236 memset(&ars_cap, 0, sizeof(ars_cap));
237 ars_cap.address = phys;
238 ars_cap.length = len;
239 noio_flag = memalloc_noio_save();
240 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap,
241 sizeof(ars_cap), &cmd_rc);
242 memalloc_noio_restore(noio_flag);
243 if (rc < 0)
244 return rc;
245 if (cmd_rc < 0)
246 return cmd_rc;
247 clear_err_unit = ars_cap.clear_err_unit;
248 if (!clear_err_unit || !is_power_of_2(clear_err_unit))
249 return -ENXIO;
250
251 mask = clear_err_unit - 1;
252 if ((phys | len) & mask)
253 return -ENXIO;
254 memset(&clear_err, 0, sizeof(clear_err));
255 clear_err.address = phys;
256 clear_err.length = len;
257 noio_flag = memalloc_noio_save();
258 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err,
259 sizeof(clear_err), &cmd_rc);
260 memalloc_noio_restore(noio_flag);
261 if (rc < 0)
262 return rc;
263 if (cmd_rc < 0)
264 return cmd_rc;
265
266 nvdimm_account_cleared_poison(nvdimm_bus, phys, clear_err.cleared);
267
268 return clear_err.cleared;
269}
270EXPORT_SYMBOL_GPL(nvdimm_clear_poison);
271
272static int nvdimm_bus_match(struct device *dev, struct device_driver *drv);
273
274static struct bus_type nvdimm_bus_type = {
275 .name = "nd",
276 .uevent = nvdimm_bus_uevent,
277 .match = nvdimm_bus_match,
278 .probe = nvdimm_bus_probe,
279 .remove = nvdimm_bus_remove,
280 .shutdown = nvdimm_bus_shutdown,
281};
282
283static void nvdimm_bus_release(struct device *dev)
284{
285 struct nvdimm_bus *nvdimm_bus;
286
287 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
288 ida_free(&nd_ida, nvdimm_bus->id);
289 kfree(nvdimm_bus);
290}
291
292static const struct device_type nvdimm_bus_dev_type = {
293 .release = nvdimm_bus_release,
294 .groups = nvdimm_bus_attribute_groups,
295};
296
297bool is_nvdimm_bus(struct device *dev)
298{
299 return dev->type == &nvdimm_bus_dev_type;
300}
301
302struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev)
303{
304 struct device *dev;
305
306 for (dev = nd_dev; dev; dev = dev->parent)
307 if (is_nvdimm_bus(dev))
308 break;
309 dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n");
310 if (dev)
311 return to_nvdimm_bus(dev);
312 return NULL;
313}
314
315struct nvdimm_bus *to_nvdimm_bus(struct device *dev)
316{
317 struct nvdimm_bus *nvdimm_bus;
318
319 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
320 WARN_ON(!is_nvdimm_bus(dev));
321 return nvdimm_bus;
322}
323EXPORT_SYMBOL_GPL(to_nvdimm_bus);
324
325struct nvdimm_bus *nvdimm_to_bus(struct nvdimm *nvdimm)
326{
327 return to_nvdimm_bus(nvdimm->dev.parent);
328}
329EXPORT_SYMBOL_GPL(nvdimm_to_bus);
330
331static struct lock_class_key nvdimm_bus_key;
332
333struct nvdimm_bus *nvdimm_bus_register(struct device *parent,
334 struct nvdimm_bus_descriptor *nd_desc)
335{
336 struct nvdimm_bus *nvdimm_bus;
337 int rc;
338
339 nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL);
340 if (!nvdimm_bus)
341 return NULL;
342 INIT_LIST_HEAD(&nvdimm_bus->list);
343 INIT_LIST_HEAD(&nvdimm_bus->mapping_list);
344 init_waitqueue_head(&nvdimm_bus->wait);
345 nvdimm_bus->id = ida_alloc(&nd_ida, GFP_KERNEL);
346 if (nvdimm_bus->id < 0) {
347 kfree(nvdimm_bus);
348 return NULL;
349 }
350 mutex_init(&nvdimm_bus->reconfig_mutex);
351 badrange_init(&nvdimm_bus->badrange);
352 nvdimm_bus->nd_desc = nd_desc;
353 nvdimm_bus->dev.parent = parent;
354 nvdimm_bus->dev.type = &nvdimm_bus_dev_type;
355 nvdimm_bus->dev.groups = nd_desc->attr_groups;
356 nvdimm_bus->dev.bus = &nvdimm_bus_type;
357 nvdimm_bus->dev.of_node = nd_desc->of_node;
358 device_initialize(&nvdimm_bus->dev);
359 lockdep_set_class(&nvdimm_bus->dev.mutex, &nvdimm_bus_key);
360 device_set_pm_not_required(&nvdimm_bus->dev);
361 rc = dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id);
362 if (rc)
363 goto err;
364
365 rc = device_add(&nvdimm_bus->dev);
366 if (rc) {
367 dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc);
368 goto err;
369 }
370
371 return nvdimm_bus;
372 err:
373 put_device(&nvdimm_bus->dev);
374 return NULL;
375}
376EXPORT_SYMBOL_GPL(nvdimm_bus_register);
377
378void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus)
379{
380 if (!nvdimm_bus)
381 return;
382 device_unregister(&nvdimm_bus->dev);
383}
384EXPORT_SYMBOL_GPL(nvdimm_bus_unregister);
385
386static int child_unregister(struct device *dev, void *data)
387{
388 /*
389 * the singular ndctl class device per bus needs to be
390 * "device_destroy"ed, so skip it here
391 *
392 * i.e. remove classless children
393 */
394 if (dev->class)
395 return 0;
396
397 if (is_nvdimm(dev))
398 nvdimm_delete(to_nvdimm(dev));
399 else
400 nd_device_unregister(dev, ND_SYNC);
401
402 return 0;
403}
404
405static void free_badrange_list(struct list_head *badrange_list)
406{
407 struct badrange_entry *bre, *next;
408
409 list_for_each_entry_safe(bre, next, badrange_list, list) {
410 list_del(&bre->list);
411 kfree(bre);
412 }
413 list_del_init(badrange_list);
414}
415
416static void nd_bus_remove(struct device *dev)
417{
418 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
419
420 mutex_lock(&nvdimm_bus_list_mutex);
421 list_del_init(&nvdimm_bus->list);
422 mutex_unlock(&nvdimm_bus_list_mutex);
423
424 wait_event(nvdimm_bus->wait,
425 atomic_read(&nvdimm_bus->ioctl_active) == 0);
426
427 nd_synchronize();
428 device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister);
429
430 spin_lock(&nvdimm_bus->badrange.lock);
431 free_badrange_list(&nvdimm_bus->badrange.list);
432 spin_unlock(&nvdimm_bus->badrange.lock);
433
434 nvdimm_bus_destroy_ndctl(nvdimm_bus);
435}
436
437static int nd_bus_probe(struct device *dev)
438{
439 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
440 int rc;
441
442 rc = nvdimm_bus_create_ndctl(nvdimm_bus);
443 if (rc)
444 return rc;
445
446 mutex_lock(&nvdimm_bus_list_mutex);
447 list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list);
448 mutex_unlock(&nvdimm_bus_list_mutex);
449
450 /* enable bus provider attributes to look up their local context */
451 dev_set_drvdata(dev, nvdimm_bus->nd_desc);
452
453 return 0;
454}
455
456static struct nd_device_driver nd_bus_driver = {
457 .probe = nd_bus_probe,
458 .remove = nd_bus_remove,
459 .drv = {
460 .name = "nd_bus",
461 .suppress_bind_attrs = true,
462 .bus = &nvdimm_bus_type,
463 .owner = THIS_MODULE,
464 .mod_name = KBUILD_MODNAME,
465 },
466};
467
468static int nvdimm_bus_match(struct device *dev, struct device_driver *drv)
469{
470 struct nd_device_driver *nd_drv = to_nd_device_driver(drv);
471
472 if (is_nvdimm_bus(dev) && nd_drv == &nd_bus_driver)
473 return true;
474
475 return !!test_bit(to_nd_device_type(dev), &nd_drv->type);
476}
477
478static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain);
479
480void nd_synchronize(void)
481{
482 async_synchronize_full_domain(&nd_async_domain);
483}
484EXPORT_SYMBOL_GPL(nd_synchronize);
485
486static void nd_async_device_register(void *d, async_cookie_t cookie)
487{
488 struct device *dev = d;
489
490 if (device_add(dev) != 0) {
491 dev_err(dev, "%s: failed\n", __func__);
492 put_device(dev);
493 }
494 put_device(dev);
495 if (dev->parent)
496 put_device(dev->parent);
497}
498
499static void nd_async_device_unregister(void *d, async_cookie_t cookie)
500{
501 struct device *dev = d;
502
503 /* flush bus operations before delete */
504 nvdimm_bus_lock(dev);
505 nvdimm_bus_unlock(dev);
506
507 device_unregister(dev);
508 put_device(dev);
509}
510
511static void __nd_device_register(struct device *dev, bool sync)
512{
513 if (!dev)
514 return;
515
516 /*
517 * Ensure that region devices always have their NUMA node set as
518 * early as possible. This way we are able to make certain that
519 * any memory associated with the creation and the creation
520 * itself of the region is associated with the correct node.
521 */
522 if (is_nd_region(dev))
523 set_dev_node(dev, to_nd_region(dev)->numa_node);
524
525 dev->bus = &nvdimm_bus_type;
526 device_set_pm_not_required(dev);
527 if (dev->parent) {
528 get_device(dev->parent);
529 if (dev_to_node(dev) == NUMA_NO_NODE)
530 set_dev_node(dev, dev_to_node(dev->parent));
531 }
532 get_device(dev);
533
534 if (sync)
535 nd_async_device_register(dev, 0);
536 else
537 async_schedule_dev_domain(nd_async_device_register, dev,
538 &nd_async_domain);
539}
540
541void nd_device_register(struct device *dev)
542{
543 __nd_device_register(dev, false);
544}
545EXPORT_SYMBOL(nd_device_register);
546
547void nd_device_register_sync(struct device *dev)
548{
549 __nd_device_register(dev, true);
550}
551
552void nd_device_unregister(struct device *dev, enum nd_async_mode mode)
553{
554 bool killed;
555
556 switch (mode) {
557 case ND_ASYNC:
558 /*
559 * In the async case this is being triggered with the
560 * device lock held and the unregistration work needs to
561 * be moved out of line iff this is thread has won the
562 * race to schedule the deletion.
563 */
564 if (!kill_device(dev))
565 return;
566
567 get_device(dev);
568 async_schedule_domain(nd_async_device_unregister, dev,
569 &nd_async_domain);
570 break;
571 case ND_SYNC:
572 /*
573 * In the sync case the device is being unregistered due
574 * to a state change of the parent. Claim the kill state
575 * to synchronize against other unregistration requests,
576 * or otherwise let the async path handle it if the
577 * unregistration was already queued.
578 */
579 device_lock(dev);
580 killed = kill_device(dev);
581 device_unlock(dev);
582
583 if (!killed)
584 return;
585
586 nd_synchronize();
587 device_unregister(dev);
588 break;
589 }
590}
591EXPORT_SYMBOL(nd_device_unregister);
592
593/**
594 * __nd_driver_register() - register a region or a namespace driver
595 * @nd_drv: driver to register
596 * @owner: automatically set by nd_driver_register() macro
597 * @mod_name: automatically set by nd_driver_register() macro
598 */
599int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner,
600 const char *mod_name)
601{
602 struct device_driver *drv = &nd_drv->drv;
603
604 if (!nd_drv->type) {
605 pr_debug("driver type bitmask not set (%ps)\n",
606 __builtin_return_address(0));
607 return -EINVAL;
608 }
609
610 if (!nd_drv->probe) {
611 pr_debug("%s ->probe() must be specified\n", mod_name);
612 return -EINVAL;
613 }
614
615 drv->bus = &nvdimm_bus_type;
616 drv->owner = owner;
617 drv->mod_name = mod_name;
618
619 return driver_register(drv);
620}
621EXPORT_SYMBOL(__nd_driver_register);
622
623void nvdimm_check_and_set_ro(struct gendisk *disk)
624{
625 struct device *dev = disk_to_dev(disk)->parent;
626 struct nd_region *nd_region = to_nd_region(dev->parent);
627 int disk_ro = get_disk_ro(disk);
628
629 /* catch the disk up with the region ro state */
630 if (disk_ro == nd_region->ro)
631 return;
632
633 dev_info(dev, "%s read-%s, marking %s read-%s\n",
634 dev_name(&nd_region->dev), nd_region->ro ? "only" : "write",
635 disk->disk_name, nd_region->ro ? "only" : "write");
636 set_disk_ro(disk, nd_region->ro);
637}
638EXPORT_SYMBOL(nvdimm_check_and_set_ro);
639
640static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
641 char *buf)
642{
643 return sprintf(buf, ND_DEVICE_MODALIAS_FMT "\n",
644 to_nd_device_type(dev));
645}
646static DEVICE_ATTR_RO(modalias);
647
648static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
649 char *buf)
650{
651 return sprintf(buf, "%s\n", dev->type->name);
652}
653static DEVICE_ATTR_RO(devtype);
654
655static struct attribute *nd_device_attributes[] = {
656 &dev_attr_modalias.attr,
657 &dev_attr_devtype.attr,
658 NULL,
659};
660
661/*
662 * nd_device_attribute_group - generic attributes for all devices on an nd bus
663 */
664const struct attribute_group nd_device_attribute_group = {
665 .attrs = nd_device_attributes,
666};
667
668static ssize_t numa_node_show(struct device *dev,
669 struct device_attribute *attr, char *buf)
670{
671 return sprintf(buf, "%d\n", dev_to_node(dev));
672}
673static DEVICE_ATTR_RO(numa_node);
674
675static int nvdimm_dev_to_target_node(struct device *dev)
676{
677 struct device *parent = dev->parent;
678 struct nd_region *nd_region = NULL;
679
680 if (is_nd_region(dev))
681 nd_region = to_nd_region(dev);
682 else if (parent && is_nd_region(parent))
683 nd_region = to_nd_region(parent);
684
685 if (!nd_region)
686 return NUMA_NO_NODE;
687 return nd_region->target_node;
688}
689
690static ssize_t target_node_show(struct device *dev,
691 struct device_attribute *attr, char *buf)
692{
693 return sprintf(buf, "%d\n", nvdimm_dev_to_target_node(dev));
694}
695static DEVICE_ATTR_RO(target_node);
696
697static struct attribute *nd_numa_attributes[] = {
698 &dev_attr_numa_node.attr,
699 &dev_attr_target_node.attr,
700 NULL,
701};
702
703static umode_t nd_numa_attr_visible(struct kobject *kobj, struct attribute *a,
704 int n)
705{
706 struct device *dev = container_of(kobj, typeof(*dev), kobj);
707
708 if (!IS_ENABLED(CONFIG_NUMA))
709 return 0;
710
711 if (a == &dev_attr_target_node.attr &&
712 nvdimm_dev_to_target_node(dev) == NUMA_NO_NODE)
713 return 0;
714
715 return a->mode;
716}
717
718/*
719 * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus
720 */
721const struct attribute_group nd_numa_attribute_group = {
722 .attrs = nd_numa_attributes,
723 .is_visible = nd_numa_attr_visible,
724};
725
726static void ndctl_release(struct device *dev)
727{
728 kfree(dev);
729}
730
731static struct lock_class_key nvdimm_ndctl_key;
732
733int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus)
734{
735 dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id);
736 struct device *dev;
737 int rc;
738
739 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
740 if (!dev)
741 return -ENOMEM;
742 device_initialize(dev);
743 lockdep_set_class(&dev->mutex, &nvdimm_ndctl_key);
744 device_set_pm_not_required(dev);
745 dev->class = nd_class;
746 dev->parent = &nvdimm_bus->dev;
747 dev->devt = devt;
748 dev->release = ndctl_release;
749 rc = dev_set_name(dev, "ndctl%d", nvdimm_bus->id);
750 if (rc)
751 goto err;
752
753 rc = device_add(dev);
754 if (rc) {
755 dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %d\n",
756 nvdimm_bus->id, rc);
757 goto err;
758 }
759 return 0;
760
761err:
762 put_device(dev);
763 return rc;
764}
765
766void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus)
767{
768 device_destroy(nd_class, MKDEV(nvdimm_bus_major, nvdimm_bus->id));
769}
770
771static const struct nd_cmd_desc __nd_cmd_dimm_descs[] = {
772 [ND_CMD_IMPLEMENTED] = { },
773 [ND_CMD_SMART] = {
774 .out_num = 2,
775 .out_sizes = { 4, 128, },
776 },
777 [ND_CMD_SMART_THRESHOLD] = {
778 .out_num = 2,
779 .out_sizes = { 4, 8, },
780 },
781 [ND_CMD_DIMM_FLAGS] = {
782 .out_num = 2,
783 .out_sizes = { 4, 4 },
784 },
785 [ND_CMD_GET_CONFIG_SIZE] = {
786 .out_num = 3,
787 .out_sizes = { 4, 4, 4, },
788 },
789 [ND_CMD_GET_CONFIG_DATA] = {
790 .in_num = 2,
791 .in_sizes = { 4, 4, },
792 .out_num = 2,
793 .out_sizes = { 4, UINT_MAX, },
794 },
795 [ND_CMD_SET_CONFIG_DATA] = {
796 .in_num = 3,
797 .in_sizes = { 4, 4, UINT_MAX, },
798 .out_num = 1,
799 .out_sizes = { 4, },
800 },
801 [ND_CMD_VENDOR] = {
802 .in_num = 3,
803 .in_sizes = { 4, 4, UINT_MAX, },
804 .out_num = 3,
805 .out_sizes = { 4, 4, UINT_MAX, },
806 },
807 [ND_CMD_CALL] = {
808 .in_num = 2,
809 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
810 .out_num = 1,
811 .out_sizes = { UINT_MAX, },
812 },
813};
814
815const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd)
816{
817 if (cmd < ARRAY_SIZE(__nd_cmd_dimm_descs))
818 return &__nd_cmd_dimm_descs[cmd];
819 return NULL;
820}
821EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc);
822
823static const struct nd_cmd_desc __nd_cmd_bus_descs[] = {
824 [ND_CMD_IMPLEMENTED] = { },
825 [ND_CMD_ARS_CAP] = {
826 .in_num = 2,
827 .in_sizes = { 8, 8, },
828 .out_num = 4,
829 .out_sizes = { 4, 4, 4, 4, },
830 },
831 [ND_CMD_ARS_START] = {
832 .in_num = 5,
833 .in_sizes = { 8, 8, 2, 1, 5, },
834 .out_num = 2,
835 .out_sizes = { 4, 4, },
836 },
837 [ND_CMD_ARS_STATUS] = {
838 .out_num = 3,
839 .out_sizes = { 4, 4, UINT_MAX, },
840 },
841 [ND_CMD_CLEAR_ERROR] = {
842 .in_num = 2,
843 .in_sizes = { 8, 8, },
844 .out_num = 3,
845 .out_sizes = { 4, 4, 8, },
846 },
847 [ND_CMD_CALL] = {
848 .in_num = 2,
849 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
850 .out_num = 1,
851 .out_sizes = { UINT_MAX, },
852 },
853};
854
855const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd)
856{
857 if (cmd < ARRAY_SIZE(__nd_cmd_bus_descs))
858 return &__nd_cmd_bus_descs[cmd];
859 return NULL;
860}
861EXPORT_SYMBOL_GPL(nd_cmd_bus_desc);
862
863u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd,
864 const struct nd_cmd_desc *desc, int idx, void *buf)
865{
866 if (idx >= desc->in_num)
867 return UINT_MAX;
868
869 if (desc->in_sizes[idx] < UINT_MAX)
870 return desc->in_sizes[idx];
871
872 if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA && idx == 2) {
873 struct nd_cmd_set_config_hdr *hdr = buf;
874
875 return hdr->in_length;
876 } else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) {
877 struct nd_cmd_vendor_hdr *hdr = buf;
878
879 return hdr->in_length;
880 } else if (cmd == ND_CMD_CALL) {
881 struct nd_cmd_pkg *pkg = buf;
882
883 return pkg->nd_size_in;
884 }
885
886 return UINT_MAX;
887}
888EXPORT_SYMBOL_GPL(nd_cmd_in_size);
889
890u32 nd_cmd_out_size(struct nvdimm *nvdimm, int cmd,
891 const struct nd_cmd_desc *desc, int idx, const u32 *in_field,
892 const u32 *out_field, unsigned long remainder)
893{
894 if (idx >= desc->out_num)
895 return UINT_MAX;
896
897 if (desc->out_sizes[idx] < UINT_MAX)
898 return desc->out_sizes[idx];
899
900 if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && idx == 1)
901 return in_field[1];
902 else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2)
903 return out_field[1];
904 else if (!nvdimm && cmd == ND_CMD_ARS_STATUS && idx == 2) {
905 /*
906 * Per table 9-276 ARS Data in ACPI 6.1, out_field[1] is
907 * "Size of Output Buffer in bytes, including this
908 * field."
909 */
910 if (out_field[1] < 4)
911 return 0;
912 /*
913 * ACPI 6.1 is ambiguous if 'status' is included in the
914 * output size. If we encounter an output size that
915 * overshoots the remainder by 4 bytes, assume it was
916 * including 'status'.
917 */
918 if (out_field[1] - 4 == remainder)
919 return remainder;
920 return out_field[1] - 8;
921 } else if (cmd == ND_CMD_CALL) {
922 struct nd_cmd_pkg *pkg = (struct nd_cmd_pkg *) in_field;
923
924 return pkg->nd_size_out;
925 }
926
927
928 return UINT_MAX;
929}
930EXPORT_SYMBOL_GPL(nd_cmd_out_size);
931
932void wait_nvdimm_bus_probe_idle(struct device *dev)
933{
934 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
935
936 do {
937 if (nvdimm_bus->probe_active == 0)
938 break;
939 nvdimm_bus_unlock(dev);
940 device_unlock(dev);
941 wait_event(nvdimm_bus->wait,
942 nvdimm_bus->probe_active == 0);
943 device_lock(dev);
944 nvdimm_bus_lock(dev);
945 } while (true);
946}
947
948static int nd_pmem_forget_poison_check(struct device *dev, void *data)
949{
950 struct nd_cmd_clear_error *clear_err =
951 (struct nd_cmd_clear_error *)data;
952 struct nd_btt *nd_btt = is_nd_btt(dev) ? to_nd_btt(dev) : NULL;
953 struct nd_pfn *nd_pfn = is_nd_pfn(dev) ? to_nd_pfn(dev) : NULL;
954 struct nd_dax *nd_dax = is_nd_dax(dev) ? to_nd_dax(dev) : NULL;
955 struct nd_namespace_common *ndns = NULL;
956 struct nd_namespace_io *nsio;
957 resource_size_t offset = 0, end_trunc = 0, start, end, pstart, pend;
958
959 if (nd_dax || !dev->driver)
960 return 0;
961
962 start = clear_err->address;
963 end = clear_err->address + clear_err->cleared - 1;
964
965 if (nd_btt || nd_pfn || nd_dax) {
966 if (nd_btt)
967 ndns = nd_btt->ndns;
968 else if (nd_pfn)
969 ndns = nd_pfn->ndns;
970 else if (nd_dax)
971 ndns = nd_dax->nd_pfn.ndns;
972
973 if (!ndns)
974 return 0;
975 } else
976 ndns = to_ndns(dev);
977
978 nsio = to_nd_namespace_io(&ndns->dev);
979 pstart = nsio->res.start + offset;
980 pend = nsio->res.end - end_trunc;
981
982 if ((pstart >= start) && (pend <= end))
983 return -EBUSY;
984
985 return 0;
986
987}
988
989static int nd_ns_forget_poison_check(struct device *dev, void *data)
990{
991 return device_for_each_child(dev, data, nd_pmem_forget_poison_check);
992}
993
994/* set_config requires an idle interleave set */
995static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus,
996 struct nvdimm *nvdimm, unsigned int cmd, void *data)
997{
998 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
999
1000 /* ask the bus provider if it would like to block this request */
1001 if (nd_desc->clear_to_send) {
1002 int rc = nd_desc->clear_to_send(nd_desc, nvdimm, cmd, data);
1003
1004 if (rc)
1005 return rc;
1006 }
1007
1008 /* require clear error to go through the pmem driver */
1009 if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR)
1010 return device_for_each_child(&nvdimm_bus->dev, data,
1011 nd_ns_forget_poison_check);
1012
1013 if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA)
1014 return 0;
1015
1016 /* prevent label manipulation while the kernel owns label updates */
1017 wait_nvdimm_bus_probe_idle(&nvdimm_bus->dev);
1018 if (atomic_read(&nvdimm->busy))
1019 return -EBUSY;
1020 return 0;
1021}
1022
1023static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
1024 int read_only, unsigned int ioctl_cmd, unsigned long arg)
1025{
1026 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
1027 const struct nd_cmd_desc *desc = NULL;
1028 unsigned int cmd = _IOC_NR(ioctl_cmd);
1029 struct device *dev = &nvdimm_bus->dev;
1030 void __user *p = (void __user *) arg;
1031 char *out_env = NULL, *in_env = NULL;
1032 const char *cmd_name, *dimm_name;
1033 u32 in_len = 0, out_len = 0;
1034 unsigned int func = cmd;
1035 unsigned long cmd_mask;
1036 struct nd_cmd_pkg pkg;
1037 int rc, i, cmd_rc;
1038 void *buf = NULL;
1039 u64 buf_len = 0;
1040
1041 if (nvdimm) {
1042 desc = nd_cmd_dimm_desc(cmd);
1043 cmd_name = nvdimm_cmd_name(cmd);
1044 cmd_mask = nvdimm->cmd_mask;
1045 dimm_name = dev_name(&nvdimm->dev);
1046 } else {
1047 desc = nd_cmd_bus_desc(cmd);
1048 cmd_name = nvdimm_bus_cmd_name(cmd);
1049 cmd_mask = nd_desc->cmd_mask;
1050 dimm_name = "bus";
1051 }
1052
1053 /* Validate command family support against bus declared support */
1054 if (cmd == ND_CMD_CALL) {
1055 unsigned long *mask;
1056
1057 if (copy_from_user(&pkg, p, sizeof(pkg)))
1058 return -EFAULT;
1059
1060 if (nvdimm) {
1061 if (pkg.nd_family > NVDIMM_FAMILY_MAX)
1062 return -EINVAL;
1063 mask = &nd_desc->dimm_family_mask;
1064 } else {
1065 if (pkg.nd_family > NVDIMM_BUS_FAMILY_MAX)
1066 return -EINVAL;
1067 mask = &nd_desc->bus_family_mask;
1068 }
1069
1070 if (!test_bit(pkg.nd_family, mask))
1071 return -EINVAL;
1072 }
1073
1074 if (!desc ||
1075 (desc->out_num + desc->in_num == 0) ||
1076 cmd > ND_CMD_CALL ||
1077 !test_bit(cmd, &cmd_mask))
1078 return -ENOTTY;
1079
1080 /* fail write commands (when read-only) */
1081 if (read_only)
1082 switch (cmd) {
1083 case ND_CMD_VENDOR:
1084 case ND_CMD_SET_CONFIG_DATA:
1085 case ND_CMD_ARS_START:
1086 case ND_CMD_CLEAR_ERROR:
1087 case ND_CMD_CALL:
1088 dev_dbg(dev, "'%s' command while read-only.\n",
1089 nvdimm ? nvdimm_cmd_name(cmd)
1090 : nvdimm_bus_cmd_name(cmd));
1091 return -EPERM;
1092 default:
1093 break;
1094 }
1095
1096 /* process an input envelope */
1097 in_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1098 if (!in_env)
1099 return -ENOMEM;
1100 for (i = 0; i < desc->in_num; i++) {
1101 u32 in_size, copy;
1102
1103 in_size = nd_cmd_in_size(nvdimm, cmd, desc, i, in_env);
1104 if (in_size == UINT_MAX) {
1105 dev_err(dev, "%s:%s unknown input size cmd: %s field: %d\n",
1106 __func__, dimm_name, cmd_name, i);
1107 rc = -ENXIO;
1108 goto out;
1109 }
1110 if (in_len < ND_CMD_MAX_ENVELOPE)
1111 copy = min_t(u32, ND_CMD_MAX_ENVELOPE - in_len, in_size);
1112 else
1113 copy = 0;
1114 if (copy && copy_from_user(&in_env[in_len], p + in_len, copy)) {
1115 rc = -EFAULT;
1116 goto out;
1117 }
1118 in_len += in_size;
1119 }
1120
1121 if (cmd == ND_CMD_CALL) {
1122 func = pkg.nd_command;
1123 dev_dbg(dev, "%s, idx: %llu, in: %u, out: %u, len %llu\n",
1124 dimm_name, pkg.nd_command,
1125 in_len, out_len, buf_len);
1126 }
1127
1128 /* process an output envelope */
1129 out_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1130 if (!out_env) {
1131 rc = -ENOMEM;
1132 goto out;
1133 }
1134
1135 for (i = 0; i < desc->out_num; i++) {
1136 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i,
1137 (u32 *) in_env, (u32 *) out_env, 0);
1138 u32 copy;
1139
1140 if (out_size == UINT_MAX) {
1141 dev_dbg(dev, "%s unknown output size cmd: %s field: %d\n",
1142 dimm_name, cmd_name, i);
1143 rc = -EFAULT;
1144 goto out;
1145 }
1146 if (out_len < ND_CMD_MAX_ENVELOPE)
1147 copy = min_t(u32, ND_CMD_MAX_ENVELOPE - out_len, out_size);
1148 else
1149 copy = 0;
1150 if (copy && copy_from_user(&out_env[out_len],
1151 p + in_len + out_len, copy)) {
1152 rc = -EFAULT;
1153 goto out;
1154 }
1155 out_len += out_size;
1156 }
1157
1158 buf_len = (u64) out_len + (u64) in_len;
1159 if (buf_len > ND_IOCTL_MAX_BUFLEN) {
1160 dev_dbg(dev, "%s cmd: %s buf_len: %llu > %d\n", dimm_name,
1161 cmd_name, buf_len, ND_IOCTL_MAX_BUFLEN);
1162 rc = -EINVAL;
1163 goto out;
1164 }
1165
1166 buf = vmalloc(buf_len);
1167 if (!buf) {
1168 rc = -ENOMEM;
1169 goto out;
1170 }
1171
1172 if (copy_from_user(buf, p, buf_len)) {
1173 rc = -EFAULT;
1174 goto out;
1175 }
1176
1177 device_lock(dev);
1178 nvdimm_bus_lock(dev);
1179 rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, func, buf);
1180 if (rc)
1181 goto out_unlock;
1182
1183 rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, &cmd_rc);
1184 if (rc < 0)
1185 goto out_unlock;
1186
1187 if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR && cmd_rc >= 0) {
1188 struct nd_cmd_clear_error *clear_err = buf;
1189
1190 nvdimm_account_cleared_poison(nvdimm_bus, clear_err->address,
1191 clear_err->cleared);
1192 }
1193
1194 if (copy_to_user(p, buf, buf_len))
1195 rc = -EFAULT;
1196
1197out_unlock:
1198 nvdimm_bus_unlock(dev);
1199 device_unlock(dev);
1200out:
1201 kfree(in_env);
1202 kfree(out_env);
1203 vfree(buf);
1204 return rc;
1205}
1206
1207enum nd_ioctl_mode {
1208 BUS_IOCTL,
1209 DIMM_IOCTL,
1210};
1211
1212static int match_dimm(struct device *dev, void *data)
1213{
1214 long id = (long) data;
1215
1216 if (is_nvdimm(dev)) {
1217 struct nvdimm *nvdimm = to_nvdimm(dev);
1218
1219 return nvdimm->id == id;
1220 }
1221
1222 return 0;
1223}
1224
1225static long nd_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
1226 enum nd_ioctl_mode mode)
1227
1228{
1229 struct nvdimm_bus *nvdimm_bus, *found = NULL;
1230 long id = (long) file->private_data;
1231 struct nvdimm *nvdimm = NULL;
1232 int rc, ro;
1233
1234 ro = ((file->f_flags & O_ACCMODE) == O_RDONLY);
1235 mutex_lock(&nvdimm_bus_list_mutex);
1236 list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) {
1237 if (mode == DIMM_IOCTL) {
1238 struct device *dev;
1239
1240 dev = device_find_child(&nvdimm_bus->dev,
1241 file->private_data, match_dimm);
1242 if (!dev)
1243 continue;
1244 nvdimm = to_nvdimm(dev);
1245 found = nvdimm_bus;
1246 } else if (nvdimm_bus->id == id) {
1247 found = nvdimm_bus;
1248 }
1249
1250 if (found) {
1251 atomic_inc(&nvdimm_bus->ioctl_active);
1252 break;
1253 }
1254 }
1255 mutex_unlock(&nvdimm_bus_list_mutex);
1256
1257 if (!found)
1258 return -ENXIO;
1259
1260 nvdimm_bus = found;
1261 rc = __nd_ioctl(nvdimm_bus, nvdimm, ro, cmd, arg);
1262
1263 if (nvdimm)
1264 put_device(&nvdimm->dev);
1265 if (atomic_dec_and_test(&nvdimm_bus->ioctl_active))
1266 wake_up(&nvdimm_bus->wait);
1267
1268 return rc;
1269}
1270
1271static long bus_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1272{
1273 return nd_ioctl(file, cmd, arg, BUS_IOCTL);
1274}
1275
1276static long dimm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1277{
1278 return nd_ioctl(file, cmd, arg, DIMM_IOCTL);
1279}
1280
1281static int nd_open(struct inode *inode, struct file *file)
1282{
1283 long minor = iminor(inode);
1284
1285 file->private_data = (void *) minor;
1286 return 0;
1287}
1288
1289static const struct file_operations nvdimm_bus_fops = {
1290 .owner = THIS_MODULE,
1291 .open = nd_open,
1292 .unlocked_ioctl = bus_ioctl,
1293 .compat_ioctl = compat_ptr_ioctl,
1294 .llseek = noop_llseek,
1295};
1296
1297static const struct file_operations nvdimm_fops = {
1298 .owner = THIS_MODULE,
1299 .open = nd_open,
1300 .unlocked_ioctl = dimm_ioctl,
1301 .compat_ioctl = compat_ptr_ioctl,
1302 .llseek = noop_llseek,
1303};
1304
1305int __init nvdimm_bus_init(void)
1306{
1307 int rc;
1308
1309 rc = bus_register(&nvdimm_bus_type);
1310 if (rc)
1311 return rc;
1312
1313 rc = register_chrdev(0, "ndctl", &nvdimm_bus_fops);
1314 if (rc < 0)
1315 goto err_bus_chrdev;
1316 nvdimm_bus_major = rc;
1317
1318 rc = register_chrdev(0, "dimmctl", &nvdimm_fops);
1319 if (rc < 0)
1320 goto err_dimm_chrdev;
1321 nvdimm_major = rc;
1322
1323 nd_class = class_create("nd");
1324 if (IS_ERR(nd_class)) {
1325 rc = PTR_ERR(nd_class);
1326 goto err_class;
1327 }
1328
1329 rc = driver_register(&nd_bus_driver.drv);
1330 if (rc)
1331 goto err_nd_bus;
1332
1333 return 0;
1334
1335 err_nd_bus:
1336 class_destroy(nd_class);
1337 err_class:
1338 unregister_chrdev(nvdimm_major, "dimmctl");
1339 err_dimm_chrdev:
1340 unregister_chrdev(nvdimm_bus_major, "ndctl");
1341 err_bus_chrdev:
1342 bus_unregister(&nvdimm_bus_type);
1343
1344 return rc;
1345}
1346
1347void nvdimm_bus_exit(void)
1348{
1349 driver_unregister(&nd_bus_driver.drv);
1350 class_destroy(nd_class);
1351 unregister_chrdev(nvdimm_bus_major, "ndctl");
1352 unregister_chrdev(nvdimm_major, "dimmctl");
1353 bus_unregister(&nvdimm_bus_type);
1354 ida_destroy(&nd_ida);
1355}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4 */
5#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6#include <linux/libnvdimm.h>
7#include <linux/sched/mm.h>
8#include <linux/vmalloc.h>
9#include <linux/uaccess.h>
10#include <linux/module.h>
11#include <linux/blkdev.h>
12#include <linux/fcntl.h>
13#include <linux/async.h>
14#include <linux/genhd.h>
15#include <linux/ndctl.h>
16#include <linux/sched.h>
17#include <linux/slab.h>
18#include <linux/cpu.h>
19#include <linux/fs.h>
20#include <linux/io.h>
21#include <linux/mm.h>
22#include <linux/nd.h>
23#include "nd-core.h"
24#include "nd.h"
25#include "pfn.h"
26
27int nvdimm_major;
28static int nvdimm_bus_major;
29struct class *nd_class;
30static DEFINE_IDA(nd_ida);
31
32static int to_nd_device_type(struct device *dev)
33{
34 if (is_nvdimm(dev))
35 return ND_DEVICE_DIMM;
36 else if (is_memory(dev))
37 return ND_DEVICE_REGION_PMEM;
38 else if (is_nd_blk(dev))
39 return ND_DEVICE_REGION_BLK;
40 else if (is_nd_dax(dev))
41 return ND_DEVICE_DAX_PMEM;
42 else if (is_nd_region(dev->parent))
43 return nd_region_to_nstype(to_nd_region(dev->parent));
44
45 return 0;
46}
47
48static int nvdimm_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
49{
50 return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT,
51 to_nd_device_type(dev));
52}
53
54static struct module *to_bus_provider(struct device *dev)
55{
56 /* pin bus providers while regions are enabled */
57 if (is_nd_region(dev)) {
58 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
59
60 return nvdimm_bus->nd_desc->module;
61 }
62 return NULL;
63}
64
65static void nvdimm_bus_probe_start(struct nvdimm_bus *nvdimm_bus)
66{
67 nvdimm_bus_lock(&nvdimm_bus->dev);
68 nvdimm_bus->probe_active++;
69 nvdimm_bus_unlock(&nvdimm_bus->dev);
70}
71
72static void nvdimm_bus_probe_end(struct nvdimm_bus *nvdimm_bus)
73{
74 nvdimm_bus_lock(&nvdimm_bus->dev);
75 if (--nvdimm_bus->probe_active == 0)
76 wake_up(&nvdimm_bus->wait);
77 nvdimm_bus_unlock(&nvdimm_bus->dev);
78}
79
80static int nvdimm_bus_probe(struct device *dev)
81{
82 struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
83 struct module *provider = to_bus_provider(dev);
84 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
85 int rc;
86
87 if (!try_module_get(provider))
88 return -ENXIO;
89
90 dev_dbg(&nvdimm_bus->dev, "START: %s.probe(%s)\n",
91 dev->driver->name, dev_name(dev));
92
93 nvdimm_bus_probe_start(nvdimm_bus);
94 debug_nvdimm_lock(dev);
95 rc = nd_drv->probe(dev);
96 debug_nvdimm_unlock(dev);
97
98 if ((rc == 0 || rc == -EOPNOTSUPP) &&
99 dev->parent && is_nd_region(dev->parent))
100 nd_region_advance_seeds(to_nd_region(dev->parent), dev);
101 nvdimm_bus_probe_end(nvdimm_bus);
102
103 dev_dbg(&nvdimm_bus->dev, "END: %s.probe(%s) = %d\n", dev->driver->name,
104 dev_name(dev), rc);
105
106 if (rc != 0)
107 module_put(provider);
108 return rc;
109}
110
111static int nvdimm_bus_remove(struct device *dev)
112{
113 struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
114 struct module *provider = to_bus_provider(dev);
115 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
116 int rc = 0;
117
118 if (nd_drv->remove) {
119 debug_nvdimm_lock(dev);
120 rc = nd_drv->remove(dev);
121 debug_nvdimm_unlock(dev);
122 }
123
124 dev_dbg(&nvdimm_bus->dev, "%s.remove(%s) = %d\n", dev->driver->name,
125 dev_name(dev), rc);
126 module_put(provider);
127 return rc;
128}
129
130static void nvdimm_bus_shutdown(struct device *dev)
131{
132 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
133 struct nd_device_driver *nd_drv = NULL;
134
135 if (dev->driver)
136 nd_drv = to_nd_device_driver(dev->driver);
137
138 if (nd_drv && nd_drv->shutdown) {
139 nd_drv->shutdown(dev);
140 dev_dbg(&nvdimm_bus->dev, "%s.shutdown(%s)\n",
141 dev->driver->name, dev_name(dev));
142 }
143}
144
145void nd_device_notify(struct device *dev, enum nvdimm_event event)
146{
147 nd_device_lock(dev);
148 if (dev->driver) {
149 struct nd_device_driver *nd_drv;
150
151 nd_drv = to_nd_device_driver(dev->driver);
152 if (nd_drv->notify)
153 nd_drv->notify(dev, event);
154 }
155 nd_device_unlock(dev);
156}
157EXPORT_SYMBOL(nd_device_notify);
158
159void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event)
160{
161 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);
162
163 if (!nvdimm_bus)
164 return;
165
166 /* caller is responsible for holding a reference on the device */
167 nd_device_notify(&nd_region->dev, event);
168}
169EXPORT_SYMBOL_GPL(nvdimm_region_notify);
170
171struct clear_badblocks_context {
172 resource_size_t phys, cleared;
173};
174
175static int nvdimm_clear_badblocks_region(struct device *dev, void *data)
176{
177 struct clear_badblocks_context *ctx = data;
178 struct nd_region *nd_region;
179 resource_size_t ndr_end;
180 sector_t sector;
181
182 /* make sure device is a region */
183 if (!is_memory(dev))
184 return 0;
185
186 nd_region = to_nd_region(dev);
187 ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1;
188
189 /* make sure we are in the region */
190 if (ctx->phys < nd_region->ndr_start
191 || (ctx->phys + ctx->cleared) > ndr_end)
192 return 0;
193
194 sector = (ctx->phys - nd_region->ndr_start) / 512;
195 badblocks_clear(&nd_region->bb, sector, ctx->cleared / 512);
196
197 if (nd_region->bb_state)
198 sysfs_notify_dirent(nd_region->bb_state);
199
200 return 0;
201}
202
203static void nvdimm_clear_badblocks_regions(struct nvdimm_bus *nvdimm_bus,
204 phys_addr_t phys, u64 cleared)
205{
206 struct clear_badblocks_context ctx = {
207 .phys = phys,
208 .cleared = cleared,
209 };
210
211 device_for_each_child(&nvdimm_bus->dev, &ctx,
212 nvdimm_clear_badblocks_region);
213}
214
215static void nvdimm_account_cleared_poison(struct nvdimm_bus *nvdimm_bus,
216 phys_addr_t phys, u64 cleared)
217{
218 if (cleared > 0)
219 badrange_forget(&nvdimm_bus->badrange, phys, cleared);
220
221 if (cleared > 0 && cleared / 512)
222 nvdimm_clear_badblocks_regions(nvdimm_bus, phys, cleared);
223}
224
225long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
226 unsigned int len)
227{
228 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
229 struct nvdimm_bus_descriptor *nd_desc;
230 struct nd_cmd_clear_error clear_err;
231 struct nd_cmd_ars_cap ars_cap;
232 u32 clear_err_unit, mask;
233 unsigned int noio_flag;
234 int cmd_rc, rc;
235
236 if (!nvdimm_bus)
237 return -ENXIO;
238
239 nd_desc = nvdimm_bus->nd_desc;
240 /*
241 * if ndctl does not exist, it's PMEM_LEGACY and
242 * we want to just pretend everything is handled.
243 */
244 if (!nd_desc->ndctl)
245 return len;
246
247 memset(&ars_cap, 0, sizeof(ars_cap));
248 ars_cap.address = phys;
249 ars_cap.length = len;
250 noio_flag = memalloc_noio_save();
251 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap,
252 sizeof(ars_cap), &cmd_rc);
253 memalloc_noio_restore(noio_flag);
254 if (rc < 0)
255 return rc;
256 if (cmd_rc < 0)
257 return cmd_rc;
258 clear_err_unit = ars_cap.clear_err_unit;
259 if (!clear_err_unit || !is_power_of_2(clear_err_unit))
260 return -ENXIO;
261
262 mask = clear_err_unit - 1;
263 if ((phys | len) & mask)
264 return -ENXIO;
265 memset(&clear_err, 0, sizeof(clear_err));
266 clear_err.address = phys;
267 clear_err.length = len;
268 noio_flag = memalloc_noio_save();
269 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err,
270 sizeof(clear_err), &cmd_rc);
271 memalloc_noio_restore(noio_flag);
272 if (rc < 0)
273 return rc;
274 if (cmd_rc < 0)
275 return cmd_rc;
276
277 nvdimm_account_cleared_poison(nvdimm_bus, phys, clear_err.cleared);
278
279 return clear_err.cleared;
280}
281EXPORT_SYMBOL_GPL(nvdimm_clear_poison);
282
283static int nvdimm_bus_match(struct device *dev, struct device_driver *drv);
284
285static struct bus_type nvdimm_bus_type = {
286 .name = "nd",
287 .uevent = nvdimm_bus_uevent,
288 .match = nvdimm_bus_match,
289 .probe = nvdimm_bus_probe,
290 .remove = nvdimm_bus_remove,
291 .shutdown = nvdimm_bus_shutdown,
292};
293
294static void nvdimm_bus_release(struct device *dev)
295{
296 struct nvdimm_bus *nvdimm_bus;
297
298 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
299 ida_simple_remove(&nd_ida, nvdimm_bus->id);
300 kfree(nvdimm_bus);
301}
302
303bool is_nvdimm_bus(struct device *dev)
304{
305 return dev->release == nvdimm_bus_release;
306}
307
308struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev)
309{
310 struct device *dev;
311
312 for (dev = nd_dev; dev; dev = dev->parent)
313 if (is_nvdimm_bus(dev))
314 break;
315 dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n");
316 if (dev)
317 return to_nvdimm_bus(dev);
318 return NULL;
319}
320
321struct nvdimm_bus *to_nvdimm_bus(struct device *dev)
322{
323 struct nvdimm_bus *nvdimm_bus;
324
325 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
326 WARN_ON(!is_nvdimm_bus(dev));
327 return nvdimm_bus;
328}
329EXPORT_SYMBOL_GPL(to_nvdimm_bus);
330
331struct nvdimm_bus *nvdimm_to_bus(struct nvdimm *nvdimm)
332{
333 return to_nvdimm_bus(nvdimm->dev.parent);
334}
335EXPORT_SYMBOL_GPL(nvdimm_to_bus);
336
337struct nvdimm_bus *nvdimm_bus_register(struct device *parent,
338 struct nvdimm_bus_descriptor *nd_desc)
339{
340 struct nvdimm_bus *nvdimm_bus;
341 int rc;
342
343 nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL);
344 if (!nvdimm_bus)
345 return NULL;
346 INIT_LIST_HEAD(&nvdimm_bus->list);
347 INIT_LIST_HEAD(&nvdimm_bus->mapping_list);
348 init_waitqueue_head(&nvdimm_bus->wait);
349 nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL);
350 if (nvdimm_bus->id < 0) {
351 kfree(nvdimm_bus);
352 return NULL;
353 }
354 mutex_init(&nvdimm_bus->reconfig_mutex);
355 badrange_init(&nvdimm_bus->badrange);
356 nvdimm_bus->nd_desc = nd_desc;
357 nvdimm_bus->dev.parent = parent;
358 nvdimm_bus->dev.release = nvdimm_bus_release;
359 nvdimm_bus->dev.groups = nd_desc->attr_groups;
360 nvdimm_bus->dev.bus = &nvdimm_bus_type;
361 nvdimm_bus->dev.of_node = nd_desc->of_node;
362 dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id);
363 rc = device_register(&nvdimm_bus->dev);
364 if (rc) {
365 dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc);
366 goto err;
367 }
368
369 return nvdimm_bus;
370 err:
371 put_device(&nvdimm_bus->dev);
372 return NULL;
373}
374EXPORT_SYMBOL_GPL(nvdimm_bus_register);
375
376void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus)
377{
378 if (!nvdimm_bus)
379 return;
380 device_unregister(&nvdimm_bus->dev);
381}
382EXPORT_SYMBOL_GPL(nvdimm_bus_unregister);
383
384static int child_unregister(struct device *dev, void *data)
385{
386 /*
387 * the singular ndctl class device per bus needs to be
388 * "device_destroy"ed, so skip it here
389 *
390 * i.e. remove classless children
391 */
392 if (dev->class)
393 return 0;
394
395 if (is_nvdimm(dev)) {
396 struct nvdimm *nvdimm = to_nvdimm(dev);
397 bool dev_put = false;
398
399 /* We are shutting down. Make state frozen artificially. */
400 nvdimm_bus_lock(dev);
401 set_bit(NVDIMM_SECURITY_FROZEN, &nvdimm->sec.flags);
402 if (test_and_clear_bit(NDD_WORK_PENDING, &nvdimm->flags))
403 dev_put = true;
404 nvdimm_bus_unlock(dev);
405 cancel_delayed_work_sync(&nvdimm->dwork);
406 if (dev_put)
407 put_device(dev);
408 }
409 nd_device_unregister(dev, ND_SYNC);
410
411 return 0;
412}
413
414static void free_badrange_list(struct list_head *badrange_list)
415{
416 struct badrange_entry *bre, *next;
417
418 list_for_each_entry_safe(bre, next, badrange_list, list) {
419 list_del(&bre->list);
420 kfree(bre);
421 }
422 list_del_init(badrange_list);
423}
424
425static int nd_bus_remove(struct device *dev)
426{
427 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
428
429 mutex_lock(&nvdimm_bus_list_mutex);
430 list_del_init(&nvdimm_bus->list);
431 mutex_unlock(&nvdimm_bus_list_mutex);
432
433 wait_event(nvdimm_bus->wait,
434 atomic_read(&nvdimm_bus->ioctl_active) == 0);
435
436 nd_synchronize();
437 device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister);
438
439 spin_lock(&nvdimm_bus->badrange.lock);
440 free_badrange_list(&nvdimm_bus->badrange.list);
441 spin_unlock(&nvdimm_bus->badrange.lock);
442
443 nvdimm_bus_destroy_ndctl(nvdimm_bus);
444
445 return 0;
446}
447
448static int nd_bus_probe(struct device *dev)
449{
450 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
451 int rc;
452
453 rc = nvdimm_bus_create_ndctl(nvdimm_bus);
454 if (rc)
455 return rc;
456
457 mutex_lock(&nvdimm_bus_list_mutex);
458 list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list);
459 mutex_unlock(&nvdimm_bus_list_mutex);
460
461 /* enable bus provider attributes to look up their local context */
462 dev_set_drvdata(dev, nvdimm_bus->nd_desc);
463
464 return 0;
465}
466
467static struct nd_device_driver nd_bus_driver = {
468 .probe = nd_bus_probe,
469 .remove = nd_bus_remove,
470 .drv = {
471 .name = "nd_bus",
472 .suppress_bind_attrs = true,
473 .bus = &nvdimm_bus_type,
474 .owner = THIS_MODULE,
475 .mod_name = KBUILD_MODNAME,
476 },
477};
478
479static int nvdimm_bus_match(struct device *dev, struct device_driver *drv)
480{
481 struct nd_device_driver *nd_drv = to_nd_device_driver(drv);
482
483 if (is_nvdimm_bus(dev) && nd_drv == &nd_bus_driver)
484 return true;
485
486 return !!test_bit(to_nd_device_type(dev), &nd_drv->type);
487}
488
489static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain);
490
491void nd_synchronize(void)
492{
493 async_synchronize_full_domain(&nd_async_domain);
494}
495EXPORT_SYMBOL_GPL(nd_synchronize);
496
497static void nd_async_device_register(void *d, async_cookie_t cookie)
498{
499 struct device *dev = d;
500
501 if (device_add(dev) != 0) {
502 dev_err(dev, "%s: failed\n", __func__);
503 put_device(dev);
504 }
505 put_device(dev);
506 if (dev->parent)
507 put_device(dev->parent);
508}
509
510static void nd_async_device_unregister(void *d, async_cookie_t cookie)
511{
512 struct device *dev = d;
513
514 /* flush bus operations before delete */
515 nvdimm_bus_lock(dev);
516 nvdimm_bus_unlock(dev);
517
518 device_unregister(dev);
519 put_device(dev);
520}
521
522void __nd_device_register(struct device *dev)
523{
524 if (!dev)
525 return;
526
527 /*
528 * Ensure that region devices always have their NUMA node set as
529 * early as possible. This way we are able to make certain that
530 * any memory associated with the creation and the creation
531 * itself of the region is associated with the correct node.
532 */
533 if (is_nd_region(dev))
534 set_dev_node(dev, to_nd_region(dev)->numa_node);
535
536 dev->bus = &nvdimm_bus_type;
537 if (dev->parent) {
538 get_device(dev->parent);
539 if (dev_to_node(dev) == NUMA_NO_NODE)
540 set_dev_node(dev, dev_to_node(dev->parent));
541 }
542 get_device(dev);
543
544 async_schedule_dev_domain(nd_async_device_register, dev,
545 &nd_async_domain);
546}
547
548void nd_device_register(struct device *dev)
549{
550 device_initialize(dev);
551 __nd_device_register(dev);
552}
553EXPORT_SYMBOL(nd_device_register);
554
555void nd_device_unregister(struct device *dev, enum nd_async_mode mode)
556{
557 bool killed;
558
559 switch (mode) {
560 case ND_ASYNC:
561 /*
562 * In the async case this is being triggered with the
563 * device lock held and the unregistration work needs to
564 * be moved out of line iff this is thread has won the
565 * race to schedule the deletion.
566 */
567 if (!kill_device(dev))
568 return;
569
570 get_device(dev);
571 async_schedule_domain(nd_async_device_unregister, dev,
572 &nd_async_domain);
573 break;
574 case ND_SYNC:
575 /*
576 * In the sync case the device is being unregistered due
577 * to a state change of the parent. Claim the kill state
578 * to synchronize against other unregistration requests,
579 * or otherwise let the async path handle it if the
580 * unregistration was already queued.
581 */
582 nd_device_lock(dev);
583 killed = kill_device(dev);
584 nd_device_unlock(dev);
585
586 if (!killed)
587 return;
588
589 nd_synchronize();
590 device_unregister(dev);
591 break;
592 }
593}
594EXPORT_SYMBOL(nd_device_unregister);
595
596/**
597 * __nd_driver_register() - register a region or a namespace driver
598 * @nd_drv: driver to register
599 * @owner: automatically set by nd_driver_register() macro
600 * @mod_name: automatically set by nd_driver_register() macro
601 */
602int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner,
603 const char *mod_name)
604{
605 struct device_driver *drv = &nd_drv->drv;
606
607 if (!nd_drv->type) {
608 pr_debug("driver type bitmask not set (%ps)\n",
609 __builtin_return_address(0));
610 return -EINVAL;
611 }
612
613 if (!nd_drv->probe) {
614 pr_debug("%s ->probe() must be specified\n", mod_name);
615 return -EINVAL;
616 }
617
618 drv->bus = &nvdimm_bus_type;
619 drv->owner = owner;
620 drv->mod_name = mod_name;
621
622 return driver_register(drv);
623}
624EXPORT_SYMBOL(__nd_driver_register);
625
626int nvdimm_revalidate_disk(struct gendisk *disk)
627{
628 struct device *dev = disk_to_dev(disk)->parent;
629 struct nd_region *nd_region = to_nd_region(dev->parent);
630 int disk_ro = get_disk_ro(disk);
631
632 /*
633 * Upgrade to read-only if the region is read-only preserve as
634 * read-only if the disk is already read-only.
635 */
636 if (disk_ro || nd_region->ro == disk_ro)
637 return 0;
638
639 dev_info(dev, "%s read-only, marking %s read-only\n",
640 dev_name(&nd_region->dev), disk->disk_name);
641 set_disk_ro(disk, 1);
642
643 return 0;
644
645}
646EXPORT_SYMBOL(nvdimm_revalidate_disk);
647
648static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
649 char *buf)
650{
651 return sprintf(buf, ND_DEVICE_MODALIAS_FMT "\n",
652 to_nd_device_type(dev));
653}
654static DEVICE_ATTR_RO(modalias);
655
656static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
657 char *buf)
658{
659 return sprintf(buf, "%s\n", dev->type->name);
660}
661static DEVICE_ATTR_RO(devtype);
662
663static struct attribute *nd_device_attributes[] = {
664 &dev_attr_modalias.attr,
665 &dev_attr_devtype.attr,
666 NULL,
667};
668
669/*
670 * nd_device_attribute_group - generic attributes for all devices on an nd bus
671 */
672struct attribute_group nd_device_attribute_group = {
673 .attrs = nd_device_attributes,
674};
675EXPORT_SYMBOL_GPL(nd_device_attribute_group);
676
677static ssize_t numa_node_show(struct device *dev,
678 struct device_attribute *attr, char *buf)
679{
680 return sprintf(buf, "%d\n", dev_to_node(dev));
681}
682static DEVICE_ATTR_RO(numa_node);
683
684static struct attribute *nd_numa_attributes[] = {
685 &dev_attr_numa_node.attr,
686 NULL,
687};
688
689static umode_t nd_numa_attr_visible(struct kobject *kobj, struct attribute *a,
690 int n)
691{
692 if (!IS_ENABLED(CONFIG_NUMA))
693 return 0;
694
695 return a->mode;
696}
697
698/*
699 * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus
700 */
701struct attribute_group nd_numa_attribute_group = {
702 .attrs = nd_numa_attributes,
703 .is_visible = nd_numa_attr_visible,
704};
705EXPORT_SYMBOL_GPL(nd_numa_attribute_group);
706
707int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus)
708{
709 dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id);
710 struct device *dev;
711
712 dev = device_create(nd_class, &nvdimm_bus->dev, devt, nvdimm_bus,
713 "ndctl%d", nvdimm_bus->id);
714
715 if (IS_ERR(dev))
716 dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %ld\n",
717 nvdimm_bus->id, PTR_ERR(dev));
718 return PTR_ERR_OR_ZERO(dev);
719}
720
721void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus)
722{
723 device_destroy(nd_class, MKDEV(nvdimm_bus_major, nvdimm_bus->id));
724}
725
726static const struct nd_cmd_desc __nd_cmd_dimm_descs[] = {
727 [ND_CMD_IMPLEMENTED] = { },
728 [ND_CMD_SMART] = {
729 .out_num = 2,
730 .out_sizes = { 4, 128, },
731 },
732 [ND_CMD_SMART_THRESHOLD] = {
733 .out_num = 2,
734 .out_sizes = { 4, 8, },
735 },
736 [ND_CMD_DIMM_FLAGS] = {
737 .out_num = 2,
738 .out_sizes = { 4, 4 },
739 },
740 [ND_CMD_GET_CONFIG_SIZE] = {
741 .out_num = 3,
742 .out_sizes = { 4, 4, 4, },
743 },
744 [ND_CMD_GET_CONFIG_DATA] = {
745 .in_num = 2,
746 .in_sizes = { 4, 4, },
747 .out_num = 2,
748 .out_sizes = { 4, UINT_MAX, },
749 },
750 [ND_CMD_SET_CONFIG_DATA] = {
751 .in_num = 3,
752 .in_sizes = { 4, 4, UINT_MAX, },
753 .out_num = 1,
754 .out_sizes = { 4, },
755 },
756 [ND_CMD_VENDOR] = {
757 .in_num = 3,
758 .in_sizes = { 4, 4, UINT_MAX, },
759 .out_num = 3,
760 .out_sizes = { 4, 4, UINT_MAX, },
761 },
762 [ND_CMD_CALL] = {
763 .in_num = 2,
764 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
765 .out_num = 1,
766 .out_sizes = { UINT_MAX, },
767 },
768};
769
770const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd)
771{
772 if (cmd < ARRAY_SIZE(__nd_cmd_dimm_descs))
773 return &__nd_cmd_dimm_descs[cmd];
774 return NULL;
775}
776EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc);
777
778static const struct nd_cmd_desc __nd_cmd_bus_descs[] = {
779 [ND_CMD_IMPLEMENTED] = { },
780 [ND_CMD_ARS_CAP] = {
781 .in_num = 2,
782 .in_sizes = { 8, 8, },
783 .out_num = 4,
784 .out_sizes = { 4, 4, 4, 4, },
785 },
786 [ND_CMD_ARS_START] = {
787 .in_num = 5,
788 .in_sizes = { 8, 8, 2, 1, 5, },
789 .out_num = 2,
790 .out_sizes = { 4, 4, },
791 },
792 [ND_CMD_ARS_STATUS] = {
793 .out_num = 3,
794 .out_sizes = { 4, 4, UINT_MAX, },
795 },
796 [ND_CMD_CLEAR_ERROR] = {
797 .in_num = 2,
798 .in_sizes = { 8, 8, },
799 .out_num = 3,
800 .out_sizes = { 4, 4, 8, },
801 },
802 [ND_CMD_CALL] = {
803 .in_num = 2,
804 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
805 .out_num = 1,
806 .out_sizes = { UINT_MAX, },
807 },
808};
809
810const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd)
811{
812 if (cmd < ARRAY_SIZE(__nd_cmd_bus_descs))
813 return &__nd_cmd_bus_descs[cmd];
814 return NULL;
815}
816EXPORT_SYMBOL_GPL(nd_cmd_bus_desc);
817
818u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd,
819 const struct nd_cmd_desc *desc, int idx, void *buf)
820{
821 if (idx >= desc->in_num)
822 return UINT_MAX;
823
824 if (desc->in_sizes[idx] < UINT_MAX)
825 return desc->in_sizes[idx];
826
827 if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA && idx == 2) {
828 struct nd_cmd_set_config_hdr *hdr = buf;
829
830 return hdr->in_length;
831 } else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) {
832 struct nd_cmd_vendor_hdr *hdr = buf;
833
834 return hdr->in_length;
835 } else if (cmd == ND_CMD_CALL) {
836 struct nd_cmd_pkg *pkg = buf;
837
838 return pkg->nd_size_in;
839 }
840
841 return UINT_MAX;
842}
843EXPORT_SYMBOL_GPL(nd_cmd_in_size);
844
845u32 nd_cmd_out_size(struct nvdimm *nvdimm, int cmd,
846 const struct nd_cmd_desc *desc, int idx, const u32 *in_field,
847 const u32 *out_field, unsigned long remainder)
848{
849 if (idx >= desc->out_num)
850 return UINT_MAX;
851
852 if (desc->out_sizes[idx] < UINT_MAX)
853 return desc->out_sizes[idx];
854
855 if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && idx == 1)
856 return in_field[1];
857 else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2)
858 return out_field[1];
859 else if (!nvdimm && cmd == ND_CMD_ARS_STATUS && idx == 2) {
860 /*
861 * Per table 9-276 ARS Data in ACPI 6.1, out_field[1] is
862 * "Size of Output Buffer in bytes, including this
863 * field."
864 */
865 if (out_field[1] < 4)
866 return 0;
867 /*
868 * ACPI 6.1 is ambiguous if 'status' is included in the
869 * output size. If we encounter an output size that
870 * overshoots the remainder by 4 bytes, assume it was
871 * including 'status'.
872 */
873 if (out_field[1] - 4 == remainder)
874 return remainder;
875 return out_field[1] - 8;
876 } else if (cmd == ND_CMD_CALL) {
877 struct nd_cmd_pkg *pkg = (struct nd_cmd_pkg *) in_field;
878
879 return pkg->nd_size_out;
880 }
881
882
883 return UINT_MAX;
884}
885EXPORT_SYMBOL_GPL(nd_cmd_out_size);
886
887void wait_nvdimm_bus_probe_idle(struct device *dev)
888{
889 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
890
891 do {
892 if (nvdimm_bus->probe_active == 0)
893 break;
894 nvdimm_bus_unlock(dev);
895 nd_device_unlock(dev);
896 wait_event(nvdimm_bus->wait,
897 nvdimm_bus->probe_active == 0);
898 nd_device_lock(dev);
899 nvdimm_bus_lock(dev);
900 } while (true);
901}
902
903static int nd_pmem_forget_poison_check(struct device *dev, void *data)
904{
905 struct nd_cmd_clear_error *clear_err =
906 (struct nd_cmd_clear_error *)data;
907 struct nd_btt *nd_btt = is_nd_btt(dev) ? to_nd_btt(dev) : NULL;
908 struct nd_pfn *nd_pfn = is_nd_pfn(dev) ? to_nd_pfn(dev) : NULL;
909 struct nd_dax *nd_dax = is_nd_dax(dev) ? to_nd_dax(dev) : NULL;
910 struct nd_namespace_common *ndns = NULL;
911 struct nd_namespace_io *nsio;
912 resource_size_t offset = 0, end_trunc = 0, start, end, pstart, pend;
913
914 if (nd_dax || !dev->driver)
915 return 0;
916
917 start = clear_err->address;
918 end = clear_err->address + clear_err->cleared - 1;
919
920 if (nd_btt || nd_pfn || nd_dax) {
921 if (nd_btt)
922 ndns = nd_btt->ndns;
923 else if (nd_pfn)
924 ndns = nd_pfn->ndns;
925 else if (nd_dax)
926 ndns = nd_dax->nd_pfn.ndns;
927
928 if (!ndns)
929 return 0;
930 } else
931 ndns = to_ndns(dev);
932
933 nsio = to_nd_namespace_io(&ndns->dev);
934 pstart = nsio->res.start + offset;
935 pend = nsio->res.end - end_trunc;
936
937 if ((pstart >= start) && (pend <= end))
938 return -EBUSY;
939
940 return 0;
941
942}
943
944static int nd_ns_forget_poison_check(struct device *dev, void *data)
945{
946 return device_for_each_child(dev, data, nd_pmem_forget_poison_check);
947}
948
949/* set_config requires an idle interleave set */
950static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus,
951 struct nvdimm *nvdimm, unsigned int cmd, void *data)
952{
953 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
954
955 /* ask the bus provider if it would like to block this request */
956 if (nd_desc->clear_to_send) {
957 int rc = nd_desc->clear_to_send(nd_desc, nvdimm, cmd, data);
958
959 if (rc)
960 return rc;
961 }
962
963 /* require clear error to go through the pmem driver */
964 if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR)
965 return device_for_each_child(&nvdimm_bus->dev, data,
966 nd_ns_forget_poison_check);
967
968 if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA)
969 return 0;
970
971 /* prevent label manipulation while the kernel owns label updates */
972 wait_nvdimm_bus_probe_idle(&nvdimm_bus->dev);
973 if (atomic_read(&nvdimm->busy))
974 return -EBUSY;
975 return 0;
976}
977
978static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
979 int read_only, unsigned int ioctl_cmd, unsigned long arg)
980{
981 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
982 const struct nd_cmd_desc *desc = NULL;
983 unsigned int cmd = _IOC_NR(ioctl_cmd);
984 struct device *dev = &nvdimm_bus->dev;
985 void __user *p = (void __user *) arg;
986 char *out_env = NULL, *in_env = NULL;
987 const char *cmd_name, *dimm_name;
988 u32 in_len = 0, out_len = 0;
989 unsigned int func = cmd;
990 unsigned long cmd_mask;
991 struct nd_cmd_pkg pkg;
992 int rc, i, cmd_rc;
993 void *buf = NULL;
994 u64 buf_len = 0;
995
996 if (nvdimm) {
997 desc = nd_cmd_dimm_desc(cmd);
998 cmd_name = nvdimm_cmd_name(cmd);
999 cmd_mask = nvdimm->cmd_mask;
1000 dimm_name = dev_name(&nvdimm->dev);
1001 } else {
1002 desc = nd_cmd_bus_desc(cmd);
1003 cmd_name = nvdimm_bus_cmd_name(cmd);
1004 cmd_mask = nd_desc->cmd_mask;
1005 dimm_name = "bus";
1006 }
1007
1008 if (cmd == ND_CMD_CALL) {
1009 if (copy_from_user(&pkg, p, sizeof(pkg)))
1010 return -EFAULT;
1011 }
1012
1013 if (!desc || (desc->out_num + desc->in_num == 0) ||
1014 !test_bit(cmd, &cmd_mask))
1015 return -ENOTTY;
1016
1017 /* fail write commands (when read-only) */
1018 if (read_only)
1019 switch (cmd) {
1020 case ND_CMD_VENDOR:
1021 case ND_CMD_SET_CONFIG_DATA:
1022 case ND_CMD_ARS_START:
1023 case ND_CMD_CLEAR_ERROR:
1024 case ND_CMD_CALL:
1025 dev_dbg(dev, "'%s' command while read-only.\n",
1026 nvdimm ? nvdimm_cmd_name(cmd)
1027 : nvdimm_bus_cmd_name(cmd));
1028 return -EPERM;
1029 default:
1030 break;
1031 }
1032
1033 /* process an input envelope */
1034 in_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1035 if (!in_env)
1036 return -ENOMEM;
1037 for (i = 0; i < desc->in_num; i++) {
1038 u32 in_size, copy;
1039
1040 in_size = nd_cmd_in_size(nvdimm, cmd, desc, i, in_env);
1041 if (in_size == UINT_MAX) {
1042 dev_err(dev, "%s:%s unknown input size cmd: %s field: %d\n",
1043 __func__, dimm_name, cmd_name, i);
1044 rc = -ENXIO;
1045 goto out;
1046 }
1047 if (in_len < ND_CMD_MAX_ENVELOPE)
1048 copy = min_t(u32, ND_CMD_MAX_ENVELOPE - in_len, in_size);
1049 else
1050 copy = 0;
1051 if (copy && copy_from_user(&in_env[in_len], p + in_len, copy)) {
1052 rc = -EFAULT;
1053 goto out;
1054 }
1055 in_len += in_size;
1056 }
1057
1058 if (cmd == ND_CMD_CALL) {
1059 func = pkg.nd_command;
1060 dev_dbg(dev, "%s, idx: %llu, in: %u, out: %u, len %llu\n",
1061 dimm_name, pkg.nd_command,
1062 in_len, out_len, buf_len);
1063 }
1064
1065 /* process an output envelope */
1066 out_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1067 if (!out_env) {
1068 rc = -ENOMEM;
1069 goto out;
1070 }
1071
1072 for (i = 0; i < desc->out_num; i++) {
1073 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i,
1074 (u32 *) in_env, (u32 *) out_env, 0);
1075 u32 copy;
1076
1077 if (out_size == UINT_MAX) {
1078 dev_dbg(dev, "%s unknown output size cmd: %s field: %d\n",
1079 dimm_name, cmd_name, i);
1080 rc = -EFAULT;
1081 goto out;
1082 }
1083 if (out_len < ND_CMD_MAX_ENVELOPE)
1084 copy = min_t(u32, ND_CMD_MAX_ENVELOPE - out_len, out_size);
1085 else
1086 copy = 0;
1087 if (copy && copy_from_user(&out_env[out_len],
1088 p + in_len + out_len, copy)) {
1089 rc = -EFAULT;
1090 goto out;
1091 }
1092 out_len += out_size;
1093 }
1094
1095 buf_len = (u64) out_len + (u64) in_len;
1096 if (buf_len > ND_IOCTL_MAX_BUFLEN) {
1097 dev_dbg(dev, "%s cmd: %s buf_len: %llu > %d\n", dimm_name,
1098 cmd_name, buf_len, ND_IOCTL_MAX_BUFLEN);
1099 rc = -EINVAL;
1100 goto out;
1101 }
1102
1103 buf = vmalloc(buf_len);
1104 if (!buf) {
1105 rc = -ENOMEM;
1106 goto out;
1107 }
1108
1109 if (copy_from_user(buf, p, buf_len)) {
1110 rc = -EFAULT;
1111 goto out;
1112 }
1113
1114 nd_device_lock(dev);
1115 nvdimm_bus_lock(dev);
1116 rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, func, buf);
1117 if (rc)
1118 goto out_unlock;
1119
1120 rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, &cmd_rc);
1121 if (rc < 0)
1122 goto out_unlock;
1123
1124 if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR && cmd_rc >= 0) {
1125 struct nd_cmd_clear_error *clear_err = buf;
1126
1127 nvdimm_account_cleared_poison(nvdimm_bus, clear_err->address,
1128 clear_err->cleared);
1129 }
1130
1131 if (copy_to_user(p, buf, buf_len))
1132 rc = -EFAULT;
1133
1134out_unlock:
1135 nvdimm_bus_unlock(dev);
1136 nd_device_unlock(dev);
1137out:
1138 kfree(in_env);
1139 kfree(out_env);
1140 vfree(buf);
1141 return rc;
1142}
1143
1144enum nd_ioctl_mode {
1145 BUS_IOCTL,
1146 DIMM_IOCTL,
1147};
1148
1149static int match_dimm(struct device *dev, void *data)
1150{
1151 long id = (long) data;
1152
1153 if (is_nvdimm(dev)) {
1154 struct nvdimm *nvdimm = to_nvdimm(dev);
1155
1156 return nvdimm->id == id;
1157 }
1158
1159 return 0;
1160}
1161
1162static long nd_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
1163 enum nd_ioctl_mode mode)
1164
1165{
1166 struct nvdimm_bus *nvdimm_bus, *found = NULL;
1167 long id = (long) file->private_data;
1168 struct nvdimm *nvdimm = NULL;
1169 int rc, ro;
1170
1171 ro = ((file->f_flags & O_ACCMODE) == O_RDONLY);
1172 mutex_lock(&nvdimm_bus_list_mutex);
1173 list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) {
1174 if (mode == DIMM_IOCTL) {
1175 struct device *dev;
1176
1177 dev = device_find_child(&nvdimm_bus->dev,
1178 file->private_data, match_dimm);
1179 if (!dev)
1180 continue;
1181 nvdimm = to_nvdimm(dev);
1182 found = nvdimm_bus;
1183 } else if (nvdimm_bus->id == id) {
1184 found = nvdimm_bus;
1185 }
1186
1187 if (found) {
1188 atomic_inc(&nvdimm_bus->ioctl_active);
1189 break;
1190 }
1191 }
1192 mutex_unlock(&nvdimm_bus_list_mutex);
1193
1194 if (!found)
1195 return -ENXIO;
1196
1197 nvdimm_bus = found;
1198 rc = __nd_ioctl(nvdimm_bus, nvdimm, ro, cmd, arg);
1199
1200 if (nvdimm)
1201 put_device(&nvdimm->dev);
1202 if (atomic_dec_and_test(&nvdimm_bus->ioctl_active))
1203 wake_up(&nvdimm_bus->wait);
1204
1205 return rc;
1206}
1207
1208static long bus_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1209{
1210 return nd_ioctl(file, cmd, arg, BUS_IOCTL);
1211}
1212
1213static long dimm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1214{
1215 return nd_ioctl(file, cmd, arg, DIMM_IOCTL);
1216}
1217
1218static int nd_open(struct inode *inode, struct file *file)
1219{
1220 long minor = iminor(inode);
1221
1222 file->private_data = (void *) minor;
1223 return 0;
1224}
1225
1226static const struct file_operations nvdimm_bus_fops = {
1227 .owner = THIS_MODULE,
1228 .open = nd_open,
1229 .unlocked_ioctl = bus_ioctl,
1230 .compat_ioctl = bus_ioctl,
1231 .llseek = noop_llseek,
1232};
1233
1234static const struct file_operations nvdimm_fops = {
1235 .owner = THIS_MODULE,
1236 .open = nd_open,
1237 .unlocked_ioctl = dimm_ioctl,
1238 .compat_ioctl = dimm_ioctl,
1239 .llseek = noop_llseek,
1240};
1241
1242int __init nvdimm_bus_init(void)
1243{
1244 int rc;
1245
1246 rc = bus_register(&nvdimm_bus_type);
1247 if (rc)
1248 return rc;
1249
1250 rc = register_chrdev(0, "ndctl", &nvdimm_bus_fops);
1251 if (rc < 0)
1252 goto err_bus_chrdev;
1253 nvdimm_bus_major = rc;
1254
1255 rc = register_chrdev(0, "dimmctl", &nvdimm_fops);
1256 if (rc < 0)
1257 goto err_dimm_chrdev;
1258 nvdimm_major = rc;
1259
1260 nd_class = class_create(THIS_MODULE, "nd");
1261 if (IS_ERR(nd_class)) {
1262 rc = PTR_ERR(nd_class);
1263 goto err_class;
1264 }
1265
1266 rc = driver_register(&nd_bus_driver.drv);
1267 if (rc)
1268 goto err_nd_bus;
1269
1270 return 0;
1271
1272 err_nd_bus:
1273 class_destroy(nd_class);
1274 err_class:
1275 unregister_chrdev(nvdimm_major, "dimmctl");
1276 err_dimm_chrdev:
1277 unregister_chrdev(nvdimm_bus_major, "ndctl");
1278 err_bus_chrdev:
1279 bus_unregister(&nvdimm_bus_type);
1280
1281 return rc;
1282}
1283
1284void nvdimm_bus_exit(void)
1285{
1286 driver_unregister(&nd_bus_driver.drv);
1287 class_destroy(nd_class);
1288 unregister_chrdev(nvdimm_bus_major, "ndctl");
1289 unregister_chrdev(nvdimm_major, "dimmctl");
1290 bus_unregister(&nvdimm_bus_type);
1291 ida_destroy(&nd_ida);
1292}